mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-08 10:10:18 -10:00
Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Nakul Bharti
|
746e92e675 | ||
|
Nakul Bharti
|
3f88aad2cc | ||
|
Nakul Bharti
|
0eb52d1a07 | ||
|
Nakul Bharti
|
7c0cb98e9b | ||
|
Nakul Bharti
|
f6f8786531 | ||
|
Nakul Bharti
|
c041c5b1e4 | ||
|
Nakul Bharti
|
e647e92e67 | ||
|
Nakul Bharti
|
68049c5985 | ||
|
Nakul Bharti
|
6cdfcffc6d | ||
|
Nakul Bharti
|
151115fad2 | ||
|
Nakul Bharti
|
6b79ff1a68 | ||
|
Nakul Bharti
|
bdcbf5d1ae | ||
|
Nakul Bharti
|
190fb46436 | ||
|
Nakul Bharti
|
97f09fa509 | ||
|
Nakul Bharti
|
f51ae596be | ||
|
Nakul Bharti
|
91456915e9 | ||
|
Nakul Bharti
|
9666293e9f | ||
|
Nakul Bharti
|
a5555a72c2 | ||
|
Nakul Bharti
|
dfe70666a3 |
No files matched your search
@@ -17,4 +17,4 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: crate-ci/typos@v1.51.1
|
||||
- uses: crate-ci/typos@v1.49.0
|
||||
@@ -1,7 +1,7 @@
|
||||
|
||||
|
||||
<div align="center">
|
||||
|
||||
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README.md">`English`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_CN.md">`中文`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_KR.md">`Korean`</a> •
|
||||
@@ -9,9 +9,8 @@
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_FA.md">`فارسی`</a>
|
||||
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a>
|
||||
|
||||
</div>
|
||||
|
||||
<p align="center">
|
||||
@@ -111,7 +110,7 @@ Browse the full Nuclei [**`documentation here`**](https://docs.projectdiscovery.
|
||||
|
||||
### Installation
|
||||
|
||||
`nuclei` requires **go >= 1.27.3** to install successfully. Run the following command to get the repo:
|
||||
`nuclei` requires **go >= 1.24.2** to install successfully. Run the following command to get the repo:
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -177,7 +176,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
@@ -250,11 +249,9 @@ CONFIGURATIONS:
|
||||
-lna, -restrict-local-network-access blocks connections to the local / private network
|
||||
-i, -interface string network interface to use for network scan
|
||||
-at, -attack-type string type of payload combinations to perform (batteringram,pitchfork,clusterbomb)
|
||||
-pro, -payload-random-order iterate payload combinations in a random order instead of sequentially
|
||||
-sip, -source-ip string source ip address to use for network scan
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
@@ -323,10 +320,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-stp, -strict-probe skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
|
||||
-tf, -tech-filter opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
@@ -361,11 +355,10 @@ DEBUG:
|
||||
-hc, -health-check run diagnostic check up
|
||||
|
||||
UPDATE:
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-utc, -update-templates-channel string nuclei-templates update channel (stable, latest) (default "stable")
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
@@ -687,7 +680,7 @@ Thanks to all the amazing [**`community contributors for sending PRs`**](https:/
|
||||
<br>
|
||||
|
||||
<div align="center">
|
||||
|
||||
|
||||
<sub>**`nuclei`** is distributed under [**MIT License**](https://github.com/projectdiscovery/nuclei/blob/main/LICENSE.md)</sub>
|
||||
|
||||
</div>
|
||||
+7
-13
@@ -9,8 +9,7 @@
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_FA.md">`فارسی`</a>
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -111,7 +110,7 @@ _针对安全团队和企业，我们提供了构建在 Nuclei OSS 之上的云
|
||||
|
||||
### 安装
|
||||
|
||||
`nuclei` 需要 **go >= 1.27.3** 才能成功安装。运行以下命令获取仓库：
|
||||
`nuclei` 需要 **go >= 1.24.2** 才能成功安装。运行以下命令获取仓库：
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -175,7 +174,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
@@ -251,7 +250,6 @@ CONFIGURATIONS:
|
||||
-sip, -source-ip string source ip address to use for network scan
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
@@ -318,10 +316,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-stp, -strict-probe skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
|
||||
-tf, -tech-filter opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
@@ -355,11 +350,10 @@ DEBUG:
|
||||
-hc, -health-check run diagnostic check up
|
||||
|
||||
UPDATE:
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-utc, -update-templates-channel string nuclei-templates update channel (stable, latest) (default "stable")
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
|
||||
+7
-13
@@ -9,8 +9,7 @@
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_FA.md">`فارسی`</a>
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -111,7 +110,7 @@ Consulta la [**`documentación completa de Nuclei aquí`**](https://docs.project
|
||||
|
||||
### Instalación
|
||||
|
||||
`nuclei` requiere **go >= 1.27.3** para instalarse correctamente. Ejecuta el siguiente comando para obtener el repositorio:
|
||||
`nuclei` requiere **go >= 1.24.2** para instalarse correctamente. Ejecuta el siguiente comando para obtener el repositorio:
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -175,7 +174,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
@@ -251,7 +250,6 @@ CONFIGURATIONS:
|
||||
-sip, -source-ip string source ip address to use for network scan
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
@@ -318,10 +316,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-stp, -strict-probe skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
|
||||
-tf, -tech-filter opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
@@ -355,11 +350,10 @@ DEBUG:
|
||||
-hc, -health-check run diagnostic check up
|
||||
|
||||
UPDATE:
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-utc, -update-templates-channel string nuclei-templates update channel (stable, latest) (default "stable")
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
|
||||
-728
@@ -1,728 +0,0 @@
|
||||
|
||||
|
||||
<div align="center">
|
||||
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README.md">`English`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_CN.md">`中文`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_KR.md">`Korean`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ID.md">`Indonesia`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_FA.md">`فارسی`</a>
|
||||
|
||||
</div>
|
||||
|
||||
<p align="center">
|
||||
|
||||
<a href="https://docs.projectdiscovery.io/tools/nuclei/overview?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme"><img src="https://img.shields.io/badge/Documentation-%23000000.svg?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJub25lIiBzdHJva2U9IiNmZmZmZmYiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIiBjbGFzcz0ibHVjaWRlIGx1Y2lkZS1ib29rLW9wZW4iPjxwYXRoIGQ9Ik0xMiA3djE0Ii8+PHBhdGggZD0iTTMgMThhMSAxIDAgMCAxLTEtMVY0YTEgMSAwIDAgMSAxLTFoNWE0IDQgMCAwIDEgNCA0IDQgNCAwIDAgMSA0LTRoNWExIDEgMCAwIDEgMSAxdjEzYTEgMSAwIDAgMS0xIDFoLTZhMyAzIDAgMCAwLTMgMyAzIDMgMCAwIDAtMy0zeiIvPjwvc3ZnPg==&logoColor=white" alt="Documentation Badge"></a>
|
||||
|
||||
<a href="https://github.com/projectdiscovery/nuclei-templates"><img src="https://img.shields.io/badge/Templates Library-%23000000.svg?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxNiIgaGVpZ2h0PSIxNiIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJub25lIiBzdHJva2U9IiNmZmZmZmYiIHN0cm9rZS13aWR0aD0iMS41IiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIGNsYXNzPSJsdWNpZGUgbHVjaWRlLXNoaWVsZCI+PHBhdGggZD0iTTIwIDEzYzAgNS0zLjUgNy41LTcuNjYgOC45NWExIDEgMCAwIDEtLjY3LS4wMUM3LjUgMjAuNSA0IDE4IDQgMTNWNmExIDEgMCAwIDEgMS0xYzIgMCA0LjUtMS4yIDYuMjQtMi43MmExLjE3IDEuMTcgMCAwIDEgMS41MiAwQzE0LjUxIDMuODEgMTcgNSAxOSA1YTEgMSAwIDAgMSAxIDF6Ii8+PC9zdmc+&logoColor=white"></a>
|
||||
|
||||
<a href="https://discord.gg/projectdiscovery?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme"><img src="https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white" alt="Discord Badge"></a>
|
||||
|
||||
|
||||
<hr>
|
||||
</p>
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
**Nuclei یک اسکنر آسیبپذیری مدرن و با کارایی بالاست که از قالبهای ساده بر پایه YAML استفاده میکند. این ابزار به شما امکان میدهد سناریوهای سفارشی برای شناسایی آسیبپذیریها طراحی کنید که شرایط واقعی را شبیهسازی میکنند و منجر به صفر شدن خطاهای مثبت کاذب (گزارشهای اشتباه) میشوند.**
|
||||
|
||||
- فرمت ساده YAML برای ساخت و سفارشیسازی قالبهای آسیبپذیری.
|
||||
- مشارکت توسط هزاران متخصص امنیت جهت مقابله با آسیبپذیریهای نوظهور و رایج.
|
||||
- کاهش خطاهای مثبت کاذب با شبیهسازی مراحل واقعی جهت تایید یک آسیبپذیری.
|
||||
- پردازش موازی فوقسریع اسکنها و خوشهبندی درخواستها.
|
||||
- قابلیت ادغام در خطوط لوله CI/CD برای شناسایی آسیبپذیریها و تستهای رگرسیون.
|
||||
- پشتیبانی از پروتکلهای متعدد مانند TCP، DNS، HTTP، SSL، WHOIS، جاوااسکریپت، کد و موارد دیگر.
|
||||
- قابلیت یکپارچهسازی با Jira، Splunk، GitHub، Elastic و GitLab.
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
## فهرست مطالب
|
||||
|
||||
- [**`شروع کار`**](#شروع-کار)
|
||||
- [_`۱. ابزار خط فرمان Nuclei`_](#۱-ابزار-خط-فرمان-nuclei)
|
||||
- [_`۲. نسخههای حرفهای و سازمانی`_](#۲-نسخههای-حرفهای-و-سازمانی)
|
||||
- [**`مستندات`**](#مستندات)
|
||||
- [_`دستورات خط فرمان`_](#دستورات-خط-فرمان)
|
||||
- [_`اسکن یک هدف واحد`_](#اسکن-یک-هدف)
|
||||
- [_`اسکن اهداف متعدد`_](#اسکن-چندین-هدف)
|
||||
- [_`اسکن شبکه`_](#اسکن-شبکه)
|
||||
- [_`اسکن با قالب سفارشی شما`_](#اسکن-با-استفاده-از-قالب-سفارشی-خودتان)
|
||||
- [_`اتصال Nuclei به ProjectDiscovery`_](#اتصال-nuclei-به-projectdiscovery)
|
||||
- [**`قالبهای Nuclei، جامعه کاربری و پاداشها`**](#قالبهای-nuclei-جامعه-و-پاداشها-) 💎
|
||||
- [**`ماموریت ما`**](#ماموریت-ما)
|
||||
- [**`مشارکتکنندگان`**](#مشارکتکنندگان-) ❤
|
||||
- [**`مجوز استفاده`**](#license)
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
## شروع کار
|
||||
|
||||
### ۱. ابزار خط فرمان Nuclei
|
||||
|
||||
_ابزار Nuclei را روی سیستم خود نصب کنید. کار خود را با دنبال کردن راهنمای نصب در [**`اینجا`**](https://docs.projectdiscovery.io/tools/nuclei/install?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme) آغاز کنید. علاوه بر این، ما [**`یک سطح ابری رایگان`**](https://cloud.projectdiscovery.io/sign-up) ارائه میدهیم که همراه با محدودیتهای رایگان ماهانه سخاوتمندانهای عرضه میشود:_
|
||||
|
||||
- ذخیرهسازی و بصریسازی یافتههای آسیبپذیری شما
|
||||
- نوشتن و مدیریت قالبهای Nuclei شما
|
||||
- دسترسی به آخرین قالبهای Nuclei
|
||||
- شناسایی و ذخیرهسازی اهداف شما
|
||||
|
||||
> [!Important]
|
||||
> |**این پروژه در حال توسعه فعال است**. انتظار تغییرات اساسی در نسخههای جدید را داشته باشید. لطفاً قبل از بهروزرسانی، لیست تغییرات نسخه (Changelog) را بررسی کنید.|
|
||||
> |:--------------------------------|
|
||||
> | این پروژه در درجه اول برای استفاده به عنوان یک ابزار خط فرمان (CLI) مستقل ساخته شده است. **اجرای Nuclei به عنوان یک سرویس ممکن است خطرات امنیتی به همراه داشته باشد.** توصیه میشود با احتیاط و با در نظر گرفتن تدابیر امنیتی اضافی از آن استفاده کنید. |
|
||||
|
||||
<br>
|
||||
|
||||
### ۲. نسخههای حرفهای و سازمانی
|
||||
|
||||
_برای تیمهای امنیتی و سازمانها، ما یک سرویس میزبانیشده در ابر بر پایه نسخه متنباز (OSS) ابزار Nuclei ارائه میدهیم؛ این سرویس بهطور دقیق بهینهسازی شده است تا به شما کمک کند اسکنهای آسیبپذیری را بهصورت مداوم، در مقیاس بزرگ و همراه با تیم و گردشهای کاری فعلی خود اجرا کنید:_
|
||||
|
||||
- اسکنهای ۵۰ برابر سریعتر
|
||||
- اسکن در مقیاس بزرگ با دقت بسیار بالا
|
||||
- یکپارچهسازی با سرویسهای ابری (AWS، GCP، Azure، Cloudflare، Fastly، Terraform، Kubernetes)
|
||||
- اتصال به Jira، Slack، Linear، رابطهای برنامهنویسی (APIs) و وبهوکها (Webhooks)
|
||||
- گزارشدهیهای مدیریتی و سازگاری با استانداردهای امنیتی (Compliance)
|
||||
- بهعلاوه: اسکن بیدرنگ (Real-time)، پشتیبانی از SAML SSO، پلتفرم منطبق با استاندارد SOC 2 (همراه با گزینههای میزبانی در اتحادیه اروپا و آمریکا)، فضاهای کاری مشترک تیمی و موارد دیگر
|
||||
- ما بهطور مداوم در حال [**`افزودن قابلیتهای جدید`**](https://feedback.projectdiscovery.io/changelog) هستیم!
|
||||
- **ایدهآل برای:** نفوذگران قانونمند (Pentesters)، تیمهای امنیتی و سازمانها
|
||||
|
||||
در صورتی که سازمان بزرگی دارید و نیازمندیهای شما پیچیده است، میتوانید در [**`نسخه پرو ثبتنام کنید`**](https://projectdiscovery.io/pricing?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme) یا [**`با تیم ما گفتگو کنید`**](https://projectdiscovery.io/request-demo?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme).
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
## مستندات
|
||||
|
||||
[**`مستندات کامل`**](https://docs.projectdiscovery.io/tools/nuclei/running) ابزار Nuclei را در اینجا مرور کنید. اگر بهتازگی کار با Nuclei را شروع کردهاید، [**`مجموعه ویدیوهای آموزشی پایهای ما در یوتیوب`**](https://www.youtube.com/playlist?list=PLZRbR9aMzTTpItEdeNSulo8bYsvil80Rl) را بررسی کنید.
|
||||
|
||||
<div align="center">
|
||||
|
||||
<a href="https://www.youtube.com/watch?v=b5qMyQvL1ZA&list=PLZRbR9aMzTTpItEdeNSulo8bYsvil80Rl&utm_source=github&utm_medium=web&utm_campaign=nuclei_readme" target="_blank"><img src="/static/nuclei-getting-started.png" width="350px" alt="Nuclei Getting Started Video Tutorial"></a> <a href="https://www.youtube.com/watch?v=nFXygQdtjyw&utm_source=github&utm_medium=web&utm_campaign=nuclei_readme" target="_blank"><img src="/static/nuclei-write-your-first-template.png" width="350px" alt="Writing Your First Nuclei Template Video Tutorial"></a>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<br>
|
||||
|
||||
### نصب
|
||||
|
||||
برای نصب موفقیتآمیز `nuclei` به نسخه **go >= 1.24.2** نیاز است. دستور زیر را برای دریافت مخزن اجرا کنید:
|
||||
|
||||
<div dir="ltr">
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
```
|
||||
</div>
|
||||
|
||||
برای اطلاعات بیشتر درباره نحوه نصب nuclei، به آدرس `https://docs.projectdiscovery.io/tools/nuclei/install` مراجعه کنید.
|
||||
|
||||
### دستورات خط فرمان
|
||||
|
||||
برای نمایش تمامی پارامترها و سوییچهای این ابزار:
|
||||
|
||||
<div dir="ltr">
|
||||
|
||||
```sh
|
||||
nuclei -h
|
||||
```
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<details>
|
||||
<summary>گسترش تمام فلگهای راهنما</summary>
|
||||
|
||||
<div dir="ltr">
|
||||
|
||||
```yaml
|
||||
Nuclei is a fast, template based vulnerability scanner focusing
|
||||
on extensive configurability, massive extensibility and ease of use.
|
||||
|
||||
Usage:
|
||||
./nuclei [flags]
|
||||
|
||||
Flags:
|
||||
TARGET:
|
||||
-u, -target string[] target URLs/hosts to scan
|
||||
-l, -list string path to file containing a list of target URLs/hosts to scan (one per line)
|
||||
-targets-inline string inline multiline target list (for use in template profiles)
|
||||
-eh, -exclude-hosts string[] hosts to exclude to scan from the input list (ip, cidr, hostname)
|
||||
-resume string resume scan from and save to specified file (clustering will be disabled)
|
||||
-sa, -scan-all-ips scan all the IP's associated with dns record
|
||||
-iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4)
|
||||
-dun, -disable-url-normalization keep targets exactly as supplied instead of collapsing urls that address the same resource
|
||||
-mup, -max-urls-per-pattern int scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all)
|
||||
|
||||
TARGET-FORMAT:
|
||||
-im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
|
||||
-ro, -required-only use only required fields in input format when generating requests
|
||||
-sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file
|
||||
-vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode)
|
||||
-vfp, -var-file-paths string[] list of yaml file contained vars to inject into yaml input
|
||||
|
||||
TEMPLATES:
|
||||
-nt, -new-templates run only new templates added in latest nuclei-templates release
|
||||
-ntv, -new-templates-version string[] run new templates added in specific version
|
||||
-as, -automatic-scan automatic web scan using wappalyzer technology detection to tags mapping
|
||||
-t, -templates string[] list of template or template directory to run (comma-separated, file)
|
||||
-turl, -template-url string[] template url or list containing template urls to run (comma-separated, file)
|
||||
-ai, -prompt string generate and run template using ai prompt
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
-tgl list all available tags
|
||||
-sign signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable
|
||||
-code enable loading code protocol-based templates
|
||||
-dut, -disable-unsigned-templates disable running unsigned templates or templates with mismatched signature
|
||||
-esc, -enable-self-contained enable loading self-contained templates
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
-llm-max-calls int maximum llm calls per scan (0 for unlimited) (default 5000)
|
||||
-llm-concurrency int maximum concurrent llm calls (default 4)
|
||||
-llm-cache cache llm responses within a scan (default true)
|
||||
|
||||
FILTERING:
|
||||
-a, -author string[] templates to run based on authors (comma-separated, file)
|
||||
-tags string[] templates to run based on tags (comma-separated, file)
|
||||
-etags, -exclude-tags string[] templates to exclude based on tags (comma-separated, file)
|
||||
-itags, -include-tags string[] tags to be executed even if they are excluded either by default or configuration
|
||||
-id, -template-id string[] templates to run based on template ids (comma-separated, file, allow-wildcard)
|
||||
-eid, -exclude-id string[] templates to exclude based on template ids (comma-separated, file)
|
||||
-it, -include-templates string[] path to template file or directory to be executed even if they are excluded either by default or configuration
|
||||
-et, -exclude-templates string[] path to template file or directory to exclude (comma-separated, file)
|
||||
-em, -exclude-matchers string[] template matchers to exclude in result
|
||||
-s, -severity value[] templates to run based on severity. Possible values: info, low, medium, high, critical, unknown
|
||||
-es, -exclude-severity value[] templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown
|
||||
-pt, -type value[] templates to run based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code, javascript
|
||||
-ept, -exclude-type value[] templates to exclude based on protocol type. Possible values: dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code, javascript
|
||||
-tc, -template-condition string[] templates to run based on expression condition
|
||||
|
||||
OUTPUT:
|
||||
-o, -output string output file to write found issues/vulnerabilities
|
||||
-sresp, -store-resp store all request/response passed through nuclei to output directory
|
||||
-srd, -store-resp-dir string store all request/response passed through nuclei to custom directory (default "output")
|
||||
-silent display findings only
|
||||
-nc, -no-color disable output content coloring (ANSI escape codes)
|
||||
-j, -jsonl write output in JSONL(ines) format
|
||||
-irr, -include-rr -omit-raw include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true)
|
||||
-or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)
|
||||
-ot, -omit-template omit encoded template in the JSON, JSONL output
|
||||
-nm, -no-meta disable printing result metadata in cli output
|
||||
-ts, -timestamp enables printing timestamp in cli output
|
||||
-rdb, -report-db string nuclei reporting database (always use this to persist report data)
|
||||
-ms, -matcher-status display match failure status
|
||||
-me, -markdown-export string directory to export results in markdown format
|
||||
-se, -sarif-export string file to export results in SARIF format
|
||||
-je, -json-export string file to export results in JSON format
|
||||
-jle, -jsonl-export string file to export results in JSONL(ine) format
|
||||
-pe, -pdf-export string file to export results in PDF format
|
||||
-rd, -redact string[] redact given list of keys from query parameter, request header and body
|
||||
|
||||
CONFIGURATIONS:
|
||||
-config string path to the nuclei configuration file
|
||||
-tp, -profile string template profile config file to run
|
||||
-tpl, -profile-list list community template profiles
|
||||
-fr, -follow-redirects enable following redirects for http templates
|
||||
-fhr, -follow-host-redirects follow redirects on the same host
|
||||
-mr, -max-redirects int max number of redirects to follow for http templates (default 10)
|
||||
-dr, -disable-redirects disable redirects for http templates
|
||||
-rc, -report-config string nuclei reporting module configuration file
|
||||
-H, -header string[] custom header/cookie to include in all http request in header:value format (cli, file)
|
||||
-V, -var value custom vars in key=value format
|
||||
-r, -resolvers string file containing resolver list for nuclei
|
||||
-sr, -system-resolvers use system DNS resolving as error fallback
|
||||
-dc, -disable-clustering disable clustering of requests
|
||||
-passive enable passive HTTP response processing mode
|
||||
-fh2, -force-http2 force http2 connection on requests
|
||||
-ev, -env-vars enable environment variables to be used in template
|
||||
-cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts
|
||||
-ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts
|
||||
-ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts
|
||||
-sml, -show-match-line show match lines for file templates, works with extractors only
|
||||
-ztls use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default
|
||||
-sni string tls sni hostname to use (default: input domain name)
|
||||
-dka, -dialer-keep-alive value keep-alive duration for network requests.
|
||||
-lfa, -allow-local-file-access allow all templates in the run to read any file on the system (not just payloads)
|
||||
-lna, -restrict-local-network-access blocks connections to the local / private network
|
||||
-i, -interface string network interface to use for network scan
|
||||
-at, -attack-type string type of payload combinations to perform (batteringram,pitchfork,clusterbomb)
|
||||
-pro, -payload-random-order iterate payload combinations in a random order instead of sequentially
|
||||
-sip, -source-ip string source ip address to use for network scan
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
|
||||
INTERACTSH:
|
||||
-iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me)
|
||||
-itoken, -interactsh-token string authentication token for self-hosted interactsh server
|
||||
-interactions-cache-size int number of requests to keep in the interactions cache (default 5000)
|
||||
-interactions-eviction int number of seconds to wait before evicting requests from cache (default 60)
|
||||
-interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5)
|
||||
-interactions-cooldown-period int extra time for interaction polling before exiting (default 5)
|
||||
-ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates
|
||||
|
||||
FUZZING:
|
||||
-ft, -fuzzing-type string overrides fuzzing type set in template (replace, prefix, postfix, infix)
|
||||
-fm, -fuzzing-mode string overrides fuzzing mode set in template (multiple, single)
|
||||
-fuzz enable loading fuzzing templates (Deprecated: use -dast instead)
|
||||
-dast enable / run dast (fuzz) nuclei templates
|
||||
-dts, -dast-server enable dast server mode (live fuzzing)
|
||||
-dtr, -dast-report write dast scan report to file
|
||||
-dtst, -dast-server-token string dast server token (optional)
|
||||
-dtsa, -dast-server-address string dast server address (default "localhost:9055")
|
||||
-dtp, -dast-proxy enable dast proxy mode (fuzz live traffic proxied through nuclei)
|
||||
-dtpa, -dast-proxy-address string dast proxy listen address (default "127.0.0.1:9056")
|
||||
-dtpau, -dast-proxy-auth string dast proxy basic auth in user:pass format (required for non-loopback address)
|
||||
-dfp, -display-fuzz-points display fuzz points in the output for debugging
|
||||
-fuzz-param-frequency int frequency of uninteresting parameters for fuzzing before skipping (default 10)
|
||||
-fa, -fuzz-aggression string fuzzing aggression level controls payload count for fuzz (low, medium, high) (default "low")
|
||||
-cs, -fuzz-scope string[] in scope url regex to be followed by fuzzer
|
||||
-cos, -fuzz-out-scope string[] out of scope url regex to be excluded by fuzzer
|
||||
|
||||
UNCOVER:
|
||||
-uc, -uncover enable uncover engine
|
||||
-uq, -uncover-query string[] uncover search query
|
||||
-ue, -uncover-engine string[] uncover search engine (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google) (default shodan)
|
||||
-uf, -uncover-field string uncover fields to return (ip,port,host) (default "ip:port")
|
||||
-ul, -uncover-limit int uncover results to return (default 100)
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
-timeout int time to wait in seconds before timeout (default 10)
|
||||
-retries int number of times to retry a failed request (default 1)
|
||||
-ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443)
|
||||
-mhe, -max-host-error int max errors for a host before skipping from scan (default 30)
|
||||
-te, -track-error string[] adds given error to max-host-error watchlist (standard, file)
|
||||
-nmhe, -no-mhe disable skipping host from scan based on errors
|
||||
-project use a project folder to avoid sending same request multiple times
|
||||
-project-path string set a specific project path (default "/tmp")
|
||||
-spm, -stop-at-first-match stop processing HTTP requests after the first match (may break template/workflow logic)
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-stp, -strict-probe skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
|
||||
-tf, -tech-filter opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
HEADLESS:
|
||||
-headless enable templates that require headless browser support (root user on Linux will disable sandbox)
|
||||
-page-timeout int seconds to wait for each page in headless mode (default 20)
|
||||
-sb, -show-browser show the browser on the screen when running templates with headless mode
|
||||
-ho, -headless-options string[] start headless chrome with additional options
|
||||
-sc, -system-chrome use local installed Chrome browser instead of nuclei installed
|
||||
-cdpe, -cdp-endpoint string use remote browser via Chrome DevTools Protocol (CDP) endpoint
|
||||
-lha, -list-headless-action list available headless actions
|
||||
|
||||
DEBUG:
|
||||
-debug show all requests and responses
|
||||
-dreq, -debug-req show all sent requests
|
||||
-dresp, -debug-resp show all received responses
|
||||
-p, -proxy string[] list of http/socks5 proxy to use (comma separated or file input)
|
||||
-pi, -proxy-internal proxy all internal requests
|
||||
-rp, -replay-proxy string http/socks5 proxy to re-send requests that produced a finding through (e.g. burp)
|
||||
-ldf, -list-dsl-function list all supported DSL function signatures
|
||||
-tlog, -trace-log string file to write sent requests trace log
|
||||
-elog, -error-log string file to write sent requests error log
|
||||
-version show nuclei version
|
||||
-hm, -hang-monitor enable nuclei hang monitoring
|
||||
-v, -verbose show verbose output
|
||||
-profile-mem string generate memory (heap) profile & trace files
|
||||
-vv display templates loaded for scan
|
||||
-svd, -show-var-dump show variables dump for debugging
|
||||
-vdl, -var-dump-limit int limit the number of characters displayed in var dump (default 255)
|
||||
-ep, -enable-pprof enable pprof debugging server
|
||||
-tv, -templates-version shows the version of the installed nuclei-templates
|
||||
-hc, -health-check run diagnostic check up
|
||||
|
||||
UPDATE:
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-utc, -update-templates-channel string nuclei-templates update channel (stable, latest) (default "stable")
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
|
||||
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
|
||||
|
||||
STATISTICS:
|
||||
-stats display statistics about the running scan
|
||||
-sj, -stats-json display statistics in JSONL(ines) format
|
||||
-si, -stats-interval int number of seconds to wait between showing a statistics update (default 5)
|
||||
-mp, -metrics-port int port to expose nuclei metrics on (default 9092)
|
||||
-hps, -http-stats enable http status capturing (experimental)
|
||||
|
||||
CLOUD:
|
||||
-auth configure projectdiscovery cloud (pdcp) api key (default true)
|
||||
-tid, -team-id string upload scan results to given team id (optional) (default "none")
|
||||
-cup, -cloud-upload upload scan results to pdcp dashboard [DEPRECATED use -dashboard]
|
||||
-sid, -scan-id string upload scan results to existing scan id (optional)
|
||||
-sname, -scan-name string scan name to set (optional)
|
||||
-pd, -dashboard upload / view nuclei results in projectdiscovery cloud (pdcp) UI dashboard
|
||||
-pdu, -dashboard-upload string upload / view nuclei results file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard
|
||||
|
||||
AUTHENTICATION:
|
||||
-sf, -secret-file string[] path to config file containing secrets for nuclei authenticated scan
|
||||
-ps, -prefetch-secrets prefetch secrets from the secrets file
|
||||
# NOTE: Headers in secrets files preserve exact casing (useful for case-sensitive APIs)
|
||||
|
||||
|
||||
EXAMPLES:
|
||||
Run nuclei on single host:
|
||||
$ nuclei -target example.com
|
||||
|
||||
Run nuclei with specific template directories:
|
||||
$ nuclei -target example.com -t http/cves/ -t ssl
|
||||
|
||||
Run nuclei against a list of hosts:
|
||||
$ nuclei -list hosts.txt
|
||||
|
||||
Run nuclei with a JSON output:
|
||||
$ nuclei -target example.com -json-export output.json
|
||||
|
||||
Run nuclei with sorted Markdown outputs (with environment variables):
|
||||
$ MARKDOWN_EXPORT_SORT_MODE=template nuclei -target example.com -markdown-export nuclei_report/
|
||||
|
||||
Additional documentation is available at: https://docs.projectdiscovery.io/getting-started/running
|
||||
|
||||
```
|
||||
|
||||
</div>
|
||||
|
||||
مستندات تکمیلی در این آدرس در دسترس است: [**`docs.projectdiscovery.io/getting-started/running`**](https://docs.projectdiscovery.io/getting-started/running?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme)
|
||||
|
||||
</details>
|
||||
|
||||
### اسکن یک هدف
|
||||
|
||||
برای انجام یک اسکن سریع روی یک وباپلیکیشن:
|
||||
|
||||
<div dir="ltr">
|
||||
|
||||
```sh
|
||||
nuclei -target https://example.com
|
||||
```
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
### اسکن چندین هدف
|
||||
|
||||
Nuclei میتواند با ارائهٔ فهرستی از اهداف، اسکن گروهی را انجام دهد. میتوانید از فایلی استفاده کنید که شامل چندین URL است.
|
||||
|
||||
<div dir="ltr">
|
||||
|
||||
```sh
|
||||
nuclei -list urls.txt
|
||||
```
|
||||
</div>
|
||||
|
||||
### اسکن شبکه
|
||||
|
||||
این کار کل زیرشبکه را برای مشکلات مرتبط با شبکه، مانند پورتهای باز یا سرویسهای پیکربندی نادرست، اسکن میکند.
|
||||
|
||||
<div dir="ltr">
|
||||
|
||||
```sh
|
||||
nuclei -target 192.168.1.0/24
|
||||
```
|
||||
</div>
|
||||
|
||||
### اسکن با استفاده از قالب سفارشی خودتان
|
||||
|
||||
برای نوشتن و استفاده از قالب خودتان، یک فایل `.yaml` با قوانین مشخص ایجاد کنید، سپس آن را بهصورت زیر استفاده کنید.
|
||||
|
||||
<div dir="ltr">
|
||||
|
||||
```sh
|
||||
nuclei -u https://example.com -t /path/to/your-template.yaml
|
||||
```
|
||||
</div>
|
||||
|
||||
### اتصال Nuclei به ProjectDiscovery
|
||||
|
||||
میتوانید اسکنها را روی دستگاه خود اجرا کرده و نتایج را برای تحلیل و رفع مشکلات بیشتر، در پلتفرم ابری بارگذاری کنید.
|
||||
|
||||
<div dir="ltr">
|
||||
|
||||
```sh
|
||||
nuclei -target https://example.com -dashboard
|
||||
```
|
||||
</div>
|
||||
|
||||
> [!NOTE]
|
||||
> این قابلیت کاملاً رایگان است و به هیچگونه اشتراکی نیاز ندارد. برای راهنمایی دقیقتر، به [**`مستندات`**](https://docs.projectdiscovery.io/cloud/scanning/overview?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme) مراجعه کنید.
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
## قالبهای Nuclei، جامعه و پاداشها 💎
|
||||
|
||||
[**قالبهای Nuclei**](https://github.com/projectdiscovery/nuclei-templates) بر اساس مفهوم فایلهای قالب مبتنی بر YAML طراحی شدهاند که نحوه ارسال و پردازش درخواستها را تعریف میکنند. این موضوع امکان توسعهپذیری آسان Nuclei را فراهم میکند. قالبها با استفاده از YAML نوشته میشوند که قالبی ساده و قابلخواندن برای انسان ارائه میدهد تا بتوان فرآیند اجرا را بهسرعت تعریف کرد.
|
||||
|
||||
**همین حالا بهصورت آنلاین با ویرایشگر رایگان قالبهای Nuclei مجهز به هوش مصنوعی ما امتحان کنید؛** [**`اینجا کلیک کنید`**](https://cloud.projectdiscovery.io/templates).
|
||||
|
||||
قالبهای Nuclei روشی ساده و یکپارچه برای شناسایی و گزارش آسیبپذیریها ارائه میدهند و جزئیات ضروری مانند سطح شدت و روشهای شناسایی را در بر میگیرند. این ابزار متنباز که توسط جامعه توسعه داده میشود، واکنش به تهدیدات را سرعت میبخشد و در دنیای امنیت سایبری بهطور گسترده شناخته شده است. قالبهای Nuclei بهطور فعال توسط هزاران پژوهشگر امنیتی در سراسر جهان توسعه داده میشوند. ما دو برنامه برای مشارکتکنندگان خود داریم: [**`Pioneers`**](https://projectdiscovery.io/pioneers) و [**`💎 پاداشها`**](https://github.com/projectdiscovery/nuclei-templates/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22%F0%9F%92%8E%20Bounty%22).
|
||||
|
||||
<p align="right">
|
||||
<a href="/static/nuclei-templates-teamcity.png" target="_blank"><img src="/static/nuclei-templates-teamcity.png" width="1200px" alt="نمونه قالب Nuclei برای شناسایی پیکربندی نادرست TeamCity" /></a>
|
||||
</p>
|
||||
|
||||
### مثالها
|
||||
|
||||
برای مشاهده موارد استفاده و ایدهها، به [**مستندات ما**](https://docs.projectdiscovery.io/templates/introduction) مراجعه کنید.
|
||||
|
||||
| قالب Nuclei | مورد استفاده |
|
||||
| ---: | ---: |
|
||||
| **[CVE-2021-44228 (Log4Shell)](https://cloud.projectdiscovery.io/public/CVE-2021-45046)** | شناسایی CVEهای شناختهشده |
|
||||
| **[Blind SQL Injection via OOB](https://cloud.projectdiscovery.io/public/CVE-2024-22120)** | شناسایی آسیبپذیریهای Out-of-Band |
|
||||
| **[Generic SQL Injection](https://cloud.projectdiscovery.io/public/CVE-2022-34265)** | شناسایی SQL Injection |
|
||||
| **[Reflected XSS Detection](https://cloud.projectdiscovery.io/public/CVE-2023-4173)** | Cross-Site Scripting (XSS) |
|
||||
| **[Default Credentials Check](https://cloud.projectdiscovery.io/public/airflow-default-login)** | رمزهای عبور پیشفرض یا ضعیف |
|
||||
| **[Sensitive File Disclosure](https://cloud.projectdiscovery.io/public/airflow-configuration-exposure)** | فایلهای محرمانه یا افشای دادهها |
|
||||
| **[Open Redirect Detection](https://cloud.projectdiscovery.io/public/open-redirect)** | شناسایی Open Redirect |
|
||||
| **[Subdomain Takeover Templates](https://cloud.projectdiscovery.io/public/azure-takeover-detection)** | شناسایی Subdomain Takeover |
|
||||
| **[Unprotected Jenkins Console](https://cloud.projectdiscovery.io/public/unauthenticated-jenkins)** | پیکربندیهای نادرست امنیتی |
|
||||
| **[SSL Certificate Expiry](https://cloud.projectdiscovery.io/public/expired-ssl)** | پیکربندیهای ضعیف SSL/TLS |
|
||||
| **[Open S3 Bucket Detection](https://cloud.projectdiscovery.io/public/s3-public-read-acp)** | سرویسهای ابری با پیکربندی نادرست |
|
||||
| **[RCE Detection Templates](https://cloud.projectdiscovery.io/public/CVE-2024-29824)** | آسیبپذیریهای اجرای کد از راه دور |
|
||||
| **[Path Traversal Detection](https://cloud.projectdiscovery.io/public/oracle-fatwire-lfi)** | حملات Directory Traversal |
|
||||
| **[Local/Remote File Inclusion](https://cloud.projectdiscovery.io/public/CVE-2023-6977)** | آسیبپذیریهای File Inclusion |
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
## ماموریت ما
|
||||
|
||||
اسکنرهای سنتی آسیبپذیری دههها پیش ساخته شدهاند. آنها متنبسته، فوقالعاده کند و وابسته به فروشندگان هستند. مهاجمان امروزی، برخلاف گذشته که بهرهبرداری از آسیبپذیریها سالها طول میکشید، ظرف چند روز پس از انتشار CVEهای جدید، آنها را در سراسر اینترنت بهصورت گسترده مورد سوءاستفاده قرار میدهند. این تغییر، نیازمند رویکردی کاملاً متفاوت برای مقابله با اکسپلویتهای پرطرفدار در اینترنت است.
|
||||
|
||||
ما Nuclei را برای حل این چالش ساختیم. ما کل چارچوب موتور اسکن را متنباز و قابل سفارشیسازی کردیم تا جامعه امنیتی جهانی بتواند با یکدیگر همکاری کرده و با بردارهای حمله و آسیبپذیریهای پرطرفدار در اینترنت مقابله کند. امروزه Nuclei توسط شرکتهای Fortune 500، سازمانهای دولتی و دانشگاهها استفاده و توسعه داده میشود.
|
||||
|
||||
شما نیز میتوانید با مشارکت در کد ما، [**`کتابخانه قالبها`**](https://github.com/projectdiscovery/nuclei-templates)، یا [**`پیوستن به تیم ما`**](https://projectdiscovery.io/) مشارکت کنید.
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
## مشارکتکنندگان :heart:
|
||||
|
||||
از تمام مشارکتکنندگان فوقالعاده [**`جامعه که با ارسال PRها`**](https://github.com/projectdiscovery/nuclei/graphs/contributors) و بهروز نگهداشتن این پروژه از آن حمایت میکنند، سپاسگزاریم. :heart:
|
||||
|
||||
<p align="left">
|
||||
<a href="https://github.com/Ice3man543"><img src="https://avatars.githubusercontent.com/u/22318055?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/apps/dependabot"><img src="https://avatars.githubusercontent.com/in/29110?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/ehsandeep"><img src="https://avatars.githubusercontent.com/u/8293321?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Mzack9999"><img src="https://avatars.githubusercontent.com/u/13421144?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/forgedhallpass"><img src="https://avatars.githubusercontent.com/u/13679401?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/tarunKoyalwar"><img src="https://avatars.githubusercontent.com/u/45962551?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/manuelbua"><img src="https://avatars.githubusercontent.com/u/819314?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/actions-user"><img src="https://avatars.githubusercontent.com/u/65916846?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/dogancanbakir"><img src="https://avatars.githubusercontent.com/u/65292895?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/parrasajad"><img src="https://avatars.githubusercontent.com/u/16835787?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/vzamanillo"><img src="https://avatars.githubusercontent.com/u/10209695?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/ShubhamRasal"><img src="https://avatars.githubusercontent.com/u/45902122?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/RamanaReddy0M"><img src="https://avatars.githubusercontent.com/u/90540245?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/LuitelSamikshya"><img src="https://avatars.githubusercontent.com/u/85764322?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/kchason"><img src="https://avatars.githubusercontent.com/u/1111099?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/pmareke"><img src="https://avatars.githubusercontent.com/u/3502075?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/dwisiswant0"><img src="https://avatars.githubusercontent.com/u/25837540?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/xm1k3"><img src="https://avatars.githubusercontent.com/u/73166077?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/5amu"><img src="https://avatars.githubusercontent.com/u/39925709?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/ehrishirajsharma"><img src="https://avatars.githubusercontent.com/u/35542790?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/zerodivisi0n"><img src="https://avatars.githubusercontent.com/u/687694?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/geeknik"><img src="https://avatars.githubusercontent.com/u/466878?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/TerminalFi"><img src="https://avatars.githubusercontent.com/u/32599364?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/KaulSe"><img src="https://avatars.githubusercontent.com/u/45340011?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/sullo"><img src="https://avatars.githubusercontent.com/u/1474884?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/wdahlenburg"><img src="https://avatars.githubusercontent.com/u/4451504?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/ghost"><img src="https://avatars.githubusercontent.com/u/10137?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Nishan8583"><img src="https://avatars.githubusercontent.com/u/20457968?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/jdk2588"><img src="https://avatars.githubusercontent.com/u/985054?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/nothinux"><img src="https://avatars.githubusercontent.com/u/17433202?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/CodFrm"><img src="https://avatars.githubusercontent.com/u/22783163?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/CasperGN"><img src="https://avatars.githubusercontent.com/u/5549643?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/ankh2054"><img src="https://avatars.githubusercontent.com/u/6784287?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/revblock"><img src="https://avatars.githubusercontent.com/u/72813848?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/cn-kali-team"><img src="https://avatars.githubusercontent.com/u/30642514?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/EndPositive"><img src="https://avatars.githubusercontent.com/u/25148195?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/jimen0"><img src="https://avatars.githubusercontent.com/u/6826244?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/xstevens"><img src="https://avatars.githubusercontent.com/u/69216?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/mjkim610"><img src="https://avatars.githubusercontent.com/u/17107206?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/organiccrap"><img src="https://avatars.githubusercontent.com/u/376317?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/lu4nx"><img src="https://avatars.githubusercontent.com/u/3006875?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/souvikhazra1"><img src="https://avatars.githubusercontent.com/u/13842393?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/tovask"><img src="https://avatars.githubusercontent.com/u/22732484?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Marmelatze"><img src="https://avatars.githubusercontent.com/u/199681?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/doug-threatmate"><img src="https://avatars.githubusercontent.com/u/127235272?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/yabeow"><img src="https://avatars.githubusercontent.com/u/21117771?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/olearycrew"><img src="https://avatars.githubusercontent.com/u/6044920?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/gano3s"><img src="https://avatars.githubusercontent.com/u/2551605?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/alizmhdi"><img src="https://avatars.githubusercontent.com/u/79321261?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/hackerpain"><img src="https://avatars.githubusercontent.com/u/61242234?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/lc"><img src="https://avatars.githubusercontent.com/u/19563282?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/savushkin-yauheni"><img src="https://avatars.githubusercontent.com/u/5173352?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/74616e696d"><img src="https://avatars.githubusercontent.com/u/97121933?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/edoardottt"><img src="https://avatars.githubusercontent.com/u/35783570?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/zt2"><img src="https://avatars.githubusercontent.com/u/7644862?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/moonD4rk"><img src="https://avatars.githubusercontent.com/u/24284231?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/wk8"><img src="https://avatars.githubusercontent.com/u/2536231?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/mikerott"><img src="https://avatars.githubusercontent.com/u/857712?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/toufik-airane"><img src="https://avatars.githubusercontent.com/u/5610269?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/hktalent"><img src="https://avatars.githubusercontent.com/u/18223385?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/jturner"><img src="https://avatars.githubusercontent.com/u/1825202?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/gaby"><img src="https://avatars.githubusercontent.com/u/835733?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/vavkamil"><img src="https://avatars.githubusercontent.com/u/47953210?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/leonjza"><img src="https://avatars.githubusercontent.com/u/1148127?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/mionskowski-form3"><img src="https://avatars.githubusercontent.com/u/91873652?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/chenrui333"><img src="https://avatars.githubusercontent.com/u/1580956?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/iamargus95"><img src="https://avatars.githubusercontent.com/u/77744293?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/shashikarsiddharth"><img src="https://avatars.githubusercontent.com/u/60960197?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/trypa11"><img src="https://avatars.githubusercontent.com/u/67585616?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Zeokat"><img src="https://avatars.githubusercontent.com/u/1313154?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/alban-stourbe-wmx"><img src="https://avatars.githubusercontent.com/u/159776828?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/anykno"><img src="https://avatars.githubusercontent.com/u/2528207?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/ronaudinho"><img src="https://avatars.githubusercontent.com/u/10264710?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/boy-hack"><img src="https://avatars.githubusercontent.com/u/18695984?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/iuliu8899"><img src="https://avatars.githubusercontent.com/u/31680027?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/debasishbsws"><img src="https://avatars.githubusercontent.com/u/65381620?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/denysvitali-niantic"><img src="https://avatars.githubusercontent.com/u/157139422?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/fail-open"><img src="https://avatars.githubusercontent.com/u/72417455?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Xc1Ym"><img src="https://avatars.githubusercontent.com/u/29765332?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/XTeam-Wing"><img src="https://avatars.githubusercontent.com/u/25416365?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Weltolk"><img src="https://avatars.githubusercontent.com/u/40228052?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/tonghuaroot"><img src="https://avatars.githubusercontent.com/u/23011166?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/praetorian-thendrickson"><img src="https://avatars.githubusercontent.com/u/69640071?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/S0obi"><img src="https://avatars.githubusercontent.com/u/4180104?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/skahn007gl"><img src="https://avatars.githubusercontent.com/u/144735608?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/shouichi"><img src="https://avatars.githubusercontent.com/u/99586?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/seb-elttam"><img src="https://avatars.githubusercontent.com/u/111818823?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/AdallomRoy"><img src="https://avatars.githubusercontent.com/u/4046118?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/rotemreiss"><img src="https://avatars.githubusercontent.com/u/9288082?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/oscarintherocks"><img src="https://avatars.githubusercontent.com/u/1785821?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/xxcdd"><img src="https://avatars.githubusercontent.com/u/42600601?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/chen2aaron"><img src="https://avatars.githubusercontent.com/u/9978183?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/voidz0r"><img src="https://avatars.githubusercontent.com/u/1032286?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/vince-isec"><img src="https://avatars.githubusercontent.com/u/149686094?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/true13"><img src="https://avatars.githubusercontent.com/u/18207552?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/skhalsa-sigsci"><img src="https://avatars.githubusercontent.com/u/68570441?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/ShuBo6"><img src="https://avatars.githubusercontent.com/u/41125338?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/seeyarh"><img src="https://avatars.githubusercontent.com/u/16869800?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/securibee"><img src="https://avatars.githubusercontent.com/u/51520913?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/sduc"><img src="https://avatars.githubusercontent.com/u/2879617?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/scottdharvey"><img src="https://avatars.githubusercontent.com/u/25498254?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/rykkard"><img src="https://avatars.githubusercontent.com/u/51889048?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/monitor403"><img src="https://avatars.githubusercontent.com/u/45124775?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/mlec1"><img src="https://avatars.githubusercontent.com/u/42201667?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/meme-lord"><img src="https://avatars.githubusercontent.com/u/17912559?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/LazyMaple"><img src="https://avatars.githubusercontent.com/u/12314941?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/lvyaoting"><img src="https://avatars.githubusercontent.com/u/166296299?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/llussy"><img src="https://avatars.githubusercontent.com/u/18432966?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/linchizhen"><img src="https://avatars.githubusercontent.com/u/170242051?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/kiokuless"><img src="https://avatars.githubusercontent.com/u/110003596?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Jarnpher553"><img src="https://avatars.githubusercontent.com/u/10233873?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/c-f"><img src="https://avatars.githubusercontent.com/u/35263248?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/hanghuge"><img src="https://avatars.githubusercontent.com/u/166206050?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/testwill"><img src="https://avatars.githubusercontent.com/u/8717479?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/galoget"><img src="https://avatars.githubusercontent.com/u/8353133?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/fudancoder"><img src="https://avatars.githubusercontent.com/u/171416994?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/revolunet"><img src="https://avatars.githubusercontent.com/u/124937?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/jsoref"><img src="https://avatars.githubusercontent.com/u/2119212?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/MachadoOtto"><img src="https://avatars.githubusercontent.com/u/93268441?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/jonathanwalker"><img src="https://avatars.githubusercontent.com/u/14978093?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/nHurD"><img src="https://avatars.githubusercontent.com/u/233374?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/jessekelly881"><img src="https://avatars.githubusercontent.com/u/22938931?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/JaneX8"><img src="https://avatars.githubusercontent.com/u/5116641?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/eltociear"><img src="https://avatars.githubusercontent.com/u/22633385?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/atomiczsec"><img src="https://avatars.githubusercontent.com/u/75549184?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/M-Faheem-Khan"><img src="https://avatars.githubusercontent.com/u/17150767?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/denandz"><img src="https://avatars.githubusercontent.com/u/5291556?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/tibbon"><img src="https://avatars.githubusercontent.com/u/82880?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/dany74q"><img src="https://avatars.githubusercontent.com/u/2129762?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/0x123456789"><img src="https://avatars.githubusercontent.com/u/36066426?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/danigoland"><img src="https://avatars.githubusercontent.com/u/15079567?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/ChrisMandich"><img src="https://avatars.githubusercontent.com/u/14286797?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/austintraver"><img src="https://avatars.githubusercontent.com/u/25112463?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/socialsister"><img src="https://avatars.githubusercontent.com/u/155628741?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Anemys"><img src="https://avatars.githubusercontent.com/u/51196227?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/andreangelucci"><img src="https://avatars.githubusercontent.com/u/18552197?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/AlexS778"><img src="https://avatars.githubusercontent.com/u/98418121?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/noraj"><img src="https://avatars.githubusercontent.com/u/16578570?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/akkuman"><img src="https://avatars.githubusercontent.com/u/7813511?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/zrquan"><img src="https://avatars.githubusercontent.com/u/33086594?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/1efty"><img src="https://avatars.githubusercontent.com/u/18194777?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/rsrdesarrollo"><img src="https://avatars.githubusercontent.com/u/5142014?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/razin99"><img src="https://avatars.githubusercontent.com/u/44442082?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/MetzinAround"><img src="https://avatars.githubusercontent.com/u/65838556?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/vil02"><img src="https://avatars.githubusercontent.com/u/65706193?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/mrschyte"><img src="https://avatars.githubusercontent.com/u/8571?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/PeterDaveHello"><img src="https://avatars.githubusercontent.com/u/3691490?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/parthmalhotra"><img src="https://avatars.githubusercontent.com/u/28601533?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/owenrumney"><img src="https://avatars.githubusercontent.com/u/3049157?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Ovi3"><img src="https://avatars.githubusercontent.com/u/29408109?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/Bisstocuz"><img src="https://avatars.githubusercontent.com/u/42398278?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/daffainfo"><img src="https://avatars.githubusercontent.com/u/36522826?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/mhmdiaa"><img src="https://avatars.githubusercontent.com/u/19687798?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/MiryangJung"><img src="https://avatars.githubusercontent.com/u/48237511?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/0xmin"><img src="https://avatars.githubusercontent.com/u/44919834?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/maikthulhu"><img src="https://avatars.githubusercontent.com/u/680830?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/sttlr"><img src="https://avatars.githubusercontent.com/u/40246850?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/iamRjarpan"><img src="https://avatars.githubusercontent.com/u/45498226?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/leoloobeek"><img src="https://avatars.githubusercontent.com/u/8801754?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/KristinnVikar"><img src="https://avatars.githubusercontent.com/u/93918469?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/kant01ne"><img src="https://avatars.githubusercontent.com/u/5072452?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/KeisukeYamashita"><img src="https://avatars.githubusercontent.com/u/23056537?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
<a href="https://github.com/1hehaq"><img src="https://avatars.githubusercontent.com/u/162917546?v=4" width="50" height="50" alt="" style="max-width: 100%;"></a>
|
||||
</p>
|
||||
|
||||
<br>
|
||||
<br>
|
||||
<br>
|
||||
|
||||
<div align="center">
|
||||
|
||||
<sub>**`nuclei`** تحت [**مجوز MIT**](https://github.com/projectdiscovery/nuclei/blob/main/LICENSE.md) منتشر شده است.</sub>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
+7
-13
@@ -9,8 +9,7 @@
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_FA.md">`فارسی`</a>
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -111,7 +110,7 @@ Telusuri [**`dokumentasi Nuclei lengkap di sini`**](https://docs.projectdiscover
|
||||
|
||||
### Instalasi
|
||||
|
||||
`nuclei` memerlukan **go >= 1.27.3** agar dapat diinstal dengan sukses. Jalankan perintah berikut untuk mengambil repo:
|
||||
`nuclei` memerlukan **go >= 1.24.2** agar dapat diinstal dengan sukses. Jalankan perintah berikut untuk mengambil repo:
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -175,7 +174,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
@@ -251,7 +250,6 @@ CONFIGURATIONS:
|
||||
-sip, -source-ip string source ip address to use for network scan
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
@@ -318,10 +316,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-stp, -strict-probe skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
|
||||
-tf, -tech-filter opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
@@ -355,11 +350,10 @@ DEBUG:
|
||||
-hc, -health-check run diagnostic check up
|
||||
|
||||
UPDATE:
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-utc, -update-templates-channel string nuclei-templates update channel (stable, latest) (default "stable")
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
|
||||
+7
-13
@@ -9,8 +9,7 @@
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_FA.md">`فارسی`</a>
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -111,7 +110,7 @@ Nuclei の完全な[**`ドキュメントはこちら`**](https://docs.projectdi
|
||||
|
||||
### インストール
|
||||
|
||||
`nuclei` のインストールには **go >= 1.27.3** が必要です。以下のコマンドを実行してリポジトリを取得してください:
|
||||
`nuclei` のインストールには **go >= 1.24.2** が必要です。以下のコマンドを実行してリポジトリを取得してください:
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -175,7 +174,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
@@ -251,7 +250,6 @@ CONFIGURATIONS:
|
||||
-sip, -source-ip string source ip address to use for network scan
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
@@ -318,10 +316,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-stp, -strict-probe skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
|
||||
-tf, -tech-filter opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
@@ -355,11 +350,10 @@ DEBUG:
|
||||
-hc, -health-check run diagnostic check up
|
||||
|
||||
UPDATE:
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-utc, -update-templates-channel string nuclei-templates update channel (stable, latest) (default "stable")
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
|
||||
+7
-13
@@ -9,8 +9,7 @@
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_FA.md">`فارسی`</a>
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -111,7 +110,7 @@ Nuclei 전체 [**`문서를 여기에서`**](https://docs.projectdiscovery.io/to
|
||||
|
||||
### 설치
|
||||
|
||||
`nuclei` 설치에는 **go >= 1.27.3** 가 필요합니다. 다음 명령어를 실행하여 저장소를 가져옵니다:
|
||||
`nuclei` 설치에는 **go >= 1.24.2** 가 필요합니다. 다음 명령어를 실행하여 저장소를 가져옵니다:
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -175,7 +174,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
@@ -251,7 +250,6 @@ CONFIGURATIONS:
|
||||
-sip, -source-ip string source ip address to use for network scan
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
@@ -318,10 +316,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-stp, -strict-probe skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
|
||||
-tf, -tech-filter opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
@@ -355,11 +350,10 @@ DEBUG:
|
||||
-hc, -health-check run diagnostic check up
|
||||
|
||||
UPDATE:
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-utc, -update-templates-channel string nuclei-templates update channel (stable, latest) (default "stable")
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
|
||||
+7
-13
@@ -9,8 +9,7 @@
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_FA.md">`فارسی`</a>
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -111,7 +110,7 @@ Acesse a [**`documentação completa do Nuclei aqui`**](https://docs.projectdisc
|
||||
|
||||
### Instalação
|
||||
|
||||
O `nuclei` requer **go >= 1.27.3** para instalar com sucesso. Execute o seguinte comando para obter o repositório:
|
||||
O `nuclei` requer **go >= 1.24.2** para instalar com sucesso. Execute o seguinte comando para obter o repositório:
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -175,7 +174,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
@@ -251,7 +250,6 @@ CONFIGURATIONS:
|
||||
-sip, -source-ip string source ip address to use for network scan
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
@@ -318,10 +316,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-stp, -strict-probe skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
|
||||
-tf, -tech-filter opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
@@ -355,11 +350,10 @@ DEBUG:
|
||||
-hc, -health-check run diagnostic check up
|
||||
|
||||
UPDATE:
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-utc, -update-templates-channel string nuclei-templates update channel (stable, latest) (default "stable")
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
|
||||
+7
-13
@@ -9,8 +9,7 @@
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a> •
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_FA.md">`فارسی`</a>
|
||||
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_TR.md">`Türkçe`</a>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -111,7 +110,7 @@ Nuclei'nin tam [**`dokümantasyonuna buradan`**](https://docs.projectdiscovery.i
|
||||
|
||||
### Kurulum
|
||||
|
||||
`nuclei` yüklemek için **go >= 1.27.3** gerektirir. Repoyu almak için aşağıdaki komutu çalıştırın:
|
||||
`nuclei` yüklemek için **go >= 1.24.2** gerektirir. Repoyu almak için aşağıdaki komutu çalıştırın:
|
||||
|
||||
```sh
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
|
||||
@@ -175,7 +174,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
@@ -251,7 +250,6 @@ CONFIGURATIONS:
|
||||
-sip, -source-ip string source ip address to use for network scan
|
||||
-rsr, -response-size-read int max response size to read in bytes
|
||||
-rss, -response-size-save int max response size to read in bytes (default 1048576)
|
||||
-rqss, -request-size-save int max request body size to save in bytes (0 to save the whole body) (default 32768)
|
||||
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
|
||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||
-hae, -http-api-endpoint string experimental http api endpoint
|
||||
@@ -318,10 +316,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-stp, -strict-probe skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
|
||||
-tf, -tech-filter opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
@@ -355,11 +350,10 @@ DEBUG:
|
||||
-hc, -health-check run diagnostic check up
|
||||
|
||||
UPDATE:
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-utc, -update-templates-channel string nuclei-templates update channel (stable, latest) (default "stable")
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
-up, -update update nuclei engine to the latest released version
|
||||
-ut, -update-templates update nuclei-templates to latest released version
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
|
||||
@@ -1011,7 +1011,6 @@ Part Definitions:
|
||||
- <code>headers_from_response</code> - HTTP response headers in name:value format
|
||||
- <code>tls_version</code> - TLS version negotiated for the HTTP connection
|
||||
- <code>cipher</code> - TLS cipher suite negotiated for the HTTP connection
|
||||
- <code>key_exchange</code> - TLS key exchange group negotiated for the HTTP connection
|
||||
- <code>sni</code> - SNI value used in the TLS handshake
|
||||
- <code>subject_cn</code> - Leaf certificate subject common name
|
||||
- <code>subject_dn</code> - Leaf certificate subject distinguished name
|
||||
@@ -4145,7 +4144,6 @@ Part Definitions:
|
||||
- <code>issuer_cn</code> - Issuer CN is the common name of the certificate issuer
|
||||
- <code>issuer_dn</code> - Issuer DN is the distinguished name of the certificate issuer
|
||||
- <code>issuer_org</code> - Issuer organization is the organization of the certificate issuer
|
||||
- <code>key_exchange</code> - Key exchange is the TLS key exchange group negotiated in the handshake
|
||||
- <code>not_after</code> - Timestamp after which the remote cert expires
|
||||
- <code>not_before</code> - Timestamp before which the certificate is not valid
|
||||
- <code>probe_status</code> - Probe status indicates if the probe was successful
|
||||
|
||||
@@ -32,8 +32,6 @@ AllowdTypes = "AllowdTypes"
|
||||
ExludedDastTmplStats = "ExludedDastTmplStats"
|
||||
|
||||
[default.extend-words]
|
||||
# SMTP greeting command
|
||||
HELO = "HELO"
|
||||
# External package API (fileutil.IsWriteable)
|
||||
Writeable = "Writeable" # TODO(dwisiswant0): Fix in upstream.
|
||||
# Variable name used for flow annotations
|
||||
|
||||
+1
-9
@@ -244,8 +244,6 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// readConfig defines the cli flags and loads them from the command line and
|
||||
// config files.
|
||||
func readConfig() *goflags.FlagSet {
|
||||
var selectedFiles configuration.Selection
|
||||
// when true updates nuclei binary to latest version
|
||||
@@ -309,7 +307,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVarP(&options.EnableGlobalMatchersTemplates, "enable-global-matchers", "egm", false, "enable loading global matchers templates"),
|
||||
flagSet.BoolVar(&options.EnableFileTemplates, "file", false, "enable loading file templates"),
|
||||
flagSet.BoolVar(&options.EnableLLM, "llm", false, "enable llm matchers and extractors (semantic matching)"),
|
||||
flagSet.StringVar(&options.LLMProvider, "llm-provider", "openai", "llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter)"),
|
||||
flagSet.StringVar(&options.LLMProvider, "llm-provider", "openai", "llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together)"),
|
||||
flagSet.StringVar(&options.LLMBaseURL, "llm-base-url", "", "openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)"),
|
||||
flagSet.StringVar(&options.LLMModel, "llm-model", "", "model used for semantic matching"),
|
||||
flagSet.IntVar(&options.LLMTimeout, "llm-timeout", 30, "time in seconds to wait for a single llm call"),
|
||||
@@ -385,11 +383,9 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVarP(&options.RestrictLocalNetworkAccess, "restrict-local-network-access", "lna", false, "blocks connections to the local / private network"),
|
||||
flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"),
|
||||
flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"),
|
||||
flagSet.BoolVarP(&options.PayloadRandomOrder, "payload-random-order", "pro", false, "iterate payload combinations in a random order instead of sequentially"),
|
||||
flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
|
||||
flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
|
||||
flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"),
|
||||
flagSet.IntVarP(&options.RequestSaveSize, "request-size-save", "rqss", 32*unitutils.Kilo, "max request body size to save in bytes (0 to save the whole body)"),
|
||||
flagSet.CallbackVar(func() { resetRequested = true }, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
|
||||
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
|
||||
flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
|
||||
@@ -466,10 +462,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
scanstrategy.TemplateSpray.String(): goflags.EnumVariable(2),
|
||||
}),
|
||||
flagSet.DurationVarP(&options.InputReadTimeout, "input-read-timeout", "irt", time.Duration(3*time.Minute), "timeout on input read"),
|
||||
flagSet.BoolVar(&options.ResponseCache, "response-cache", false, "reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)"),
|
||||
flagSet.BoolVarP(&options.DisableHTTPProbe, "no-httpx", "nh", false, "disable httpx probing for non-url input"),
|
||||
flagSet.BoolVarP(&options.StrictProbe, "strict-probe", "stp", false, "skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)"),
|
||||
flagSet.BoolVarP(&options.TechFilter, "tech-filter", "tf", false, "opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)"),
|
||||
flagSet.BoolVar(&options.PreflightPortScan, "preflight-portscan", false, "run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)"),
|
||||
flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "disable stdin processing"),
|
||||
flagSet.BoolVar(&options.EnableHTTPCache, "http-cache", false, "enable HTTP cache (RFC 9111) for HTTP requests"),
|
||||
@@ -510,7 +503,6 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.CreateGroup("update", "Update",
|
||||
flagSet.BoolVarP(&updateNucleiBinary, "update", "up", false, "update nuclei engine to the latest released version"),
|
||||
flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"),
|
||||
flagSet.StringVarP(&options.UpdateTemplatesChannel, "update-templates-channel", "utc", installer.TemplatesChannelStable, "nuclei-templates update channel (stable, latest)"),
|
||||
flagSet.StringVarP(&options.NewTemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"),
|
||||
flagSet.BoolVarP(&disableUpdateCheck, "disable-update-check", "duc", false, "disable automatic nuclei/templates update check"),
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/projectdiscovery/nuclei/v3
|
||||
|
||||
go 1.27.3
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
carvel.dev/ytt v0.52.0
|
||||
@@ -76,7 +76,7 @@ require (
|
||||
github.com/praetorian-inc/fingerprintx v1.1.15
|
||||
github.com/projectdiscovery/clistats v0.1.7
|
||||
github.com/projectdiscovery/dsl v0.8.24
|
||||
github.com/projectdiscovery/fastdialer v0.6.0
|
||||
github.com/projectdiscovery/fastdialer v0.5.26
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2
|
||||
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c
|
||||
github.com/projectdiscovery/goflags v0.2.1
|
||||
@@ -92,7 +92,7 @@ require (
|
||||
github.com/projectdiscovery/interactsh v1.4.1
|
||||
github.com/projectdiscovery/mapcidr v1.1.97
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
|
||||
github.com/projectdiscovery/networkpolicy v0.1.56
|
||||
github.com/projectdiscovery/networkpolicy v0.1.55
|
||||
github.com/projectdiscovery/ratelimit v0.0.94
|
||||
github.com/projectdiscovery/rawhttp v0.2.0
|
||||
github.com/projectdiscovery/rdap v0.9.0
|
||||
@@ -102,7 +102,7 @@ require (
|
||||
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
|
||||
github.com/projectdiscovery/uncover v1.2.1
|
||||
github.com/projectdiscovery/useragent v0.0.109
|
||||
github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5
|
||||
github.com/projectdiscovery/utils v0.11.11-0.20261020211236-cde16c1381ec
|
||||
github.com/projectdiscovery/wappalyzergo v0.3.5
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.7
|
||||
github.com/redis/go-redis/v9 v9.19.0
|
||||
@@ -263,7 +263,7 @@ require (
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.9.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.22.5 // indirect
|
||||
@@ -349,7 +349,7 @@ require (
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/moby/api v1.54.2 // indirect
|
||||
github.com/moby/moby/client v0.4.1 // indirect
|
||||
github.com/moby/sys/user v0.4.1 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -445,11 +445,11 @@ require (
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
|
||||
go.opentelemetry.io/otel v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
go.uber.org/zap/exp v0.3.0 // indirect
|
||||
|
||||
@@ -470,8 +470,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
@@ -848,8 +848,8 @@ github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg=
|
||||
github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY=
|
||||
github.com/moby/moby/client v0.4.1/go.mod h1:z52C9O2POPOsnxZAy//WtKcQ32P+jT/NGeXu/7nfjGQ=
|
||||
github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0=
|
||||
github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y=
|
||||
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@@ -937,8 +937,8 @@ github.com/projectdiscovery/clistats v0.1.7 h1:NXf4lUzx1QcbJd6QycNwwYHTv2FsW9Mch
|
||||
github.com/projectdiscovery/clistats v0.1.7/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8=
|
||||
github.com/projectdiscovery/dsl v0.8.24 h1:rN2w6LSIjA16qKGRjyj70oxdwXGaspe6sAjEkP7F2fk=
|
||||
github.com/projectdiscovery/dsl v0.8.24/go.mod h1:V+EludvS5PS19UNMlM3CuBwZ+sFnMaQhCFGxysDUanQ=
|
||||
github.com/projectdiscovery/fastdialer v0.6.0 h1:4ZeFQ6LyRH4D8e3qO/yzqIP4Cf9Fr0DduaVpJD6ENnE=
|
||||
github.com/projectdiscovery/fastdialer v0.6.0/go.mod h1:FpHwa9S5PUgfPbh+irOm+cdU6AO631XidxkhPwPv7yY=
|
||||
github.com/projectdiscovery/fastdialer v0.5.26 h1:wCCsopLsO1EF1V1+9Wqkxj1bMm6sRo8NdlzwE2Ib2Ss=
|
||||
github.com/projectdiscovery/fastdialer v0.5.26/go.mod h1:dZzEPnpsxqCiD3IkM5CN3RglW3l1gXJNyTkxqjPZIbs=
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
|
||||
github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
|
||||
@@ -977,8 +977,8 @@ github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8Qu
|
||||
github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA=
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
|
||||
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.56 h1:M/L421HFuN707oX0tLTzKZY46uRyaKwxAeyi0q8PlxI=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.56/go.mod h1:N5bXm8L3UsDw51CeA1nhTsGEbQMbgL5lXTiIU013xd0=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.55 h1:D30pipxOdWnSq3r7sX0kQvGyb18sUgopuda1j9NAmnQ=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.55/go.mod h1:N5bXm8L3UsDw51CeA1nhTsGEbQMbgL5lXTiIU013xd0=
|
||||
github.com/projectdiscovery/ratelimit v0.0.94 h1:2qciSuUxarn73YGICZeprZtjrwyAH+0iVpdKzKCXoXE=
|
||||
github.com/projectdiscovery/ratelimit v0.0.94/go.mod h1:GAerX6s+eWRuACedHRv2/xmMtlW+VahZmbRstQh0sS0=
|
||||
github.com/projectdiscovery/rawhttp v0.2.0 h1:TGiHC5hydL97/zoi0WjEuNrfO6Gjtg9LeXndUf1Cq3M=
|
||||
@@ -999,8 +999,8 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
|
||||
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
|
||||
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
|
||||
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
|
||||
github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5 h1:AOspf7ScbwZr8A2YnjCFd4ZJAMSFqzfEaIJIbJPuu10=
|
||||
github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
|
||||
github.com/projectdiscovery/utils v0.11.11-0.20261020211236-cde16c1381ec h1:VhJiivJR8ljv5Qk3QZQC9og50wspEBB5qkNfdyqNhws=
|
||||
github.com/projectdiscovery/utils v0.11.11-0.20261020211236-cde16c1381ec/go.mod h1:T7KzyElivWJiPP+B4sXqkp1CcFe6O/P0qBn/MjDNa74=
|
||||
github.com/projectdiscovery/wappalyzergo v0.3.5 h1:UZradYF1GIcruSIV5wwo/Coph9/VnM17oajLD9eXR2U=
|
||||
github.com/projectdiscovery/wappalyzergo v0.3.5/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
|
||||
@@ -1284,20 +1284,20 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.6
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo=
|
||||
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
|
||||
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw=
|
||||
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
|
||||
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
|
||||
go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns=
|
||||
go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
|
||||
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
|
||||
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA=
|
||||
go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
|
||||
@@ -1,445 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
syncutil "github.com/projectdiscovery/utils/sync"
|
||||
)
|
||||
|
||||
const hostGroupProbeWorkers = 100
|
||||
|
||||
// hostGroup is a set of targets that share the same open-port / HTTP signature.
|
||||
// Used for savings estimates / logging; execution itself is a single spray pass
|
||||
// with a per-(template,target) filter derived from the same probe data.
|
||||
type hostGroup struct {
|
||||
key string
|
||||
hosts []*contextargs.MetaInput
|
||||
openPorts map[string]struct{}
|
||||
httpOK bool
|
||||
explicitPort string // operator host:port; network dials this, not template defaults
|
||||
}
|
||||
|
||||
type hostProbeResult struct {
|
||||
input *contextargs.MetaInput
|
||||
openPorts []string
|
||||
httpOK bool
|
||||
explicitPort string
|
||||
}
|
||||
|
||||
// hostReachabilityIndex maps each input to its probed reachability so the
|
||||
// engine can skip impossible template×target pairs in a single Execute pass.
|
||||
type hostReachabilityIndex struct {
|
||||
byInput map[string]hostReachability
|
||||
}
|
||||
|
||||
type hostReachability struct {
|
||||
openPorts map[string]struct{}
|
||||
httpOK bool
|
||||
// explicitPort is the operator-specified port from host:port input (if any).
|
||||
// Network execution keeps this port (UseNetworkPort does not replace bare
|
||||
// host:port), so reachability must not require the template's default port.
|
||||
explicitPort string
|
||||
}
|
||||
|
||||
// Allow reports whether template may produce a finding on input (lossless).
|
||||
func (idx *hostReachabilityIndex) Allow(t *templates.Template, mi *contextargs.MetaInput) bool {
|
||||
if idx == nil || t == nil || mi == nil {
|
||||
return true
|
||||
}
|
||||
h, ok := idx.byInput[mi.Input]
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
return templateAllowedOnHost(t, h.httpOK, h.openPorts, h.explicitPort)
|
||||
}
|
||||
|
||||
func templateAllowedOnHost(t *templates.Template, httpOK bool, openPorts map[string]struct{}, explicitPort string) bool {
|
||||
if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) {
|
||||
return true
|
||||
}
|
||||
if isWebTemplate(t) {
|
||||
return httpOK
|
||||
}
|
||||
ports, ok := tcpNetworkOnlyPorts(t)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
// host:port inputs: dial the operator-chosen port at execution time, not the
|
||||
// template default (see contextargs.UseNetworkPort). If that port is
|
||||
// reachable, network templates may run against it.
|
||||
if explicitPort != "" {
|
||||
if _, open := openPorts[explicitPort]; open {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, p := range ports {
|
||||
if _, open := openPorts[p]; open {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// estimateTemplateRequests mirrors core.getRequestCount for scheduling math.
|
||||
func estimateTemplateRequests(tpls []*templates.Template) int {
|
||||
count := 0
|
||||
for _, t := range tpls {
|
||||
if len(t.Workflows) > 0 {
|
||||
continue
|
||||
}
|
||||
if t.TotalRequests > 0 {
|
||||
count += t.TotalRequests
|
||||
continue
|
||||
}
|
||||
count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func hostGroupKey(openPorts []string, httpOK bool) string {
|
||||
ports := append([]string(nil), openPorts...)
|
||||
sort.Strings(ports)
|
||||
http := "nohttp"
|
||||
if httpOK {
|
||||
http = "http"
|
||||
}
|
||||
if len(ports) == 0 {
|
||||
return http + "|-"
|
||||
}
|
||||
return http + "|" + strings.Join(ports, ",")
|
||||
}
|
||||
|
||||
func isWebTemplate(t *templates.Template) bool {
|
||||
return len(t.RequestsHTTP)+len(t.RequestsWithHTTP)+len(t.RequestsHeadless)+len(t.RequestsWebsocket) > 0 &&
|
||||
len(t.RequestsNetwork)+len(t.RequestsWithTCP)+len(t.RequestsDNS)+len(t.RequestsFile)+
|
||||
len(t.RequestsSSL)+len(t.RequestsCode)+len(t.RequestsJavascript)+len(t.Workflows) == 0
|
||||
}
|
||||
|
||||
func isUniversalTemplate(t *templates.Template) bool {
|
||||
if t.SelfContained || len(t.Workflows) > 0 {
|
||||
return true
|
||||
}
|
||||
if _, ok := tcpNetworkOnlyPorts(t); ok {
|
||||
return false
|
||||
}
|
||||
if isWebTemplate(t) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// templatesForHostGroup returns templates that can produce findings on this group.
|
||||
func templatesForHostGroup(all []*templates.Template, g hostGroup) []*templates.Template {
|
||||
out := make([]*templates.Template, 0, len(all))
|
||||
for _, t := range all {
|
||||
if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) {
|
||||
continue
|
||||
}
|
||||
if templateAllowedOnHost(t, g.httpOK, g.openPorts, g.explicitPort) {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func partitionTemplates(all []*templates.Template) (web, network, universal, selfContained []*templates.Template) {
|
||||
for _, t := range all {
|
||||
switch {
|
||||
case t.SelfContained:
|
||||
selfContained = append(selfContained, t)
|
||||
case len(t.Workflows) > 0:
|
||||
universal = append(universal, t)
|
||||
case isWebTemplate(t):
|
||||
web = append(web, t)
|
||||
case isUniversalTemplate(t):
|
||||
universal = append(universal, t)
|
||||
default:
|
||||
network = append(network, t)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// buildHostReachability probes each target once, then returns an index for
|
||||
// single-pass filtering plus host groups for logging/estimates.
|
||||
//
|
||||
// HTTP reachability prefers the already-populated InputHelper.InputsHTTP map
|
||||
// (from initializeTemplatesHTTPInput) so we do not re-run httpx.
|
||||
// When probePorts is false, only explicit input ports are recorded (HTTP-only mode).
|
||||
func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *input.Helper, probePorts bool) (*hostReachabilityIndex, []hostGroup, error) {
|
||||
if r.inputProvider == nil {
|
||||
return nil, nil, fmt.Errorf("no input provider")
|
||||
}
|
||||
|
||||
var portsToProbe []string
|
||||
if probePorts {
|
||||
// Only concrete network-template ports. Do not pull 80/443 from HTTP
|
||||
// templates via portsPopularityFromTemplates — httpOK already comes from
|
||||
// scheme / InputsHTTP, and spraying web ports across every host inflates
|
||||
// probe cost and flakes under parallel dial load.
|
||||
seen := map[string]struct{}{}
|
||||
for _, t := range tpls {
|
||||
ps, ok := tcpNetworkOnlyPorts(t)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, p := range ps {
|
||||
if _, dup := seen[p]; dup {
|
||||
continue
|
||||
}
|
||||
seen[p] = struct{}{}
|
||||
portsToProbe = append(portsToProbe, p)
|
||||
}
|
||||
}
|
||||
sort.Strings(portsToProbe)
|
||||
}
|
||||
|
||||
var targets []*contextargs.MetaInput
|
||||
r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool {
|
||||
targets = append(targets, mi.Clone())
|
||||
return true
|
||||
})
|
||||
if len(targets) == 0 {
|
||||
return &hostReachabilityIndex{byInput: map[string]hostReachability{}}, nil, nil
|
||||
}
|
||||
|
||||
// Only build an httpx client if we still need live HTTP probes.
|
||||
needLiveHTTPProbe := false
|
||||
for _, mi := range targets {
|
||||
if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") {
|
||||
continue
|
||||
}
|
||||
if httpHelper != nil && httpHelper.InputsHTTPProbed {
|
||||
continue // already probed (hit or miss) during initializeTemplatesHTTPInput
|
||||
}
|
||||
needLiveHTTPProbe = true
|
||||
break
|
||||
}
|
||||
|
||||
var httpClient *httpx.HTTPX
|
||||
if needLiveHTTPProbe {
|
||||
dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
|
||||
if dialers != nil {
|
||||
httpxOptions := httpx.DefaultOptions
|
||||
if r.options.AliveHttpProxy != "" {
|
||||
httpxOptions.Proxy = r.options.AliveHttpProxy
|
||||
} else if r.options.AliveSocksProxy != "" {
|
||||
httpxOptions.Proxy = r.options.AliveSocksProxy
|
||||
}
|
||||
httpxOptions.RetryMax = r.options.Retries
|
||||
if r.options.Timeout > 0 {
|
||||
httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
|
||||
}
|
||||
httpxOptions.NetworkPolicy = dialers.NetworkPolicy
|
||||
if c, err := httpx.New(&httpxOptions); err == nil {
|
||||
httpClient = c
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
workers := hostGroupProbeWorkers
|
||||
// Fan out one task per host:port so closed-port timeouts do not serialize per host.
|
||||
type probeJob struct {
|
||||
hostIdx int
|
||||
host string
|
||||
port string
|
||||
}
|
||||
var jobs []probeJob
|
||||
hostMeta := make([]struct {
|
||||
mi *contextargs.MetaInput
|
||||
host string
|
||||
explicitPort string
|
||||
}, len(targets))
|
||||
|
||||
for i, mi := range targets {
|
||||
host, explicitPort := hostAndExplicitPort(mi.Target())
|
||||
if host == "" {
|
||||
host, explicitPort = hostAndExplicitPort(mi.Input)
|
||||
}
|
||||
hostMeta[i].mi = mi
|
||||
hostMeta[i].host = host
|
||||
hostMeta[i].explicitPort = explicitPort
|
||||
if !probePorts {
|
||||
continue
|
||||
}
|
||||
// Always probe the operator-specified port (may differ from template
|
||||
// defaults). Do not spray unrelated template ports onto host:port inputs.
|
||||
if explicitPort != "" && isNumericPort(explicitPort) {
|
||||
jobs = append(jobs, probeJob{hostIdx: i, host: host, port: explicitPort})
|
||||
continue
|
||||
}
|
||||
for _, p := range portsToProbe {
|
||||
jobs = append(jobs, probeJob{hostIdx: i, host: host, port: p})
|
||||
}
|
||||
}
|
||||
|
||||
if len(jobs) < workers {
|
||||
workers = len(jobs)
|
||||
}
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
|
||||
openByHost := make([]map[string]struct{}, len(targets))
|
||||
for i := range openByHost {
|
||||
openByHost[i] = map[string]struct{}{}
|
||||
// Only assume explicit ports are open when we are not probing them.
|
||||
if !probePorts {
|
||||
if ep := hostMeta[i].explicitPort; ep != "" && isNumericPort(ep) {
|
||||
openByHost[i][ep] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(jobs) > 0 {
|
||||
swg, err := syncutil.New(syncutil.WithSize(workers))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var mu sync.Mutex
|
||||
for _, job := range jobs {
|
||||
swg.Add()
|
||||
go func(job probeJob) {
|
||||
defer swg.Done()
|
||||
switch r.probe(job.host, job.port) {
|
||||
case portOpen, portUnknown:
|
||||
mu.Lock()
|
||||
openByHost[job.hostIdx][job.port] = struct{}{}
|
||||
mu.Unlock()
|
||||
}
|
||||
}(job)
|
||||
}
|
||||
swg.Wait()
|
||||
}
|
||||
|
||||
results := make([]hostProbeResult, len(targets))
|
||||
for i, meta := range hostMeta {
|
||||
open := openByHost[i]
|
||||
ports := make([]string, 0, len(open))
|
||||
for p := range open {
|
||||
ports = append(ports, p)
|
||||
}
|
||||
sort.Strings(ports)
|
||||
results[i] = hostProbeResult{
|
||||
input: meta.mi,
|
||||
openPorts: ports,
|
||||
httpOK: resolveHTTPReachability(meta.mi, open, httpHelper, httpClient),
|
||||
explicitPort: meta.explicitPort,
|
||||
}
|
||||
}
|
||||
|
||||
idx := &hostReachabilityIndex{byInput: make(map[string]hostReachability, len(results))}
|
||||
grouped := map[string]*hostGroup{}
|
||||
order := make([]string, 0)
|
||||
for _, res := range results {
|
||||
if res.input == nil {
|
||||
continue
|
||||
}
|
||||
portsSet := make(map[string]struct{}, len(res.openPorts))
|
||||
for _, p := range res.openPorts {
|
||||
portsSet[p] = struct{}{}
|
||||
}
|
||||
idx.byInput[res.input.Input] = hostReachability{
|
||||
openPorts: portsSet,
|
||||
httpOK: res.httpOK,
|
||||
explicitPort: res.explicitPort,
|
||||
}
|
||||
|
||||
key := hostGroupKey(res.openPorts, res.httpOK)
|
||||
if res.explicitPort != "" {
|
||||
// Keep explicit host:port inputs in their own estimate bucket so
|
||||
// network-template fan-out is not attributed to bare hosts.
|
||||
key = key + "|ep:" + res.explicitPort
|
||||
}
|
||||
g, ok := grouped[key]
|
||||
if !ok {
|
||||
g = &hostGroup{
|
||||
key: key,
|
||||
openPorts: portsSet,
|
||||
httpOK: res.httpOK,
|
||||
explicitPort: res.explicitPort,
|
||||
}
|
||||
grouped[key] = g
|
||||
order = append(order, key)
|
||||
}
|
||||
g.hosts = append(g.hosts, res.input)
|
||||
}
|
||||
|
||||
out := make([]hostGroup, 0, len(order))
|
||||
for _, key := range order {
|
||||
out = append(out, *grouped[key])
|
||||
}
|
||||
return idx, out, nil
|
||||
}
|
||||
|
||||
func resolveHTTPReachability(mi *contextargs.MetaInput, open map[string]struct{}, helper *input.Helper, client *httpx.HTTPX) bool {
|
||||
if mi == nil {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") {
|
||||
return true
|
||||
}
|
||||
// Reuse httpx results from initializeTemplatesHTTPInput when available.
|
||||
if helper != nil && helper.InputsHTTP != nil {
|
||||
if probed, ok := helper.InputsHTTP.Get(mi.Input); ok && len(probed) > 0 {
|
||||
return true
|
||||
}
|
||||
// Empty/skipped probe map (e.g. MultiFormat) must not force non-HTTP.
|
||||
if !helper.InputsHTTPProbed {
|
||||
// fall through to live probe / open ports
|
||||
} else {
|
||||
// Map was built for all inputs: absence means not HTTP.
|
||||
return false
|
||||
}
|
||||
}
|
||||
if client != nil {
|
||||
return utils.ProbeURL(mi.Input, client) != ""
|
||||
}
|
||||
if _, ok := open["80"]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := open["443"]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// estimateGroupedExecutions returns baseline and filtered template×host counts.
|
||||
func estimateGroupedExecutions(all []*templates.Template, groups []hostGroup, hostCount int) (baseline, filtered int) {
|
||||
baseline = estimateTemplateRequests(all) * hostCount
|
||||
_, _, universal, selfContained := partitionTemplates(all)
|
||||
once := append([]*templates.Template{}, selfContained...)
|
||||
once = append(once, universal...)
|
||||
filtered = estimateTemplateRequests(once) * hostCount
|
||||
for _, g := range groups {
|
||||
filtered += estimateTemplateRequests(templatesForHostGroup(all, g)) * len(g.hosts)
|
||||
}
|
||||
return baseline, filtered
|
||||
}
|
||||
|
||||
// countReachabilityStats returns concrete network template count and distinct numeric ports to probe.
|
||||
func countReachabilityStats(tpls []*templates.Template) (concreteNetwork int, portsToProbe int) {
|
||||
ports := map[string]struct{}{}
|
||||
for _, t := range tpls {
|
||||
ps, ok := tcpNetworkOnlyPorts(t)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
concreteNetwork++
|
||||
for _, p := range ps {
|
||||
ports[p] = struct{}{}
|
||||
}
|
||||
}
|
||||
return concreteNetwork, len(ports)
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHostGroupKey(t *testing.T) {
|
||||
// Lexicographic sort: "443" < "80".
|
||||
require.Equal(t, "http|443,80", hostGroupKey([]string{"443", "80"}, true))
|
||||
require.Equal(t, "nohttp|22", hostGroupKey([]string{"22"}, false))
|
||||
require.Equal(t, "nohttp|-", hostGroupKey(nil, false))
|
||||
}
|
||||
|
||||
func TestTemplatesForHostGroup(t *testing.T) {
|
||||
web := &templates.Template{
|
||||
ID: "web",
|
||||
RequestsHTTP: []*http.Request{{}},
|
||||
}
|
||||
ftp := &templates.Template{
|
||||
ID: "ftp",
|
||||
RequestsNetwork: []*network.Request{{
|
||||
Port: "21",
|
||||
}},
|
||||
}
|
||||
ssh := &templates.Template{
|
||||
ID: "ssh",
|
||||
RequestsNetwork: []*network.Request{{
|
||||
Port: "22",
|
||||
}},
|
||||
}
|
||||
all := []*templates.Template{web, ftp, ssh}
|
||||
|
||||
webGroup := hostGroup{
|
||||
key: "http|80,443",
|
||||
httpOK: true,
|
||||
openPorts: map[string]struct{}{"80": {}, "443": {}},
|
||||
}
|
||||
got := templatesForHostGroup(all, webGroup)
|
||||
require.Len(t, got, 1)
|
||||
require.Equal(t, "web", got[0].ID)
|
||||
|
||||
ftpGroup := hostGroup{
|
||||
key: "nohttp|21",
|
||||
httpOK: false,
|
||||
openPorts: map[string]struct{}{"21": {}},
|
||||
}
|
||||
got = templatesForHostGroup(all, ftpGroup)
|
||||
require.Len(t, got, 1)
|
||||
require.Equal(t, "ftp", got[0].ID)
|
||||
|
||||
sshGroup := hostGroup{
|
||||
key: "nohttp|22",
|
||||
httpOK: false,
|
||||
openPorts: map[string]struct{}{"22": {}},
|
||||
}
|
||||
got = templatesForHostGroup(all, sshGroup)
|
||||
require.Len(t, got, 1)
|
||||
require.Equal(t, "ssh", got[0].ID)
|
||||
}
|
||||
|
||||
func TestPartitionTemplates(t *testing.T) {
|
||||
web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
|
||||
ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}}
|
||||
self := &templates.Template{ID: "self", SelfContained: true}
|
||||
dynNet := &templates.Template{
|
||||
ID: "dyn",
|
||||
RequestsNetwork: []*network.Request{{Port: ""}},
|
||||
}
|
||||
|
||||
w, n, u, s := partitionTemplates([]*templates.Template{web, ftp, self, dynNet})
|
||||
require.Len(t, w, 1)
|
||||
require.Equal(t, "web", w[0].ID)
|
||||
require.Len(t, n, 1)
|
||||
require.Equal(t, "ftp", n[0].ID)
|
||||
require.Len(t, u, 1)
|
||||
require.Equal(t, "dyn", u[0].ID)
|
||||
require.Len(t, s, 1)
|
||||
require.Equal(t, "self", s[0].ID)
|
||||
}
|
||||
|
||||
func TestEstimateGroupedVsBaseline(t *testing.T) {
|
||||
web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}, TotalRequests: 1}
|
||||
ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}, TotalRequests: 1}
|
||||
ssh := &templates.Template{ID: "ssh", RequestsNetwork: []*network.Request{{Port: "22"}}, TotalRequests: 1}
|
||||
all := []*templates.Template{web, ftp, ssh}
|
||||
|
||||
groups := []hostGroup{
|
||||
{key: "http|80", httpOK: true, openPorts: map[string]struct{}{"80": {}}, hosts: metaInputs("h1", "h2")},
|
||||
{key: "nohttp|21", httpOK: false, openPorts: map[string]struct{}{"21": {}}, hosts: metaInputs("h3")},
|
||||
{key: "nohttp|22", httpOK: false, openPorts: map[string]struct{}{"22": {}}, hosts: metaInputs("h4")},
|
||||
}
|
||||
|
||||
baseline, filtered := estimateGroupedExecutions(all, groups, 4)
|
||||
require.Equal(t, 12, baseline)
|
||||
// web*2 + ftp*1 + ssh*1 = 4
|
||||
require.Equal(t, 4, filtered)
|
||||
require.Equal(t, 8, baseline-filtered)
|
||||
}
|
||||
|
||||
func TestHostReachabilityAllow(t *testing.T) {
|
||||
web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
|
||||
ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}}
|
||||
redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
|
||||
idx := &hostReachabilityIndex{byInput: map[string]hostReachability{
|
||||
"h1": {httpOK: true, openPorts: map[string]struct{}{"80": {}}},
|
||||
"h2": {httpOK: false, openPorts: map[string]struct{}{"21": {}}},
|
||||
"127.0.0.1:19637": {httpOK: false, openPorts: map[string]struct{}{"19637": {}}, explicitPort: "19637"},
|
||||
}}
|
||||
require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "h1"}))
|
||||
require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "h2"}))
|
||||
require.False(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h1"}))
|
||||
require.True(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h2"}))
|
||||
require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "unknown"})) // lossless
|
||||
// Explicit host:port must allow network templates even when template default
|
||||
// port (6379) differs from the operator-specified port (19637).
|
||||
require.True(t, idx.Allow(redis, &contextargs.MetaInput{Input: "127.0.0.1:19637"}))
|
||||
require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "127.0.0.1:19637"}))
|
||||
}
|
||||
|
||||
func TestTemplateAllowedOnHostUniversal(t *testing.T) {
|
||||
dnsish := &templates.Template{ID: "dns", RequestsDNS: nil} // no concrete network ports
|
||||
// force universal via empty network dynamic port
|
||||
dyn := &templates.Template{ID: "dyn", RequestsNetwork: []*network.Request{{Port: ""}}}
|
||||
require.True(t, templateAllowedOnHost(dyn, false, map[string]struct{}{}, ""))
|
||||
_ = dnsish
|
||||
self := &templates.Template{ID: "self", SelfContained: true}
|
||||
require.True(t, templateAllowedOnHost(self, false, nil, ""))
|
||||
}
|
||||
|
||||
func TestTemplateAllowedOnHostExplicitPort(t *testing.T) {
|
||||
redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
|
||||
web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
|
||||
open := map[string]struct{}{"19637": {}}
|
||||
require.True(t, templateAllowedOnHost(redis, false, open, "19637"))
|
||||
require.False(t, templateAllowedOnHost(redis, false, open, "")) // bare host: need 6379
|
||||
require.False(t, templateAllowedOnHost(web, false, open, "19637"))
|
||||
// Closed explicit port: openPorts empty => network templates denied.
|
||||
require.False(t, templateAllowedOnHost(redis, false, map[string]struct{}{}, "19637"))
|
||||
}
|
||||
|
||||
func TestTemplatesForHostGroupExplicitPort(t *testing.T) {
|
||||
redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
|
||||
ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}}
|
||||
web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
|
||||
g := hostGroup{
|
||||
httpOK: false,
|
||||
openPorts: map[string]struct{}{"19637": {}},
|
||||
explicitPort: "19637",
|
||||
}
|
||||
got := templatesForHostGroup([]*templates.Template{redis, ftp, web}, g)
|
||||
require.Len(t, got, 2)
|
||||
ids := map[string]bool{got[0].ID: true, got[1].ID: true}
|
||||
require.True(t, ids["redis"] && ids["ftp"])
|
||||
}
|
||||
|
||||
func TestCountReachabilityStats(t *testing.T) {
|
||||
web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
|
||||
redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
|
||||
pg := &templates.Template{ID: "pg", RequestsNetwork: []*network.Request{{Port: "5432,5432"}}}
|
||||
dyn := &templates.Template{ID: "dyn", RequestsNetwork: []*network.Request{{Port: ""}}}
|
||||
concrete, ports := countReachabilityStats([]*templates.Template{web, redis, pg, dyn})
|
||||
require.Equal(t, 2, concrete)
|
||||
require.Equal(t, 2, ports) // 6379 + 5432
|
||||
}
|
||||
|
||||
func TestResolveHTTPReachability(t *testing.T) {
|
||||
require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "http://web"}, nil, nil, nil))
|
||||
require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "https://web"}, nil, nil, nil))
|
||||
require.False(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "redis1"}, map[string]struct{}{}, nil, nil))
|
||||
require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"80": {}}, nil, nil))
|
||||
require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"443": {}}, nil, nil))
|
||||
}
|
||||
|
||||
func TestResolveHTTPReachabilitySkippedProbeFailOpen(t *testing.T) {
|
||||
// MultiFormat leaves an empty InputsHTTP map without probing — must not
|
||||
// force non-HTTP when open ports still suggest HTTP.
|
||||
helper := &input.Helper{InputsHTTPProbed: false}
|
||||
require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"80": {}}, helper, nil))
|
||||
}
|
||||
|
||||
func TestTemplatesForHostGroupSkipsUniversal(t *testing.T) {
|
||||
web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
|
||||
univ := &templates.Template{ID: "univ", RequestsNetwork: []*network.Request{{Port: ""}}}
|
||||
g := hostGroup{httpOK: true, openPorts: map[string]struct{}{"80": {}}}
|
||||
got := templatesForHostGroup([]*templates.Template{web, univ}, g)
|
||||
require.Len(t, got, 1)
|
||||
require.Equal(t, "web", got[0].ID)
|
||||
}
|
||||
|
||||
func TestEstimateGroupedIncludesUniversalsOnce(t *testing.T) {
|
||||
web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}, TotalRequests: 1}
|
||||
univ := &templates.Template{ID: "univ", RequestsNetwork: []*network.Request{{Port: ""}}, TotalRequests: 1}
|
||||
groups := []hostGroup{
|
||||
{httpOK: true, openPorts: map[string]struct{}{"80": {}}, hosts: metaInputs("h1")},
|
||||
}
|
||||
baseline, filtered := estimateGroupedExecutions([]*templates.Template{web, univ}, groups, 1)
|
||||
require.Equal(t, 2, baseline)
|
||||
// universal counted once for all hosts + web for http group
|
||||
require.Equal(t, 2, filtered)
|
||||
}
|
||||
|
||||
func metaInputs(hosts ...string) []*contextargs.MetaInput {
|
||||
out := make([]*contextargs.MetaInput, 0, len(hosts))
|
||||
for _, h := range hosts {
|
||||
mi := contextargs.NewMetaInput()
|
||||
mi.Input = h
|
||||
out = append(out, mi)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/hmap/store/hybrid"
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
@@ -69,15 +68,12 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
|
||||
}
|
||||
|
||||
swg.Add()
|
||||
go func(target *contextargs.MetaInput) {
|
||||
go func(input *contextargs.MetaInput) {
|
||||
defer swg.Done()
|
||||
|
||||
// a pinned target is probed through its ip, and each ip keeps its
|
||||
// own result, as servers of one host may serve different schemes
|
||||
ctx := contextargs.WithCustomIP(context.Background(), target)
|
||||
if result := utils.ProbeURLWithContext(ctx, target.Input, httpxClient); result != "" {
|
||||
if result := utils.ProbeURL(input.Input, httpxClient); result != "" {
|
||||
count.Add(1)
|
||||
_ = hm.Set(input.ProbeKey(target.Input, target.CustomIP), []byte(result))
|
||||
_ = hm.Set(input.Input, []byte(result))
|
||||
}
|
||||
}(value)
|
||||
return true
|
||||
|
||||
+19
-24
@@ -39,30 +39,25 @@ func GetAuthTmplStore(opts *types.Options, catalog catalog.Catalog, execOpts *pr
|
||||
}
|
||||
tmpls = append(tmpls, data...)
|
||||
}
|
||||
// The auth store only wants the secret-file templates. Copy first so
|
||||
// this scoping cannot clear Templates, ExcludeTags, or the other filters
|
||||
// on the caller's options. The CLI already copies; the SDK passes the
|
||||
// engine options directly.
|
||||
scoped := opts.Copy()
|
||||
scoped.Templates = tmpls
|
||||
scoped.Workflows = nil
|
||||
scoped.RemoteTemplateDomainList = nil
|
||||
scoped.TemplateURLs = nil
|
||||
scoped.WorkflowURLs = nil
|
||||
scoped.ExcludedTemplates = nil
|
||||
scoped.Tags = nil
|
||||
scoped.ExcludeTags = nil
|
||||
scoped.IncludeTemplates = nil
|
||||
scoped.Authors = nil
|
||||
scoped.Severities = nil
|
||||
scoped.ExcludeSeverities = nil
|
||||
scoped.IncludeTags = nil
|
||||
scoped.IncludeIds = nil
|
||||
scoped.ExcludeIds = nil
|
||||
scoped.Protocols = nil
|
||||
scoped.ExcludeProtocols = nil
|
||||
scoped.IncludeConditions = nil
|
||||
cfg := loader.NewConfig(scoped, catalog, execOpts)
|
||||
opts.Templates = tmpls
|
||||
opts.Workflows = nil
|
||||
opts.RemoteTemplateDomainList = nil
|
||||
opts.TemplateURLs = nil
|
||||
opts.WorkflowURLs = nil
|
||||
opts.ExcludedTemplates = nil
|
||||
opts.Tags = nil
|
||||
opts.ExcludeTags = nil
|
||||
opts.IncludeTemplates = nil
|
||||
opts.Authors = nil
|
||||
opts.Severities = nil
|
||||
opts.ExcludeSeverities = nil
|
||||
opts.IncludeTags = nil
|
||||
opts.IncludeIds = nil
|
||||
opts.ExcludeIds = nil
|
||||
opts.Protocols = nil
|
||||
opts.ExcludeProtocols = nil
|
||||
opts.IncludeConditions = nil
|
||||
cfg := loader.NewConfig(opts, catalog, execOpts)
|
||||
cfg.StoreId = loader.AuthStoreId
|
||||
store, err := loader.New(cfg)
|
||||
if err != nil {
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/server/proxy"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/installer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
@@ -150,7 +149,7 @@ func ParseOptions(options *types.Options) {
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateOptions validates the configuration options passed
|
||||
// validateOptions validates the configuration options passed
|
||||
func ValidateOptions(options *types.Options) error {
|
||||
if err := validateOptions.Struct(options); err != nil {
|
||||
if _, ok := err.(*validator.InvalidValidationError); ok {
|
||||
@@ -176,11 +175,6 @@ func ValidateOptions(options *types.Options) error {
|
||||
if options.ShouldFollowHTTPRedirects() && options.DisableRedirects {
|
||||
return errors.New("both follow redirects and disable redirects specified")
|
||||
}
|
||||
switch options.UpdateTemplatesChannel {
|
||||
case "", installer.TemplatesChannelStable, installer.TemplatesChannelLatest:
|
||||
default:
|
||||
return fmt.Errorf("invalid templates update channel %q, must be %q or %q", options.UpdateTemplatesChannel, installer.TemplatesChannelStable, installer.TemplatesChannelLatest)
|
||||
}
|
||||
// loading the proxy server list from file or cli and test the connectivity
|
||||
if err := loadProxyServers(options); err != nil {
|
||||
return err
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/installer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -61,15 +60,6 @@ func TestParseHeadlessOptionalArguments(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateOptionsUpdateTemplatesChannel checks that only known templates
|
||||
// update channels are accepted.
|
||||
func TestValidateOptionsUpdateTemplatesChannel(t *testing.T) {
|
||||
for _, channel := range []string{"", installer.TemplatesChannelStable, installer.TemplatesChannelLatest} {
|
||||
require.NoError(t, ValidateOptions(&types.Options{UpdateTemplatesChannel: channel}), "channel %q", channel)
|
||||
}
|
||||
require.ErrorContains(t, ValidateOptions(&types.Options{UpdateTemplatesChannel: "nightly"}), `invalid templates update channel "nightly"`)
|
||||
}
|
||||
|
||||
func TestReadsTargetsFromStdin(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
)
|
||||
|
||||
// planScope narrows the templates run on each target by the scan plan filters
|
||||
// (strict-probe reachability, tech filter), on top of the per-target profiles
|
||||
// in base. It is a protocols.TargetScope, so the engine skips, counts and
|
||||
// narrows clusters for these pairs the same way it does for profiles.
|
||||
type planScope struct {
|
||||
base protocols.TargetScope
|
||||
allow func(*templates.Template, *contextargs.MetaInput) bool
|
||||
// byPath maps a template path to its template. A path shared by several
|
||||
// templates, or empty, is left out, so such templates are never filtered.
|
||||
byPath map[string]*templates.Template
|
||||
|
||||
mu sync.RWMutex
|
||||
selections map[planKey]*planSelection
|
||||
}
|
||||
|
||||
// planKey identifies one selection: the target and the profile selection it
|
||||
// narrows. Profile selections are comparable by contract.
|
||||
type planKey struct {
|
||||
input string
|
||||
ip string
|
||||
base protocols.TemplateSelection
|
||||
}
|
||||
|
||||
// planSelection is the selection of one target.
|
||||
type planSelection struct {
|
||||
base protocols.TemplateSelection
|
||||
scope *planScope
|
||||
input *contextargs.MetaInput
|
||||
}
|
||||
|
||||
var _ protocols.TargetScope = (*planScope)(nil)
|
||||
|
||||
func newPlanScope(base protocols.TargetScope, allow func(*templates.Template, *contextargs.MetaInput) bool, tpls []*templates.Template) *planScope {
|
||||
byPath := make(map[string]*templates.Template, len(tpls))
|
||||
shared := make(map[string]struct{})
|
||||
for _, t := range tpls {
|
||||
if t == nil || t.Path == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := byPath[t.Path]; ok {
|
||||
shared[t.Path] = struct{}{}
|
||||
continue
|
||||
}
|
||||
byPath[t.Path] = t
|
||||
}
|
||||
for path := range shared {
|
||||
delete(byPath, path)
|
||||
}
|
||||
return &planScope{
|
||||
base: base,
|
||||
allow: allow,
|
||||
byPath: byPath,
|
||||
selections: make(map[planKey]*planSelection),
|
||||
}
|
||||
}
|
||||
|
||||
// For returns the selection for input. It is the same value for the same
|
||||
// input, so the engine can group targets by selection.
|
||||
func (s *planScope) For(input *contextargs.MetaInput) protocols.TemplateSelection {
|
||||
var base protocols.TemplateSelection
|
||||
if s.base != nil {
|
||||
base = s.base.For(input)
|
||||
}
|
||||
if input == nil {
|
||||
return base
|
||||
}
|
||||
key := planKey{input: input.Input, ip: input.CustomIP, base: base}
|
||||
s.mu.RLock()
|
||||
sel, ok := s.selections[key]
|
||||
s.mu.RUnlock()
|
||||
if ok {
|
||||
return sel
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if sel, ok := s.selections[key]; ok {
|
||||
return sel
|
||||
}
|
||||
sel = &planSelection{base: base, scope: s, input: input.Clone()}
|
||||
s.selections[key] = sel
|
||||
return sel
|
||||
}
|
||||
|
||||
// Allows reports whether the profiles and the plan filters select the template.
|
||||
func (p *planSelection) Allows(templatePath string) bool {
|
||||
if p.base != nil && !p.base.Allows(templatePath) {
|
||||
return false
|
||||
}
|
||||
t, ok := p.scope.byPath[templatePath]
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
return p.scope.allow(t, p.input)
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// profileScope selects by path, like per-target profiles do.
|
||||
type profileScope struct{ sel *profileSelection }
|
||||
|
||||
func (p profileScope) For(*contextargs.MetaInput) protocols.TemplateSelection { return p.sel }
|
||||
|
||||
type profileSelection struct{ allowed map[string]bool }
|
||||
|
||||
func (p *profileSelection) Allows(path string) bool { return p.allowed[path] }
|
||||
|
||||
func TestPlanScope(t *testing.T) {
|
||||
web := &templates.Template{ID: "web", Path: "/t/web.yaml"}
|
||||
net := &templates.Template{ID: "net", Path: "/t/net.yaml"}
|
||||
dupA := &templates.Template{ID: "dup-a", Path: "/t/dup.yaml"}
|
||||
dupB := &templates.Template{ID: "dup-b", Path: "/t/dup.yaml"}
|
||||
// the filter drops the network template on host-a only
|
||||
allow := func(tpl *templates.Template, mi *contextargs.MetaInput) bool {
|
||||
return mi.Input != "host-a" || tpl.ID == "web"
|
||||
}
|
||||
input := func(raw string) *contextargs.MetaInput {
|
||||
mi := contextargs.NewMetaInput()
|
||||
mi.Input = raw
|
||||
return mi
|
||||
}
|
||||
a, b := input("host-a"), input("host-b")
|
||||
|
||||
scope := newPlanScope(nil, allow, []*templates.Template{web, net, dupA, dupB})
|
||||
selA := scope.For(a)
|
||||
require.Same(t, selA, scope.For(input("host-a")), "one selection per target, so the engine can group targets")
|
||||
require.True(t, web.SelectedBy(selA))
|
||||
require.False(t, net.SelectedBy(selA))
|
||||
require.True(t, dupA.SelectedBy(selA), "a path shared by several templates is never filtered")
|
||||
require.True(t, net.SelectedBy(scope.For(b)))
|
||||
require.True(t, (&templates.Template{Path: "/t/unknown.yaml"}).SelectedBy(selA), "unknown templates are never filtered")
|
||||
|
||||
// The plan narrows the profiles, never widens them.
|
||||
profiles := profileScope{sel: &profileSelection{allowed: map[string]bool{"/t/net.yaml": true}}}
|
||||
scope = newPlanScope(profiles, allow, []*templates.Template{web, net})
|
||||
require.False(t, web.SelectedBy(scope.For(b)), "a template the profile leaves out stays out")
|
||||
require.True(t, net.SelectedBy(scope.For(b)))
|
||||
require.False(t, net.SelectedBy(scope.For(a)))
|
||||
}
|
||||
@@ -14,25 +14,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
HTTP_PROXY_ENV = "HTTP_PROXY"
|
||||
HTTPS_PROXY_ENV = "HTTPS_PROXY"
|
||||
HTTP_PROXY_ENV = "HTTP_PROXY"
|
||||
)
|
||||
|
||||
var proxyEnvVars = []string{
|
||||
"HTTP_PROXY",
|
||||
"HTTPS_PROXY",
|
||||
"http_proxy",
|
||||
"https_proxy",
|
||||
"ALL_PROXY",
|
||||
"all_proxy",
|
||||
}
|
||||
|
||||
func setProxyEnv(proxyURL string) {
|
||||
for _, env := range proxyEnvVars {
|
||||
_ = os.Setenv(env, proxyURL)
|
||||
}
|
||||
}
|
||||
|
||||
// loadProxyServers load list of proxy servers from file or comma separated
|
||||
func loadProxyServers(options *types.Options) error {
|
||||
if len(options.Proxy) == 0 {
|
||||
@@ -69,19 +53,15 @@ func loadProxyServers(options *types.Options) error {
|
||||
return errkit.Wrapf(err, "failed to parse proxy got %v", err)
|
||||
}
|
||||
if options.ProxyInternal {
|
||||
setProxyEnv(proxyURL.String())
|
||||
_ = os.Setenv(HTTP_PROXY_ENV, proxyURL.String())
|
||||
}
|
||||
switch proxyURL.Scheme {
|
||||
case proxyutils.HTTP, proxyutils.HTTPS:
|
||||
if options.Logger != nil {
|
||||
options.Logger.Verbose().Msgf("Using %s as proxy server", proxyURL.String())
|
||||
}
|
||||
options.Logger.Verbose().Msgf("Using %s as proxy server", proxyURL.String())
|
||||
options.AliveHttpProxy = proxyURL.String()
|
||||
case proxyutils.SOCKS5:
|
||||
options.AliveSocksProxy = proxyURL.String()
|
||||
if options.Logger != nil {
|
||||
options.Logger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String())
|
||||
}
|
||||
options.Logger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,14 +1,8 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -20,78 +14,3 @@ func TestValidateReplayProxy(t *testing.T) {
|
||||
require.Error(t, validateReplayProxy(invalid), invalid)
|
||||
}
|
||||
}
|
||||
|
||||
type envState struct {
|
||||
val string
|
||||
set bool
|
||||
}
|
||||
|
||||
func saveAndRestoreProxyEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
original := make(map[string]envState, len(proxyEnvVars))
|
||||
for _, env := range proxyEnvVars {
|
||||
val, set := os.LookupEnv(env)
|
||||
original[env] = envState{val: val, set: set}
|
||||
_ = os.Unsetenv(env)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
for env, state := range original {
|
||||
if state.set {
|
||||
_ = os.Setenv(env, state.val)
|
||||
} else {
|
||||
_ = os.Unsetenv(env)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSetProxyEnv(t *testing.T) {
|
||||
saveAndRestoreProxyEnv(t)
|
||||
|
||||
testProxy := "http://127.0.0.1:8080"
|
||||
setProxyEnv(testProxy)
|
||||
|
||||
for _, env := range proxyEnvVars {
|
||||
require.Equal(t, testProxy, os.Getenv(env), "expected %s to be set", env)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadProxyServers_ProxyInternal(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
t.Run("proxy internal enabled sets all env vars", func(t *testing.T) {
|
||||
saveAndRestoreProxyEnv(t)
|
||||
|
||||
options := &types.Options{
|
||||
Proxy: goflags.StringSlice{server.URL},
|
||||
ProxyInternal: true,
|
||||
Timeout: 5,
|
||||
Logger: &gologger.Logger{},
|
||||
}
|
||||
err := loadProxyServers(options)
|
||||
require.NoError(t, err)
|
||||
for _, env := range proxyEnvVars {
|
||||
require.Equal(t, server.URL, os.Getenv(env), "expected %s to be set", env)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("proxy internal disabled does not set env vars", func(t *testing.T) {
|
||||
saveAndRestoreProxyEnv(t)
|
||||
|
||||
options := &types.Options{
|
||||
Proxy: goflags.StringSlice{server.URL},
|
||||
ProxyInternal: false,
|
||||
Timeout: 5,
|
||||
Logger: &gologger.Logger{},
|
||||
}
|
||||
err := loadProxyServers(options)
|
||||
require.NoError(t, err)
|
||||
for _, env := range proxyEnvVars {
|
||||
require.Empty(t, os.Getenv(env), "expected %s not to be set", env)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,199 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
)
|
||||
|
||||
type probeResult int
|
||||
|
||||
const (
|
||||
portClosed probeResult = iota // connection refused / host unreachable
|
||||
portOpen // connect succeeded
|
||||
portUnknown // timeout / filtered — treat as possibly open
|
||||
)
|
||||
|
||||
const reachabilityProbeTimeout = 750 * time.Millisecond
|
||||
|
||||
// strictProbeEnabled reports whether the lossless reachability prune
|
||||
// applies to this run. It is limited to standard target inputs; request-shaped
|
||||
// (DAST / multi-format) inputs are never pruned, because probing a request
|
||||
// template as a live service is meaningless and could drop coverage.
|
||||
func (r *Runner) strictProbeEnabled() bool {
|
||||
return r.options.StrictProbe &&
|
||||
r.inputProvider != nil &&
|
||||
r.inputProvider.InputType() != provider.MultiFormatInputProvider
|
||||
}
|
||||
|
||||
// noHTTPServiceReachable reports whether NO target actually speaks HTTP(S).
|
||||
//
|
||||
// When it returns true, web templates (HTTP/headless/websocket) can be excluded
|
||||
// losslessly: a port that does not serve HTTP has no HTTP application, so no
|
||||
// HTTP template can produce a real finding there. Unlike a bare TCP-open check,
|
||||
// this catches the open-but-non-HTTP case (e.g. redis on 6379): the port is
|
||||
// open, but it is not an HTTP service, so HTTP templates are pure waste.
|
||||
//
|
||||
// It reuses nuclei's own httpx probe (the same one used for input routing), so
|
||||
// the determination is consistent with the engine and respects the network
|
||||
// policy (the probe dials only through the policy-configured httpx client).
|
||||
// Conservative: an explicit http/https target, any scheme that responds, or the
|
||||
// inability to probe safely all keep web templates in place.
|
||||
// Skipped when -no-httpx is set so we do not probe or prune against user intent.
|
||||
func (r *Runner) noHTTPServiceReachable() bool {
|
||||
if r.inputProvider == nil || r.options.DisableHTTPProbe {
|
||||
return false
|
||||
}
|
||||
dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
|
||||
if dialers == nil {
|
||||
return false // cannot probe within policy — keep web templates
|
||||
}
|
||||
httpxOptions := httpx.DefaultOptions
|
||||
if r.options.AliveHttpProxy != "" {
|
||||
httpxOptions.Proxy = r.options.AliveHttpProxy
|
||||
} else if r.options.AliveSocksProxy != "" {
|
||||
httpxOptions.Proxy = r.options.AliveSocksProxy
|
||||
}
|
||||
httpxOptions.RetryMax = r.options.Retries
|
||||
if r.options.Timeout > 0 {
|
||||
httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
|
||||
}
|
||||
httpxOptions.NetworkPolicy = dialers.NetworkPolicy
|
||||
client, err := httpx.New(&httpxOptions)
|
||||
if err != nil {
|
||||
return false // cannot probe — keep web templates
|
||||
}
|
||||
|
||||
anyHTTP := false
|
||||
r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool {
|
||||
if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") {
|
||||
anyHTTP = true
|
||||
return false
|
||||
}
|
||||
if utils.ProbeURL(mi.Input, client) != "" {
|
||||
anyHTTP = true
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return !anyHTTP
|
||||
}
|
||||
|
||||
// tcpNetworkOnlyPorts returns the concrete numeric TCP ports of a single-protocol
|
||||
// network template, or ok=false if the template is ineligible for port pruning.
|
||||
func tcpNetworkOnlyPorts(t *templates.Template) ([]string, bool) {
|
||||
if len(t.RequestsNetwork)+len(t.RequestsWithTCP) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
// must be network-only (no other protocol request could still be reachable)
|
||||
otherReqs := len(t.RequestsHTTP) + len(t.RequestsWithHTTP) + len(t.RequestsHeadless) +
|
||||
len(t.RequestsDNS) + len(t.RequestsFile) + len(t.RequestsSSL) +
|
||||
len(t.RequestsCode) + len(t.RequestsJavascript) + len(t.Workflows)
|
||||
if otherReqs > 0 {
|
||||
return nil, false
|
||||
}
|
||||
reqs := make([]*network.Request, 0, len(t.RequestsNetwork)+len(t.RequestsWithTCP))
|
||||
reqs = append(reqs, t.RequestsNetwork...)
|
||||
reqs = append(reqs, t.RequestsWithTCP...)
|
||||
|
||||
var ports []string
|
||||
for _, req := range reqs {
|
||||
for _, addr := range req.Address {
|
||||
if strings.Contains(strings.ToLower(addr), "udp://") {
|
||||
return nil, false // UDP: a TCP probe cannot prove unreachability
|
||||
}
|
||||
}
|
||||
pp := splitPorts(req.Port)
|
||||
if len(pp) == 0 {
|
||||
return nil, false // empty/dynamic port (uses input port) — keep
|
||||
}
|
||||
for _, p := range pp {
|
||||
if !isNumericPort(p) {
|
||||
return nil, false // service name or template var — keep
|
||||
}
|
||||
ports = append(ports, p)
|
||||
}
|
||||
}
|
||||
if len(ports) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
return sliceutil.Dedupe(ports), true
|
||||
}
|
||||
|
||||
func isNumericPort(p string) bool {
|
||||
n, err := strconv.Atoi(p)
|
||||
return err == nil && n > 0 && n < 65536
|
||||
}
|
||||
|
||||
// hostAndExplicitPort splits a target into host and explicit port (port empty if
|
||||
// the target is a bare host).
|
||||
func hostAndExplicitPort(input string) (string, string) {
|
||||
if strings.Contains(input, "://") {
|
||||
if u, err := url.Parse(input); err == nil && u.Hostname() != "" {
|
||||
return u.Hostname(), u.Port()
|
||||
}
|
||||
}
|
||||
if host, port, err := net.SplitHostPort(input); err == nil {
|
||||
return host, port
|
||||
}
|
||||
return input, ""
|
||||
}
|
||||
|
||||
// probe classifies reachability of host:port WITHIN the scan's network policy.
|
||||
// Security first: it never dials a target the policy forbids and never bypasses
|
||||
// the sandbox — all probing goes through the execution's fastdialer, which
|
||||
// enforces the same DenyList (‑lna, exclude-targets, private/metadata ranges,
|
||||
// DNS-rebind protection) as the scan itself.
|
||||
//
|
||||
// It reconciles security with losslessness by mapping every uncertain case to
|
||||
// portUnknown ("keep, don't dial"): a policy-denied host, a missing dialer, or a
|
||||
// timeout all leave templates in place. Pruning happens only on a definitive,
|
||||
// policy-permitted, refused connection.
|
||||
func (r *Runner) probe(host, port string) probeResult {
|
||||
if !protocolstate.IsHostAllowed(r.options.ExecutionId, host) {
|
||||
// Policy denies this host: do not dial (security), do not prune (lossless).
|
||||
return portUnknown
|
||||
}
|
||||
dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
|
||||
if dialers == nil || dialers.Fastdialer == nil {
|
||||
// No policy-enforcing dialer available: never fall back to a raw dial.
|
||||
return portUnknown
|
||||
}
|
||||
return classifyDial(dialers.Fastdialer.Dial, net.JoinHostPort(host, port), reachabilityProbeTimeout)
|
||||
}
|
||||
|
||||
type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
|
||||
// classifyDial performs one connect via the supplied (policy-enforcing) dialer.
|
||||
func classifyDial(dial dialFunc, addr string, timeout time.Duration) probeResult {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
conn, err := dial(ctx, "tcp", addr)
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
return portOpen
|
||||
}
|
||||
if ne, ok := err.(net.Error); ok && ne.Timeout() {
|
||||
return portUnknown
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
// Only definitive negatives prune. DNS blips / temporary dialer errors must
|
||||
// stay unknown so mixed-fleet probing under load remains lossless.
|
||||
if strings.Contains(msg, "refused") || strings.Contains(msg, "unreachable") ||
|
||||
strings.Contains(msg, "no route to host") || strings.Contains(msg, "network is unreachable") {
|
||||
return portClosed
|
||||
}
|
||||
return portUnknown
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
httpproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
jsproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/javascript"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
)
|
||||
|
||||
func TestHostAndExplicitPort(t *testing.T) {
|
||||
cases := []struct{ in, host, port string }{
|
||||
{"127.0.0.1", "127.0.0.1", ""},
|
||||
{"127.0.0.1:6379", "127.0.0.1", "6379"},
|
||||
{"http://example.com:8080", "example.com", "8080"},
|
||||
{"https://example.com", "example.com", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
h, p := hostAndExplicitPort(c.in)
|
||||
if h != c.host || p != c.port {
|
||||
t.Errorf("hostAndExplicitPort(%q) = (%q,%q), want (%q,%q)", c.in, h, p, c.host, c.port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsNumericPort(t *testing.T) {
|
||||
for _, p := range []string{"80", "6379", "65535", "1"} {
|
||||
if !isNumericPort(p) {
|
||||
t.Errorf("isNumericPort(%q) = false, want true", p)
|
||||
}
|
||||
}
|
||||
for _, p := range []string{"ftp", "{{Port}}", "0", "70000", "", "80a"} {
|
||||
if isNumericPort(p) {
|
||||
t.Errorf("isNumericPort(%q) = true, want false", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestTCPNetworkOnlyPorts locks in the losslessness guards: only single-protocol
|
||||
// network templates with concrete numeric TCP ports are eligible for pruning.
|
||||
func TestTCPNetworkOnlyPorts(t *testing.T) {
|
||||
netTmpl := func(addr []string, port string) *templates.Template {
|
||||
return &templates.Template{RequestsNetwork: []*network.Request{{Address: addr, Port: port}}}
|
||||
}
|
||||
|
||||
t.Run("concrete tcp port eligible", func(t *testing.T) {
|
||||
ports, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "6379,6380"))
|
||||
if !ok || len(ports) != 2 {
|
||||
t.Fatalf("want ok with 2 ports, got ok=%v ports=%v", ok, ports)
|
||||
}
|
||||
})
|
||||
t.Run("tls address still tcp", func(t *testing.T) {
|
||||
if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"tls://{{Hostname}}"}, "6379")); !ok {
|
||||
t.Fatal("tls:// should remain eligible (still TCP)")
|
||||
}
|
||||
})
|
||||
t.Run("udp address rejected", func(t *testing.T) {
|
||||
if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"udp://{{Hostname}}"}, "161")); ok {
|
||||
t.Fatal("udp:// must be ineligible — a TCP probe cannot prove UDP unreachability")
|
||||
}
|
||||
})
|
||||
t.Run("service-name port rejected", func(t *testing.T) {
|
||||
if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "ftp")); ok {
|
||||
t.Fatal("service-name port must be ineligible")
|
||||
}
|
||||
})
|
||||
t.Run("dynamic port rejected", func(t *testing.T) {
|
||||
if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "{{Port}}")); ok {
|
||||
t.Fatal("template-var port must be ineligible")
|
||||
}
|
||||
})
|
||||
t.Run("empty port rejected", func(t *testing.T) {
|
||||
if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "")); ok {
|
||||
t.Fatal("empty port (uses input port) must be ineligible")
|
||||
}
|
||||
})
|
||||
t.Run("mixed protocol rejected", func(t *testing.T) {
|
||||
mixed := netTmpl([]string{"{{Hostname}}"}, "6379")
|
||||
mixed.RequestsHTTP = []*httpproto.Request{{}}
|
||||
if _, ok := tcpNetworkOnlyPorts(mixed); ok {
|
||||
t.Fatal("template with an HTTP request must be ineligible")
|
||||
}
|
||||
})
|
||||
t.Run("javascript protocol rejected", func(t *testing.T) {
|
||||
js := &templates.Template{RequestsJavascript: []*jsproto.Request{{}}}
|
||||
if _, ok := tcpNetworkOnlyPorts(js); ok {
|
||||
t.Fatal("javascript template must be ineligible (opaque dial protocol)")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestClassifyDial(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Skip("cannot listen:", err)
|
||||
}
|
||||
defer func() { _ = ln.Close() }()
|
||||
d := &net.Dialer{}
|
||||
if got := classifyDial(d.DialContext, ln.Addr().String(), reachabilityProbeTimeout); got != portOpen {
|
||||
t.Errorf("classifyDial(open) = %v, want portOpen", got)
|
||||
}
|
||||
// an unused low port on loopback should refuse quickly
|
||||
if got := classifyDial(d.DialContext, "127.0.0.1:1", reachabilityProbeTimeout); got != portClosed {
|
||||
t.Errorf("classifyDial(closed) = %v, want portClosed", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyDialDNSFailureIsUnknown(t *testing.T) {
|
||||
// NXDOMAIN / resolution failure must not prune (lossless under load).
|
||||
dial := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return nil, &net.OpError{Op: "dial", Net: network, Err: &net.DNSError{Err: "no such host", Name: "no-such-host.invalid", IsNotFound: true}}
|
||||
}
|
||||
got := classifyDial(dial, "no-such-host.invalid:6379", reachabilityProbeTimeout)
|
||||
if got != portUnknown {
|
||||
t.Fatalf("classifyDial(dns-fail) = %v, want portUnknown", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyDialTimeoutIsUnknown(t *testing.T) {
|
||||
// A dropped connect runs into the timeout and must not prune.
|
||||
dial := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
<-ctx.Done()
|
||||
return nil, &net.OpError{Op: "dial", Net: network, Err: ctx.Err()}
|
||||
}
|
||||
got := classifyDial(dial, "192.0.2.1:65535", 50*time.Millisecond)
|
||||
if got != portUnknown {
|
||||
t.Fatalf("classifyDial(timeout) = %v, want portUnknown", got)
|
||||
}
|
||||
}
|
||||
+13
-224
@@ -3,8 +3,6 @@ package runner
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
@@ -45,8 +43,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core"
|
||||
scanplan "github.com/projectdiscovery/nuclei/v3/pkg/core/plan"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/external/customtemplates"
|
||||
fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input"
|
||||
@@ -70,18 +66,14 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
ptrutil "github.com/projectdiscovery/utils/ptr"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -94,7 +86,7 @@ var (
|
||||
// Runner is a client for running the enumeration process.
|
||||
type Runner struct {
|
||||
output output.Writer
|
||||
interactsh interactsh.IClient
|
||||
interactsh *interactsh.Client
|
||||
options *types.Options
|
||||
projectFile *projectfile.ProjectFile
|
||||
catalog catalog.Catalog
|
||||
@@ -169,9 +161,7 @@ func New(options *types.Options) (*Runner, error) {
|
||||
tm := &installer.TemplateManager{
|
||||
CustomTemplates: ctm,
|
||||
DisablePublicTemplates: options.PublicTemplateDisableDownload,
|
||||
Channel: options.UpdateTemplatesChannel,
|
||||
}
|
||||
templatesVersion := config.DefaultConfig.TemplateVersion
|
||||
if err := tm.FreshInstallIfNotExists(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to install nuclei templates: %s\n", err)
|
||||
}
|
||||
@@ -188,12 +178,7 @@ func New(options *types.Options) (*Runner, error) {
|
||||
if options.UpdateTemplates {
|
||||
// we automatically check for updates unless explicitly disabled
|
||||
// this print statement is only to inform the user that there are no updates
|
||||
noUpdates := !config.DefaultConfig.NeedsTemplateUpdate()
|
||||
if tm.Channel == installer.TemplatesChannelLatest {
|
||||
// the latest channel updates without a new release
|
||||
noUpdates = config.DefaultConfig.TemplateVersion == templatesVersion
|
||||
}
|
||||
if noUpdates {
|
||||
if !config.DefaultConfig.NeedsTemplateUpdate() {
|
||||
runner.Logger.Info().Msgf("No new updates found for nuclei templates")
|
||||
}
|
||||
|
||||
@@ -238,7 +223,7 @@ func New(options *types.Options) (*Runner, error) {
|
||||
runner.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
|
||||
|
||||
var httpclient *retryablehttp.Client
|
||||
if options.ProxyInternal && (options.AliveHttpProxy != "" || options.AliveSocksProxy != "") {
|
||||
if options.ProxyInternal && options.AliveHttpProxy != "" || options.AliveSocksProxy != "" {
|
||||
var err error
|
||||
|
||||
httpclient, err = httpclientpool.Get(options, &httpclientpool.Configuration{}, "")
|
||||
@@ -437,7 +422,13 @@ func New(options *types.Options) (*Runner, error) {
|
||||
opts.HTTPClient = httpclient
|
||||
}
|
||||
if opts.HTTPClient == nil {
|
||||
opts.HTTPClient = newInteractshHTTPClient(options)
|
||||
httpOpts := retryablehttp.DefaultOptionsSingle
|
||||
httpOpts.Timeout = 20 * time.Second // for stability reasons
|
||||
if options.Timeout > 20 {
|
||||
httpOpts.Timeout = time.Duration(options.Timeout) * time.Second
|
||||
}
|
||||
// in testing it was found most of times when interactsh failed, it was due to failure in registering /polling requests
|
||||
opts.HTTPClient = retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
|
||||
}
|
||||
|
||||
interactshClient, err := interactsh.New(opts)
|
||||
@@ -695,12 +686,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
DoNotCache: r.options.DoNotCacheTemplates,
|
||||
Logger: r.Logger,
|
||||
}
|
||||
// Attach scan-scoped HTTP response cache only for opt-in -tf, and only
|
||||
// before templates compile so request executers share the same pointer.
|
||||
// Default scans leave this nil (no behavioral change).
|
||||
if r.options.UsesResponseCache() {
|
||||
executorOpts.HTTPResponseCache = httprespcache.New()
|
||||
}
|
||||
|
||||
if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {
|
||||
// Go StdLib style experimental/debug feature switch
|
||||
@@ -758,16 +743,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
|
||||
// If using input-file flags, only load http fuzzing based templates.
|
||||
loaderConfig := loader.NewConfig(r.options, r.catalog, executorOpts)
|
||||
|
||||
// Lossless reachability prune: if no web port is reachable on any target,
|
||||
// exclude web-protocol templates at load time — they cannot connect, so
|
||||
// cannot produce a finding. Reuses the standard protocol-type exclusion on
|
||||
// the loader config only, so shared options stay unchanged across runs.
|
||||
if r.strictProbeEnabled() && r.noHTTPServiceReachable() {
|
||||
loaderConfig.ExcludeProtocols = append(slices.Clone(loaderConfig.ExcludeProtocols),
|
||||
templateTypes.HTTPProtocol, templateTypes.HeadlessProtocol, templateTypes.WebsocketProtocol)
|
||||
r.Logger.Info().Msgf("reachability prune: no HTTP service on any target; excluding web-protocol templates (lossless)")
|
||||
}
|
||||
profiles := r.targetProfiles()
|
||||
if profiles != nil {
|
||||
if r.options.AutomaticScan {
|
||||
@@ -775,11 +750,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
}
|
||||
loaderConfig.TargetFilter = profiles.LoadFilter()
|
||||
}
|
||||
if r.hasPinnedTargets() {
|
||||
if err := pinnedTargetConflict(r.options); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if (!strings.EqualFold(r.options.InputFileMode, "list") || r.options.DAST) && !r.options.OfflineHTTP {
|
||||
// if input type is not list (implicitly enable fuzzing), unless passive/offlinehttp
|
||||
r.options.DAST = true
|
||||
@@ -824,11 +794,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
if err := store.Load(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Each template is parsed once per scan, so the compiled cache only holds
|
||||
// pre-compilation snapshots that nothing reads after loading.
|
||||
if parser, ok := r.parser.(*templates.Parser); ok {
|
||||
parser.PurgeCompiled()
|
||||
}
|
||||
if profiles != nil {
|
||||
profiles.Prepare(slices.Concat(store.Templates(), store.Workflows()))
|
||||
executorOpts.TargetScope = profiles
|
||||
@@ -879,15 +844,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
return errors.Wrap(err, "could not probe http input")
|
||||
}
|
||||
executorOpts.InputHelper.InputsHTTP = inputHelpers
|
||||
executorOpts.InputHelper.InputsHTTPProbed = r.inputProvider.InputType() != provider.MultiFormatInputProvider
|
||||
// Under strict-probe, skip web templates per-input for targets that
|
||||
// probed as non-HTTP (lossless), instead of falling back to raw URL.
|
||||
executorOpts.InputHelper.StrictProbe = r.strictProbeEnabled()
|
||||
if r.strictProbeEnabled() {
|
||||
r.Logger.Info().Msgf("Strict probe enabled: hosts without httpx confirmation are skipped for HTTP/headless templates")
|
||||
}
|
||||
} else if r.strictProbeEnabled() && r.options.DisableHTTPProbe {
|
||||
r.Logger.Warning().Msgf("strict-probe has no effect with -no-httpx (httpx probing is disabled)")
|
||||
}
|
||||
|
||||
inputCount := int(r.inputProvider.Count())
|
||||
@@ -1036,7 +992,7 @@ func (r *Runner) executeSmartWorkflowInput(executorOpts *protocols.ExecutorOptio
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not create automatic scan service")
|
||||
}
|
||||
if err := service.Execute(context.Background()); err != nil {
|
||||
if err := service.Execute(); err != nil {
|
||||
return nil, errors.Wrap(err, "could not execute automatic scan")
|
||||
}
|
||||
result := &atomic.Bool{}
|
||||
@@ -1062,157 +1018,15 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine)
|
||||
return nil, errors.New("no templates provided for scan")
|
||||
}
|
||||
|
||||
// pass input provider to engine
|
||||
// TODO: this should be not necessary after r.hmapInputProvider is removed + refactored
|
||||
if r.inputProvider == nil {
|
||||
return nil, errors.New("no input provider found")
|
||||
}
|
||||
|
||||
return r.executeTemplatesWithScanPlan(engine, finalTemplates)
|
||||
}
|
||||
|
||||
// executeTemplatesWithScanPlan chooses spray strategy and optionally attaches a
|
||||
// reachability / tech filter, then runs a single Execute pass. Wall clock must
|
||||
// match or beat baseline spray; probing is skipped when its cost would dominate.
|
||||
func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplates []*templates.Template) (*atomic.Bool, error) {
|
||||
defer r.attachScanPlan(engine, finalTemplates)()
|
||||
results := engine.ExecuteScanWithOpts(context.Background(), finalTemplates, r.inputProvider, r.options.DisableClustering)
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// attachScanPlan applies the scan plan to engine for one scan. It returns a
|
||||
// func that undoes the per-scan state once the scan is done.
|
||||
func (r *Runner) attachScanPlan(engine *core.Engine, finalTemplates []*templates.Template) func() {
|
||||
var undo []func()
|
||||
cleanup := func() {
|
||||
for i := len(undo) - 1; i >= 0; i-- {
|
||||
undo[i]()
|
||||
}
|
||||
}
|
||||
hostCount := int(r.inputProvider.Count())
|
||||
baselineReqs := estimateTemplateRequests(finalTemplates) * hostCount
|
||||
concreteNet, portsToProbe := countReachabilityStats(finalTemplates)
|
||||
techBound := techfilter.CountTechBound(finalTemplates)
|
||||
|
||||
planIn := scanplan.Input{
|
||||
Hosts: hostCount,
|
||||
Templates: len(finalTemplates),
|
||||
Requests: baselineReqs,
|
||||
BulkSize: r.options.BulkSize,
|
||||
Stream: r.options.Stream,
|
||||
StrictProbe: r.strictProbeEnabled(),
|
||||
PortsToProbe: portsToProbe,
|
||||
ConcreteNetworkTemplates: concreteNet,
|
||||
TechFilter: r.options.TechFilter,
|
||||
TechBoundTemplates: techBound,
|
||||
}
|
||||
p := scanplan.Decide(planIn)
|
||||
|
||||
// Resolve -ss auto before Execute so the engine placeholder is unused.
|
||||
if stringsutil.EqualFoldAny(r.options.ScanStrategy, scanstrategy.Auto.String(), "") {
|
||||
r.options.ScanStrategy = p.Strategy
|
||||
}
|
||||
|
||||
var httpHelper *input.Helper
|
||||
if opts := engine.ExecuterOptions(); opts != nil {
|
||||
httpHelper = opts.InputHelper
|
||||
}
|
||||
|
||||
var reachFilter func(*templates.Template, *contextargs.MetaInput) bool
|
||||
var techFilterFn func(*templates.Template, *contextargs.MetaInput) bool
|
||||
var techIdx *techReachabilityIndex
|
||||
filteredReqs := baselineReqs
|
||||
|
||||
if p.BuildReachability {
|
||||
idx, groups, err := r.buildHostReachability(finalTemplates, httpHelper, p.ProbePorts)
|
||||
if err != nil {
|
||||
r.Logger.Warning().Msgf("could not build host reachability index, continuing unfiltered: %s", err)
|
||||
} else {
|
||||
baseline, filtered := estimateGroupedExecutions(finalTemplates, groups, hostCount)
|
||||
filteredReqs = filtered
|
||||
saved := baseline - filtered
|
||||
pct := 0.0
|
||||
if baseline > 0 {
|
||||
pct = 100 * float64(saved) / float64(baseline)
|
||||
}
|
||||
for _, g := range groups {
|
||||
r.Logger.Info().Msgf("host-group %s: hosts=%d templates=%d", g.key, len(g.hosts), len(templatesForHostGroup(finalTemplates, g)))
|
||||
}
|
||||
r.Logger.Info().Msgf(
|
||||
"host-groups: groups=%d baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) mode=single-pass",
|
||||
len(groups), baseline, filtered, saved, pct,
|
||||
)
|
||||
if p.UseReachabilityFilter {
|
||||
reachFilter = idx.Allow
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if p.UseTechFilter {
|
||||
var err error
|
||||
var cache *httprespcache.Cache
|
||||
if opts := engine.ExecuterOptions(); opts != nil {
|
||||
if opts.HTTPResponseCache == nil {
|
||||
opts.HTTPResponseCache = httprespcache.New()
|
||||
}
|
||||
cache = opts.HTTPResponseCache
|
||||
}
|
||||
techIdx, err = r.buildTechReachability(finalTemplates, httpHelper, cache)
|
||||
if err != nil {
|
||||
r.Logger.Warning().Msgf("could not build tech filter index, continuing without tech-filter: %s", err)
|
||||
if cache != nil {
|
||||
cache.Disable()
|
||||
}
|
||||
techIdx = nil
|
||||
} else {
|
||||
base, techFiltered := estimateTechFilteredExecutions(finalTemplates, techIdx, hostCount)
|
||||
// When stacked with reachability, take the more aggressive (lower) estimate
|
||||
// as a lower bound for progress; actual AND filter may skip more.
|
||||
if techFiltered < filteredReqs {
|
||||
filteredReqs = techFiltered
|
||||
}
|
||||
saved := base - techFiltered
|
||||
pct := 0.0
|
||||
if base > 0 {
|
||||
pct = 100 * float64(saved) / float64(base)
|
||||
}
|
||||
r.Logger.Info().Msgf(
|
||||
"tech-filter: baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) bound_templates=%d",
|
||||
base, techFiltered, saved, pct, techBound,
|
||||
)
|
||||
techFilterFn = techIdx.Allow
|
||||
}
|
||||
} else if opts := engine.ExecuterOptions(); opts != nil && opts.HTTPResponseCache != nil && !r.options.ResponseCache {
|
||||
// -tf set but planner skipped (e.g. no tech-bound templates): do not
|
||||
// alter request paths via an idle response cache. Keep it when the user
|
||||
// asked for -response-cache explicitly.
|
||||
opts.HTTPResponseCache.Disable()
|
||||
}
|
||||
|
||||
p.ApplyFiltered(filteredReqs)
|
||||
if opts := engine.ExecuterOptions(); opts != nil {
|
||||
// The filters narrow the templates per target through the same
|
||||
// TargetScope as per-target profiles, on top of them. The engine then
|
||||
// skips the pairs, counts progress and narrows clusters as it does for
|
||||
// profiles.
|
||||
if filter := composeTemplateFilters(reachFilter, techFilterFn); filter != nil {
|
||||
profiles := opts.TargetScope
|
||||
opts.TargetScope = newPlanScope(profiles, filter, finalTemplates)
|
||||
undo = append(undo, func() { opts.TargetScope = profiles })
|
||||
}
|
||||
if opts.HTTPResponseCache != nil {
|
||||
undo = append(undo, func() {
|
||||
hits, misses, stores := opts.HTTPResponseCache.Stats()
|
||||
r.Logger.Info().Msgf("http-response-cache: hits=%d misses=%d stores=%d", hits, misses, stores)
|
||||
opts.HTTPResponseCache = nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
r.Logger.Info().Msgf("scan-plan: strategy=%s reach=%v tech=%v expected_req=%d reason=%s",
|
||||
r.options.ScanStrategy, p.UseReachabilityFilter, p.UseTechFilter, p.ExpectedRequests, p.Reason)
|
||||
|
||||
return cleanup
|
||||
}
|
||||
|
||||
// displayExecutionInfo prints parser stats, version info, and scan counts.
|
||||
func (r *Runner) displayExecutionInfo(store *loader.Store) {
|
||||
// Display parser stats for templates loaded into the store.
|
||||
@@ -1421,28 +1235,3 @@ func init() {
|
||||
HideAutoSaveMsg = env.GetEnvOrDefault("DISABLE_CLOUD_UPLOAD_WRN", false)
|
||||
EnableCloudUpload = env.GetEnvOrDefault("ENABLE_CLOUD_UPLOAD", false)
|
||||
}
|
||||
|
||||
func newInteractshHTTPClient(options *types.Options) *retryablehttp.Client {
|
||||
httpOpts := retryablehttp.DefaultOptionsSingle
|
||||
httpOpts.Timeout = 20 * time.Second // for stability reasons
|
||||
if options.Timeout > 20 {
|
||||
httpOpts.Timeout = time.Duration(options.Timeout) * time.Second
|
||||
}
|
||||
if dialers := protocolstate.GetDialersWithId(options.ExecutionId); dialers != nil && dialers.Fastdialer != nil {
|
||||
// The pooled default keeps ProxyFromEnvironment and the TLS config that
|
||||
// INTERACTSH_TLS_VERIFY edits. Nuclei only exports HTTP_PROXY when -pi
|
||||
// is set, and that path uses the pooled client, so -proxy without -pi
|
||||
// still dials the interactsh server directly.
|
||||
transport := retryablehttp.DefaultReusePooledTransport()
|
||||
transport.DialContext = dialers.Fastdialer.Dial
|
||||
transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return dialers.Fastdialer.DialTLS(ctx, network, addr)
|
||||
}
|
||||
httpOpts.HttpClient = &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: httpOpts.Timeout,
|
||||
}
|
||||
}
|
||||
// in testing it was found most of times when interactsh failed, it was due to failure in registering /polling requests
|
||||
return retryablehttp.NewClient(httpOpts)
|
||||
}
|
||||
@@ -1,18 +1,13 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
@@ -148,52 +143,3 @@ func TestWalkReflectStructHandlesNestedStructs(t *testing.T) {
|
||||
require.Equal(t, "2", testStruct.Struct.B)
|
||||
require.Equal(t, "true", testStruct.Struct.C)
|
||||
}
|
||||
|
||||
func TestNewInteractshHTTPClient(t *testing.T) {
|
||||
// ProxyFromEnvironment reads the environment once per process. Set it
|
||||
// before the first request. NO_PROXY keeps the local dial direct.
|
||||
for _, key := range []string{"HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "https_proxy", "no_proxy"} {
|
||||
t.Setenv(key, "")
|
||||
}
|
||||
t.Setenv("HTTPS_PROXY", "http://127.0.0.1:9")
|
||||
t.Setenv("NO_PROXY", "127.0.0.1,localhost")
|
||||
|
||||
options := &types.Options{
|
||||
ExecutionId: "test-interactsh-client-dialer",
|
||||
Timeout: 35,
|
||||
}
|
||||
require.NoError(t, protocolstate.Init(options))
|
||||
t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
|
||||
|
||||
client := newInteractshHTTPClient(options)
|
||||
require.NotNil(t, client)
|
||||
require.Equal(t, 35*time.Second, client.HTTPClient.Timeout)
|
||||
|
||||
transport, ok := client.HTTPClient.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
require.NotNil(t, transport.Proxy)
|
||||
require.NotNil(t, transport.DialContext)
|
||||
require.NotNil(t, transport.DialTLSContext)
|
||||
require.NotNil(t, transport.TLSClientConfig, "INTERACTSH_TLS_VERIFY edits this config")
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
resp, err := client.HTTPClient.Get(srv.URL)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusNoContent, resp.StatusCode)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
|
||||
host := srv.Listener.Addr().(*net.TCPAddr).IP.String()
|
||||
dialers := protocolstate.GetDialersWithId(options.ExecutionId)
|
||||
require.Equal(t, host, dialers.Fastdialer.GetDialedIP(host))
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
|
||||
require.NoError(t, err)
|
||||
proxyURL, err := transport.Proxy(req)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, proxyURL)
|
||||
require.Equal(t, "http://127.0.0.1:9", proxyURL.String())
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
// AttachScanPlan applies -strict-probe and -tech-filter to an SDK engine for
|
||||
// one scan, the same way the CLI runner does. It returns a func that undoes the
|
||||
// per-scan state; call it once the scan is done. When neither option is set it
|
||||
// does nothing, so default SDK scans keep their behavior.
|
||||
func AttachScanPlan(opts *types.Options, inputProvider provider.InputProvider, logger *gologger.Logger, engine *core.Engine, tpls []*templates.Template) func() {
|
||||
if opts == nil || engine == nil || inputProvider == nil || (!opts.StrictProbe && !opts.TechFilter) {
|
||||
return func() {}
|
||||
}
|
||||
if logger == nil {
|
||||
logger = gologger.DefaultLogger
|
||||
}
|
||||
// The SDK engine is reused across scans, so keep the options and the
|
||||
// response cache it owns as they were before this scan.
|
||||
strategy := opts.ScanStrategy
|
||||
executerOpts := engine.ExecuterOptions()
|
||||
var responseCache *httprespcache.Cache
|
||||
if executerOpts != nil {
|
||||
responseCache = executerOpts.HTTPResponseCache
|
||||
}
|
||||
r := &Runner{options: opts, inputProvider: inputProvider, Logger: logger}
|
||||
cleanup := r.attachScanPlan(engine, tpls)
|
||||
return func() {
|
||||
cleanup()
|
||||
opts.ScanStrategy = strategy
|
||||
if executerOpts != nil {
|
||||
executerOpts.HTTPResponseCache = responseCache
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Without -strict-probe or -tech-filter the SDK hook changes nothing.
|
||||
func TestAttachScanPlanNoop(t *testing.T) {
|
||||
opts := types.DefaultOptions()
|
||||
opts.ScanStrategy = "auto"
|
||||
engine := core.New(opts)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{Options: opts})
|
||||
ip := provider.NewSimpleInputProviderWithUrls(opts.ExecutionId, "http://127.0.0.1:1")
|
||||
|
||||
AttachScanPlan(opts, ip, nil, engine, nil)()
|
||||
require.Equal(t, "auto", opts.ScanStrategy)
|
||||
require.Nil(t, engine.ExecuterOptions().TargetScope)
|
||||
}
|
||||
|
||||
// The SDK engine is reused across scans, so the hook must leave its options
|
||||
// and its response cache as they were before the scan.
|
||||
func TestAttachScanPlanRestoresEngineState(t *testing.T) {
|
||||
opts := types.DefaultOptions()
|
||||
opts.ScanStrategy = "auto"
|
||||
opts.TechFilter = true
|
||||
cache := httprespcache.New()
|
||||
engine := core.New(opts)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{Options: opts, HTTPResponseCache: cache})
|
||||
ip := provider.NewSimpleInputProviderWithUrls(opts.ExecutionId, "http://127.0.0.1:1")
|
||||
|
||||
cleanup := AttachScanPlan(opts, ip, nil, engine, nil)
|
||||
require.NotEqual(t, "auto", opts.ScanStrategy, "the plan resolves the auto strategy for the scan")
|
||||
cleanup()
|
||||
require.Equal(t, "auto", opts.ScanStrategy)
|
||||
require.Same(t, cache, engine.ExecuterOptions().HTTPResponseCache)
|
||||
require.Nil(t, engine.ExecuterOptions().TargetScope)
|
||||
}
|
||||
@@ -1,9 +1,7 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/targetprofile"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
// targetProfiles returns the per-target profiles read from input, or nil when
|
||||
@@ -17,25 +15,3 @@ func (r *Runner) targetProfiles() *targetprofile.Registry {
|
||||
}
|
||||
return provider.TargetProfiles()
|
||||
}
|
||||
|
||||
// hasPinnedTargets reports whether a target line pinned a target to an ip.
|
||||
func (r *Runner) hasPinnedTargets() bool {
|
||||
provider, ok := r.inputProvider.(interface {
|
||||
PinnedTargets() bool
|
||||
})
|
||||
return ok && provider.PinnedTargets()
|
||||
}
|
||||
|
||||
// pinnedTargetConflict returns an error for an option that cannot run with
|
||||
// targets pinned to an ip.
|
||||
func pinnedTargetConflict(options *types.Options) error {
|
||||
switch {
|
||||
case options.AutomaticScan:
|
||||
return errors.New("targets pinned to an ip cannot be combined with automatic scan")
|
||||
case options.AliveHttpProxy != "" || options.AliveSocksProxy != "":
|
||||
return errors.New("targets pinned to an ip cannot be combined with a proxy, as the proxy resolves the host itself")
|
||||
case options.ReplayProxy != "":
|
||||
return errors.New("targets pinned to an ip cannot be combined with -replay-proxy, as the replay proxy resolves the host itself")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPinnedTargetConflict(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
options *types.Options
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "no conflicting option", options: &types.Options{}},
|
||||
{name: "automatic scan", options: &types.Options{AutomaticScan: true}, wantErr: true},
|
||||
{name: "http proxy", options: &types.Options{AliveHttpProxy: "http://127.0.0.1:8080"}, wantErr: true},
|
||||
{name: "socks proxy", options: &types.Options{AliveSocksProxy: "socks5://127.0.0.1:1080"}, wantErr: true},
|
||||
{name: "replay proxy", options: &types.Options{ReplayProxy: "http://127.0.0.1:8080"}, wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
require.Equal(t, tt.wantErr, pinnedTargetConflict(tt.options) != nil)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,283 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
syncutil "github.com/projectdiscovery/utils/sync"
|
||||
unitutils "github.com/projectdiscovery/utils/unit"
|
||||
wappalyzer "github.com/projectdiscovery/wappalyzergo"
|
||||
)
|
||||
|
||||
const (
|
||||
techFilterWorkers = 50
|
||||
techFilterMaxBody = 1 * unitutils.Mega
|
||||
techFilterHTTPTimeout = 5 * time.Second
|
||||
techFilterMaxRedirects = 5
|
||||
)
|
||||
|
||||
// techReachabilityIndex filters template×target pairs by product/macro tags.
|
||||
type techReachabilityIndex struct {
|
||||
byInput map[string]techfilter.HostProfile // keyed by normalizeInputKey
|
||||
|
||||
// boundTags memoizes template ID -> product tags. It is filled lazily from
|
||||
// the template execution goroutines, so every access needs the mutex.
|
||||
tagsMu sync.RWMutex
|
||||
boundTags map[string][]string
|
||||
}
|
||||
|
||||
func normalizeInputKey(raw string) string {
|
||||
return strings.TrimRight(strings.TrimSpace(raw), "/")
|
||||
}
|
||||
|
||||
// Allow is fail-open: missing profile or unbound template => true.
|
||||
func (idx *techReachabilityIndex) Allow(t *templates.Template, mi *contextargs.MetaInput) bool {
|
||||
if idx == nil || t == nil || mi == nil {
|
||||
return true
|
||||
}
|
||||
profile, ok := idx.lookup(mi.Input)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
return techfilter.AllowTags(profile, idx.tagsFor(t))
|
||||
}
|
||||
|
||||
func (idx *techReachabilityIndex) tagsFor(t *templates.Template) []string {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
idx.tagsMu.RLock()
|
||||
tags, ok := idx.boundTags[t.ID]
|
||||
idx.tagsMu.RUnlock()
|
||||
if ok {
|
||||
return tags
|
||||
}
|
||||
|
||||
tags = techfilter.TemplateProductTags(t)
|
||||
|
||||
idx.tagsMu.Lock()
|
||||
defer idx.tagsMu.Unlock()
|
||||
if idx.boundTags == nil {
|
||||
idx.boundTags = make(map[string][]string)
|
||||
}
|
||||
// Another goroutine may have won the race; keep one slice per template ID.
|
||||
if existing, ok := idx.boundTags[t.ID]; ok {
|
||||
return existing
|
||||
}
|
||||
idx.boundTags[t.ID] = tags
|
||||
return tags
|
||||
}
|
||||
|
||||
func (idx *techReachabilityIndex) lookup(raw string) (techfilter.HostProfile, bool) {
|
||||
if idx == nil || idx.byInput == nil {
|
||||
return techfilter.HostProfile{}, false
|
||||
}
|
||||
if p, ok := idx.byInput[normalizeInputKey(raw)]; ok {
|
||||
return p, true
|
||||
}
|
||||
if p, ok := idx.byInput[raw]; ok {
|
||||
return p, true
|
||||
}
|
||||
return techfilter.HostProfile{}, false
|
||||
}
|
||||
|
||||
// buildTechReachability fingerprints HTTP(S) targets and builds a tag index.
|
||||
// When cache is non-nil, fingerprint GET responses seed it so matching template
|
||||
// GETs pay no extra RTT.
|
||||
func (r *Runner) buildTechReachability(tpls []*templates.Template, httpHelper *input.Helper, cache *httprespcache.Cache) (*techReachabilityIndex, error) {
|
||||
_ = tpls
|
||||
wapp, err := wappalyzer.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Redirects must be followed. A site that answers "/" with a 302 to its real
|
||||
// entry point (WordPress -> /wp-admin/install.php, any app -> /login) returns
|
||||
// an empty body, so the probe would detect nothing and every template bound
|
||||
// to the undetected product would be pruned. Same-host only, so the probe
|
||||
// never leaves the target authority.
|
||||
httpclient, err := httpclientpool.Get(r.options, &httpclientpool.Configuration{
|
||||
DisableCookie: true,
|
||||
ResponseHeaderTimeout: techFilterHTTPTimeout,
|
||||
RedirectFlow: httpclientpool.FollowSameHostRedirect,
|
||||
MaxRedirects: techFilterMaxRedirects,
|
||||
}, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
idx := &techReachabilityIndex{
|
||||
byInput: make(map[string]techfilter.HostProfile),
|
||||
boundTags: make(map[string][]string),
|
||||
}
|
||||
var mu sync.Mutex
|
||||
sg, err := syncutil.New(syncutil.WithSize(techFilterWorkers))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fingerprinted := 0
|
||||
eligible := 0
|
||||
r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool {
|
||||
if mi == nil || mi.Input == "" {
|
||||
return true
|
||||
}
|
||||
fpURL := fingerprintURLForInput(mi.Input, httpHelper)
|
||||
if fpURL == "" {
|
||||
return true
|
||||
}
|
||||
eligible++
|
||||
sg.Add()
|
||||
go func(inputKey, url string) {
|
||||
defer sg.Done()
|
||||
profile := fingerprintTarget(wapp, httpclient, url, cache)
|
||||
key := normalizeInputKey(inputKey)
|
||||
mu.Lock()
|
||||
idx.byInput[key] = profile
|
||||
if profile.HasTags() {
|
||||
fingerprinted++
|
||||
}
|
||||
mu.Unlock()
|
||||
}(mi.Input, fpURL)
|
||||
return true
|
||||
})
|
||||
sg.Wait()
|
||||
|
||||
gologger.Info().Msgf("tech-filter: fingerprinted=%d/%d hosts with tags (eligible=%d)", fingerprinted, len(idx.byInput), eligible)
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
// fingerprintURLForInput returns an absolute http(s) URL to fingerprint, or "".
|
||||
// Bare hosts reuse InputsHTTP probed URLs when available.
|
||||
func fingerprintURLForInput(raw string, helper *input.Helper) string {
|
||||
if looksLikeHTTPTarget(raw) {
|
||||
return raw
|
||||
}
|
||||
if helper != nil && helper.InputsHTTP != nil {
|
||||
if probed, ok := helper.InputsHTTP.Get(raw); ok && len(probed) > 0 {
|
||||
return string(probed)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func looksLikeHTTPTarget(raw string) bool {
|
||||
lower := strings.ToLower(raw)
|
||||
return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")
|
||||
}
|
||||
|
||||
func fingerprintTarget(wapp *wappalyzer.Wappalyze, client *retryablehttp.Client, rawURL string, cache *httprespcache.Cache) techfilter.HostProfile {
|
||||
// Prefer a trailing-slash form that matches typical {{BaseURL}}/ template URLs.
|
||||
seedURL := rawURL
|
||||
if !strings.HasSuffix(seedURL, "/") {
|
||||
seedURL = seedURL + "/"
|
||||
}
|
||||
req, err := retryablehttp.NewRequest(http.MethodGet, seedURL, nil)
|
||||
if err != nil {
|
||||
return techfilter.HostProfile{}
|
||||
}
|
||||
// Same user agent the templates will send, so the response seeded below is
|
||||
// reachable from the cache instead of being stored under a key nothing hits.
|
||||
req.Header.Set("User-Agent", httpProtocol.StableUserAgent())
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return techfilter.HostProfile{}
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, techFilterMaxBody))
|
||||
if err != nil {
|
||||
return techfilter.HostProfile{}
|
||||
}
|
||||
if cache != nil {
|
||||
seedFingerprintResponse(cache, req, resp, body, seedURL)
|
||||
}
|
||||
info := wapp.FingerprintWithInfo(resp.Header, body)
|
||||
products := make(map[string][]string, len(info))
|
||||
for name, app := range info {
|
||||
products[name] = append([]string(nil), app.Categories...)
|
||||
}
|
||||
return techfilter.ProfileFromFingerprint(products)
|
||||
}
|
||||
|
||||
// seedFingerprintResponse stores the probe response under the URL it actually
|
||||
// belongs to, so templates can reuse the RTT.
|
||||
//
|
||||
// The probe follows redirects, so resp may describe a different URL than the one
|
||||
// requested. Seeding it under the requested URL would hand every template that
|
||||
// asks for that URL the redirect target's status, headers and body instead of
|
||||
// the redirect itself, which silently changes what they match on.
|
||||
func seedFingerprintResponse(cache *httprespcache.Cache, req *retryablehttp.Request, resp *http.Response, body []byte, seedURL string) {
|
||||
finalURL := seedURL
|
||||
if resp != nil && resp.Request != nil && resp.Request.URL != nil {
|
||||
finalURL = resp.Request.URL.String()
|
||||
}
|
||||
if finalURL == seedURL {
|
||||
cache.SeedHTTP(req.Request, resp, body, techFilterMaxBody)
|
||||
return
|
||||
}
|
||||
finalReq := req.Request.Clone(req.Context())
|
||||
finalReq.URL = resp.Request.URL
|
||||
finalReq.Host = ""
|
||||
cache.SeedHTTP(finalReq, resp, body, techFilterMaxBody)
|
||||
}
|
||||
|
||||
// estimateTechFilteredExecutions counts template×host pairs that survive the tech filter.
|
||||
func estimateTechFilteredExecutions(tpls []*templates.Template, idx *techReachabilityIndex, hostCount int) (baseline, filtered int) {
|
||||
baseline = estimateTemplateRequests(tpls) * hostCount
|
||||
if idx == nil || len(idx.byInput) == 0 {
|
||||
return baseline, baseline
|
||||
}
|
||||
filtered = 0
|
||||
for _, t := range tpls {
|
||||
reqs := t.TotalRequests
|
||||
if reqs <= 0 {
|
||||
reqs = 1
|
||||
}
|
||||
if len(t.Workflows) > 0 {
|
||||
continue
|
||||
}
|
||||
tags := idx.tagsFor(t)
|
||||
for _, profile := range idx.byInput {
|
||||
if techfilter.AllowTags(profile, tags) {
|
||||
filtered += reqs
|
||||
}
|
||||
}
|
||||
missing := hostCount - len(idx.byInput)
|
||||
if missing > 0 {
|
||||
filtered += reqs * missing
|
||||
}
|
||||
}
|
||||
return baseline, filtered
|
||||
}
|
||||
|
||||
// composeTemplateFilters ANDs optional filters; nil filters are ignored.
|
||||
func composeTemplateFilters(filters ...func(*templates.Template, *contextargs.MetaInput) bool) func(*templates.Template, *contextargs.MetaInput) bool {
|
||||
var active []func(*templates.Template, *contextargs.MetaInput) bool
|
||||
for _, f := range filters {
|
||||
if f != nil {
|
||||
active = append(active, f)
|
||||
}
|
||||
}
|
||||
if len(active) == 0 {
|
||||
return nil
|
||||
}
|
||||
return func(t *templates.Template, mi *contextargs.MetaInput) bool {
|
||||
for _, f := range active {
|
||||
if !f(t, mi) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
)
|
||||
|
||||
// tagsFor is filled lazily from the template execution goroutines. Run it under
|
||||
// -race to catch the concurrent map access that used to crash whole scans.
|
||||
func TestTechReachabilityIndexConcurrentTagsFor(t *testing.T) {
|
||||
idx := &techReachabilityIndex{
|
||||
byInput: map[string]techfilter.HostProfile{},
|
||||
boundTags: map[string][]string{},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 64; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 64; j++ {
|
||||
tpl := &templates.Template{ID: string(rune('a' + (i+j)%26))}
|
||||
_ = idx.tagsFor(tpl)
|
||||
_ = idx.Allow(tpl, &contextargs.MetaInput{Input: "http://x/"})
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
)
|
||||
|
||||
func mustReq(t *testing.T, raw string) *retryablehttp.Request {
|
||||
t.Helper()
|
||||
req, err := retryablehttp.NewRequest(http.MethodGet, raw, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("request: %v", err)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func respFor(raw string, status int) *http.Response {
|
||||
u, _ := url.Parse(raw)
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: http.StatusText(status),
|
||||
Header: http.Header{"Server": []string{"probe"}},
|
||||
Request: &http.Request{Method: http.MethodGet, URL: u},
|
||||
}
|
||||
}
|
||||
|
||||
// The probe follows redirects. The body it comes back with belongs to the
|
||||
// redirect target, so it must never be stored under the requested URL: a
|
||||
// template asking for "/" must still see the redirect, not the target page.
|
||||
func TestSeedFingerprintResponseAfterRedirect(t *testing.T) {
|
||||
const seedURL = "http://example.test/"
|
||||
const finalURL = "http://example.test/wp-admin/install.php"
|
||||
|
||||
c := httprespcache.New()
|
||||
req := mustReq(t, seedURL)
|
||||
req.Header.Set("User-Agent", "probe-ua")
|
||||
seedFingerprintResponse(c, req, respFor(finalURL, 200), []byte("install page"), seedURL)
|
||||
|
||||
lookup := req.Request.Clone(req.Context())
|
||||
if got := c.Get(httprespcache.KeyFromRequest(lookup), "template", lookup); got != nil {
|
||||
t.Fatal("the redirect target must not be cached under the requested URL")
|
||||
}
|
||||
|
||||
finalLookup := req.Request.Clone(req.Context())
|
||||
finalLookup.URL, _ = url.Parse(finalURL)
|
||||
finalLookup.Host = ""
|
||||
if got := c.Get(httprespcache.KeyFromRequest(finalLookup), "template", finalLookup); got == nil {
|
||||
t.Fatal("the response must be cached under the URL it belongs to")
|
||||
}
|
||||
}
|
||||
|
||||
// Without a redirect the response does describe the requested URL, so seeding
|
||||
// it there is what saves the round trip.
|
||||
func TestSeedFingerprintResponseWithoutRedirect(t *testing.T) {
|
||||
const seedURL = "http://example.test/"
|
||||
|
||||
c := httprespcache.New()
|
||||
req := mustReq(t, seedURL)
|
||||
req.Header.Set("User-Agent", "probe-ua")
|
||||
seedFingerprintResponse(c, req, respFor(seedURL, 200), []byte("home"), seedURL)
|
||||
|
||||
lookup := req.Request.Clone(req.Context())
|
||||
if got := c.Get(httprespcache.KeyFromRequest(lookup), "template", lookup); got == nil {
|
||||
t.Fatal("a non-redirected response must be cached under the requested URL")
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/hmap/store/hybrid"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFingerprintURLForInput(t *testing.T) {
|
||||
require.Equal(t, "http://a/", fingerprintURLForInput("http://a/", nil))
|
||||
require.Equal(t, "", fingerprintURLForInput("example.com", nil))
|
||||
|
||||
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = hm.Close() })
|
||||
require.NoError(t, hm.Set("example.com", []byte("https://example.com")))
|
||||
helper := &input.Helper{InputsHTTP: hm, InputsHTTPProbed: true}
|
||||
require.Equal(t, "https://example.com", fingerprintURLForInput("example.com", helper))
|
||||
}
|
||||
@@ -52,7 +52,7 @@ type NucleiExecutorOptions struct {
|
||||
Catalog catalog.Catalog
|
||||
IssuesClient reporting.Client
|
||||
RateLimiter *ratelimit.Limiter
|
||||
Interactsh interactsh.IClient
|
||||
Interactsh *interactsh.Client
|
||||
ProjectFile *projectfile.ProjectFile
|
||||
Browser *browserEngine.Browser
|
||||
FuzzStatsDB *stats.Tracker
|
||||
|
||||
@@ -6,7 +6,6 @@ package integration_test
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
@@ -84,8 +83,6 @@ var httpTestcases = []integrationCase{
|
||||
{Path: "protocols/http/annotation-timeout.yaml", TestCase: &annotationTimeout{}},
|
||||
{Path: "protocols/http/custom-attack-type.yaml", TestCase: &customAttackType{}},
|
||||
{Path: "protocols/http/get-all-ips.yaml", TestCase: &scanAllIPS{}},
|
||||
{Path: "protocols/http/target-line-ip.yaml", TestCase: &httpTargetLineIP{}},
|
||||
{Path: "protocols/http/target-line-ip-pipeline.yaml", TestCase: &httpTargetLineIP{}},
|
||||
{Path: "protocols/http/get-without-scheme.yaml", TestCase: &httpGetWithoutScheme{}},
|
||||
{Path: "protocols/http/cl-body-without-header.yaml", TestCase: &httpCLBodyWithoutHeader{}},
|
||||
{Path: "protocols/http/cl-body-with-header.yaml", TestCase: &httpCLBodyWithHeader{}},
|
||||
@@ -1609,70 +1606,6 @@ func (h *scanAllIPS) Execute(filePath string) error {
|
||||
return expectResultsCount(got, 1)
|
||||
}
|
||||
|
||||
// target lines pin a hostname that does not resolve to the test servers
|
||||
type httpTargetLineIP struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *httpTargetLineIP) Execute(filePath string) error {
|
||||
serve := func(addr, name string) (*httptest.Server, error) {
|
||||
listener, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprintf(w, "host=%s server=%s", r.Host, name)
|
||||
}))
|
||||
_ = ts.Listener.Close()
|
||||
ts.Listener = listener
|
||||
ts.Start()
|
||||
return ts, nil
|
||||
}
|
||||
|
||||
one, err := serve("127.0.0.1:0", "one")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer one.Close()
|
||||
_, port, err := net.SplitHostPort(one.Listener.Addr().String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lines := []string{fmt.Sprintf(`{"target": "http://pinned.test:%s", "ip": "127.0.0.1"}`, port)}
|
||||
want := []string{"one"}
|
||||
// the same host pinned to a second ip, where 127.0.0.2 is available
|
||||
if two, err := serve(net.JoinHostPort("127.0.0.2", port), "two"); err == nil {
|
||||
defer two.Close()
|
||||
lines = append(lines, fmt.Sprintf(`{"target": "http://pinned.test:%s", "ip": "127.0.0.2"}`, port))
|
||||
want = append(want, "two")
|
||||
}
|
||||
|
||||
targets, err := os.CreateTemp("", "target-lines-*.txt")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = os.Remove(targets.Name()) }()
|
||||
if _, err := targets.WriteString(strings.Join(lines, "\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = targets.Close()
|
||||
|
||||
got, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-t", filePath, "-l", targets.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := expectResultsCount(got, len(want)); err != nil {
|
||||
return err
|
||||
}
|
||||
joined := strings.Join(got, "\n")
|
||||
for _, name := range want {
|
||||
if !strings.Contains(joined, name) {
|
||||
return fmt.Errorf("no result from server %q: %v", name, got)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensure that ip|host are handled without http|https scheme
|
||||
type httpGetWithoutScheme struct{}
|
||||
|
||||
|
||||
@@ -5,12 +5,10 @@ package integration_test
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-pg/pg/v10"
|
||||
@@ -18,7 +16,6 @@ import (
|
||||
"github.com/ory/dockertest/v3"
|
||||
dc "github.com/ory/dockertest/v3/docker"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
osutils "github.com/projectdiscovery/utils/os"
|
||||
"github.com/projectdiscovery/utils/reader"
|
||||
"go.uber.org/multierr"
|
||||
@@ -37,7 +34,6 @@ var jsTestcases = []integrationCase{
|
||||
{Path: "protocols/javascript/redis-lua-script.yaml", TestCase: &javascriptRedisLuaScript{}, DisableOn: javascriptDockerDisabled, Serial: true},
|
||||
{Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: javascriptDockerDisabled, Serial: true},
|
||||
{Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}},
|
||||
{Path: "protocols/javascript/target-line-ip.yaml", TestCase: &javascriptTargetLineIP{}},
|
||||
{Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}},
|
||||
{Path: "protocols/javascript/grpc-health.yaml", TestCase: &javascriptGRPCHealth{}},
|
||||
{Path: "protocols/javascript/grpc-denied.yaml", TestCase: &javascriptGRPCDenied{}},
|
||||
@@ -471,87 +467,6 @@ func (j *networkMultiStep) Execute(filePath string) error {
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
// target lines pin a hostname that does not resolve, so the javascript
|
||||
// connection reaches a server only through the pinned ip
|
||||
type javascriptTargetLineIP struct{}
|
||||
|
||||
func (j *javascriptTargetLineIP) Execute(filePath string) error {
|
||||
serve := func(addr, banner string) (net.Listener, error) {
|
||||
listener, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = conn.Write([]byte(banner))
|
||||
_ = conn.Close()
|
||||
}
|
||||
}()
|
||||
return listener, nil
|
||||
}
|
||||
|
||||
one, err := serve("127.0.0.1:0", "one")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = one.Close() }()
|
||||
_, port, err := net.SplitHostPort(one.Addr().String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
want := map[string]string{"127.0.0.1": "one"}
|
||||
// the same host pinned to a second ip, where 127.0.0.2 is available
|
||||
if two, err := serve(net.JoinHostPort("127.0.0.2", port), "two"); err == nil {
|
||||
defer func() { _ = two.Close() }()
|
||||
want["127.0.0.2"] = "two"
|
||||
}
|
||||
|
||||
lines := make([]string, 0, len(want))
|
||||
for ip := range want {
|
||||
lines = append(lines, fmt.Sprintf(`{"target": "pinned.test:%s", "ip": %q}`, port, ip))
|
||||
}
|
||||
targets, err := os.CreateTemp("", "target-lines-*.txt")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = os.Remove(targets.Name()) }()
|
||||
if _, err := targets.WriteString(strings.Join(lines, "\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = targets.Close()
|
||||
|
||||
if err := ensureSignedIntegrationTemplates(); err != nil {
|
||||
return err
|
||||
}
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, getEnvValues(), "-t", filePath, "-l", targets.Name(), "-jsonl")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
got := make(map[string]string)
|
||||
for _, result := range results {
|
||||
var event output.ResultEvent
|
||||
if !strings.HasPrefix(result, "{") || json.Unmarshal([]byte(result), &event) != nil {
|
||||
continue
|
||||
}
|
||||
got[event.IP] = strings.Join(event.ExtractedResults, ",")
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
return fmt.Errorf("got results for %d ips, want %d: %v", len(got), len(want), got)
|
||||
}
|
||||
for ip, banner := range want {
|
||||
if got[ip] != banner {
|
||||
return fmt.Errorf("ip %s answered %q, want %q", ip, got[ip], banner)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runJavascriptDockerCase(filePath string, spec javascriptDockerSpec, expectedNumbers ...int) error {
|
||||
if len(expectedNumbers) == 0 {
|
||||
expectedNumbers = []int{1}
|
||||
|
||||
@@ -17,7 +17,7 @@ type multiProtoDynamicExtractor struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *multiProtoDynamicExtractor) Execute(templatePath string) error {
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "docs.projectdiscovery.io", debug)
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "https://docs.projectdiscovery.io/", debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6,8 +6,6 @@ package integration_test
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -46,38 +44,6 @@ func TestNetwork(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("PinnedTargetLine", func(t *testing.T) {
|
||||
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
|
||||
if err == nil && string(data) == "PING" {
|
||||
_, _ = conn.Write([]byte("PONG"))
|
||||
}
|
||||
})
|
||||
defer server.Close()
|
||||
_, port, err := net.SplitHostPort(server.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
targets := filepath.Join(t.TempDir(), "targets.txt")
|
||||
line := fmt.Sprintf(`{"target": "pinned.test:%s", "ip": "127.0.0.1"}`, port)
|
||||
if err := os.WriteFile(targets, []byte(line), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// pinned.test does not resolve: the network request reaches the server
|
||||
// only through the pinned ip
|
||||
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, "-t", "protocols/network/basic.yaml", "-l", targets)
|
||||
if err != nil {
|
||||
t.Fatalf("pinned network request failed: %v", err)
|
||||
}
|
||||
if err := expectResultsCount(results, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Hex", func(t *testing.T) {
|
||||
errState := &capturedError{}
|
||||
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
|
||||
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
id: target-line-ip-pipeline
|
||||
|
||||
info:
|
||||
name: Target Line IP Pipeline
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- raw:
|
||||
- |
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
|
||||
pipeline: true
|
||||
pipeline-concurrent-connections: 1
|
||||
pipeline-requests-per-connection: 1
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "host=pinned.test:"
|
||||
extractors:
|
||||
- type: regex
|
||||
group: 1
|
||||
regex:
|
||||
- "server=([a-z]+)"
|
||||
@@ -1,20 +0,0 @@
|
||||
id: target-line-ip
|
||||
|
||||
info:
|
||||
name: Target Line IP
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "host=pinned.test:"
|
||||
extractors:
|
||||
- type: regex
|
||||
group: 1
|
||||
regex:
|
||||
- "server=([a-z]+)"
|
||||
@@ -1,25 +0,0 @@
|
||||
id: javascript-target-line-ip
|
||||
|
||||
info:
|
||||
name: JavaScript Target Line IP
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
javascript:
|
||||
- code: |
|
||||
var m = require("nuclei/net");
|
||||
var conn = m.Open("tcp", address);
|
||||
conn.SetTimeout(timeout);
|
||||
conn.RecvString(3)
|
||||
|
||||
args:
|
||||
address: "{{Host}}:{{Port}}"
|
||||
Host: "{{Host}}"
|
||||
Port: 5431
|
||||
timeout: 3
|
||||
|
||||
extractors:
|
||||
- type: regex
|
||||
part: response
|
||||
regex:
|
||||
- "[a-z]+"
|
||||
@@ -1,71 +0,0 @@
|
||||
package nuclei_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
nuclei "github.com/projectdiscovery/nuclei/v3/lib"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestThreadSafeCachedFlowPreservesPrecondition(t *testing.T) {
|
||||
for _, positive := range []bool{false, true} {
|
||||
name := "negative"
|
||||
if positive {
|
||||
name = "positive"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "flow.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(`id: cached-flow
|
||||
info:
|
||||
name: Cached flow
|
||||
author: pdteam
|
||||
severity: info
|
||||
flow: http(1) && http(2)
|
||||
http:
|
||||
- method: GET
|
||||
path: ["{{BaseURL}}/precondition"]
|
||||
matchers:
|
||||
- type: status
|
||||
status: [200]
|
||||
internal: true
|
||||
- method: GET
|
||||
path: ["{{BaseURL}}/check"]
|
||||
matchers:
|
||||
- type: status
|
||||
status: [404]
|
||||
`), 0o600))
|
||||
var checks, findings atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/check" {
|
||||
checks.Add(1)
|
||||
}
|
||||
if positive && r.URL.Path == "/precondition" {
|
||||
w.WriteHeader(200)
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
engine, err := nuclei.NewThreadSafeNucleiEngineCtx(context.Background(), nuclei.DisableUpdateCheck())
|
||||
require.NoError(t, err)
|
||||
defer engine.Close()
|
||||
engine.GlobalResultCallback(func(*output.ResultEvent) { findings.Add(1) })
|
||||
for call := int64(1); call <= 3; call++ {
|
||||
require.NoError(t, engine.ExecuteNucleiWithOptsCtx(context.Background(), []string{server.URL}, nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{Templates: []string{path}})))
|
||||
expected := int64(0)
|
||||
if positive {
|
||||
expected = call
|
||||
}
|
||||
require.Equal(t, expected, checks.Load(), "check requests after call %d", call)
|
||||
require.Equal(t, expected, findings.Load(), "findings after call %d", call)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+1
-41
@@ -173,16 +173,6 @@ func WithResponseReadSize(responseReadSize int) NucleiSDKOptions {
|
||||
}
|
||||
}
|
||||
|
||||
// EnableResponseCache reuses identical safe GET/HEAD responses within one scan.
|
||||
// It is off by default: a template that depends on state or timing could see a
|
||||
// different result.
|
||||
func EnableResponseCache() NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.ResponseCache = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
|
||||
// Deprecated: will be removed in favour of WithGlobalRateLimitCtx in next release
|
||||
func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
|
||||
@@ -377,9 +367,7 @@ func UseOutputWriter(writer OutputWriter) NucleiSDKOptions {
|
||||
}
|
||||
}
|
||||
|
||||
// StatsWriter receives scan statistics. A writer that also implements
|
||||
// progress.TaskTracker is told how many template executions the scan runs
|
||||
// and how many are done.
|
||||
// StatsWriter
|
||||
type StatsWriter progress.Progress
|
||||
|
||||
// UseStatsWriter allows setting a custom stats writer
|
||||
@@ -766,31 +754,3 @@ func WithTemporaryDirectory(parentDir string) NucleiSDKOptions {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithPerHostRateLimit applies the rate limit per host instead of globally.
|
||||
func WithPerHostRateLimit(enabled bool) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.PerHostRateLimit = enabled
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// EnableStrictProbe skips templates whose target service is unreachable: web
|
||||
// templates on hosts that do not speak HTTP and network templates on closed
|
||||
// ports. It is the SDK form of -strict-probe.
|
||||
func EnableStrictProbe() NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.StrictProbe = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// EnableTechFilter fingerprints HTTP targets and skips product-bound templates
|
||||
// that cannot match them. It also turns on the response cache, which the
|
||||
// fingerprint requests seed. It is the SDK form of -tech-filter.
|
||||
func EnableTechFilter() NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.TechFilter = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
+1
-61
@@ -8,15 +8,9 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Update checks are disabled explicitly so this pins corrupt-file rejection on
|
||||
// its own. The ignore file is read after init's update block, which may replace
|
||||
// a corrupt file via UpdateIgnoreFile — with update checks left on, this test
|
||||
// would assert rejection or repair depending on whether the process could reach
|
||||
// the network.
|
||||
func TestNewNucleiEngineRejectsCorruptActiveIgnoreFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, config.NucleiIgnoreFileName)
|
||||
@@ -27,62 +21,8 @@ func TestNewNucleiEngineRejectsCorruptActiveIgnoreFile(t *testing.T) {
|
||||
t.Cleanup(func() { cfg.SetTemplatesDir(oldRoot) })
|
||||
cfg.SetTemplatesDir(root)
|
||||
|
||||
engine, err := NewNucleiEngineCtx(context.Background(), DisableUpdateCheck())
|
||||
engine, err := NewNucleiEngineCtx(context.Background())
|
||||
require.Nil(t, engine)
|
||||
require.ErrorContains(t, err, "error parsing")
|
||||
require.ErrorContains(t, err, strconv.Quote(path))
|
||||
}
|
||||
|
||||
// TestIgnoreFileSurvivesAuthTemplateStore pins two init orderings.
|
||||
//
|
||||
// The ignore file is read after the update block, so a file this same call
|
||||
// creates is the one that becomes the deny-list. GetAuthTmplStore also runs
|
||||
// during init whenever SecretsFile is set. It must scope a copy of the
|
||||
// options: the caller's Templates and ExcludeTags stay put, and the ignore
|
||||
// file is still applied on top.
|
||||
//
|
||||
// A static-only secrets file is enough to build that store.
|
||||
func TestIgnoreFileSurvivesAuthTemplateStore(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(root, config.NucleiIgnoreFileName),
|
||||
[]byte("tags:\n - dos\nfiles:\n - dns/soa-detect.yaml\n"),
|
||||
0o600,
|
||||
))
|
||||
|
||||
cfg := config.DefaultConfig
|
||||
oldRoot := cfg.TemplatesDirectory
|
||||
t.Cleanup(func() { cfg.SetTemplatesDir(oldRoot) })
|
||||
cfg.SetTemplatesDir(root)
|
||||
|
||||
secrets := filepath.Join(root, "secrets.yaml")
|
||||
require.NoError(t, os.WriteFile(secrets, []byte(`static:
|
||||
- type: header
|
||||
domains:
|
||||
- example.com
|
||||
headers:
|
||||
- key: x-api-key
|
||||
value: not-a-real-secret
|
||||
`), 0o600))
|
||||
|
||||
opts := types.DefaultOptions()
|
||||
opts.SecretsFile = []string{secrets}
|
||||
opts.Templates = []string{"http/kept.yaml"}
|
||||
opts.ExcludeTags = []string{"custom"}
|
||||
|
||||
engine, err := NewNucleiEngineCtx(context.Background(),
|
||||
WithOptions(opts),
|
||||
DisableUpdateCheck(),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(engine.Close)
|
||||
|
||||
require.Equal(t, []string{"http/kept.yaml"}, []string(engine.Options().Templates),
|
||||
"the auth store must not replace the caller's template list")
|
||||
require.Contains(t, []string(engine.Options().ExcludeTags), "custom",
|
||||
"the caller's own excluded tags must survive auth-store setup")
|
||||
require.Contains(t, []string(engine.Options().ExcludeTags), "dos",
|
||||
"the .nuclei-ignore deny-list must be applied after auth-store setup")
|
||||
require.Contains(t, []string(engine.Options().ExcludedTemplates), "dns/soa-detect.yaml",
|
||||
"the files section must be applied too")
|
||||
}
|
||||
+5
-9
@@ -5,7 +5,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/runner"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core"
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
@@ -46,7 +44,11 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
Parser: base.parser,
|
||||
Browser: base.browserInstance,
|
||||
DoNotCache: opts.DoNotCacheTemplates,
|
||||
// Thread-safe executes can run concurrently with different Output writers.
|
||||
// The compiled-template cache shallow-copies requests and mutates shared
|
||||
// ExecutorOptions.Output via UpdateOptions/ApplyNewEngineOptions, which
|
||||
// would otherwise route all findings to whichever call last won the race.
|
||||
DoNotCache: true,
|
||||
}
|
||||
if base.interactshClient != nil {
|
||||
u.executerOpts.InteractshScope = base.interactshClient.NewRequestScope()
|
||||
@@ -62,11 +64,6 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
|
||||
opts.RateLimitDuration = time.Second
|
||||
}
|
||||
u.executerOpts.RateLimiter = utils.GetRateLimiter(ctx, opts.RateLimit, opts.RateLimitDuration)
|
||||
// Each execution gets its own response cache, created before its
|
||||
// templates compile.
|
||||
if opts.UsesResponseCache() && u.executerOpts.HTTPResponseCache == nil {
|
||||
u.executerOpts.HTTPResponseCache = httprespcache.New()
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
@@ -247,7 +244,6 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
|
||||
engine := core.New(tmpEngine.opts)
|
||||
engine.SetExecuterOptions(unsafeOpts.executerOpts)
|
||||
|
||||
defer runner.AttachScanPlan(tmpEngine.opts, inputProvider, tmpEngine.opts.Logger, engine, store.Templates())()
|
||||
_ = engine.ExecuteScanWithOpts(ctx, store.Templates(), inputProvider, false)
|
||||
|
||||
engine.WorkPool().Wait()
|
||||
|
||||
@@ -2,14 +2,10 @@ package nuclei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -72,87 +68,3 @@ http:
|
||||
require.NotNil(t, cached)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreadSafeExecuteUsesSharedCompiledCache(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "template.yaml")
|
||||
require.NoError(t, os.WriteFile(templatePath, []byte(`id: thread-safe-shared-cache
|
||||
|
||||
info:
|
||||
name: Thread safe shared cache
|
||||
author: pdteam
|
||||
severity: info
|
||||
tags: thread-safe-shared-cache
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "thread-safe-shared-cache"
|
||||
`), 0o600))
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("thread-safe-shared-cache"))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
engine, err := NewThreadSafeNucleiEngineCtx(context.Background())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(engine.Close)
|
||||
|
||||
const executions = 4
|
||||
var waitGroup sync.WaitGroup
|
||||
errors := make([]error, executions)
|
||||
for i := range executions {
|
||||
waitGroup.Add(1)
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
errors[i] = engine.ExecuteNucleiWithOptsCtx(context.Background(), []string{server.URL},
|
||||
WithTemplatesOrWorkflows(TemplateSources{Templates: []string{templatePath}}),
|
||||
WithTemplateFilters(TemplateFilters{Tags: []string{"thread-safe-shared-cache"}}),
|
||||
WithResultCallback(func(*output.ResultEvent) {}),
|
||||
)
|
||||
}()
|
||||
}
|
||||
waitGroup.Wait()
|
||||
|
||||
for _, err := range errors {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.Equal(t, 1, engine.eng.GetParser().CompiledCount())
|
||||
}
|
||||
|
||||
func TestThreadSafeExecuteHonorsDisableTemplateCache(t *testing.T) {
|
||||
templatePath := filepath.Join(t.TempDir(), "template.yaml")
|
||||
require.NoError(t, os.WriteFile(templatePath, []byte(`id: thread-safe-disable-cache
|
||||
|
||||
info:
|
||||
name: Thread safe disable cache
|
||||
author: pdteam
|
||||
severity: info
|
||||
tags: thread-safe-disable-cache
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
`), 0o600))
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
engine, err := NewThreadSafeNucleiEngineCtx(context.Background(), DisableTemplateCache())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(engine.Close)
|
||||
|
||||
err = engine.ExecuteNucleiWithOptsCtx(context.Background(), []string{server.URL},
|
||||
WithTemplatesOrWorkflows(TemplateSources{Templates: []string{templatePath}}),
|
||||
WithTemplateFilters(TemplateFilters{Tags: []string{"thread-safe-disable-cache"}}),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, engine.eng.GetParser().CompiledCount())
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package nuclei_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
nuclei "github.com/projectdiscovery/nuclei/v3/lib"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestScanPlanOptions(t *testing.T) {
|
||||
ne, err := nuclei.NewNucleiEngineCtx(context.Background(),
|
||||
nuclei.DisableUpdateCheck(),
|
||||
nuclei.EnableStrictProbe(),
|
||||
nuclei.EnableTechFilter(),
|
||||
nuclei.EnableResponseCache(),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
defer ne.Close()
|
||||
|
||||
opts := ne.Options()
|
||||
require.True(t, opts.StrictProbe)
|
||||
require.True(t, opts.TechFilter)
|
||||
require.True(t, opts.ResponseCache)
|
||||
}
|
||||
|
||||
// sdkCacheTemplates writes two templates that both GET /same. The second also
|
||||
// GETs /other, so request clustering cannot merge them into one request.
|
||||
func sdkCacheTemplates(t *testing.T) []string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
a := `id: sdk-cache-a
|
||||
info:
|
||||
name: SDK Cache A
|
||||
author: test
|
||||
severity: info
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/same"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "cached-body"
|
||||
`
|
||||
b := `id: sdk-cache-b
|
||||
info:
|
||||
name: SDK Cache B
|
||||
author: test
|
||||
severity: info
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/same"
|
||||
- "{{BaseURL}}/other"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "cached-body"
|
||||
`
|
||||
pa, pb := filepath.Join(dir, "a.yaml"), filepath.Join(dir, "b.yaml")
|
||||
require.NoError(t, os.WriteFile(pa, []byte(a), 0o600))
|
||||
require.NoError(t, os.WriteFile(pb, []byte(b), 0o600))
|
||||
return []string{pa, pb}
|
||||
}
|
||||
|
||||
// With the SDK options for the scan plan set, both templates still run and
|
||||
// match, and the response cache sends /same once. Without them it is sent
|
||||
// twice.
|
||||
func TestSDKScanPlanAndResponseCache(t *testing.T) {
|
||||
var sameHits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/same" {
|
||||
sameHits.Add(1)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
_, _ = w.Write([]byte("cached-body"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
tpls := sdkCacheTemplates(t)
|
||||
|
||||
run := func(extra ...nuclei.NucleiSDKOptions) map[string]bool {
|
||||
sameHits.Store(0)
|
||||
opts := append([]nuclei.NucleiSDKOptions{
|
||||
nuclei.DisableUpdateCheck(),
|
||||
nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{Templates: tpls}),
|
||||
}, extra...)
|
||||
ne, err := nuclei.NewNucleiEngineCtx(context.Background(), opts...)
|
||||
require.NoError(t, err)
|
||||
defer ne.Close()
|
||||
ne.Options().RestrictLocalNetworkAccess = false
|
||||
ne.Options().Retries = 0
|
||||
ne.LoadTargets([]string{ts.URL}, false)
|
||||
|
||||
var mu sync.Mutex
|
||||
matched := map[string]bool{}
|
||||
require.NoError(t, ne.ExecuteCallbackWithCtx(context.Background(), func(event *output.ResultEvent) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
matched[event.TemplateID] = true
|
||||
}))
|
||||
return matched
|
||||
}
|
||||
|
||||
matched := run(nuclei.EnableResponseCache(), nuclei.EnableStrictProbe(), nuclei.EnableTechFilter())
|
||||
require.Equal(t, map[string]bool{"sdk-cache-a": true, "sdk-cache-b": true}, matched)
|
||||
require.Equal(t, int32(1), sameHits.Load(), "the response cache must send /same once")
|
||||
|
||||
matched = run()
|
||||
require.Equal(t, map[string]bool{"sdk-cache-a": true, "sdk-cache-b": true}, matched)
|
||||
require.Equal(t, int32(2), sameHits.Load(), "without the cache /same is sent by each template")
|
||||
}
|
||||
+4
-21
@@ -9,7 +9,6 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/runner"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
|
||||
metadataindex "github.com/projectdiscovery/nuclei/v3/pkg/catalog/index"
|
||||
@@ -71,7 +70,7 @@ type NucleiEngine struct {
|
||||
|
||||
// unexported core fields
|
||||
ctx context.Context
|
||||
interactshClient interactsh.IClient
|
||||
interactshClient *interactsh.Client
|
||||
catalog catalog.Catalog
|
||||
rateLimiter *ratelimit.Limiter
|
||||
store *loader.Store
|
||||
@@ -152,9 +151,6 @@ func (e *NucleiEngine) GetTemplates() []*templates.Template {
|
||||
if !e.templatesLoaded {
|
||||
_ = e.LoadAllTemplates()
|
||||
}
|
||||
if e.store == nil {
|
||||
return nil
|
||||
}
|
||||
return e.store.Templates()
|
||||
}
|
||||
|
||||
@@ -163,9 +159,6 @@ func (e *NucleiEngine) GetWorkflows() []*templates.Template {
|
||||
if !e.templatesLoaded {
|
||||
_ = e.LoadAllTemplates()
|
||||
}
|
||||
if e.store == nil {
|
||||
return nil
|
||||
}
|
||||
return e.store.Workflows()
|
||||
}
|
||||
|
||||
@@ -258,7 +251,7 @@ func (e *NucleiEngine) closeInternal() {
|
||||
if e.hostErrCache != nil {
|
||||
e.hostErrCache.Close()
|
||||
}
|
||||
if e.executerOpts != nil && e.executerOpts.RateLimiter != nil {
|
||||
if e.executerOpts.RateLimiter != nil {
|
||||
e.executerOpts.RateLimiter.Stop()
|
||||
}
|
||||
if e.rateLimiter != nil {
|
||||
@@ -304,11 +297,9 @@ func (e *NucleiEngine) Close() {
|
||||
// enable matcher-status option if you expect this callback to be called for all results regardless if it matched or not
|
||||
func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...func(event *output.ResultEvent)) error {
|
||||
if !e.templatesLoaded {
|
||||
if err := e.LoadAllTemplates(); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = e.LoadAllTemplates()
|
||||
}
|
||||
if e.store == nil || (len(e.store.Templates()) == 0 && len(e.store.Workflows()) == 0) {
|
||||
if len(e.store.Templates()) == 0 && len(e.store.Workflows()) == 0 {
|
||||
return ErrNoTemplatesAvailable
|
||||
}
|
||||
if e.inputProvider.Count() == 0 {
|
||||
@@ -332,10 +323,6 @@ func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...f
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
// A response cached by an earlier scan of this engine is never served
|
||||
// to this one.
|
||||
e.executerOpts.HTTPResponseCache.Reset()
|
||||
defer runner.AttachScanPlan(e.opts, e.inputProvider, e.opts.Logger, e.engine, templatesAndWorkflows)()
|
||||
_ = e.engine.ExecuteScanWithOpts(ctx, templatesAndWorkflows, e.inputProvider, false)
|
||||
}()
|
||||
|
||||
@@ -391,10 +378,6 @@ func NewNucleiEngineCtx(ctx context.Context, options ...NucleiSDKOptions) (*Nucl
|
||||
}
|
||||
}
|
||||
if err := e.init(ctx); err != nil {
|
||||
// init may already own interactsh, a browser, a temp dir, and
|
||||
// reporting. Close is safe on a partial engine: executerOpts is
|
||||
// checked, and protocolinit.Close is a no-op before Init.
|
||||
e.Close()
|
||||
return nil, err
|
||||
}
|
||||
return e, nil
|
||||
|
||||
+8
-41
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
nucleiUtils "github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
@@ -45,9 +44,6 @@ import (
|
||||
func (e *NucleiEngine) applyRequiredDefaults(ctx context.Context) {
|
||||
mockoutput := testutils.NewMockOutputWriter(e.opts.OmitTemplate)
|
||||
mockoutput.WriteCallback = func(event *output.ResultEvent) {
|
||||
if (event.Error != "" || !event.MatcherStatus) && !e.opts.MatcherStatus {
|
||||
return
|
||||
}
|
||||
if len(e.resultCallbacks) > 0 {
|
||||
for _, callback := range e.resultCallbacks {
|
||||
if callback != nil {
|
||||
@@ -188,6 +184,10 @@ func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.loadIgnoreFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if e.opts.Parser != nil {
|
||||
if op, ok := e.opts.Parser.(*templates.Parser); ok {
|
||||
e.parser = op
|
||||
@@ -211,7 +211,7 @@ func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
_ = protocolinit.Init(e.opts)
|
||||
}
|
||||
|
||||
if e.opts.ProxyInternal && (e.opts.AliveHttpProxy != "" || e.opts.AliveSocksProxy != "") {
|
||||
if e.opts.ProxyInternal && e.opts.AliveHttpProxy != "" || e.opts.AliveSocksProxy != "" {
|
||||
httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}, "")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -354,12 +354,6 @@ func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
e.executerOpts.Options.Parser = op
|
||||
}
|
||||
|
||||
// Templates compile against e.executerOpts, so the response cache must
|
||||
// exist before they load. Each scan resets it.
|
||||
if e.opts.UsesResponseCache() && e.executerOpts.HTTPResponseCache == nil {
|
||||
e.executerOpts.HTTPResponseCache = httprespcache.New()
|
||||
}
|
||||
|
||||
e.engine = core.New(e.opts)
|
||||
e.engine.SetExecuterOptions(e.executerOpts)
|
||||
|
||||
@@ -388,26 +382,6 @@ func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Read .nuclei-ignore LAST, once init has finished managing that file and
|
||||
// the options it writes into.
|
||||
//
|
||||
// It must come after the update block above: init creates or replaces the
|
||||
// ignore file there (UpdateIgnoreFile), so reading earlier reads a file this
|
||||
// same call is about to write. On a host without a pre-existing ignore file
|
||||
// — a fresh install, or any container whose filesystem was reset — the read
|
||||
// found nothing, warned, and left the engine with an empty deny-list for its
|
||||
// whole lifetime, because ExcludeTags is never re-read after init. Reading
|
||||
// afterwards also makes the corrupt-file check recoverable rather than fatal:
|
||||
// a malformed local file that IgnoreFileNeedsUpdate replaces no longer fails
|
||||
// the scan.
|
||||
//
|
||||
// GetAuthTmplStore scopes a copy of the options, so it no longer clears
|
||||
// the engine filters. This read is still what installs the deny-list, and
|
||||
// it has to follow the update block above.
|
||||
if err := e.loadIgnoreFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -415,10 +389,7 @@ type syncOnce struct {
|
||||
sync.Once
|
||||
}
|
||||
|
||||
var (
|
||||
updateCheckInstance = &syncOnce{}
|
||||
templateUpdateInstance = &syncOnce{}
|
||||
)
|
||||
var updateCheckInstance = &syncOnce{}
|
||||
|
||||
// processUpdateCheckResults processes update check results
|
||||
func (e *NucleiEngine) processUpdateCheckResults() error {
|
||||
@@ -427,12 +398,8 @@ func (e *NucleiEngine) processUpdateCheckResults() error {
|
||||
if e.onUpdateAvailableCallback != nil {
|
||||
e.onUpdateAvailableCallback(config.DefaultConfig.LatestNucleiTemplatesVersion)
|
||||
}
|
||||
tm := installer.TemplateManager{}
|
||||
err = tm.UpdateIfOutdated()
|
||||
})
|
||||
if !e.disableTemplatesAutoUpgrade {
|
||||
templateUpdateInstance.Do(func() {
|
||||
tm := installer.TemplateManager{}
|
||||
err = tm.UpdateIfOutdated()
|
||||
})
|
||||
}
|
||||
return err
|
||||
}
|
||||
-172
@@ -2,20 +2,13 @@ package nuclei_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
nuclei "github.com/projectdiscovery/nuclei/v3/lib"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -162,168 +155,3 @@ func TestIgnoreFileFilesSectionIsApplied(t *testing.T) {
|
||||
require.Contains(t, ne.Options().ExcludedTemplates, "http/fuzzing/wordpress-themes-detect.yaml",
|
||||
"files section should reach ExcludedTemplates")
|
||||
}
|
||||
|
||||
func TestSDKCustomPortTemplateDoesNotPoisonBaseHost(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprint(w, "server-ok")
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
// Allocate and immediately close a TCP listener to obtain a guaranteed closed local port.
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
closedPort := ln.Addr().(*net.TCPAddr).Port
|
||||
require.NoError(t, ln.Close())
|
||||
|
||||
tplDir := t.TempDir()
|
||||
closedPortTpl := fmt.Sprintf(`id: closed-port-parallel-probe
|
||||
info:
|
||||
name: Closed Port Parallel Probe
|
||||
author: test
|
||||
severity: info
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "http://{{Hostname}}:%d/probe?p={{val}}"
|
||||
threads: 2
|
||||
payloads:
|
||||
val:
|
||||
- "1"
|
||||
- "2"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "never-matches"
|
||||
`, closedPort)
|
||||
healthyTpl := `id: healthy-base-target-match
|
||||
info:
|
||||
name: Healthy Base Target Match
|
||||
author: test
|
||||
severity: high
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "server-ok"
|
||||
`
|
||||
closedPortPath := filepath.Join(tplDir, "01-closed-port.yaml")
|
||||
healthyPath := filepath.Join(tplDir, "02-healthy.yaml")
|
||||
require.NoError(t, os.WriteFile(closedPortPath, []byte(closedPortTpl), 0o600))
|
||||
require.NoError(t, os.WriteFile(healthyPath, []byte(healthyTpl), 0o600))
|
||||
|
||||
for run := 1; run <= 3; run++ {
|
||||
ne, err := nuclei.NewNucleiEngineCtx(
|
||||
context.Background(),
|
||||
nuclei.DisableUpdateCheck(),
|
||||
nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{
|
||||
Templates: []string{closedPortPath, healthyPath},
|
||||
}),
|
||||
nuclei.WithConcurrency(nuclei.Concurrency{
|
||||
TemplateConcurrency: 1,
|
||||
HostConcurrency: 1,
|
||||
HeadlessHostConcurrency: 1,
|
||||
HeadlessTemplateConcurrency: 1,
|
||||
JavascriptTemplateConcurrency: 1,
|
||||
TemplatePayloadConcurrency: 2,
|
||||
ProbeConcurrency: 1,
|
||||
}),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
ne.Options().RestrictLocalNetworkAccess = false
|
||||
ne.Options().Timeout = 2
|
||||
ne.Options().Retries = 0
|
||||
|
||||
ne.LoadTargets([]string{ts.URL}, false)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
matched []string
|
||||
violations []*output.ResultEvent
|
||||
)
|
||||
err = ne.ExecuteCallbackWithCtx(context.Background(), func(event *output.ResultEvent) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !event.MatcherStatus || event.Error != "" {
|
||||
violations = append(violations, event)
|
||||
}
|
||||
matched = append(matched, event.TemplateID)
|
||||
})
|
||||
ne.Close()
|
||||
|
||||
require.NoError(t, err, "run %d failed", run)
|
||||
require.Empty(t, violations, "ExecuteCallbackWithCtx must not emit non-matching or skip events when MatcherStatus is false")
|
||||
require.Equal(t, []string{"healthy-base-target-match"}, matched, "run %d should consistently match healthy base target after custom-port failure", run)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSDKSkipEventsRespectMatcherStatus(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
closedAddr := ln.Addr().String()
|
||||
require.NoError(t, ln.Close())
|
||||
|
||||
tplDir := t.TempDir()
|
||||
for i := 1; i <= 3; i++ {
|
||||
tpl := fmt.Sprintf(`id: unreachable-check-%d
|
||||
info:
|
||||
name: Unreachable Check %d
|
||||
author: test
|
||||
severity: info
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/path-%d"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "ok"
|
||||
`, i, i, i)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(tplDir, fmt.Sprintf("0%d-tpl.yaml", i)), []byte(tpl), 0o600))
|
||||
}
|
||||
|
||||
runScan := func(matcherStatus bool) []*output.ResultEvent {
|
||||
ne, err := nuclei.NewNucleiEngineCtx(
|
||||
context.Background(),
|
||||
nuclei.DisableUpdateCheck(),
|
||||
nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{
|
||||
Templates: []string{tplDir},
|
||||
}),
|
||||
nuclei.WithConcurrency(nuclei.Concurrency{
|
||||
TemplateConcurrency: 1,
|
||||
HostConcurrency: 1,
|
||||
HeadlessHostConcurrency: 1,
|
||||
HeadlessTemplateConcurrency: 1,
|
||||
JavascriptTemplateConcurrency: 1,
|
||||
TemplatePayloadConcurrency: 1,
|
||||
ProbeConcurrency: 1,
|
||||
}),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
ne.Options().RestrictLocalNetworkAccess = false
|
||||
ne.Options().Timeout = 1
|
||||
ne.Options().Retries = 0
|
||||
ne.Options().MatcherStatus = matcherStatus
|
||||
|
||||
ne.LoadTargets([]string{"http://" + closedAddr}, false)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
events []*output.ResultEvent
|
||||
)
|
||||
err = ne.ExecuteCallbackWithCtx(context.Background(), func(event *output.ResultEvent) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
events = append(events, event)
|
||||
})
|
||||
ne.Close()
|
||||
require.NoError(t, err)
|
||||
return events
|
||||
}
|
||||
|
||||
require.Empty(t, runScan(false), "skip/error events must not be emitted when MatcherStatus is false")
|
||||
require.NotEmpty(t, runScan(true), "skip/error events should be emitted when MatcherStatus is true")
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/extensions"
|
||||
@@ -33,13 +32,6 @@ func GetKnownConfigFiles() []string {
|
||||
return knownConfigFiles
|
||||
}
|
||||
|
||||
// IsKnownConfigFile returns true if the base name of fpath is exactly one of
|
||||
// the known config files. Names that only contain one of them, such as
|
||||
// "cves.json.yaml", are not considered config files.
|
||||
func IsKnownConfigFile(fpath string) bool {
|
||||
return slices.Contains(knownConfigFiles, filepath.Base(filepath.FromSlash(fpath)))
|
||||
}
|
||||
|
||||
// GetKnownMiscDirectories returns known misc directories with trailing slashes.
|
||||
//
|
||||
// The trailing slash ensures that directory matching is explicit and avoids
|
||||
@@ -86,7 +78,7 @@ func IsTemplateWithRoot(fpath, rootDir string) bool {
|
||||
fname := filepath.Base(fpath)
|
||||
fext := strings.ToLower(filepath.Ext(fpath))
|
||||
|
||||
if IsKnownConfigFile(fname) {
|
||||
if stringsutil.ContainsAny(fname, GetKnownConfigFiles()...) {
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -9,8 +9,6 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// toAbs converts a slash separated absolute path into an OS specific absolute
|
||||
// path, prefixing it with the system volume on Windows.
|
||||
func toAbs(p string) string {
|
||||
if osutils.IsWindows() {
|
||||
// Infer the drive letter from the Windows system path
|
||||
@@ -30,8 +28,6 @@ func toAbs(p string) string {
|
||||
return filepath.FromSlash(p)
|
||||
}
|
||||
|
||||
// TestIsTemplate checks template detection for supported extensions, excluded
|
||||
// directories and known config files.
|
||||
func TestIsTemplate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -93,24 +89,6 @@ func TestIsTemplate(t *testing.T) {
|
||||
rootDir: "",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "excluded config file in subdirectory",
|
||||
fpath: "nuclei-templates/TEMPLATES-STATS.json",
|
||||
rootDir: "",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "template name containing config file name",
|
||||
fpath: "http/cves.json.yaml",
|
||||
rootDir: "",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "json template name containing config file name",
|
||||
fpath: "http/contributors.json.json",
|
||||
rootDir: "",
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -121,29 +99,6 @@ func TestIsTemplate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsKnownConfigFile ensures only exact config file base names match, and
|
||||
// names that merely contain one (e.g. cves.json.yaml) do not.
|
||||
func TestIsKnownConfigFile(t *testing.T) {
|
||||
for _, fpath := range []string{
|
||||
"cves.json",
|
||||
"contributors.json",
|
||||
"TEMPLATES-STATS.json",
|
||||
toAbs("/home/user/nuclei-templates/cves.json"),
|
||||
"https://example.com/nuclei-templates/cves.json",
|
||||
} {
|
||||
require.True(t, IsKnownConfigFile(fpath), "IsKnownConfigFile(%q)", fpath)
|
||||
}
|
||||
|
||||
for _, fpath := range []string{
|
||||
"cves.json.yaml",
|
||||
"http/cves.json.yaml",
|
||||
"my-contributors.json",
|
||||
toAbs("/home/user/cves.json/test.yaml"),
|
||||
} {
|
||||
require.False(t, IsKnownConfigFile(fpath), "IsKnownConfigFile(%q)", fpath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNucleiTemplatesIndexIncludesCustomDirSiblingPrefix(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
cfgDir := t.TempDir()
|
||||
|
||||
@@ -26,7 +26,9 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
|
||||
erred := make(map[string]error)
|
||||
|
||||
for _, t := range definitions {
|
||||
if config.IsKnownConfigFile(t) {
|
||||
if stringsutil.ContainsAny(t, knownConfigFiles...) {
|
||||
// TODO: this is a temporary fix to avoid treating these files as templates
|
||||
// this should be replaced with more appropriate and robust logic
|
||||
continue
|
||||
}
|
||||
if strings.Contains(t, urlutil.SchemeSeparator) && stringsutil.ContainsAny(t, config.GetSupportTemplateFileExtensions()...) {
|
||||
@@ -50,7 +52,9 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
|
||||
// purge all false positives
|
||||
filteredTemplates := []string{}
|
||||
for _, v := range allTemplates {
|
||||
if !config.IsKnownConfigFile(v) {
|
||||
// TODO: this is a temporary fix to avoid treating these files as templates
|
||||
// this should be replaced with more appropriate and robust logic
|
||||
if !stringsutil.ContainsAny(v, knownConfigFiles...) {
|
||||
filteredTemplates = append(filteredTemplates, v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
package disk
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing/fstest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -43,8 +42,6 @@ func TestFindGlobPathMatchesUsesCanonicalContainment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFindGlobPathMatchesResolvesContainedPath ensures a glob inside the
|
||||
// templates directory resolves against the embedded FS.
|
||||
func TestFindGlobPathMatchesResolvesContainedPath(t *testing.T) {
|
||||
memFS := fstest.MapFS{
|
||||
"http/test.yaml": {Data: []byte("legit")},
|
||||
@@ -59,42 +56,3 @@ func TestFindGlobPathMatchesResolvesContainedPath(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{"http/test.yaml"}, matches)
|
||||
}
|
||||
|
||||
// TestGetTemplatesPathKnownConfigFiles ensures that only exact known config
|
||||
// file names are skipped, and templates whose names merely contain one (e.g.
|
||||
// cves.json.yaml) are still loaded.
|
||||
func TestGetTemplatesPathKnownConfigFiles(t *testing.T) {
|
||||
templatesDir := t.TempDir()
|
||||
httpDir := filepath.Join(templatesDir, "http")
|
||||
require.NoError(t, os.MkdirAll(httpDir, 0755))
|
||||
|
||||
template := filepath.Join(httpDir, "cves.json.yaml")
|
||||
for _, f := range []string{
|
||||
template,
|
||||
filepath.Join(templatesDir, "cves.json"),
|
||||
filepath.Join(templatesDir, "contributors.json"),
|
||||
filepath.Join(templatesDir, "TEMPLATES-STATS.json"),
|
||||
} {
|
||||
require.NoError(t, os.WriteFile(f, []byte("id: test"), 0644))
|
||||
}
|
||||
|
||||
cat := NewCatalog(templatesDir)
|
||||
|
||||
t.Run("direct path", func(t *testing.T) {
|
||||
paths, errs := cat.GetTemplatesPath([]string{template})
|
||||
require.Empty(t, errs)
|
||||
require.Equal(t, []string{template}, paths)
|
||||
})
|
||||
|
||||
t.Run("directory traversal", func(t *testing.T) {
|
||||
paths, errs := cat.GetTemplatesPath([]string{templatesDir})
|
||||
require.Empty(t, errs)
|
||||
require.Equal(t, []string{template}, paths)
|
||||
})
|
||||
|
||||
t.Run("config file is skipped", func(t *testing.T) {
|
||||
paths, errs := cat.GetTemplatesPath([]string{filepath.Join(templatesDir, "cves.json")})
|
||||
require.Empty(t, errs)
|
||||
require.Empty(t, paths)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
package disk
|
||||
|
||||
var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"}
|
||||
+23
-45
@@ -63,6 +63,20 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
|
||||
return &atomic.Bool{}
|
||||
}
|
||||
|
||||
if e.executerOpts.Progress != nil {
|
||||
// Notes:
|
||||
// workflow requests are not counted as they can be conditional
|
||||
// templateList count is user requested templates count (before clustering)
|
||||
// totalReqAfterClustering is total requests count after clustering
|
||||
e.executerOpts.Progress.Init(target.Count(), len(templatesList), e.expectedRequests(finalTemplates, target, totalReqAfterClustering))
|
||||
}
|
||||
|
||||
if stringsutil.EqualFoldAny(e.options.ScanStrategy, scanstrategy.Auto.String(), "") {
|
||||
// TODO: this is only a placeholder, auto scan strategy should choose scan strategy
|
||||
// based on no of hosts , templates , stream and other optimization parameters
|
||||
e.options.ScanStrategy = scanstrategy.TemplateSpray.String()
|
||||
}
|
||||
|
||||
filtered := []*templates.Template{}
|
||||
selfContained := []*templates.Template{}
|
||||
// Filter Self Contained templates since they are not bound to target
|
||||
@@ -74,22 +88,6 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
|
||||
}
|
||||
}
|
||||
|
||||
if e.executerOpts.Progress != nil {
|
||||
// Notes:
|
||||
// workflow requests are not counted as they can be conditional
|
||||
// templateList count is user requested templates count (before clustering)
|
||||
// totalReqAfterClustering is total requests count after clustering
|
||||
requests, tasks := e.expectedWork(finalTemplates, filtered, target, totalReqAfterClustering)
|
||||
e.executerOpts.Progress.Init(target.Count(), len(templatesList), requests)
|
||||
// a self-contained template runs once
|
||||
e.addTaskTotal(tasks + int64(len(selfContained)))
|
||||
}
|
||||
|
||||
if stringsutil.EqualFoldAny(e.options.ScanStrategy, scanstrategy.Auto.String(), "") {
|
||||
// Last-resort fallback: runner scan planner normally resolves auto.
|
||||
e.options.ScanStrategy = scanstrategy.TemplateSpray.String()
|
||||
}
|
||||
|
||||
// Execute All SelfContained in parallel
|
||||
e.executeAllSelfContained(ctx, selfContained, results, selfcontainedWg)
|
||||
|
||||
@@ -125,7 +123,7 @@ func (e *Engine) executeTemplateSpray(ctx context.Context, templatesList []*temp
|
||||
}
|
||||
|
||||
// resize check point - nop if there are no changes
|
||||
wp.RefreshWithConfigContext(ctx, e.GetWorkPoolConfig())
|
||||
wp.RefreshWithConfig(e.GetWorkPoolConfig())
|
||||
|
||||
templateType := template.Type()
|
||||
var wg *syncutil.AdaptiveWaitGroup
|
||||
@@ -172,10 +170,7 @@ func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*template
|
||||
default:
|
||||
}
|
||||
|
||||
if err := wp.AddWithContext(ctx); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
wp.Add()
|
||||
go func(targetval *contextargs.MetaInput) {
|
||||
defer wp.Done()
|
||||
e.executeTemplatesOnTarget(ctx, templatesList, targetval, results)
|
||||
@@ -185,40 +180,23 @@ func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*template
|
||||
return results
|
||||
}
|
||||
|
||||
// expectedWork returns the requests and the tasks the target-bound templates
|
||||
// will run. It counts only the template and target pairs that per-target
|
||||
// profiles select, so progress reflects the work that will be done. A task is
|
||||
// one template, cluster or workflow run on one target.
|
||||
func (e *Engine) expectedWork(templatesList, targetBound []*templates.Template, target provider.InputProvider, unscopedRequests int) (requests, tasks int64) {
|
||||
// expectedRequests counts only the template and target pairs that per-target
|
||||
// profiles select, so progress reflects the requests that will be sent.
|
||||
func (e *Engine) expectedRequests(templatesList []*templates.Template, target provider.InputProvider, unscoped int) int64 {
|
||||
scope := e.executerOpts.TargetScope
|
||||
if scope == nil {
|
||||
return int64(unscopedRequests), target.Count() * int64(len(targetBound))
|
||||
return int64(unscoped)
|
||||
}
|
||||
targets := make(map[protocols.TemplateSelection]int64)
|
||||
target.Iterate(func(value *contextargs.MetaInput) bool {
|
||||
targets[scope.For(value)]++
|
||||
return true
|
||||
})
|
||||
var total int64
|
||||
for selection, count := range targets {
|
||||
requests += count * int64(getSelectedRequestCount(templatesList, selection))
|
||||
tasks += count * int64(getSelectedCount(targetBound, selection))
|
||||
total += count * int64(getSelectedRequestCount(templatesList, selection))
|
||||
}
|
||||
return requests, tasks
|
||||
}
|
||||
|
||||
// getSelectedCount returns the number of templates selection selects; a nil
|
||||
// selection selects every template.
|
||||
func getSelectedCount(templates []*templates.Template, selection protocols.TemplateSelection) int {
|
||||
if selection == nil {
|
||||
return len(templates)
|
||||
}
|
||||
count := 0
|
||||
for _, template := range templates {
|
||||
if template.SelectedBy(selection) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
return total
|
||||
}
|
||||
|
||||
// returns total requests count
|
||||
|
||||
+11
-55
@@ -9,8 +9,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/normalize"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
@@ -38,7 +36,6 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
|
||||
}
|
||||
finished := e.templateExecutionStarted(template, "")
|
||||
defer func() { finished(ctx.Err()) }()
|
||||
defer e.tasksDone(1)
|
||||
var err error
|
||||
var match bool
|
||||
ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
|
||||
@@ -169,18 +166,12 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
|
||||
currentInfo.Unlock()
|
||||
|
||||
// Skip if the host has had errors
|
||||
hostDown := inScope && e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(template.Type().String(), contextargs.NewWithMetaInput(ctx, scannedValue))
|
||||
// in-scope pairs that never reach executeTemplateOnInput are done here:
|
||||
// those done by the resumed scan and those of an unresponsive host
|
||||
if inScope && (skip || hostDown) {
|
||||
e.tasksDone(1)
|
||||
}
|
||||
if hostDown {
|
||||
if inScope && e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(e.executerOpts.ProtocolType.String(), contextargs.NewWithMetaInput(ctx, scannedValue)) {
|
||||
skipEvent := &output.ResultEvent{
|
||||
TemplateID: template.ID,
|
||||
TemplatePath: template.Path,
|
||||
Info: template.Info,
|
||||
Type: template.Type().String(),
|
||||
Type: e.executerOpts.ProtocolType.String(),
|
||||
Host: scannedValue.Input,
|
||||
MatcherStatus: false,
|
||||
Error: "host was skipped as it was found unresponsive",
|
||||
@@ -219,7 +210,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
|
||||
wp := e.GetWorkPool()
|
||||
defer wp.Wait()
|
||||
|
||||
for i, tpl := range alltemplates {
|
||||
for _, tpl := range alltemplates {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -233,12 +224,12 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
|
||||
// Check whether the target has already been marked as permanently
|
||||
// unresponsive by HostErrorsCache before spawning another goroutine.
|
||||
if e.executerOpts.HostErrorsCache != nil &&
|
||||
e.executerOpts.HostErrorsCache.Check(tpl.Type().String(), contextargs.NewWithMetaInput(ctx, target)) {
|
||||
e.executerOpts.HostErrorsCache.Check(e.executerOpts.ProtocolType.String(), contextargs.NewWithMetaInput(ctx, target)) {
|
||||
skipEvent := &output.ResultEvent{
|
||||
TemplateID: tpl.ID,
|
||||
TemplatePath: tpl.Path,
|
||||
Info: tpl.Info,
|
||||
Type: tpl.Type().String(),
|
||||
Type: e.executerOpts.ProtocolType.String(),
|
||||
Host: target.Input,
|
||||
MatcherStatus: false,
|
||||
Error: "host was skipped as it was found unresponsive",
|
||||
@@ -249,13 +240,11 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
|
||||
} else if e.executerOpts.Output != nil {
|
||||
_ = e.executerOpts.Output.Write(skipEvent)
|
||||
}
|
||||
// the remaining templates are skipped for this target as well
|
||||
e.tasksDone(int64(getSelectedCount(alltemplates[i:], e.selectionFor(target))))
|
||||
break
|
||||
}
|
||||
|
||||
// resize check point - nop if there are no changes
|
||||
wp.RefreshWithConfigContext(ctx, e.GetWorkPoolConfig())
|
||||
wp.RefreshWithConfig(e.GetWorkPoolConfig())
|
||||
|
||||
var sg *syncutil.AdaptiveWaitGroup
|
||||
if tpl.Type() == types.HeadlessProtocol {
|
||||
@@ -290,46 +279,15 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
|
||||
|
||||
// inScope reports whether per-target profiles select template for input.
|
||||
func (e *Engine) inScope(template *templates.Template, input *contextargs.MetaInput) bool {
|
||||
selection := e.selectionFor(input)
|
||||
return selection == nil || template.SelectedBy(selection)
|
||||
}
|
||||
|
||||
// selectionFor returns the templates per-target profiles select for input; nil
|
||||
// selects every template.
|
||||
func (e *Engine) selectionFor(input *contextargs.MetaInput) protocols.TemplateSelection {
|
||||
if e.executerOpts.TargetScope == nil {
|
||||
return nil
|
||||
}
|
||||
return e.executerOpts.TargetScope.For(input)
|
||||
}
|
||||
|
||||
// taskTracker returns the progress when it counts tasks, nil otherwise.
|
||||
func (e *Engine) taskTracker() progress.TaskTracker {
|
||||
if e.executerOpts == nil {
|
||||
return nil
|
||||
}
|
||||
tracker, _ := e.executerOpts.Progress.(progress.TaskTracker)
|
||||
return tracker
|
||||
}
|
||||
|
||||
// addTaskTotal adds count tasks to the total when the progress counts tasks.
|
||||
func (e *Engine) addTaskTotal(count int64) {
|
||||
if tracker := e.taskTracker(); tracker != nil {
|
||||
tracker.AddToTaskTotal(count)
|
||||
}
|
||||
}
|
||||
|
||||
// tasksDone marks count tasks as done when the progress counts tasks.
|
||||
func (e *Engine) tasksDone(count int64) {
|
||||
if tracker := e.taskTracker(); tracker != nil {
|
||||
tracker.IncrementTasksBy(count)
|
||||
return true
|
||||
}
|
||||
selection := e.executerOpts.TargetScope.For(input)
|
||||
return selection == nil || template.SelectedBy(selection)
|
||||
}
|
||||
|
||||
// executeTemplateOnInput performs template execution for a single input and returns match status and error
|
||||
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) {
|
||||
// a pair skipped as a repeat of its origin is done as well
|
||||
defer e.tasksDone(1)
|
||||
// A template fixed to the target's origin makes the same requests for every
|
||||
// target that shares one, which a crawled list produces by the hundred.
|
||||
// Running it once per origin sends exactly the same traffic, minus the
|
||||
@@ -342,11 +300,9 @@ func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates
|
||||
defer func() { finish(err == nil) }()
|
||||
finished := e.templateExecutionStarted(template, value.Input)
|
||||
defer func() { finished(ctx.Err()) }()
|
||||
// every protocol dials the target's host through its pinned ip, if any
|
||||
pinnedCtx := contextargs.WithCustomIP(ctx, value)
|
||||
ctxArgs := contextargs.New(pinnedCtx)
|
||||
ctxArgs := contextargs.New(ctx)
|
||||
ctxArgs.MetaInput = value
|
||||
scanCtx := scan.NewScanContext(pinnedCtx, ctxArgs)
|
||||
scanCtx := scan.NewScanContext(ctx, ctxArgs)
|
||||
|
||||
switch template.Type() {
|
||||
case types.WorkflowProtocol:
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
inputtypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
@@ -656,113 +655,3 @@ func Test_executeTemplateWithTargets_RespectsCancellation(t *testing.T) {
|
||||
var matched atomic.Bool
|
||||
e.executeTemplateWithTargets(ctx, tpl, targets, &matched)
|
||||
}
|
||||
|
||||
func newCancelTestEngine() *Engine {
|
||||
opts := &types.Options{
|
||||
TemplateThreads: 1,
|
||||
HeadlessTemplateThreads: 1,
|
||||
BulkSize: 1,
|
||||
HeadlessBulkSize: 1,
|
||||
Logger: gologger.DefaultLogger,
|
||||
}
|
||||
e := New(opts)
|
||||
e.SetExecuterOptions(&protocols.ExecutorOptions{Logger: e.Logger, ResumeCfg: types.NewResumeCfg(), ProtocolType: tmpltypes.HTTPProtocol})
|
||||
return e
|
||||
}
|
||||
|
||||
type gateExecuter struct {
|
||||
started chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (g *gateExecuter) Compile() error { return nil }
|
||||
func (g *gateExecuter) Requests() int { return 1 }
|
||||
func (g *gateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
|
||||
g.once.Do(func() { close(g.started) })
|
||||
<-ctx.Context().Done()
|
||||
return false, ctx.Context().Err()
|
||||
}
|
||||
func (g *gateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
|
||||
_, err := g.Execute(ctx)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func TestExecuteTemplateSprayUnblocksCancelledAcquire(t *testing.T) {
|
||||
e := newCancelTestEngine()
|
||||
gate := &gateExecuter{started: make(chan struct{})}
|
||||
templatesList := []*templates.Template{{Executer: gate}, {Executer: &gateExecuter{started: make(chan struct{})}}}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "a"}}}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
e.executeTemplateSpray(ctx, templatesList, targets)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-gate.started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("first template did not start")
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("executeTemplateSpray stayed blocked after cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteHostSprayUnblocksCancelledAcquire(t *testing.T) {
|
||||
e := newCancelTestEngine()
|
||||
gate := &gateExecuter{started: make(chan struct{})}
|
||||
templatesList := []*templates.Template{{Executer: gate}}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "a"}, {Input: "b"}}}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
e.executeHostSpray(ctx, templatesList, targets)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-gate.started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("first host did not start")
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("executeHostSpray stayed blocked after cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTemplatesOnTargetUnblocksCancelledAcquire(t *testing.T) {
|
||||
e := newCancelTestEngine()
|
||||
gate := &gateExecuter{started: make(chan struct{})}
|
||||
var matched atomic.Bool
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
e.executeTemplatesOnTarget(ctx, []*templates.Template{{Executer: gate}, {Executer: &gateExecuter{started: make(chan struct{})}}}, &contextargs.MetaInput{Input: "a"}, &matched)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-gate.started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("first template did not start")
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("executeTemplatesOnTarget stayed blocked after cancel")
|
||||
}
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
package plan
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
|
||||
)
|
||||
|
||||
// Input is the scan shape the planner uses to pick strategy and reachability.
|
||||
type Input struct {
|
||||
Hosts int
|
||||
Templates int
|
||||
Requests int // template×host request estimate (baseline)
|
||||
BulkSize int
|
||||
Stream bool
|
||||
StrictProbe bool
|
||||
|
||||
// Reachability / probe budget (known before or after probing).
|
||||
PortsToProbe int
|
||||
ConcreteNetworkTemplates int
|
||||
// FilteredRequests, when > 0, is the post-filter request estimate the plan reports.
|
||||
FilteredRequests int
|
||||
// GroupCount is distinct host signatures after probing (0 = unknown).
|
||||
GroupCount int
|
||||
|
||||
// TechFilter enables product/macro tag filtering after cheap HTTP fingerprinting.
|
||||
TechFilter bool
|
||||
// TechBoundTemplates is how many templates carry product/macro tags.
|
||||
TechBoundTemplates int
|
||||
}
|
||||
|
||||
// Plan is the execution plan for a single scan pass.
|
||||
type Plan struct {
|
||||
Strategy string
|
||||
BuildReachability bool
|
||||
ProbePorts bool
|
||||
UseReachabilityFilter bool
|
||||
UseTechFilter bool
|
||||
ExpectedRequests int64
|
||||
Reason string
|
||||
}
|
||||
|
||||
// Decide chooses spray strategy and whether to spend budget on reachability probing.
|
||||
//
|
||||
// Hard bar: never choose work that is expected to regress wall clock vs baseline spray.
|
||||
// When probe cost would dominate and savings are uncertain, reachability is skipped.
|
||||
func Decide(in Input) Plan {
|
||||
p := Plan{
|
||||
Strategy: chooseStrategy(in),
|
||||
ExpectedRequests: int64(max(in.Requests, 0)),
|
||||
}
|
||||
|
||||
reachReason := "reachability=off"
|
||||
if in.StrictProbe {
|
||||
reach := decideReachability(in)
|
||||
p.BuildReachability = reach.build
|
||||
p.ProbePorts = reach.probePorts
|
||||
p.UseReachabilityFilter = reach.useFilter
|
||||
reachReason = reach.reason
|
||||
}
|
||||
|
||||
tech := decideTechFilter(in)
|
||||
p.UseTechFilter = tech.use
|
||||
p.Reason = strategyReason(in, p.Strategy) + "; " + reachReason + "; " + tech.reason
|
||||
|
||||
if in.FilteredRequests > 0 {
|
||||
p.ExpectedRequests = int64(in.FilteredRequests)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func chooseStrategy(in Input) string {
|
||||
if in.Stream || in.Hosts <= 0 || in.Templates <= 0 {
|
||||
return scanstrategy.TemplateSpray.String()
|
||||
}
|
||||
bulk := in.BulkSize
|
||||
if bulk <= 0 {
|
||||
bulk = 25
|
||||
}
|
||||
// Host-spray only when the shape strongly favors per-host locality.
|
||||
// Conservative default is template-spray (historical auto) so wall clock
|
||||
// does not regress on small / mixed benches.
|
||||
if in.Hosts <= bulk && in.Hosts <= 10 && in.Templates >= 10*in.Hosts && in.Requests >= 5000 {
|
||||
return scanstrategy.HostSpray.String()
|
||||
}
|
||||
return scanstrategy.TemplateSpray.String()
|
||||
}
|
||||
|
||||
func strategyReason(in Input, strategy string) string {
|
||||
return fmt.Sprintf("strategy=%s hosts=%d templates=%d", strategy, in.Hosts, in.Templates)
|
||||
}
|
||||
|
||||
type reachabilityDecision struct {
|
||||
build bool
|
||||
probePorts bool
|
||||
useFilter bool
|
||||
reason string
|
||||
}
|
||||
|
||||
func decideReachability(in Input) reachabilityDecision {
|
||||
if in.Hosts <= 0 || in.Templates <= 0 {
|
||||
return reachabilityDecision{reason: "reachability=skipped-empty"}
|
||||
}
|
||||
|
||||
probeCost := in.PortsToProbe * in.Hosts
|
||||
baseline := in.Requests
|
||||
if baseline <= 0 {
|
||||
baseline = in.Templates * in.Hosts
|
||||
}
|
||||
|
||||
// No concrete network ports to probe: HTTP-only index is cheap (reuse httpx map).
|
||||
if in.PortsToProbe == 0 || in.ConcreteNetworkTemplates == 0 {
|
||||
return reachabilityDecision{
|
||||
build: true,
|
||||
probePorts: false,
|
||||
useFilter: true,
|
||||
reason: "reachability=http-only",
|
||||
}
|
||||
}
|
||||
|
||||
// Probe budget: if dialing every port×host costs more than the scan itself
|
||||
// and we have no signal of a mixed fleet, skip probing to match baseline time.
|
||||
if probeCost > baseline && in.GroupCount <= 1 {
|
||||
// GroupCount==0 (unknown) or 1 (homogeneous): do not pay for probes.
|
||||
if probeCost > max(baseline*2, 500) {
|
||||
return reachabilityDecision{
|
||||
reason: fmt.Sprintf("reachability=skipped-probe-budget cost=%d baseline=%d", probeCost, baseline),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// After probing, homogeneous fleets still benefit from a cheap filter (no extra dials
|
||||
// during spray for closed-port templates that global prune may have kept).
|
||||
return reachabilityDecision{
|
||||
build: true,
|
||||
probePorts: true,
|
||||
useFilter: true,
|
||||
reason: fmt.Sprintf("reachability=full probe_cost=%d", probeCost),
|
||||
}
|
||||
}
|
||||
|
||||
type techFilterDecision struct {
|
||||
use bool
|
||||
reason string
|
||||
}
|
||||
|
||||
func decideTechFilter(in Input) techFilterDecision {
|
||||
// Opt-in only: never enable from strategy/reachability alone.
|
||||
if !in.TechFilter {
|
||||
return techFilterDecision{reason: "tech-filter=off"}
|
||||
}
|
||||
// Skip fingerprinting when nothing is tech-bound — otherwise we add
|
||||
// probe latency with zero savings (wall-clock regression).
|
||||
if in.TechBoundTemplates <= 0 {
|
||||
return techFilterDecision{reason: "tech-filter=skipped-no-bound-templates"}
|
||||
}
|
||||
return techFilterDecision{use: true, reason: fmt.Sprintf("tech-filter=on bound=%d", in.TechBoundTemplates)}
|
||||
}
|
||||
|
||||
// ApplyFiltered updates ExpectedRequests once post-probe estimates are known.
|
||||
func (p *Plan) ApplyFiltered(filtered int) {
|
||||
if filtered > 0 {
|
||||
p.ExpectedRequests = int64(filtered)
|
||||
}
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
package plan
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDecideStrategyHostSpray(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 200,
|
||||
Requests: 10000,
|
||||
BulkSize: 25,
|
||||
})
|
||||
require.Equal(t, scanstrategy.HostSpray.String(), p.Strategy)
|
||||
require.Contains(t, p.Reason, "strategy=host-spray")
|
||||
}
|
||||
|
||||
func TestDecideStrategyTemplateSpraySmallBench(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 100,
|
||||
Requests: 500,
|
||||
BulkSize: 25,
|
||||
})
|
||||
require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
|
||||
}
|
||||
|
||||
func TestDecideStrategyTemplateSprayManyHosts(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 1000,
|
||||
Templates: 50,
|
||||
Requests: 50000,
|
||||
BulkSize: 25,
|
||||
})
|
||||
require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
|
||||
}
|
||||
|
||||
func TestDecideStrategyStreamDefaultsTemplateSpray(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 100,
|
||||
Stream: true,
|
||||
BulkSize: 25,
|
||||
})
|
||||
require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
|
||||
}
|
||||
|
||||
func TestDecideReachabilityOffWithoutStrictProbe(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 60,
|
||||
Requests: 300,
|
||||
StrictProbe: false,
|
||||
PortsToProbe: 3,
|
||||
})
|
||||
require.False(t, p.BuildReachability)
|
||||
require.False(t, p.UseReachabilityFilter)
|
||||
require.Contains(t, p.Reason, "reachability=off")
|
||||
}
|
||||
|
||||
func TestDecideReachabilityHTTPOnly(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 60,
|
||||
Requests: 300,
|
||||
StrictProbe: true,
|
||||
PortsToProbe: 0,
|
||||
ConcreteNetworkTemplates: 0,
|
||||
})
|
||||
require.True(t, p.BuildReachability)
|
||||
require.False(t, p.ProbePorts)
|
||||
require.True(t, p.UseReachabilityFilter)
|
||||
require.Contains(t, p.Reason, "reachability=http-only")
|
||||
}
|
||||
|
||||
func TestDecideReachabilityProbeBudgetSkip(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 100,
|
||||
Templates: 10,
|
||||
Requests: 1000,
|
||||
StrictProbe: true,
|
||||
PortsToProbe: 50, // cost = 5000 > 2*baseline
|
||||
ConcreteNetworkTemplates: 5,
|
||||
GroupCount: 0,
|
||||
})
|
||||
require.False(t, p.BuildReachability)
|
||||
require.False(t, p.UseReachabilityFilter)
|
||||
require.Contains(t, p.Reason, "reachability=skipped-probe-budget")
|
||||
}
|
||||
|
||||
func TestDecideReachabilityProbeWithinBudget(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 100,
|
||||
Templates: 10,
|
||||
Requests: 1000,
|
||||
StrictProbe: true,
|
||||
PortsToProbe: 15, // cost = 1500: above baseline, within 2*baseline
|
||||
ConcreteNetworkTemplates: 5,
|
||||
GroupCount: 0,
|
||||
})
|
||||
require.True(t, p.BuildReachability)
|
||||
require.True(t, p.ProbePorts)
|
||||
require.True(t, p.UseReachabilityFilter)
|
||||
require.Contains(t, p.Reason, "reachability=full")
|
||||
}
|
||||
|
||||
func TestDecideReachabilityFull(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 60,
|
||||
Requests: 300,
|
||||
StrictProbe: true,
|
||||
PortsToProbe: 3, // cost = 15 << baseline
|
||||
ConcreteNetworkTemplates: 30,
|
||||
})
|
||||
require.True(t, p.BuildReachability)
|
||||
require.True(t, p.ProbePorts)
|
||||
require.True(t, p.UseReachabilityFilter)
|
||||
require.Contains(t, p.Reason, "reachability=full")
|
||||
}
|
||||
|
||||
func TestDecideReachabilityEmptyInput(t *testing.T) {
|
||||
p := Decide(Input{StrictProbe: true, Hosts: 0, Templates: 10})
|
||||
require.False(t, p.BuildReachability)
|
||||
require.Contains(t, p.Reason, "reachability=skipped-empty")
|
||||
}
|
||||
|
||||
func TestDecideReachabilityConcreteZeroUsesHTTPOnly(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 20,
|
||||
Templates: 100,
|
||||
Requests: 2000,
|
||||
StrictProbe: true,
|
||||
PortsToProbe: 5,
|
||||
ConcreteNetworkTemplates: 0, // web-only set
|
||||
})
|
||||
require.True(t, p.BuildReachability)
|
||||
require.False(t, p.ProbePorts)
|
||||
require.True(t, p.UseReachabilityFilter)
|
||||
}
|
||||
|
||||
func TestDecideHostSprayRequiresLargeRequestBudget(t *testing.T) {
|
||||
// hosts small + many templates but requests below threshold → template-spray
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 200,
|
||||
Requests: 4999,
|
||||
BulkSize: 25,
|
||||
})
|
||||
require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
|
||||
}
|
||||
|
||||
func TestDecideApplyFilteredIgnoredWhenZero(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 10,
|
||||
Requests: 50,
|
||||
StrictProbe: false,
|
||||
FilteredRequests: 0,
|
||||
})
|
||||
require.Equal(t, int64(50), p.ExpectedRequests)
|
||||
p.ApplyFiltered(0)
|
||||
require.Equal(t, int64(50), p.ExpectedRequests)
|
||||
p.ApplyFiltered(12)
|
||||
require.Equal(t, int64(12), p.ExpectedRequests)
|
||||
}
|
||||
|
||||
func TestDecideTechFilterOffByDefault(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 100,
|
||||
Requests: 500,
|
||||
TechFilter: false,
|
||||
TechBoundTemplates: 40,
|
||||
})
|
||||
require.False(t, p.UseTechFilter)
|
||||
require.Contains(t, p.Reason, "tech-filter=off")
|
||||
}
|
||||
|
||||
func TestDecideTechFilterSkippedWhenNoBound(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 100,
|
||||
Requests: 500,
|
||||
TechFilter: true,
|
||||
TechBoundTemplates: 0,
|
||||
})
|
||||
require.False(t, p.UseTechFilter)
|
||||
require.Contains(t, p.Reason, "tech-filter=skipped-no-bound-templates")
|
||||
}
|
||||
|
||||
func TestDecideTechFilterOn(t *testing.T) {
|
||||
p := Decide(Input{
|
||||
Hosts: 5,
|
||||
Templates: 100,
|
||||
Requests: 500,
|
||||
TechFilter: true,
|
||||
TechBoundTemplates: 40,
|
||||
})
|
||||
require.True(t, p.UseTechFilter)
|
||||
require.Contains(t, p.Reason, "tech-filter=on")
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// taskRecorder counts the tasks the engine schedules and completes.
|
||||
type taskRecorder struct {
|
||||
progress.Progress
|
||||
total atomic.Int64
|
||||
done atomic.Int64
|
||||
}
|
||||
|
||||
func (r *taskRecorder) Init(int64, int, int64) {}
|
||||
func (r *taskRecorder) AddToTaskTotal(delta int64) { r.total.Add(delta) }
|
||||
func (r *taskRecorder) IncrementTasksBy(count int64) { r.done.Add(count) }
|
||||
|
||||
// downHosts reports the listed inputs as unresponsive.
|
||||
type downHosts struct {
|
||||
hosterrorscache.CacheInterface
|
||||
inputs map[string]bool
|
||||
}
|
||||
|
||||
func (d *downHosts) Check(_ string, ctx *contextargs.Context) bool {
|
||||
return d.inputs[ctx.MetaInput.Input]
|
||||
}
|
||||
|
||||
// newTaskEngine returns an engine for strategy whose progress records tasks.
|
||||
// down lists the unresponsive inputs, and a nil resume starts a fresh scan.
|
||||
func newTaskEngine(strategy scanstrategy.ScanStrategy, scope protocols.TargetScope, down map[string]bool, resume *types.ResumeCfg) (*Engine, *taskRecorder) {
|
||||
options := &types.Options{
|
||||
BulkSize: 2,
|
||||
TemplateThreads: 2,
|
||||
HeadlessBulkSize: 1,
|
||||
HeadlessTemplateThreads: 1,
|
||||
ScanStrategy: strategy.String(),
|
||||
Logger: gologger.DefaultLogger,
|
||||
}
|
||||
if resume == nil {
|
||||
resume = types.NewResumeCfg()
|
||||
}
|
||||
recorder := &taskRecorder{}
|
||||
executerOpts := &protocols.ExecutorOptions{
|
||||
Options: options,
|
||||
ResumeCfg: resume,
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: recorder,
|
||||
TargetScope: scope,
|
||||
}
|
||||
if down != nil {
|
||||
executerOpts.HostErrorsCache = &downHosts{inputs: down}
|
||||
}
|
||||
engine := New(options)
|
||||
executerOpts.Logger = engine.Logger
|
||||
engine.SetExecuterOptions(executerOpts)
|
||||
return engine, recorder
|
||||
}
|
||||
|
||||
// targetsOf returns a target provider of inputs.
|
||||
func targetsOf(inputs ...string) *fakeTargetProvider {
|
||||
targets := &fakeTargetProvider{}
|
||||
for _, input := range inputs {
|
||||
targets.values = append(targets.values, &contextargs.MetaInput{Input: input})
|
||||
}
|
||||
return targets
|
||||
}
|
||||
|
||||
func TestExecuteScanCountsTasks(t *testing.T) {
|
||||
scope := scopeByInput{
|
||||
"php": {paths: []string{"thinkphp.yaml"}},
|
||||
"tomcat": {paths: []string{"tomcat.yaml", "generic.yaml"}},
|
||||
"none": {},
|
||||
}
|
||||
inputs := []string{"php", "tomcat", "none", "plain"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
scope protocols.TargetScope
|
||||
down map[string]bool
|
||||
wantRuns int
|
||||
wantTotal int64
|
||||
wantDone int64
|
||||
}{
|
||||
{
|
||||
name: "every pair",
|
||||
// 3 templates on 4 targets, and the self-contained template once
|
||||
wantRuns: 13,
|
||||
wantTotal: 13,
|
||||
wantDone: 13,
|
||||
},
|
||||
{
|
||||
name: "per-target profiles",
|
||||
scope: scope,
|
||||
// php 1, tomcat 2, none 0, plain 3, self-contained 1
|
||||
wantRuns: 7,
|
||||
wantTotal: 7,
|
||||
wantDone: 7,
|
||||
},
|
||||
{
|
||||
name: "unresponsive host",
|
||||
down: map[string]bool{"tomcat": true},
|
||||
// tomcat is skipped for all 3 templates
|
||||
wantRuns: 10,
|
||||
wantTotal: 13,
|
||||
wantDone: 13,
|
||||
},
|
||||
{
|
||||
name: "unresponsive host with per-target profiles",
|
||||
scope: scope,
|
||||
down: map[string]bool{"tomcat": true},
|
||||
// tomcat is skipped for its 2 templates
|
||||
wantRuns: 5,
|
||||
wantTotal: 7,
|
||||
wantDone: 7,
|
||||
},
|
||||
}
|
||||
|
||||
for _, strategy := range []scanstrategy.ScanStrategy{scanstrategy.TemplateSpray, scanstrategy.HostSpray} {
|
||||
for _, test := range tests {
|
||||
t.Run(strategy.String()+"/"+test.name, func(t *testing.T) {
|
||||
engine, recorder := newTaskEngine(strategy, test.scope, test.down, nil)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
runs []string
|
||||
)
|
||||
var templatesList []*templates.Template
|
||||
for _, path := range []string{"thinkphp.yaml", "tomcat.yaml", "generic.yaml"} {
|
||||
templatesList = append(templatesList, &templates.Template{
|
||||
ID: path,
|
||||
Path: path,
|
||||
TotalRequests: 1,
|
||||
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
|
||||
})
|
||||
}
|
||||
templatesList = append(templatesList, &templates.Template{
|
||||
ID: "self.yaml",
|
||||
Path: "self.yaml",
|
||||
TotalRequests: 1,
|
||||
SelfContained: true,
|
||||
Executer: &recordingExecuter{path: "self.yaml", mu: &mu, runs: &runs},
|
||||
})
|
||||
engine.ExecuteScanWithOpts(context.Background(), templatesList, targetsOf(inputs...), true)
|
||||
|
||||
require.Len(t, runs, test.wantRuns)
|
||||
require.Equal(t, test.wantTotal, recorder.total.Load(), "task total")
|
||||
require.Equal(t, test.wantDone, recorder.done.Load(), "done tasks")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteScanCountsOriginScopedTasks(t *testing.T) {
|
||||
// a {{RootURL}} template runs once per origin, and the skipped repeats
|
||||
// are done tasks too
|
||||
inputs := []string{"http://a.test/x", "http://a.test/y", "http://a.test/z", "http://b.test/x"}
|
||||
|
||||
for _, strategy := range []scanstrategy.ScanStrategy{scanstrategy.TemplateSpray, scanstrategy.HostSpray} {
|
||||
t.Run(strategy.String(), func(t *testing.T) {
|
||||
engine, recorder := newTaskEngine(strategy, nil, nil, nil)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
runs []string
|
||||
)
|
||||
templatesList := []*templates.Template{{
|
||||
ID: "root.yaml",
|
||||
Path: "root.yaml",
|
||||
TotalRequests: 1,
|
||||
RequestsHTTP: []*http.Request{{Path: []string{"{{RootURL}}/"}}},
|
||||
Executer: &recordingExecuter{path: "root.yaml", mu: &mu, runs: &runs},
|
||||
}}
|
||||
require.True(t, templatesList[0].IsOriginScoped())
|
||||
engine.ExecuteScanWithOpts(context.Background(), templatesList, targetsOf(inputs...), true)
|
||||
|
||||
require.Len(t, runs, 2, "one run per origin")
|
||||
require.EqualValues(t, 4, recorder.total.Load(), "task total")
|
||||
require.EqualValues(t, 4, recorder.done.Load(), "done tasks")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteScanCountsResumedTasks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
down map[string]bool
|
||||
wantRuns []string
|
||||
}{
|
||||
{
|
||||
name: "responsive hosts",
|
||||
wantRuns: []string{"todo.yaml@a", "todo.yaml@b"},
|
||||
},
|
||||
{
|
||||
// a resumed pair on an unresponsive host is counted once
|
||||
name: "unresponsive host",
|
||||
down: map[string]bool{"a": true},
|
||||
wantRuns: []string{"todo.yaml@b"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
resumeCfg := types.NewResumeCfg()
|
||||
resumeCfg.ResumeFrom["done.yaml"] = &types.ResumeInfo{Completed: true}
|
||||
engine, recorder := newTaskEngine(scanstrategy.TemplateSpray, nil, test.down, resumeCfg)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
runs []string
|
||||
)
|
||||
var templatesList []*templates.Template
|
||||
for _, path := range []string{"done.yaml", "todo.yaml"} {
|
||||
templatesList = append(templatesList, &templates.Template{
|
||||
ID: path,
|
||||
Path: path,
|
||||
TotalRequests: 1,
|
||||
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
|
||||
})
|
||||
}
|
||||
engine.ExecuteScanWithOpts(context.Background(), templatesList, targetsOf("a", "b"), true)
|
||||
|
||||
require.ElementsMatch(t, test.wantRuns, runs)
|
||||
require.EqualValues(t, 4, recorder.total.Load(), "task total")
|
||||
require.EqualValues(t, 4, recorder.done.Load(), "done tasks")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,343 +0,0 @@
|
||||
// Package techfilter maps Wappalyzer products/categories to template tags and
|
||||
// decides whether a template can produce a finding on a fingerprinted host.
|
||||
//
|
||||
// Fail-open rules (coverage over aggressiveness):
|
||||
// - templates with no product/macro tags always run
|
||||
// - hosts with no fingerprints always allow every template
|
||||
// - CDN/WAF-only fingerprints do not suppress product templates
|
||||
// - non-HTTP templates are not filtered here (reachability owns that)
|
||||
package techfilter
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
wappalyzer "github.com/projectdiscovery/wappalyzergo"
|
||||
)
|
||||
|
||||
// genericTags are common nuclei tags that do not bind a template to a product.
|
||||
var genericTags = map[string]struct{}{
|
||||
"cve": {}, "cnvd": {}, "edb": {}, "kev": {}, "vkev": {},
|
||||
"misconfig": {}, "exposure": {}, "config": {}, "disclosure": {},
|
||||
"tech": {}, "detect": {}, "fingerprint": {}, "favicon": {}, "waf": {},
|
||||
"http": {}, "network": {}, "dns": {}, "ssl": {}, "tls": {},
|
||||
"websocket": {}, "headless": {}, "file": {}, "code": {}, "javascript": {},
|
||||
"workflow": {}, "intrusive": {}, "dos": {}, "fuzz": {}, "token-spray": {},
|
||||
"osint": {}, "panel": {}, "login": {}, "auth": {}, "unauth": {},
|
||||
"default-login": {}, "vuln": {}, "rce": {}, "lfi": {}, "sqli": {},
|
||||
"xss": {}, "ssrf": {}, "xxe": {}, "ssti": {}, "idor": {},
|
||||
"critical": {}, "high": {}, "medium": {}, "low": {}, "info": {},
|
||||
"unknown": {}, "generic": {}, "misc": {}, "bench": {},
|
||||
// Meta / source / taxonomy tags that are not products.
|
||||
"oast": {}, "takeover": {}, "deserialization": {}, "packetstorm": {},
|
||||
"seclists": {}, "hackerone": {}, "huntr": {}, "wp-plugin": {}, "wp-theme": {},
|
||||
"wordpress-plugin": {}, "wordpress-theme": {}, "authenticated": {},
|
||||
"unauthenticated": {}, "bypass": {}, "injection": {}, "traversal": {},
|
||||
"redirect": {}, "crlf": {}, "csrf": {}, "cors": {},
|
||||
}
|
||||
|
||||
// weakFingerprintTags alone must not suppress product-bound templates
|
||||
// (e.g. cloudflare/cdn-only fingerprints).
|
||||
var weakFingerprintTags = map[string]struct{}{
|
||||
"cdn": {}, "waf": {}, "misc": {}, "security": {}, "caching": {},
|
||||
}
|
||||
|
||||
// productAliases normalize common nuclei / wappalyzer spellings to one token.
|
||||
var productAliases = map[string]string{
|
||||
"wp": "wordpress",
|
||||
"httpd": "apache",
|
||||
"apache-http-server": "apache",
|
||||
}
|
||||
|
||||
// categoryMacros maps Wappalyzer category names to short macro tags.
|
||||
var categoryMacros = map[string]string{
|
||||
"cms": "cms",
|
||||
"ecommerce": "ecommerce",
|
||||
"web servers": "webserver",
|
||||
"web-servers": "webserver",
|
||||
"web frameworks": "web-framework",
|
||||
"web-frameworks": "web-framework",
|
||||
"javascript frameworks": "js-framework",
|
||||
"javascript-frameworks": "js-framework",
|
||||
"javascript libraries": "js-library",
|
||||
"javascript-libraries": "js-library",
|
||||
"cdn": "cdn",
|
||||
"databases": "database",
|
||||
"programming languages": "language",
|
||||
"programming-languages": "language",
|
||||
"paas": "paas",
|
||||
"saas": "saas",
|
||||
"security": "security",
|
||||
"caching": "caching",
|
||||
"miscellaneous": "misc",
|
||||
}
|
||||
|
||||
// HostProfile is the set of product + macro tags inferred for one target.
|
||||
type HostProfile struct {
|
||||
Tags map[string]struct{} // empty / nil => unknown => fail-open
|
||||
|
||||
// Cats are the Wappalyzer category ids of every detected product. They
|
||||
// record what the fingerprint actually has evidence about, which is what
|
||||
// makes it safe to rule a template out. See AllowTags.
|
||||
Cats map[int]struct{}
|
||||
}
|
||||
|
||||
// HasTags reports whether fingerprinting produced at least one tag.
|
||||
func (h HostProfile) HasTags() bool {
|
||||
return len(h.Tags) > 0
|
||||
}
|
||||
|
||||
// HasSubstantiveTags reports whether the profile has a strong category macro
|
||||
// (cms, webserver, …). CDN/WAF-only fingerprints are not substantive.
|
||||
func (h HostProfile) HasSubstantiveTags() bool {
|
||||
for tag := range h.Tags {
|
||||
if isStrongMacro(tag) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isStrongMacro(tag string) bool {
|
||||
if _, weak := weakFingerprintTags[tag]; weak {
|
||||
return false
|
||||
}
|
||||
for _, m := range categoryMacros {
|
||||
if tag == m {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// NormalizeProduct turns a Wappalyzer / nuclei product name into a tag token.
|
||||
func NormalizeProduct(name string) string {
|
||||
name = strings.TrimSpace(strings.ToLower(name))
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
if i := strings.IndexByte(name, ':'); i > 0 {
|
||||
name = name[:i]
|
||||
}
|
||||
name = strings.ReplaceAll(name, "_", "-")
|
||||
name = strings.Join(strings.Fields(name), "-")
|
||||
if alias, ok := productAliases[name]; ok {
|
||||
return alias
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// NormalizeCategory turns a Wappalyzer category into a macro tag.
|
||||
func NormalizeCategory(cat string) string {
|
||||
key := strings.ToLower(strings.TrimSpace(cat))
|
||||
if key == "" {
|
||||
return ""
|
||||
}
|
||||
if m, ok := categoryMacros[key]; ok {
|
||||
return m
|
||||
}
|
||||
return strings.ReplaceAll(key, " ", "-")
|
||||
}
|
||||
|
||||
// IsGenericTag reports whether tag is not product/macro binding.
|
||||
func IsGenericTag(tag string) bool {
|
||||
tag = strings.ToLower(strings.TrimSpace(tag))
|
||||
if tag == "" {
|
||||
return true
|
||||
}
|
||||
if _, ok := genericTags[tag]; ok {
|
||||
return true
|
||||
}
|
||||
// cve2021 / cve-2021 style year tags bind no product.
|
||||
if strings.HasPrefix(tag, "cve") {
|
||||
rest := strings.TrimPrefix(tag, "cve")
|
||||
rest = strings.TrimPrefix(rest, "-")
|
||||
if rest != "" && isAllDigits(rest) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isAllDigits(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ProfileFromFingerprint builds a host profile from products and their categories.
|
||||
// products maps product name -> category names from Wappalyzer.
|
||||
func ProfileFromFingerprint(products map[string][]string) HostProfile {
|
||||
tags := make(map[string]struct{})
|
||||
observed := make(map[int]struct{})
|
||||
byProduct := productCategories()
|
||||
for product, cats := range products {
|
||||
if p := NormalizeProduct(product); p != "" {
|
||||
tags[p] = struct{}{}
|
||||
for _, id := range byProduct[p] {
|
||||
observed[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, c := range cats {
|
||||
if m := NormalizeCategory(c); m != "" {
|
||||
tags[m] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
return HostProfile{Tags: tags, Cats: observed}
|
||||
}
|
||||
|
||||
// TemplateProductTags returns product/macro tags that bind a template to a stack.
|
||||
// Empty means the template is generic and must always be allowed.
|
||||
func TemplateProductTags(t *templates.Template) []string {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
raw := t.Info.Tags.ToSlice()
|
||||
out := make([]string, 0, len(raw))
|
||||
seen := make(map[string]struct{}, len(raw))
|
||||
known := knownFingerprintTokens()
|
||||
for _, tag := range raw {
|
||||
tag = NormalizeProduct(tag)
|
||||
if tag == "" || IsGenericTag(tag) {
|
||||
continue
|
||||
}
|
||||
// A tag binds only when fingerprinting can actually emit it. Anything
|
||||
// else would never match a profile and would prune the template on
|
||||
// every host, so it must leave the template unbound.
|
||||
if _, ok := known[tag]; !ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[tag]; ok {
|
||||
continue
|
||||
}
|
||||
seen[tag] = struct{}{}
|
||||
out = append(out, tag)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// knownFingerprintTokens is every token a host profile can contain: the
|
||||
// normalized name of every Wappalyzer app plus every category macro.
|
||||
//
|
||||
// This is an allowlist on purpose. nuclei-templates carries thousands of tags
|
||||
// that name no product at all (discovery, wpscan, top-200, auth-bypass, ...).
|
||||
// Treating an unrecognized tag as a product binds the template to something no
|
||||
// fingerprint can ever report, which silently drops it on every host. Unknown
|
||||
// tags must therefore bind nothing.
|
||||
var knownFingerprintTokens = sync.OnceValue(func() map[string]struct{} {
|
||||
var data struct {
|
||||
Apps map[string]json.RawMessage `json:"apps"`
|
||||
}
|
||||
tokens := make(map[string]struct{}, 8192)
|
||||
if err := json.Unmarshal([]byte(wappalyzer.GetFingerprints()), &data); err == nil {
|
||||
for name := range data.Apps {
|
||||
if p := NormalizeProduct(name); p != "" {
|
||||
tokens[p] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, macro := range categoryMacros {
|
||||
tokens[macro] = struct{}{}
|
||||
}
|
||||
for tag := range weakFingerprintTags {
|
||||
tokens[tag] = struct{}{}
|
||||
}
|
||||
return tokens
|
||||
})
|
||||
|
||||
// IsTechBound reports whether the template is constrained to a product/macro.
|
||||
func IsTechBound(t *templates.Template) bool {
|
||||
return len(TemplateProductTags(t)) > 0
|
||||
}
|
||||
|
||||
// Allow reports whether template may run on a host with the given profile.
|
||||
// Fail-open when the host was not fingerprinted or the template is unbound.
|
||||
func Allow(profile HostProfile, t *templates.Template) bool {
|
||||
return AllowTags(profile, TemplateProductTags(t))
|
||||
}
|
||||
|
||||
// AllowTags is Allow with precomputed product tags (avoids per-pair allocation).
|
||||
func AllowTags(profile HostProfile, bound []string) bool {
|
||||
if len(bound) == 0 {
|
||||
return true
|
||||
}
|
||||
if !profile.HasTags() || !profile.HasSubstantiveTags() {
|
||||
return true
|
||||
}
|
||||
for _, tag := range bound {
|
||||
if _, ok := profile.Tags[tag]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// No positive match. Ruling the template out now would assume the
|
||||
// fingerprint is complete, and it never is: it is one page of one host, and
|
||||
// a stack is routinely invisible from it (WordPress behind nginx is the
|
||||
// ordinary case, not the exception). Absence of evidence is not evidence of
|
||||
// absence, so a template is dropped only when the fingerprint positively
|
||||
// contradicts it: something else was identified in the same category, and
|
||||
// the template's own product was not. If any bound product sits in a
|
||||
// category the fingerprint says nothing about, the template runs.
|
||||
byProduct := productCategories()
|
||||
for _, tag := range bound {
|
||||
cats, ok := byProduct[tag]
|
||||
if !ok || len(cats) == 0 {
|
||||
return true // category macro or unknown product: no contradiction possible
|
||||
}
|
||||
contradicted := false
|
||||
for _, id := range cats {
|
||||
if _, seen := profile.Cats[id]; seen {
|
||||
contradicted = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !contradicted {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// productCategories maps a normalized product token to its Wappalyzer category
|
||||
// ids. It is the evidence index: a category id present in a host profile means
|
||||
// fingerprinting actually identified something of that kind on that host.
|
||||
var productCategories = sync.OnceValue(func() map[string][]int {
|
||||
var data struct {
|
||||
Apps map[string]struct {
|
||||
Cats []int `json:"cats"`
|
||||
} `json:"apps"`
|
||||
}
|
||||
out := make(map[string][]int, 8192)
|
||||
if err := json.Unmarshal([]byte(wappalyzer.GetFingerprints()), &data); err != nil {
|
||||
return out
|
||||
}
|
||||
for name, app := range data.Apps {
|
||||
p := NormalizeProduct(name)
|
||||
if p == "" || len(app.Cats) == 0 {
|
||||
continue
|
||||
}
|
||||
// Aliases collapse several apps onto one token; union their categories.
|
||||
out[p] = append(out[p], app.Cats...)
|
||||
}
|
||||
return out
|
||||
})
|
||||
|
||||
// CountTechBound returns how many templates carry product/macro tags.
|
||||
func CountTechBound(tpls []*templates.Template) int {
|
||||
n := 0
|
||||
for _, t := range tpls {
|
||||
if IsTechBound(t) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
package techfilter
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
)
|
||||
|
||||
func TestNormalizeProductAndCategory(t *testing.T) {
|
||||
if got := NormalizeProduct("Nginx:1.25"); got != "nginx" {
|
||||
t.Fatalf("NormalizeProduct = %q", got)
|
||||
}
|
||||
if got := NormalizeProduct("wp"); got != "wordpress" {
|
||||
t.Fatalf("alias = %q", got)
|
||||
}
|
||||
if got := NormalizeCategory("Web servers"); got != "webserver" {
|
||||
t.Fatalf("category = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileFromFingerprint(t *testing.T) {
|
||||
p := ProfileFromFingerprint(map[string][]string{
|
||||
"Nginx": {"Web servers"},
|
||||
"WordPress": {"CMS"},
|
||||
})
|
||||
for _, want := range []string{"nginx", "webserver", "wordpress", "cms"} {
|
||||
if _, ok := p.Tags[want]; !ok {
|
||||
t.Fatalf("missing tag %q in %#v", want, p.Tags)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func tagged(tags ...string) *templates.Template {
|
||||
return &templates.Template{
|
||||
Info: model.Info{Tags: stringslice.StringSlice{Value: tags}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowFailOpenAndMatch(t *testing.T) {
|
||||
nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}})
|
||||
empty := HostProfile{}
|
||||
|
||||
generic := tagged("misconfig", "http")
|
||||
wp := tagged("cve", "wordpress")
|
||||
ngx := tagged("nginx", "misconfig")
|
||||
cms := tagged("cms")
|
||||
|
||||
if !Allow(nginx, generic) {
|
||||
t.Fatal("generic must always run")
|
||||
}
|
||||
if !Allow(empty, wp) {
|
||||
t.Fatal("unknown host must fail-open for bound templates")
|
||||
}
|
||||
// An nginx fingerprint says nothing about the CMS behind it, and WordPress
|
||||
// behind nginx is an ordinary stack. Absence of a CMS in the profile is not
|
||||
// evidence that there is none.
|
||||
if !Allow(nginx, wp) {
|
||||
t.Fatal("nginx says nothing about a CMS, so a wordpress template must run")
|
||||
}
|
||||
if !Allow(nginx, ngx) {
|
||||
t.Fatal("nginx template must run on nginx host")
|
||||
}
|
||||
if !Allow(nginx, cms) {
|
||||
t.Fatal("nginx says nothing about a CMS, so a cms-bound template must run")
|
||||
}
|
||||
wpHost := ProfileFromFingerprint(map[string][]string{"WordPress": {"CMS"}})
|
||||
if !Allow(wpHost, cms) {
|
||||
t.Fatal("cms macro must match wordpress host")
|
||||
}
|
||||
if !Allow(wpHost, wp) {
|
||||
t.Fatal("wordpress product must match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetaTagsAreNotTechBound(t *testing.T) {
|
||||
cases := [][]string{
|
||||
{"cve", "cve2021", "oast", "rce"},
|
||||
{"takeover", "dns"},
|
||||
{"hackerone", "packetstorm", "seclists"},
|
||||
{"wp-plugin", "wordpress-plugin"},
|
||||
}
|
||||
for _, tags := range cases {
|
||||
tpl := tagged(tags...)
|
||||
if IsTechBound(tpl) {
|
||||
t.Fatalf("tags %v must not be tech-bound, got %v", tags, TemplateProductTags(tpl))
|
||||
}
|
||||
cdnOnly := ProfileFromFingerprint(map[string][]string{"Cloudflare": {"CDN"}})
|
||||
if !Allow(cdnOnly, tpl) {
|
||||
t.Fatalf("unbound meta tags %v must run on any host", tags)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCDNOnlyFingerprintFailOpen(t *testing.T) {
|
||||
cdnOnly := ProfileFromFingerprint(map[string][]string{"Cloudflare": {"CDN"}})
|
||||
if cdnOnly.HasSubstantiveTags() {
|
||||
t.Fatalf("cdn-only profile should not be substantive: %#v", cdnOnly.Tags)
|
||||
}
|
||||
wp := tagged("wordpress")
|
||||
if !Allow(cdnOnly, wp) {
|
||||
t.Fatal("CDN-only fingerprint must fail-open for product templates")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountTechBound(t *testing.T) {
|
||||
n := CountTechBound([]*templates.Template{
|
||||
tagged("misconfig"),
|
||||
tagged("wordpress"),
|
||||
tagged("nginx", "http"),
|
||||
tagged("cve2023", "oast"),
|
||||
})
|
||||
if n != 2 {
|
||||
t.Fatalf("CountTechBound = %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A tag only binds when fingerprinting can emit it. nuclei-templates carries
|
||||
// thousands of tags that name no product; binding to one of those prunes the
|
||||
// template on every host, because no profile can ever contain it.
|
||||
func TestOnlyKnownFingerprintTokensBind(t *testing.T) {
|
||||
nonProducts := [][]string{
|
||||
{"misconfig", "headers", "generic", "vuln"}, // http-missing-security-headers
|
||||
{"tech", "discovery"}, // tech-detect
|
||||
{"waf", "tech", "misc", "discovery"}, // waf-detect
|
||||
{"wpscan"},
|
||||
{"top-100"}, {"top-200"},
|
||||
{"auth-bypass"}, {"time-based-sqli"},
|
||||
{"install"}, {"files"}, {"file-upload"},
|
||||
{"phishing"}, {"osint-social"},
|
||||
{"debug"}, {"devops"}, {"token"}, {"tokens"}, {"fpd"}, {"oss"},
|
||||
}
|
||||
nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}})
|
||||
for _, tags := range nonProducts {
|
||||
tpl := tagged(tags...)
|
||||
if got := TemplateProductTags(tpl); len(got) != 0 {
|
||||
t.Fatalf("tags %v must not bind, got %v", tags, got)
|
||||
}
|
||||
if !Allow(nginx, tpl) {
|
||||
t.Fatalf("unbound template %v must run on every host", tags)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The inversion must not disable the filter: real products still bind, and a
|
||||
// template for another stack is still pruned.
|
||||
func TestKnownProductsStillBind(t *testing.T) {
|
||||
for _, tags := range [][]string{{"wordpress"}, {"apache"}, {"joomla"}, {"cms"}} {
|
||||
if got := TemplateProductTags(tagged(tags...)); len(got) == 0 {
|
||||
t.Fatalf("tags %v name a product and must bind", tags)
|
||||
}
|
||||
}
|
||||
|
||||
wp := ProfileFromFingerprint(map[string][]string{"WordPress": {"CMS"}})
|
||||
if !Allow(wp, tagged("wordpress", "rce")) {
|
||||
t.Fatal("a wordpress template must run on a wordpress profile")
|
||||
}
|
||||
}
|
||||
|
||||
// A template is dropped only when the fingerprint contradicts it: it identified
|
||||
// something else of the same kind and not the template's own product.
|
||||
func TestPruneOnlyOnPositiveContradiction(t *testing.T) {
|
||||
nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}})
|
||||
wpHost := ProfileFromFingerprint(map[string][]string{
|
||||
"WordPress": {"CMS"}, "Nginx": {"Web servers"},
|
||||
})
|
||||
|
||||
// Contradiction: a web server was identified and it is not apache.
|
||||
if Allow(nginx, tagged("apache")) {
|
||||
t.Fatal("nginx was identified, so an apache template is contradicted")
|
||||
}
|
||||
// Contradiction: a CMS was identified and it is not joomla.
|
||||
if Allow(wpHost, tagged("joomla")) {
|
||||
t.Fatal("wordpress was identified, so a joomla template is contradicted")
|
||||
}
|
||||
// No evidence: nothing of that kind was identified at all.
|
||||
for _, tags := range [][]string{{"wordpress"}, {"joomla"}, {"cms"}, {"mysql"}} {
|
||||
if !Allow(nginx, tagged(tags...)) {
|
||||
t.Fatalf("nginx gives no evidence about %v, so it must run", tags)
|
||||
}
|
||||
}
|
||||
// One uncontradicted product is enough to keep a multi-product template.
|
||||
if !Allow(nginx, tagged("apache", "wordpress")) {
|
||||
t.Fatal("wordpress is uncontradicted, so the template must run")
|
||||
}
|
||||
// A positive match always wins.
|
||||
if !Allow(wpHost, tagged("wordpress")) {
|
||||
t.Fatal("a positive match must run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKnownFingerprintTokensVocabulary(t *testing.T) {
|
||||
tokens := knownFingerprintTokens()
|
||||
if len(tokens) < 5000 {
|
||||
t.Fatalf("expected the wappalyzer vocabulary, got %d tokens", len(tokens))
|
||||
}
|
||||
for _, want := range []string{"wordpress", "nginx", "apache", "cms", "webserver"} {
|
||||
if _, ok := tokens[want]; !ok {
|
||||
t.Fatalf("token %q missing from the vocabulary", want)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"discovery", "headers", "wpscan", "top-200"} {
|
||||
if _, ok := tokens[bad]; ok {
|
||||
t.Fatalf("token %q is not a product and must not be in the vocabulary", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,9 +132,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
|
||||
}
|
||||
|
||||
for _, subtemplate := range matcher.Subtemplates {
|
||||
if err := swg.AddWithContext(ctx.Context()); err != nil {
|
||||
return
|
||||
}
|
||||
swg.Add()
|
||||
|
||||
go func(subtemplate *workflows.WorkflowTemplate) {
|
||||
defer swg.Done()
|
||||
@@ -163,9 +161,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
|
||||
}
|
||||
if len(template.Subtemplates) > 0 && firstMatched {
|
||||
for _, subtemplate := range template.Subtemplates {
|
||||
if err := swg.AddWithContext(ctx.Context()); err != nil {
|
||||
return err
|
||||
}
|
||||
swg.Add()
|
||||
|
||||
go func(template *workflows.WorkflowTemplate) {
|
||||
// create a new context with the same input but with unset callbacks
|
||||
|
||||
@@ -2,9 +2,7 @@ package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
@@ -303,61 +301,6 @@ func TestWorkflowsSameStepExecutersRefreshTemplateContext(t *testing.T) {
|
||||
require.Equal(t, "foo", secondWhoami, "could not refresh workflow context into later executers in the same step")
|
||||
}
|
||||
|
||||
func TestWorkflowSubtemplatesUnblockCancelledAcquire(t *testing.T) {
|
||||
progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0)
|
||||
var started atomic.Int32
|
||||
block := &mockExecuter{result: true, executeScanHook: func(ctx *scan.ScanContext) {
|
||||
started.Add(1)
|
||||
<-ctx.Context().Done()
|
||||
}}
|
||||
var waitingStarted atomic.Bool
|
||||
waiting := &mockExecuter{result: true, executeScanHook: func(ctx *scan.ScanContext) {
|
||||
waitingStarted.Store(true)
|
||||
<-ctx.Context().Done()
|
||||
}}
|
||||
|
||||
// Two slots: the two blockers occupy them so the third AddWithContext waits.
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 2}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, outputs: []*output.InternalWrappedEvent{
|
||||
{OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}},
|
||||
}}, Options: &protocols.ExecutorOptions{Progress: progressBar}},
|
||||
}, Subtemplates: []*workflows.WorkflowTemplate{
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{Executer: block, Options: &protocols.ExecutorOptions{Progress: progressBar}}}},
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{Executer: block, Options: &protocols.ExecutorOptions{Progress: progressBar}}}},
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{Executer: waiting, Options: &protocols.ExecutorOptions{Progress: progressBar}}}},
|
||||
}},
|
||||
}}
|
||||
|
||||
engine := &Engine{}
|
||||
parent, cancel := context.WithCancel(context.Background())
|
||||
input := contextargs.NewWithInput(parent, "https://test.com")
|
||||
ctx := scan.NewScanContext(parent, input)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
engine.executeWorkflow(ctx, workflow)
|
||||
}()
|
||||
|
||||
deadline := time.After(2 * time.Second)
|
||||
for started.Load() < 2 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("blocking subtemplates did not occupy both slots")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("workflow stayed blocked after cancel")
|
||||
}
|
||||
require.False(t, waitingStarted.Load(), "third subtemplate must not start while slots are held")
|
||||
}
|
||||
|
||||
type mockExecuter struct {
|
||||
result bool
|
||||
executeHook func(input *contextargs.MetaInput)
|
||||
|
||||
@@ -62,10 +62,6 @@ func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.Adaptive
|
||||
}
|
||||
|
||||
func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) {
|
||||
w.RefreshWithConfigContext(context.Background(), config)
|
||||
}
|
||||
|
||||
func (w *WorkPool) RefreshWithConfigContext(ctx context.Context, config WorkPoolConfig) {
|
||||
if w.config.TypeConcurrency != config.TypeConcurrency {
|
||||
w.config.TypeConcurrency = config.TypeConcurrency
|
||||
}
|
||||
@@ -78,7 +74,7 @@ func (w *WorkPool) RefreshWithConfigContext(ctx context.Context, config WorkPool
|
||||
if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency {
|
||||
w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency
|
||||
}
|
||||
w.Refresh(ctx)
|
||||
w.Refresh(context.Background())
|
||||
}
|
||||
|
||||
func (w *WorkPool) Refresh(ctx context.Context) {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
)
|
||||
|
||||
@@ -148,7 +147,10 @@ func (b *Body) Rebuild() (*retryablehttp.Request, error) {
|
||||
return nil, errors.Wrap(err, "could not encode body")
|
||||
}
|
||||
cloned := b.req.Clone(context.Background())
|
||||
httpbody.SetString(cloned, encoded)
|
||||
err = cloned.SetBodyString(encoded)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not set body")
|
||||
}
|
||||
cloned.Header.Set("Content-Length", strconv.Itoa(len(encoded)))
|
||||
return cloned, nil
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -46,7 +45,6 @@ func TestBodyComponent(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
require.IsType(t, &httpbody.Body{}, rebuilt.Body, "rebuilt body must not be copied per read")
|
||||
newBody, err := io.ReadAll(rebuilt.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newFuzzTestInteractshClient(t *testing.T) interactsh.IClient {
|
||||
func newFuzzTestInteractshClient(t *testing.T) *interactsh.Client {
|
||||
t.Helper()
|
||||
|
||||
client, err := interactsh.New(&interactsh.Options{
|
||||
|
||||
@@ -43,7 +43,6 @@ https://d.example
|
||||
| Key | Selects |
|
||||
| --- | --- |
|
||||
| `target` | the target (URL, host, IP, CIDR or ASN), required |
|
||||
| `ip` | the address to connect to instead of resolving the target's host, see below |
|
||||
| `profile` | a template profile ID from the templates `profiles` directory (paths are rejected); the line's other keys override it |
|
||||
| `templates` | template paths, relative to the templates directory |
|
||||
| `tags`, `exclude-tags`, `include-tags`, `author`, `template-id`, `exclude-id`, `severity`, `exclude-severity`, `type`, `exclude-type` | same as the CLI flags |
|
||||
@@ -55,23 +54,6 @@ https://d.example
|
||||
- Unknown keys fail the run rather than widening the scan. Profile keys that configure the whole scan, such as `code` or `var`, are ignored per target.
|
||||
- Not supported with `-automatic-scan`. Global matchers still apply to every target.
|
||||
|
||||
### Pinning a target to an IP
|
||||
|
||||
`ip` sends the target's traffic to one address, like `curl --resolve`: the hostname is still used for the Host header, SNI and template variables such as `{{Hostname}}`, while connections to that hostname go to the ip. This scans a vhost on a given server, or an origin behind a CDN.
|
||||
|
||||
```jsonl
|
||||
{"target": "https://app.example", "ip": "192.0.2.10"}
|
||||
{"target": "https://app.example", "ip": "192.0.2.11", "tags": ["cve"]}
|
||||
```
|
||||
|
||||
- Each `target` and `ip` pair is a separate target: the same host can be listed with several ips, and each pair has its own selection. `{{ip}}` and the `ip` field of results hold the pinned ip.
|
||||
- The target must have a hostname; IP, CIDR and ASN targets cannot be pinned. `-sa` and `-iv` do not apply to a pinned target.
|
||||
- A pinned ip excluded with `-eh`, or a local one with `-lna`, is skipped.
|
||||
- `-mup` counts the urls of each ip apart, so a url listed with several ips is kept on each of them.
|
||||
- HTTP, network, websocket, javascript and ssl templates connect to the ip. Headless templates and `unsafe: true` raw requests do not.
|
||||
- Pinned targets cannot be combined with `-proxy`, `-replay-proxy` or `-automatic-scan`.
|
||||
- `-u` splits its value on commas unless the value is quoted, so pass target lines in a file (`-l`) or on stdin.
|
||||
|
||||
## Formats
|
||||
|
||||
Fuzzing and DAST take the request shape (method, body, params) from the input, not from CLI flags, so a bare URL is always fuzzed as a `GET`. To fuzz anything else, feed a request-shaped input with `-im`:
|
||||
|
||||
@@ -53,25 +53,7 @@ For unsupported media types, no appropriate content type is found for the body.
|
||||
|
||||
This module converts each operation into one or more example HTTP requests. Each request is dumped into a string format, accompanied by its method, URL, headers, and body. These are send as a callback for further processing.
|
||||
|
||||
### Security Schemes
|
||||
|
||||
The OpenAPI input supports HTTP basic and bearer authentication, API keys in headers, query parameters or cookies, and OAuth2/OpenID Connect access tokens. Security requirements can be declared globally or per operation.
|
||||
|
||||
For OAuth2 (`type: oauth2`) and OpenID Connect (`type: openIdConnect`), supply an existing access token as the full `Authorization` header value:
|
||||
|
||||
```sh
|
||||
nuclei -l openapi.yaml -im openapi -var 'Authorization=Bearer YOUR_ACCESS_TOKEN'
|
||||
```
|
||||
|
||||
The same variable is used for HTTP bearer authentication. For basic authentication, provide the full `Basic ...` value; for API keys, use the parameter name specified by the security scheme.
|
||||
|
||||
Nuclei does not perform OAuth authorization flows, discover OpenID Connect providers, acquire tokens, or refresh them. Obtain an access token with the scopes required by the API before running the scan. The provided header value is used unchanged.
|
||||
|
||||
Security requirement objects are alternatives. The generator selects the first supported alternative, in document order, for which all required credentials are supplied, and includes every scheme in that object. An empty object (`{}`) permits anonymous access. If no supported alternative has all its credentials, the first supported alternative is used to report missing values; credentials from different alternatives are not combined.
|
||||
|
||||
Operations declaring `security: []` can be generated without the global token. Absent, nil or empty credential values cause affected requests to be skipped. Normal parameter validation reports missing credentials even when another required parameter is also missing. With `-skip-format-validation`, requests with missing credentials are silently skipped and the remaining operations are generated without inventing placeholder credentials.
|
||||
|
||||
_Please note: Other OpenAPI features such as responses, links and callbacks are not currently handled by the module._
|
||||
_Please note: This document does not cover other features of OpenAPI specification like responses, security schemes, links, callbacks, etc. as these are not currently handled by the module._
|
||||
|
||||
## Postman Collection file
|
||||
|
||||
|
||||
@@ -36,7 +36,33 @@ func GenerateRequestsFromSchema(schema *openapi3.T, opts formats.InputFormatOpti
|
||||
if len(schema.Servers) == 0 {
|
||||
return errors.New("no servers found in openapi schema")
|
||||
}
|
||||
globalParams, globalSecurityErr := selectSecurityParameters(schema, &schema.Security, opts.Variables)
|
||||
|
||||
// new set of globalParams obtained from security schemes
|
||||
globalParams := openapi3.NewParameters()
|
||||
|
||||
if len(schema.Security) > 0 {
|
||||
params, err := GetGlobalParamsForSecurityRequirement(schema, &schema.Security)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalParams = append(globalParams, params...)
|
||||
}
|
||||
|
||||
// validate global param requirements
|
||||
for _, param := range globalParams {
|
||||
if val, ok := opts.Variables[param.Value.Name]; ok {
|
||||
param.Value.Example = val
|
||||
} else {
|
||||
// if missing check for validation
|
||||
if opts.SkipFormatValidation {
|
||||
gologger.Verbose().Msgf("openapi: skipping all requests due to missing global auth parameter: %s\n", param.Value.Name)
|
||||
return nil
|
||||
} else {
|
||||
// fatal error
|
||||
gologger.Fatal().Msgf("openapi: missing global auth parameter: %s\n", param.Value.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
missingVarMap := make(map[string]struct{})
|
||||
optionalVarMap := make(map[string]struct{})
|
||||
@@ -66,10 +92,6 @@ func GenerateRequestsFromSchema(schema *openapi3.T, opts formats.InputFormatOpti
|
||||
requestPath = serverPath + path
|
||||
}
|
||||
for method, ov := range ops {
|
||||
// An override need not use an unsupported global security scheme.
|
||||
if ov.Security == nil && globalSecurityErr != nil {
|
||||
return globalSecurityErr
|
||||
}
|
||||
if err := generateRequestsFromOp(&generateReqOptions{
|
||||
requiredOnly: opts.RequiredOnly,
|
||||
method: method,
|
||||
@@ -126,7 +148,7 @@ type generateReqOptions struct {
|
||||
// post request generation callback
|
||||
callback formats.ParseReqRespCallback
|
||||
|
||||
// Selected global authentication parameters, used only without an override.
|
||||
// global parameters
|
||||
globalParams openapi3.Parameters
|
||||
// requestparams map
|
||||
reqParams openapi3.Parameters
|
||||
@@ -153,42 +175,21 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
|
||||
}
|
||||
// add existing req params
|
||||
reqParams = append(reqParams, opts.op.Parameters...)
|
||||
// Select one complete requirement for this operation before adding its
|
||||
// authentication parameters. Operation security overrides global security.
|
||||
authParams := opts.globalParams
|
||||
// check for endpoint specific auth
|
||||
if opts.op.Security != nil {
|
||||
authParams, err = selectSecurityParameters(opts.schema, opts.op.Security, opts.opts.Variables)
|
||||
params, err := GetGlobalParamsForSecurityRequirement(opts.schema, opts.op.Security)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
reqParams = append(reqParams, authParams...)
|
||||
authParameters := make(map[*openapi3.Parameter]struct{}, len(authParams))
|
||||
for _, parameter := range authParams {
|
||||
authParameters[parameter.Value] = struct{}{}
|
||||
}
|
||||
|
||||
// Collect missing credentials before ordinary parameter validation can return
|
||||
// early. Still validate ordinary parameters so their diagnostics are retained.
|
||||
missingAuth := false
|
||||
for _, parameter := range authParams {
|
||||
value := parameter.Value
|
||||
if types.ToString(opts.opts.Variables[value.Name]) != "" {
|
||||
continue
|
||||
}
|
||||
missingAuth = true
|
||||
if opts.missingParamValueCallback != nil {
|
||||
opts.missingParamValueCallback(value, opts)
|
||||
}
|
||||
reqParams = append(reqParams, params...)
|
||||
} else {
|
||||
reqParams = append(reqParams, opts.globalParams...)
|
||||
}
|
||||
|
||||
query := url.Values{}
|
||||
for _, parameter := range reqParams {
|
||||
value := parameter.Value
|
||||
|
||||
if _, isAuth := authParameters[value]; missingAuth && isAuth {
|
||||
continue
|
||||
}
|
||||
if value.Schema == nil || value.Schema.Value == nil {
|
||||
continue
|
||||
}
|
||||
@@ -260,9 +261,6 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
|
||||
req.AddCookie(&http.Cookie{Name: value.Name, Value: types.ToString(paramValue)})
|
||||
}
|
||||
}
|
||||
if missingAuth {
|
||||
return nil
|
||||
}
|
||||
req.URL.RawQuery = query.Encode()
|
||||
req.URL.Path = opts.requestPath
|
||||
|
||||
@@ -273,7 +271,7 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
|
||||
var val interface{}
|
||||
|
||||
if value.Schema == nil || value.Schema.Value == nil {
|
||||
val, _ = generateExampleFromSchema(generateEmptySchemaValue(content))
|
||||
val = generateEmptySchemaValue(content)
|
||||
} else {
|
||||
var err error
|
||||
|
||||
@@ -293,25 +291,15 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
|
||||
cloned.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
case "application/xml":
|
||||
var marshalled []byte
|
||||
var err error
|
||||
switch v := val.(type) {
|
||||
case map[string]interface{}:
|
||||
marshalled, err = mxj.Map(v).Xml()
|
||||
case string:
|
||||
marshalled = []byte(v)
|
||||
default:
|
||||
err = errors.Errorf("unsupported xml example type %T", val)
|
||||
}
|
||||
values := mxj.Map(val.(map[string]interface{}))
|
||||
|
||||
if err == nil {
|
||||
if marshalled, err := values.Xml(); err == nil {
|
||||
// body = string(marshalled)
|
||||
cloned.Body = io.NopCloser(bytes.NewReader(marshalled))
|
||||
cloned.ContentLength = int64(len(marshalled))
|
||||
cloned.Header.Set("Content-Type", "application/xml")
|
||||
} else {
|
||||
gologger.Warning().Msgf("openapi: could not encode xml")
|
||||
continue
|
||||
}
|
||||
case "application/x-www-form-urlencoded":
|
||||
if values, ok := val.(map[string]interface{}); ok {
|
||||
@@ -357,7 +345,7 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
|
||||
// use two strings
|
||||
str = "string1\nstring2"
|
||||
}
|
||||
if value.Schema != nil && generic.EqualsAny(value.Schema.Value.Format, "binary", "byte") {
|
||||
if value.Schema != nil && generic.EqualsAny(value.Schema.Value.Format, "bindary", "byte") {
|
||||
cloned.Body = io.NopCloser(bytes.NewReader([]byte(str)))
|
||||
cloned.ContentLength = int64(len(str))
|
||||
cloned.Header.Set("Content-Type", "application/octet-stream")
|
||||
@@ -403,79 +391,45 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetGlobalParamsForSecurityRequirement returns parameters for the first supported
|
||||
// security alternative, or no parameters when anonymous access is permitted.
|
||||
// GetGlobalParamsForSecurityRequirement returns the global parameters for a security requirement
|
||||
func GetGlobalParamsForSecurityRequirement(schema *openapi3.T, requirement *openapi3.SecurityRequirements) ([]*openapi3.ParameterRef, error) {
|
||||
return selectSecurityParameters(schema, requirement, nil)
|
||||
}
|
||||
|
||||
// selectSecurityParameters chooses the first fully supplied alternative. If none
|
||||
// is supplied, return the first supported alternative for missing-value reporting.
|
||||
func selectSecurityParameters(schema *openapi3.T, requirement *openapi3.SecurityRequirements, variables map[string]interface{}) ([]*openapi3.ParameterRef, error) {
|
||||
if requirement == nil || len(*requirement) == 0 {
|
||||
return nil, nil
|
||||
globalParams := openapi3.NewParameters()
|
||||
if len(schema.Components.SecuritySchemes) == 0 {
|
||||
return nil, errkit.Newf("security requirements (%+v) without any security schemes found in openapi file", schema.Security)
|
||||
}
|
||||
var fallback []*openapi3.ParameterRef
|
||||
var firstError error
|
||||
found := false
|
||||
// this api is protected for each security scheme pull its corresponding scheme
|
||||
schemaLabel:
|
||||
for _, security := range *requirement {
|
||||
if len(security) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
params := openapi3.NewParameters()
|
||||
complete, supported := true, true
|
||||
// Map order must not change which schemes are emitted or reported missing.
|
||||
for _, name := range mapsutil.GetSortedKeys(security) {
|
||||
var scheme *openapi3.SecuritySchemeRef
|
||||
if schema.Components != nil {
|
||||
scheme = schema.Components.SecuritySchemes[name]
|
||||
}
|
||||
if scheme == nil || scheme.Value == nil {
|
||||
if firstError == nil {
|
||||
firstError = errkit.Newf("security scheme (%s) not found in openapi file", name)
|
||||
for name := range security {
|
||||
if scheme, ok := schema.Components.SecuritySchemes[name]; ok {
|
||||
found = true
|
||||
param, err := GenerateParameterFromSecurityScheme(scheme)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
}
|
||||
supported = false
|
||||
break
|
||||
globalParams = append(globalParams, &openapi3.ParameterRef{Value: param})
|
||||
continue schemaLabel
|
||||
}
|
||||
param, err := GenerateParameterFromSecurityScheme(scheme)
|
||||
if err != nil {
|
||||
if firstError == nil {
|
||||
firstError = err
|
||||
}
|
||||
supported = false
|
||||
break
|
||||
}
|
||||
params = append(params, &openapi3.ParameterRef{Value: param})
|
||||
complete = complete && types.ToString(variables[param.Name]) != ""
|
||||
}
|
||||
if !supported {
|
||||
continue
|
||||
}
|
||||
if complete {
|
||||
return params, nil
|
||||
}
|
||||
if fallback == nil {
|
||||
fallback = params
|
||||
if !found && len(security) > 1 {
|
||||
// if this is case then both security schemes are required
|
||||
return nil, errkit.Newf("security requirement (%+v) not found in openapi file", security)
|
||||
}
|
||||
}
|
||||
if fallback != nil {
|
||||
return fallback, nil
|
||||
if !found {
|
||||
return nil, errkit.Newf("security requirement (%+v) not found in openapi file", requirement)
|
||||
}
|
||||
return nil, firstError
|
||||
|
||||
return globalParams, nil
|
||||
}
|
||||
|
||||
// GenerateParameterFromSecurityScheme generates an example from a schema object
|
||||
func GenerateParameterFromSecurityScheme(scheme *openapi3.SecuritySchemeRef) (*openapi3.Parameter, error) {
|
||||
if !generic.EqualsAny(scheme.Value.Type, "http", "apiKey", "oauth2", "openIdConnect") {
|
||||
if !generic.EqualsAny(scheme.Value.Type, "http", "apiKey") {
|
||||
return nil, errkit.Newf("unsupported security scheme type (%s) found in openapi file", scheme.Value.Type)
|
||||
}
|
||||
if generic.EqualsAny(scheme.Value.Type, "oauth2", "openIdConnect") {
|
||||
// Access tokens are supplied by the caller through the Authorization variable.
|
||||
// Request generation does not acquire or refresh tokens from the provider.
|
||||
h := openapi3.NewHeaderParameter(DEFAULT_HTTP_SCHEME_HEADER).WithSchema(openapi3.NewStringSchema())
|
||||
h.Required = true
|
||||
h.Description = globalAuth
|
||||
return h, nil
|
||||
}
|
||||
if scheme.Value.Type == "http" {
|
||||
// check scheme
|
||||
if !generic.EqualsAny(scheme.Value.Scheme, "basic", "bearer") {
|
||||
@@ -489,12 +443,12 @@ func GenerateParameterFromSecurityScheme(scheme *openapi3.SecuritySchemeRef) (*o
|
||||
// create parameters using the scheme
|
||||
switch scheme.Value.Scheme {
|
||||
case "basic":
|
||||
h := openapi3.NewHeaderParameter(headerName).WithSchema(openapi3.NewStringSchema())
|
||||
h := openapi3.NewHeaderParameter(headerName)
|
||||
h.Required = true
|
||||
h.Description = globalAuth // differentiator for normal variables and global auth
|
||||
return h, nil
|
||||
case "bearer":
|
||||
h := openapi3.NewHeaderParameter(headerName).WithSchema(openapi3.NewStringSchema())
|
||||
h := openapi3.NewHeaderParameter(headerName)
|
||||
h.Required = true
|
||||
h.Description = globalAuth // differentiator for normal variables and global auth
|
||||
return h, nil
|
||||
@@ -512,17 +466,17 @@ func GenerateParameterFromSecurityScheme(scheme *openapi3.SecuritySchemeRef) (*o
|
||||
// create parameters using the scheme
|
||||
switch scheme.Value.In {
|
||||
case "query":
|
||||
q := openapi3.NewQueryParameter(scheme.Value.Name).WithSchema(openapi3.NewStringSchema())
|
||||
q := openapi3.NewQueryParameter(scheme.Value.Name)
|
||||
q.Required = true
|
||||
q.Description = globalAuth // differentiator for normal variables and global auth
|
||||
return q, nil
|
||||
case "header":
|
||||
h := openapi3.NewHeaderParameter(scheme.Value.Name).WithSchema(openapi3.NewStringSchema())
|
||||
h := openapi3.NewHeaderParameter(scheme.Value.Name)
|
||||
h.Required = true
|
||||
h.Description = globalAuth // differentiator for normal variables and global auth
|
||||
return h, nil
|
||||
case "cookie":
|
||||
c := openapi3.NewCookieParameter(scheme.Value.Name).WithSchema(openapi3.NewStringSchema())
|
||||
c := openapi3.NewCookieParameter(scheme.Value.Name)
|
||||
c.Required = true
|
||||
c.Description = globalAuth // differentiator for normal variables and global auth
|
||||
return c, nil
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
package openapi
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestOpenAPIOctetStreamContentType(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
schema string
|
||||
contentType string
|
||||
}{
|
||||
{name: "binary format", schema: "{type: string, format: binary}", contentType: "application/octet-stream"},
|
||||
{name: "byte format", schema: "{type: string, format: byte}", contentType: "application/octet-stream"},
|
||||
{name: "plain string", schema: "{type: string}", contentType: "text/plain"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
spec := `openapi: 3.0.0
|
||||
info: {title: t, version: "1"}
|
||||
servers: [{url: "http://example.com"}]
|
||||
paths:
|
||||
/upload:
|
||||
post:
|
||||
requestBody:
|
||||
content:
|
||||
application/octet-stream: {schema: ` + tt.schema + `}
|
||||
responses:
|
||||
"200": {description: ok}
|
||||
`
|
||||
var contentTypes []string
|
||||
err := New().Parse(strings.NewReader(spec), func(rr *types.RequestResponse) bool {
|
||||
value, _ := rr.Request.Headers.Get("Content-Type")
|
||||
contentTypes = append(contentTypes, value)
|
||||
return false
|
||||
}, "spec.yaml")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{tt.contentType}, contentTypes)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,350 +0,0 @@
|
||||
package openapi
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/getkin/kin-openapi/openapi3"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
|
||||
httpTypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGenerateRequestsWithSecuritySchemes(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
scheme string
|
||||
variable string
|
||||
value string
|
||||
in string
|
||||
}{
|
||||
{
|
||||
name: "oauth2", scheme: `{"type":"oauth2","flows":{"clientCredentials":{"tokenUrl":"https://example.com/token","scopes":{"read":"Read access"}}}}`,
|
||||
variable: "Authorization", value: "Bearer supplied-token", in: "header",
|
||||
},
|
||||
{
|
||||
name: "openIdConnect", scheme: `{"type":"openIdConnect","openIdConnectUrl":"https://example.com/.well-known/openid-configuration"}`,
|
||||
variable: "Authorization", value: "Bearer supplied-token", in: "header",
|
||||
},
|
||||
{
|
||||
name: "bearer", scheme: `{"type":"http","scheme":"bearer"}`,
|
||||
variable: "Authorization", value: "Bearer supplied-token", in: "header",
|
||||
},
|
||||
{
|
||||
name: "basic", scheme: `{"type":"http","scheme":"basic"}`,
|
||||
variable: "Authorization", value: "Basic dXNlcjpwYXNz", in: "header",
|
||||
},
|
||||
{
|
||||
name: "apiKey-header", scheme: `{"type":"apiKey","name":"X-API-Key","in":"header"}`,
|
||||
variable: "X-API-Key", value: "supplied-key", in: "header",
|
||||
},
|
||||
{
|
||||
name: "apiKey-query", scheme: `{"type":"apiKey","name":"api_key","in":"query"}`,
|
||||
variable: "api_key", value: "supplied-key", in: "query",
|
||||
},
|
||||
{
|
||||
name: "apiKey-cookie", scheme: `{"type":"apiKey","name":"session","in":"cookie"}`,
|
||||
variable: "session", value: "supplied-key", in: "cookie",
|
||||
},
|
||||
} {
|
||||
for _, scope := range []string{"global", "operation"} {
|
||||
t.Run(test.name+"/"+scope, func(t *testing.T) {
|
||||
security := `"security":[{"auth":[]}],`
|
||||
if test.name == "oauth2" || test.name == "openIdConnect" {
|
||||
security = `"security":[{"auth":["read"]}],`
|
||||
}
|
||||
globalSecurity, operationSecurity := security, ""
|
||||
if scope == "operation" {
|
||||
globalSecurity, operationSecurity = "", security
|
||||
}
|
||||
document := fmt.Sprintf(`{
|
||||
"openapi":"3.0.3",
|
||||
"info":{"title":"Authentication test","version":"1.0"},
|
||||
"servers":[{"url":"https://example.com"}],
|
||||
"components":{"securitySchemes":{"auth":%s}},
|
||||
%s
|
||||
"paths":{"/protected":{"get":{%s"responses":{"200":{"description":"OK"}}}}}
|
||||
}`, test.scheme, globalSecurity, operationSecurity)
|
||||
parser := New()
|
||||
parser.SetOptions(formats.InputFormatOptions{
|
||||
RequiredOnly: true,
|
||||
Variables: map[string]interface{}{test.variable: test.value},
|
||||
})
|
||||
var requests []*http.Request
|
||||
err := parser.Parse(strings.NewReader(document), func(rr *httpTypes.RequestResponse) bool {
|
||||
req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(rr.Request.Raw)))
|
||||
require.NoError(t, err)
|
||||
requests = append(requests, req)
|
||||
return false
|
||||
}, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, requests, 1)
|
||||
req := requests[0]
|
||||
t.Cleanup(func() { require.NoError(t, req.Body.Close()) })
|
||||
require.Equal(t, "/protected", req.URL.Path)
|
||||
switch test.in {
|
||||
case "header":
|
||||
require.Equal(t, test.value, req.Header.Get(test.variable))
|
||||
case "query":
|
||||
require.Equal(t, test.value, req.URL.Query().Get(test.variable))
|
||||
case "cookie":
|
||||
cookie, err := req.Cookie(test.variable)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.value, cookie.Value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateRequestsOAuthOperationOverride(t *testing.T) {
|
||||
parser := New()
|
||||
parser.SetOptions(formats.InputFormatOptions{
|
||||
Variables: map[string]interface{}{
|
||||
"Authorization": "Bearer supplied-token",
|
||||
"X-API-Key": "supplied-key",
|
||||
},
|
||||
})
|
||||
document := `{
|
||||
"openapi":"3.0.3",
|
||||
"info":{"title":"Authentication override test","version":"1.0"},
|
||||
"servers":[{"url":"https://example.com"}],
|
||||
"components":{"securitySchemes":{
|
||||
"oauth":{"type":"oauth2","flows":{"clientCredentials":{"tokenUrl":"https://example.com/token","scopes":{}}}},
|
||||
"key":{"type":"apiKey","name":"X-API-Key","in":"header"}
|
||||
}},
|
||||
"security":[{"oauth":[]}],
|
||||
"paths":{
|
||||
"/protected":{"get":{"security":[{"key":[]}],"responses":{"200":{"description":"OK"}}}},
|
||||
"/public":{"get":{"security":[],"responses":{"200":{"description":"OK"}}}}
|
||||
}
|
||||
}`
|
||||
paths := make(map[string]bool)
|
||||
err := parser.Parse(strings.NewReader(document), func(rr *httpTypes.RequestResponse) bool {
|
||||
paths[rr.URL.Path] = true
|
||||
key, _ := rr.Request.Headers.Get("X-Api-Key")
|
||||
if rr.URL.Path == "/protected" {
|
||||
require.Equal(t, "supplied-key", key)
|
||||
} else {
|
||||
require.Empty(t, key)
|
||||
}
|
||||
auth, _ := rr.Request.Headers.Get("Authorization")
|
||||
require.Empty(t, auth)
|
||||
return false
|
||||
}, "")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, map[string]bool{"/protected": true, "/public": true}, paths)
|
||||
}
|
||||
|
||||
func TestGenerateRequestsOAuthRequiresToken(t *testing.T) {
|
||||
for _, skipValidation := range []bool{false, true} {
|
||||
for _, schemeType := range []string{"oauth2", "openIdConnect"} {
|
||||
t.Run(fmt.Sprintf("%s/skip-validation=%t", schemeType, skipValidation), func(t *testing.T) {
|
||||
param, err := GenerateParameterFromSecurityScheme(&openapi3.SecuritySchemeRef{
|
||||
Value: &openapi3.SecurityScheme{Type: schemeType},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
var missing []string
|
||||
err = generateRequestsFromOp(&generateReqOptions{
|
||||
method: http.MethodGet,
|
||||
pathURL: "https://example.com",
|
||||
requestPath: "/protected",
|
||||
op: &openapi3.Operation{},
|
||||
opts: formats.InputFormatOptions{SkipFormatValidation: skipValidation},
|
||||
globalParams: openapi3.Parameters{&openapi3.ParameterRef{Value: param}},
|
||||
missingParamValueCallback: func(param *openapi3.Parameter, _ *generateReqOptions) {
|
||||
missing = append(missing, param.Name)
|
||||
},
|
||||
callback: func(_ *httpTypes.RequestResponse) bool {
|
||||
t.Fatal("must not generate an authenticated request without a token")
|
||||
return false
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{"Authorization"}, missing)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func oauthSecurityDocument(paths string) string {
|
||||
return fmt.Sprintf(`{
|
||||
"openapi":"3.0.3",
|
||||
"info":{"title":"Authentication scope test","version":"1.0"},
|
||||
"servers":[{"url":"https://example.com"}],
|
||||
"components":{"securitySchemes":{
|
||||
"oauth":{"type":"oauth2","flows":{"clientCredentials":{"tokenUrl":"https://example.com/token","scopes":{}}}},
|
||||
"key":{"type":"apiKey","name":"X-API-Key","in":"header"}
|
||||
}},
|
||||
"security":[{"oauth":[]}],
|
||||
"paths":{%s}
|
||||
}`, paths)
|
||||
}
|
||||
|
||||
func TestGenerateRequestsOAuthWithoutGlobalToken(t *testing.T) {
|
||||
for _, skipValidation := range []bool{true, false} {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
paths string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "public-only",
|
||||
paths: `"/public":{"get":{"security":[],"responses":{"200":{"description":"OK"}}}}`,
|
||||
want: []string{"/public"},
|
||||
},
|
||||
{
|
||||
name: "mixed-security",
|
||||
paths: `
|
||||
"/protected":{"get":{"responses":{"200":{"description":"OK"}}}},
|
||||
"/public":{"get":{"security":[],"responses":{"200":{"description":"OK"}}}},
|
||||
"/key":{"get":{"security":[{"key":[]}],"responses":{"200":{"description":"OK"}}}}
|
||||
`,
|
||||
want: []string{"/public", "/key"},
|
||||
},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s/skip-validation=%t", test.name, skipValidation), func(t *testing.T) {
|
||||
parser := New()
|
||||
parser.SetOptions(formats.InputFormatOptions{
|
||||
SkipFormatValidation: skipValidation,
|
||||
Variables: map[string]interface{}{"X-API-Key": "supplied-key"},
|
||||
})
|
||||
var paths []string
|
||||
err := parser.Parse(strings.NewReader(oauthSecurityDocument(test.paths)), func(rr *httpTypes.RequestResponse) bool {
|
||||
paths = append(paths, rr.URL.Path)
|
||||
auth, _ := rr.Request.Headers.Get("Authorization")
|
||||
require.Empty(t, auth)
|
||||
key, _ := rr.Request.Headers.Get("X-Api-Key")
|
||||
if rr.URL.Path == "/key" {
|
||||
require.Equal(t, "supplied-key", key)
|
||||
} else {
|
||||
require.Empty(t, key)
|
||||
}
|
||||
return false
|
||||
}, "")
|
||||
require.NoError(t, err)
|
||||
require.ElementsMatch(t, test.want, paths)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateRequestsOAuthMissingTokenCLI(t *testing.T) {
|
||||
const helperEnv = "NUCLEI_TEST_OPENAPI_MISSING_TOKEN"
|
||||
if os.Getenv(helperEnv) == "1" {
|
||||
config.CurrentAppMode = config.AppModeCLI
|
||||
document := oauthSecurityDocument(`
|
||||
"/protected/{id}":{"get":{
|
||||
"parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],
|
||||
"responses":{"200":{"description":"OK"}}
|
||||
}},
|
||||
"/public":{"get":{"security":[],"responses":{"200":{"description":"OK"}}}}
|
||||
`)
|
||||
err := New().Parse(strings.NewReader(document), func(rr *httpTypes.RequestResponse) bool {
|
||||
fmt.Println("generated:", rr.URL.Path)
|
||||
return false
|
||||
}, "")
|
||||
require.NoError(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestGenerateRequestsOAuthMissingTokenCLI$")
|
||||
cmd.Env = append(os.Environ(), helperEnv+"=1")
|
||||
cmd.Dir = t.TempDir()
|
||||
output, err := cmd.CombinedOutput()
|
||||
var exitErr *exec.ExitError
|
||||
require.ErrorAs(t, err, &exitErr)
|
||||
require.Equal(t, 1, exitErr.ExitCode())
|
||||
require.Contains(t, string(output), "generated: /public")
|
||||
require.NotContains(t, string(output), "generated: /protected")
|
||||
parameters, err := os.ReadFile(filepath.Join(cmd.Dir, formats.DefaultVarDumpFileName))
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(parameters), "Authorization=")
|
||||
require.Contains(t, string(parameters), "id=")
|
||||
}
|
||||
|
||||
func TestGenerateRequestsCredentials(t *testing.T) {
|
||||
for _, auth := range []struct {
|
||||
scheme string
|
||||
variable string
|
||||
}{
|
||||
{scheme: "oauth", variable: "Authorization"},
|
||||
{scheme: "key", variable: "X-API-Key"},
|
||||
} {
|
||||
for _, scope := range []string{"global", "operation"} {
|
||||
for _, skipValidation := range []bool{false, true} {
|
||||
for _, credential := range []struct {
|
||||
name string
|
||||
present bool
|
||||
value interface{}
|
||||
wantProtected bool
|
||||
}{
|
||||
{name: "absent"},
|
||||
{name: "nil", present: true},
|
||||
{name: "empty", present: true, value: ""},
|
||||
{name: "supplied", present: true, value: "opaque-token", wantProtected: true},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s/%s/%s/skip-validation=%t", auth.scheme, scope, credential.name, skipValidation), func(t *testing.T) {
|
||||
document := oauthSecurityDocument(`
|
||||
"/protected":{"get":{"responses":{"200":{"description":"OK"}}}},
|
||||
"/public":{"get":{"security":[],"responses":{"200":{"description":"OK"}}}}
|
||||
`)
|
||||
schema, err := openapi3.NewLoader().LoadFromData([]byte(document))
|
||||
require.NoError(t, err)
|
||||
security := openapi3.SecurityRequirements{{auth.scheme: []string{}}}
|
||||
if scope == "global" {
|
||||
schema.Security = security
|
||||
} else {
|
||||
schema.Security = nil
|
||||
schema.Paths.Map()["/protected"].Get.Security = &security
|
||||
}
|
||||
variables := map[string]interface{}{}
|
||||
if credential.present {
|
||||
variables[auth.variable] = credential.value
|
||||
}
|
||||
var paths []string
|
||||
err = GenerateRequestsFromSchema(schema, formats.InputFormatOptions{
|
||||
Variables: variables,
|
||||
SkipFormatValidation: skipValidation,
|
||||
}, func(rr *httpTypes.RequestResponse) bool {
|
||||
paths = append(paths, rr.URL.Path)
|
||||
for _, header := range []string{"Authorization", "X-API-Key"} {
|
||||
value, _ := rr.Request.Headers.Get(http.CanonicalHeaderKey(header))
|
||||
if rr.URL.Path == "/protected" && header == auth.variable && credential.wantProtected {
|
||||
require.Equal(t, credential.value, value)
|
||||
} else {
|
||||
require.Empty(t, value)
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
require.NoError(t, err)
|
||||
wantPaths := []string{"/public"}
|
||||
if credential.wantProtected {
|
||||
wantPaths = append(wantPaths, "/protected")
|
||||
}
|
||||
require.ElementsMatch(t, wantPaths, paths)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateParameterUnsupportedSecurityScheme(t *testing.T) {
|
||||
_, err := GenerateParameterFromSecurityScheme(&openapi3.SecuritySchemeRef{
|
||||
Value: &openapi3.SecurityScheme{Type: "mutualTLS"},
|
||||
})
|
||||
require.ErrorContains(t, err, "unsupported security scheme type (mutualTLS)")
|
||||
}
|
||||
@@ -68,42 +68,3 @@ func TestOpenAPIParser(t *testing.T) {
|
||||
require.ElementsMatch(t, urls, methodToURLs[method], "invalid urls for method %s", method)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPIRequestBodies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
bodies []string
|
||||
}{
|
||||
{name: "xml without schema", content: "application/xml: {}", bodies: []string{`<?xml version="1.0"?><root/>`}},
|
||||
{name: "xml string schema", content: "application/xml: {schema: {type: string}}", bodies: []string{"string"}},
|
||||
{name: "xml array schema", content: "application/xml: {schema: {type: array, items: {type: string}}}", bodies: nil},
|
||||
{name: "json without schema", content: "application/json: {}", bodies: []string{"{}"}},
|
||||
{name: "text without schema", content: "text/plain: {}", bodies: []string{"string"}},
|
||||
{name: "octet-stream without schema", content: "application/octet-stream: {}", bodies: []string{"string1\nstring2"}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
spec := `openapi: 3.0.0
|
||||
info: {title: t, version: "1"}
|
||||
servers: [{url: "http://example.com"}]
|
||||
paths:
|
||||
/items:
|
||||
post:
|
||||
requestBody:
|
||||
content:
|
||||
` + tt.content + `
|
||||
responses:
|
||||
"200": {description: ok}
|
||||
`
|
||||
var bodies []string
|
||||
err := New().Parse(strings.NewReader(spec), func(rr *types.RequestResponse) bool {
|
||||
bodies = append(bodies, rr.Request.Body)
|
||||
return false
|
||||
}, "spec.yaml")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.bodies, bodies)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
package openapi
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/getkin/kin-openapi/openapi3"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
|
||||
httpTypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGenerateRequestsSecurityAlternatives(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, security string
|
||||
variables map[string]interface{}
|
||||
wantCount int
|
||||
wantAuth string
|
||||
wantKey string
|
||||
}{
|
||||
{"first alternative", `[{"oauth":[]},{"key":[]}]`, map[string]interface{}{"Authorization": "Bearer token"}, 1, "Bearer token", ""},
|
||||
{"second alternative", `[{"oauth":[]},{"key":[]}]`, map[string]interface{}{"X-API-Key": "key"}, 1, "", "key"},
|
||||
{"both alternatives available", `[{"oauth":[]},{"key":[]}]`, map[string]interface{}{"Authorization": "Bearer token", "X-API-Key": "key"}, 1, "Bearer token", ""},
|
||||
{"neither alternative available", `[{"oauth":[]},{"key":[]}]`, nil, 0, "", ""},
|
||||
{"combined complete", `[{"oauth":[],"key":[]}]`, map[string]interface{}{"Authorization": "Bearer token", "X-API-Key": "key"}, 1, "Bearer token", "key"},
|
||||
{"combined missing key", `[{"oauth":[],"key":[]}]`, map[string]interface{}{"Authorization": "Bearer token"}, 0, "", ""},
|
||||
{"combined missing token", `[{"oauth":[],"key":[]}]`, map[string]interface{}{"X-API-Key": "key"}, 0, "", ""},
|
||||
{"complete alternative after incomplete combined", `[{"oauth":[],"key":[]},{"key":[]}]`, map[string]interface{}{"X-API-Key": "key"}, 1, "", "key"},
|
||||
{"anonymous alternative first", `[{}, {"oauth":[]}]`, map[string]interface{}{"Authorization": "Bearer token"}, 1, "", ""},
|
||||
{"anonymous alternative last", `[{"oauth":[]},{}]`, nil, 1, "", ""},
|
||||
{"empty security list", `[]`, nil, 1, "", ""},
|
||||
{"unsupported alternative first", `[{"unsupported":[]},{"key":[]}]`, map[string]interface{}{"X-API-Key": "key"}, 1, "", "key"},
|
||||
} {
|
||||
for _, scope := range []string{"global", "operation"} {
|
||||
for _, skip := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/%s/skip=%t", tc.name, scope, skip), func(t *testing.T) {
|
||||
global, operation := `"security":`+tc.security+`,`, ""
|
||||
if scope == "operation" {
|
||||
global, operation = "", global
|
||||
}
|
||||
document := fmt.Sprintf(`{
|
||||
"openapi":"3.0.3","info":{"title":"Security alternatives","version":"1"},
|
||||
"servers":[{"url":"https://example.com"}],
|
||||
"components":{"securitySchemes":{
|
||||
"oauth":{"type":"oauth2","flows":{"clientCredentials":{"tokenUrl":"https://example.com/token","scopes":{}}}},
|
||||
"key":{"type":"apiKey","in":"header","name":"X-API-Key"},
|
||||
"unsupported":{"type":"http","scheme":"digest"}
|
||||
}},%s"paths":{"/protected":{"get":{%s"responses":{"200":{"description":"OK"}}}}}
|
||||
}`, global, operation)
|
||||
parser := New()
|
||||
parser.SetOptions(formats.InputFormatOptions{SkipFormatValidation: skip, Variables: tc.variables})
|
||||
var requests []*http.Request
|
||||
err := parser.Parse(strings.NewReader(document), func(rr *httpTypes.RequestResponse) bool {
|
||||
req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(rr.Request.Raw)))
|
||||
require.NoError(t, err)
|
||||
requests = append(requests, req)
|
||||
return false
|
||||
}, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, requests, tc.wantCount)
|
||||
for _, req := range requests {
|
||||
t.Cleanup(func() { require.NoError(t, req.Body.Close()) })
|
||||
require.Equal(t, tc.wantAuth, req.Header.Get("Authorization"))
|
||||
require.Equal(t, tc.wantKey, req.Header.Get("X-API-Key"))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateRequestsOrdinaryParameterAuthDescription(t *testing.T) {
|
||||
for _, skip := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("skip=%t", skip), func(t *testing.T) {
|
||||
parser := New()
|
||||
parser.SetOptions(formats.InputFormatOptions{SkipFormatValidation: skip})
|
||||
const document = `{
|
||||
"openapi":"3.0.3","info":{"title":"Public operation","version":"1"},
|
||||
"servers":[{"url":"https://example.com"}],
|
||||
"paths":{"/public":{"get":{
|
||||
"parameters":[{"name":"display","in":"query","required":false,"description":"globalAuth","schema":{"type":"string","default":"compact"}}],
|
||||
"responses":{"200":{"description":"OK"}}
|
||||
}}}
|
||||
}`
|
||||
count := 0
|
||||
err := parser.Parse(strings.NewReader(document), func(rr *httpTypes.RequestResponse) bool {
|
||||
count++
|
||||
require.Equal(t, "display=compact", rr.URL.RawQuery)
|
||||
return false
|
||||
}, "")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateRequestsPublicOverrideUnsupportedGlobalSecurity(t *testing.T) {
|
||||
const document = `{
|
||||
"openapi":"3.0.3","info":{"title":"Public override","version":"1"},
|
||||
"servers":[{"url":"https://example.com"}],
|
||||
"components":{"securitySchemes":{"auth":{"type":"http","scheme":"digest"}}},
|
||||
"security":[{"auth":[]}],
|
||||
"paths":{"/public":{"get":{"security":[],"responses":{"200":{"description":"OK"}}}}}
|
||||
}`
|
||||
count := 0
|
||||
err := New().Parse(strings.NewReader(document), func(rr *httpTypes.RequestResponse) bool {
|
||||
count++
|
||||
return false
|
||||
}, "")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestSecuritySelectionMissingAndUnsupportedSchemes(t *testing.T) {
|
||||
schema := &openapi3.T{Components: &openapi3.Components{SecuritySchemes: openapi3.SecuritySchemes{
|
||||
"key": {Value: &openapi3.SecurityScheme{Type: "apiKey", In: "header", Name: "X-API-Key"}},
|
||||
"oauth": {Value: &openapi3.SecurityScheme{Type: "oauth2"}},
|
||||
}}}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
requirements openapi3.SecurityRequirements
|
||||
wantNames []string
|
||||
wantError bool
|
||||
}{
|
||||
{"missing combined credentials", openapi3.SecurityRequirements{{"oauth": nil, "key": nil}}, []string{"X-API-Key", "Authorization"}, false},
|
||||
{"first alternative diagnostics", openapi3.SecurityRequirements{{"oauth": nil}, {"key": nil}}, []string{"Authorization"}, false},
|
||||
{"unknown scheme", openapi3.SecurityRequirements{{"unknown": nil}}, nil, true},
|
||||
{"known and unknown in same object", openapi3.SecurityRequirements{{"key": nil, "unknown": nil}}, nil, true},
|
||||
{"known alternative after unknown", openapi3.SecurityRequirements{{"unknown": nil}, {"key": nil}}, []string{"X-API-Key"}, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
params, err := selectSecurityParameters(schema, &tc.requirements, nil)
|
||||
if tc.wantError {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
var names []string
|
||||
for _, param := range params {
|
||||
names = append(names, param.Value.Name)
|
||||
}
|
||||
require.Equal(t, tc.wantNames, names)
|
||||
})
|
||||
}
|
||||
// Anonymous requirements do not need a components/securitySchemes section.
|
||||
params, err := GetGlobalParamsForSecurityRequirement(&openapi3.T{}, &openapi3.SecurityRequirements{{}})
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, params)
|
||||
}
|
||||
@@ -89,12 +89,6 @@ func (f *Fingerprinter) Enabled() bool {
|
||||
// pattern. A value that is not a URL is always accepted: it has no path to
|
||||
// group by, and guessing would lose targets.
|
||||
func (f *Fingerprinter) Accept(rawURL string) bool {
|
||||
return f.AcceptOn(rawURL, "")
|
||||
}
|
||||
|
||||
// AcceptOn is Accept for a target connected to through ip. Each ip of a host
|
||||
// is a server of its own, so it keeps its own count for every pattern.
|
||||
func (f *Fingerprinter) AcceptOn(rawURL, ip string) bool {
|
||||
if !f.Enabled() {
|
||||
return true
|
||||
}
|
||||
@@ -102,9 +96,6 @@ func (f *Fingerprinter) AcceptOn(rawURL, ip string) bool {
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if ip != "" {
|
||||
pattern += "@" + ip
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
@@ -80,14 +80,6 @@ func TestAcceptKeepsUpToTheCap(t *testing.T) {
|
||||
require.True(t, f.Accept("http://acme.test/admin"), "a different pattern has its own budget")
|
||||
}
|
||||
|
||||
func TestAcceptOnCountsEachIPApart(t *testing.T) {
|
||||
f := NewFingerprinter(1)
|
||||
require.True(t, f.AcceptOn("http://acme.test/user/1", "192.0.2.10"))
|
||||
require.True(t, f.AcceptOn("http://acme.test/user/1", "192.0.2.11"), "another ip is another server")
|
||||
require.False(t, f.AcceptOn("http://acme.test/user/2", "192.0.2.10"), "past the cap of its ip")
|
||||
require.True(t, f.Accept("http://acme.test/user/2"), "the host without an ip has its own budget")
|
||||
}
|
||||
|
||||
// Off by default: dropping a target means never scanning it, so it has to be
|
||||
// asked for.
|
||||
func TestDisabledFingerprinterKeepsEverything(t *testing.T) {
|
||||
|
||||
@@ -54,7 +54,6 @@ type ListInputProvider struct {
|
||||
hostMapStream *filekv.FileDB
|
||||
hostMapStreamOnce sync.Once
|
||||
profiles *targetprofile.Registry
|
||||
pinned bool
|
||||
// normalizeURLs collapses targets that address the same resource onto one
|
||||
// form before they are deduplicated.
|
||||
normalizeURLs bool
|
||||
@@ -206,8 +205,7 @@ func (i *ListInputProvider) set(executionId string, value string, selection *tar
|
||||
// scan all ips
|
||||
dialers := protocolstate.GetDialersWithId(executionId)
|
||||
if dialers == nil {
|
||||
gologger.Error().Msgf("dialers with executionId %s not found", executionId)
|
||||
return
|
||||
panic("dialers with executionId " + executionId + " not found")
|
||||
}
|
||||
|
||||
dnsData, err := dialers.Fastdialer.GetDNSData(urlx.Hostname())
|
||||
@@ -244,8 +242,7 @@ func (i *ListInputProvider) set(executionId string, value string, selection *tar
|
||||
if i.ipOptions.IPV6 {
|
||||
dialers := protocolstate.GetDialersWithId(executionId)
|
||||
if dialers == nil {
|
||||
gologger.Error().Msgf("dialers with executionId %s not found", executionId)
|
||||
return
|
||||
panic("dialers with executionId " + executionId + " not found")
|
||||
}
|
||||
|
||||
dnsData, err := dialers.Fastdialer.GetDNSData(urlx.Hostname())
|
||||
@@ -406,15 +403,11 @@ func (i *ListInputProvider) scanInputFromReader(executionId string, reader io.Re
|
||||
func (i *ListInputProvider) addInput(executionId string, item string) error {
|
||||
var selection *targetprofile.Selection
|
||||
if targetprofile.IsLine(item) {
|
||||
line, err := i.profiles.ParseLine(item)
|
||||
target, parsed, err := i.profiles.ParseLine(item)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not parse target line")
|
||||
}
|
||||
if line.IP != "" {
|
||||
i.setPinned(executionId, line)
|
||||
return nil
|
||||
}
|
||||
item, selection = line.Target, line.Selection
|
||||
item, selection = target, parsed
|
||||
}
|
||||
|
||||
switch {
|
||||
@@ -428,29 +421,6 @@ func (i *ListInputProvider) addInput(executionId string, item string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// setPinned stores a target pinned to an ip. Its hostname is kept for the Host
|
||||
// header, SNI and template variables, and connections go to the ip, so the ip
|
||||
// is checked against the network policy (-eh, -lna) here.
|
||||
func (i *ListInputProvider) setPinned(executionId string, line targetprofile.Line) {
|
||||
if dialers := protocolstate.GetDialersWithId(executionId); dialers != nil && dialers.NetworkPolicy != nil {
|
||||
if !dialers.NetworkPolicy.Validate(line.IP) {
|
||||
gologger.Warning().Msgf("Skipping %s: ip %s is not allowed by the network policy", line.Target, line.IP)
|
||||
i.skippedCount++
|
||||
return
|
||||
}
|
||||
}
|
||||
metaInput := contextargs.NewMetaInput()
|
||||
metaInput.Input = line.Target
|
||||
metaInput.CustomIP = line.IP
|
||||
i.setItem(metaInput, line.Selection)
|
||||
i.pinned = true
|
||||
}
|
||||
|
||||
// PinnedTargets reports whether any target was pinned to an ip.
|
||||
func (i *ListInputProvider) PinnedTargets() bool {
|
||||
return i.pinned
|
||||
}
|
||||
|
||||
// isExcluded checks if a URL is in the exclusion list
|
||||
func (i *ListInputProvider) isExcluded(URL string) bool {
|
||||
metaInput := contextargs.NewMetaInput()
|
||||
@@ -497,8 +467,7 @@ func (i *ListInputProvider) Del(executionId string, value string) {
|
||||
// scan all ips
|
||||
dialers := protocolstate.GetDialersWithId(executionId)
|
||||
if dialers == nil {
|
||||
gologger.Error().Msgf("dialers with executionId %s not found", executionId)
|
||||
return
|
||||
panic("dialers with executionId " + executionId + " not found")
|
||||
}
|
||||
|
||||
dnsData, err := dialers.Fastdialer.GetDNSData(urlx.Hostname())
|
||||
@@ -535,8 +504,7 @@ func (i *ListInputProvider) Del(executionId string, value string) {
|
||||
if i.ipOptions.IPV6 {
|
||||
dialers := protocolstate.GetDialersWithId(executionId)
|
||||
if dialers == nil {
|
||||
gologger.Error().Msgf("dialers with executionId %s not found", executionId)
|
||||
return
|
||||
panic("dialers with executionId " + executionId + " not found")
|
||||
}
|
||||
|
||||
dnsData, err := dialers.Fastdialer.GetDNSData(urlx.Hostname())
|
||||
@@ -588,19 +556,19 @@ func (i *ListInputProvider) setItem(metaInput *contextargs.MetaInput, selection
|
||||
if _, ok := i.hostMap.Get(dedupKey); ok {
|
||||
i.dupeCount++
|
||||
if i.profiles != nil {
|
||||
i.profiles.Merge(metaInput, selection)
|
||||
i.profiles.Merge(metaInput.Input, selection)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Past the per-pattern cap the shape is already covered by the targets kept
|
||||
// for it, so scanning another of the same shape buys nothing.
|
||||
if i.patterns != nil && !i.patterns.AcceptOn(metaInput.Input, metaInput.CustomIP) {
|
||||
if i.patterns != nil && !i.patterns.Accept(metaInput.Input) {
|
||||
i.patternSkippedCount++
|
||||
return
|
||||
}
|
||||
|
||||
if i.profiles != nil {
|
||||
i.profiles.Bind(metaInput, selection)
|
||||
i.profiles.Bind(metaInput.Input, selection)
|
||||
}
|
||||
i.inputCount++ // tracks target count
|
||||
// the entry is keyed by the collapsed form and carries the original, which
|
||||
|
||||
@@ -242,24 +242,6 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
|
||||
require.ElementsMatchf(t, tt.expected, got, "could not get correct ips for hostname %v", tt.hostname)
|
||||
input.Close()
|
||||
}
|
||||
|
||||
// -mup 1 keeps the url on each ip, as every ip is a server of its own
|
||||
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
||||
require.NoError(t, err)
|
||||
input := &ListInputProvider{
|
||||
hostMap: hm,
|
||||
ipOptions: &ipOptions{ScanAllIPs: true, IPV4: true, IPV6: true},
|
||||
patterns: normalize.NewFingerprinter(1),
|
||||
}
|
||||
defer input.Close()
|
||||
input.Set(defaultOpts.ExecutionId, "https://scanme.sh/user/1")
|
||||
input.Set(defaultOpts.ExecutionId, "https://scanme.sh/user/2")
|
||||
var got []string
|
||||
input.Iterate(func(value *contextargs.MetaInput) bool {
|
||||
got = append(got, value.Input+" "+value.CustomIP)
|
||||
return true
|
||||
})
|
||||
require.ElementsMatch(t, []string{"https://scanme.sh/user/1 128.199.158.128", "https://scanme.sh/user/1 2400:6180:0:d0::91:1001"}, got)
|
||||
}
|
||||
|
||||
func Test_expandASNInputValue(t *testing.T) {
|
||||
|
||||
@@ -5,11 +5,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/hmap/store/hybrid"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/normalize"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/targetprofile"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -37,74 +34,3 @@ func TestTargetLines(t *testing.T) {
|
||||
err = input.scanInputFromReader("", strings.NewReader(`{"target": "192.0.2.30", "tagz": ["x"]}`))
|
||||
require.ErrorContains(t, err, "unknown keys tagz")
|
||||
}
|
||||
|
||||
func TestPinnedTargetLines(t *testing.T) {
|
||||
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
||||
require.NoError(t, err)
|
||||
// pinned lines never resolve the target, even with -sa
|
||||
input := &ListInputProvider{hostMap: hm, ipOptions: &ipOptions{ScanAllIPs: true, IPV4: true}, profiles: targetprofile.NewRegistry(t.TempDir())}
|
||||
defer input.Close()
|
||||
|
||||
lines := strings.Join([]string{
|
||||
`{"target": "https://a.example", "ip": "192.0.2.10", "tags": ["thinkphp"]}`,
|
||||
`{"target": "https://a.example", "ip": "192.0.2.11"}`,
|
||||
`{"target": "https://a.example", "ip": "::ffff:192.0.2.11"}`,
|
||||
}, "\n")
|
||||
require.NoError(t, input.scanInputFromReader("", strings.NewReader(lines)))
|
||||
|
||||
require.True(t, input.PinnedTargets())
|
||||
require.EqualValues(t, 2, input.Count(), "the IPv4-mapped ip is a duplicate")
|
||||
var got []string
|
||||
input.Iterate(func(value *contextargs.MetaInput) bool {
|
||||
got = append(got, value.Input+" "+value.CustomIP)
|
||||
return true
|
||||
})
|
||||
require.ElementsMatch(t, []string{"https://a.example 192.0.2.10", "https://a.example 192.0.2.11"}, got)
|
||||
require.NotNil(t, input.profiles.For(&contextargs.MetaInput{Input: "https://a.example", CustomIP: "192.0.2.10"}))
|
||||
require.Nil(t, input.profiles.For(&contextargs.MetaInput{Input: "https://a.example", CustomIP: "192.0.2.11"}))
|
||||
}
|
||||
|
||||
func TestPinnedTargetLinesKeepEveryIPUnderPatternCap(t *testing.T) {
|
||||
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
||||
require.NoError(t, err)
|
||||
// -mup 1
|
||||
input := &ListInputProvider{hostMap: hm, ipOptions: &ipOptions{IPV4: true}, profiles: targetprofile.NewRegistry(t.TempDir()), patterns: normalize.NewFingerprinter(1)}
|
||||
defer input.Close()
|
||||
|
||||
lines := strings.Join([]string{
|
||||
`{"target": "https://a.example/user/1", "ip": "192.0.2.10"}`,
|
||||
`{"target": "https://a.example/user/1", "ip": "192.0.2.11"}`,
|
||||
`{"target": "https://a.example/user/2", "ip": "192.0.2.10"}`,
|
||||
}, "\n")
|
||||
require.NoError(t, input.scanInputFromReader("", strings.NewReader(lines)))
|
||||
|
||||
var got []string
|
||||
input.Iterate(func(value *contextargs.MetaInput) bool {
|
||||
got = append(got, value.Input+" "+value.CustomIP)
|
||||
return true
|
||||
})
|
||||
require.ElementsMatch(t, []string{"https://a.example/user/1 192.0.2.10", "https://a.example/user/1 192.0.2.11"}, got)
|
||||
require.EqualValues(t, 1, input.patternSkippedCount, "user/2 has the shape of user/1 on the same ip")
|
||||
}
|
||||
|
||||
func TestPinnedTargetLinesFollowNetworkPolicy(t *testing.T) {
|
||||
options := types.DefaultOptions()
|
||||
options.ExecutionId = "pinned-target-lines"
|
||||
options.ExcludeTargets = []string{"192.0.2.0/28"}
|
||||
require.NoError(t, protocolstate.Init(options))
|
||||
t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
|
||||
|
||||
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
||||
require.NoError(t, err)
|
||||
input := &ListInputProvider{hostMap: hm, ipOptions: &ipOptions{IPV4: true}, profiles: targetprofile.NewRegistry(t.TempDir())}
|
||||
defer input.Close()
|
||||
|
||||
lines := strings.Join([]string{
|
||||
`{"target": "https://a.example", "ip": "192.0.2.10"}`,
|
||||
`{"target": "https://a.example", "ip": "192.0.2.20"}`,
|
||||
}, "\n")
|
||||
require.NoError(t, input.scanInputFromReader(options.ExecutionId, strings.NewReader(lines)))
|
||||
|
||||
require.EqualValues(t, 1, input.Count())
|
||||
require.EqualValues(t, 1, input.skippedCount, "the excluded ip is skipped")
|
||||
}
|
||||
@@ -22,7 +22,7 @@ type Registry struct {
|
||||
templatesDir string
|
||||
|
||||
mu sync.RWMutex
|
||||
targets map[targetKey]*Selection
|
||||
targets map[string]*Selection
|
||||
selections map[string]*Selection
|
||||
rules map[string]*rule
|
||||
profiles map[string]selector
|
||||
@@ -38,47 +38,23 @@ var _ protocols.TargetScope = (*Registry)(nil)
|
||||
func NewRegistry(templatesDir string) *Registry {
|
||||
return &Registry{
|
||||
templatesDir: templatesDir,
|
||||
targets: make(map[targetKey]*Selection),
|
||||
targets: make(map[string]*Selection),
|
||||
selections: make(map[string]*Selection),
|
||||
rules: make(map[string]*rule),
|
||||
profiles: make(map[string]selector),
|
||||
}
|
||||
}
|
||||
|
||||
// Line is a parsed target line.
|
||||
type Line struct {
|
||||
Target string
|
||||
// IP is the address the target is pinned to, empty when it is not.
|
||||
IP string
|
||||
// Selection is nil when the line selects every template.
|
||||
Selection *Selection
|
||||
}
|
||||
|
||||
// targetKey identifies a stored input: the same target pinned to different
|
||||
// ips is scanned, and selected, separately.
|
||||
type targetKey struct {
|
||||
input string
|
||||
ip string
|
||||
}
|
||||
|
||||
func keyOf(input *contextargs.MetaInput) targetKey {
|
||||
return targetKey{input: input.Input, ip: input.CustomIP}
|
||||
}
|
||||
|
||||
// ParseLine parses a target line into its target, ip and selection.
|
||||
func (r *Registry) ParseLine(line string) (Line, error) {
|
||||
target, profile, ip, sel, err := parseLine(line)
|
||||
// ParseLine parses a target line into its target and selection.
|
||||
func (r *Registry) ParseLine(line string) (string, *Selection, error) {
|
||||
target, profile, sel, err := parseLine(line)
|
||||
if err != nil {
|
||||
return Line{}, err
|
||||
}
|
||||
if profile == "" && len(sel) == 0 {
|
||||
// nothing narrows the templates, e.g. a line that only sets an ip
|
||||
return Line{Target: target, IP: ip}, nil
|
||||
return "", nil, err
|
||||
}
|
||||
if profile != "" {
|
||||
base, err := r.profile(profile)
|
||||
if err != nil {
|
||||
return Line{}, fmt.Errorf("target %q: %w", target, err)
|
||||
return "", nil, fmt.Errorf("target %q: %w", target, err)
|
||||
}
|
||||
sel = base.merge(sel)
|
||||
}
|
||||
@@ -88,50 +64,48 @@ func (r *Registry) ParseLine(line string) (Line, error) {
|
||||
|
||||
compiled, err := r.internRule(sel)
|
||||
if err != nil {
|
||||
return Line{}, fmt.Errorf("target %q: %w", target, err)
|
||||
return "", nil, fmt.Errorf("target %q: %w", target, err)
|
||||
}
|
||||
return Line{Target: target, IP: ip, Selection: r.internSelection([]*rule{compiled})}, nil
|
||||
return target, r.internSelection([]*rule{compiled}), nil
|
||||
}
|
||||
|
||||
// Bind records the selection of a newly stored input; nil means unrestricted.
|
||||
func (r *Registry) Bind(input *contextargs.MetaInput, selection *Selection) {
|
||||
func (r *Registry) Bind(input string, selection *Selection) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
key := keyOf(input)
|
||||
if selection == nil {
|
||||
r.unrestricted = true
|
||||
if _, ok := r.targets[key]; ok {
|
||||
r.targets[key] = nil
|
||||
if _, ok := r.targets[input]; ok {
|
||||
r.targets[input] = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
current, ok := r.targets[key]
|
||||
current, ok := r.targets[input]
|
||||
switch {
|
||||
case !ok:
|
||||
r.targets[key] = selection
|
||||
r.targets[input] = selection
|
||||
case current != nil:
|
||||
r.targets[key] = r.union(current, selection)
|
||||
r.targets[input] = r.union(current, selection)
|
||||
}
|
||||
}
|
||||
|
||||
// Merge records the selection of an input that was already stored, so a
|
||||
// target listed more than once runs the union of its selections and is
|
||||
// scanned once.
|
||||
func (r *Registry) Merge(input *contextargs.MetaInput, selection *Selection) {
|
||||
func (r *Registry) Merge(input string, selection *Selection) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
key := keyOf(input)
|
||||
current, ok := r.targets[key]
|
||||
current, ok := r.targets[input]
|
||||
switch {
|
||||
case !ok || current == nil:
|
||||
// already unrestricted
|
||||
case selection == nil:
|
||||
r.unrestricted = true
|
||||
r.targets[key] = nil
|
||||
r.targets[input] = nil
|
||||
default:
|
||||
r.targets[key] = r.union(current, selection)
|
||||
r.targets[input] = r.union(current, selection)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,7 +153,7 @@ func (r *Registry) Prepare(tpls []*templates.Template) {
|
||||
// For returns the selection of input, or nil when every template applies.
|
||||
func (r *Registry) For(input *contextargs.MetaInput) protocols.TemplateSelection {
|
||||
r.mu.RLock()
|
||||
selection := r.targets[keyOf(input)]
|
||||
selection := r.targets[input.Input]
|
||||
r.mu.RUnlock()
|
||||
|
||||
// a nil *Selection must not become a non-nil interface
|
||||
|
||||
@@ -49,14 +49,10 @@ func selected(t *testing.T, r *Registry, input string) []string {
|
||||
|
||||
func bindLine(t *testing.T, r *Registry, line string) string {
|
||||
t.Helper()
|
||||
parsed, err := r.ParseLine(line)
|
||||
target, selection, err := r.ParseLine(line)
|
||||
require.NoError(t, err)
|
||||
r.Bind(&contextargs.MetaInput{Input: parsed.Target, CustomIP: parsed.IP}, parsed.Selection)
|
||||
return parsed.Target
|
||||
}
|
||||
|
||||
func input(target string) *contextargs.MetaInput {
|
||||
return &contextargs.MetaInput{Input: target}
|
||||
r.Bind(target, selection)
|
||||
return target
|
||||
}
|
||||
|
||||
func TestParseLineRejectsInvalidLines(t *testing.T) {
|
||||
@@ -69,62 +65,14 @@ func TestParseLineRejectsInvalidLines(t *testing.T) {
|
||||
"invalid type": `{"target": "a.com", "type": ["smtp"]}`,
|
||||
"non string value": `{"target": "a.com", "tags": [1]}`,
|
||||
"missing profile": `{"target": "a.com", "profile": "does-not-exist"}`,
|
||||
"non string ip": `{"target": "a.com", "ip": 10}`,
|
||||
"invalid ip": `{"target": "a.com", "ip": "10.0.0"}`,
|
||||
"ip with zone": `{"target": "a.com", "ip": "fe80::1%eth0"}`,
|
||||
"unspecified ip": `{"target": "a.com", "ip": "0.0.0.0"}`,
|
||||
"ip target": `{"target": "https://192.0.2.1", "ip": "192.0.2.10"}`,
|
||||
"cidr target": `{"target": "192.0.2.0/24", "ip": "192.0.2.10"}`,
|
||||
"asn target": `{"target": "AS13335", "ip": "192.0.2.10"}`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := r.ParseLine(line)
|
||||
_, _, err := r.ParseLine(line)
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLineIP(t *testing.T) {
|
||||
r := NewRegistry(templatesDir)
|
||||
for line, want := range map[string]Line{
|
||||
`{"target": "https://a.com", "ip": "192.0.2.10"}`: {Target: "https://a.com", IP: "192.0.2.10"},
|
||||
`{"target": "a.com:8443", "ip": " 2001:DB8::1 "}`: {Target: "a.com:8443", IP: "2001:db8::1"},
|
||||
`{"target": "a.com", "ip": "::ffff:192.0.2.10"}`: {Target: "a.com", IP: "192.0.2.10"},
|
||||
`{"target": "a.com"}`: {Target: "a.com"},
|
||||
`{"target": "https://a.com", "ip": "192.0.2.10", "tags": []}`: {Target: "https://a.com", IP: "192.0.2.10"},
|
||||
} {
|
||||
t.Run(line, func(t *testing.T) {
|
||||
parsed, err := r.ParseLine(line)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, want, parsed)
|
||||
})
|
||||
}
|
||||
require.False(t, r.Scoped(), "lines that only set an ip do not scope the scan")
|
||||
require.Nil(t, r.LoadFilter())
|
||||
}
|
||||
|
||||
func TestPinnedTargetsKeepTheirSelection(t *testing.T) {
|
||||
r := NewRegistry(templatesDir)
|
||||
bindLine(t, r, `{"target": "https://a.com", "ip": "192.0.2.10", "tags": ["thinkphp"]}`)
|
||||
bindLine(t, r, `{"target": "https://a.com", "ip": "192.0.2.11", "tags": ["tomcat"]}`)
|
||||
bindLine(t, r, `{"target": "https://a.com", "ip": "192.0.2.12"}`)
|
||||
r.Prepare(allTemplates)
|
||||
|
||||
selectedFor := func(ip string) []string {
|
||||
selection := r.For(&contextargs.MetaInput{Input: "https://a.com", CustomIP: ip})
|
||||
var ids []string
|
||||
for _, tpl := range allTemplates {
|
||||
if selection == nil || selection.Allows(tpl.Path) {
|
||||
ids = append(ids, tpl.ID)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
require.Equal(t, []string{"thinkphp-rce"}, selectedFor("192.0.2.10"))
|
||||
require.Equal(t, []string{"tomcat-panel", "CVE-2020-1938"}, selectedFor("192.0.2.11"))
|
||||
require.Equal(t, []string{"thinkphp-rce", "tomcat-panel", "CVE-2020-1938", "git-config"}, selectedFor("192.0.2.12"))
|
||||
}
|
||||
|
||||
func TestProfileMustBeAnID(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, "profiles"), 0o755))
|
||||
@@ -137,14 +85,14 @@ func TestProfileMustBeAnID(t *testing.T) {
|
||||
t.Run(profile, func(t *testing.T) {
|
||||
line, err := json.Marshal(map[string]string{"target": "a.com", "profile": profile})
|
||||
require.NoError(t, err)
|
||||
_, err = r.ParseLine(string(line))
|
||||
_, _, err = r.ParseLine(string(line))
|
||||
require.ErrorContains(t, err, "profile ID")
|
||||
})
|
||||
}
|
||||
|
||||
parsed, err := r.ParseLine(`{"target": "a.com", "profile": "php"}`)
|
||||
_, selection, err := r.ParseLine(`{"target": "a.com", "profile": "php"}`)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{"thinkphp"}, parsed.Selection.rules[0].filter.Tags)
|
||||
require.Equal(t, []string{"thinkphp"}, selection.rules[0].filter.Tags)
|
||||
}
|
||||
|
||||
func TestSelectionFilters(t *testing.T) {
|
||||
@@ -181,38 +129,37 @@ func TestProfileWithInlineOverride(t *testing.T) {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "profiles", "php.yml"), []byte(profile), 0o600))
|
||||
|
||||
r := NewRegistry(dir)
|
||||
parsed, err := r.ParseLine(`{"target": "https://a.com", "profile": "php", "severity": "critical"}`)
|
||||
target, selection, err := r.ParseLine(`{"target": "https://a.com", "profile": "php", "severity": "critical"}`)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, parsed.Selection.rules, 1)
|
||||
require.Equal(t, []string{"thinkphp"}, parsed.Selection.rules[0].filter.Tags)
|
||||
require.Equal(t, []severity.Severity{severity.Critical}, parsed.Selection.rules[0].filter.Severities)
|
||||
require.Equal(t, "https://a.com", parsed.Target)
|
||||
require.Len(t, selection.rules, 1)
|
||||
require.Equal(t, []string{"thinkphp"}, selection.rules[0].filter.Tags)
|
||||
require.Equal(t, []severity.Severity{severity.Critical}, selection.rules[0].filter.Severities)
|
||||
require.Equal(t, "https://a.com", target)
|
||||
}
|
||||
|
||||
func TestDuplicateTargetsMerge(t *testing.T) {
|
||||
r := NewRegistry(templatesDir)
|
||||
phpLine, err := r.ParseLine(`{"target": "https://a.com", "tags": ["thinkphp"]}`)
|
||||
_, php, err := r.ParseLine(`{"target": "https://a.com", "tags": ["thinkphp"]}`)
|
||||
require.NoError(t, err)
|
||||
tomcatLine, err := r.ParseLine(`{"target": "https://a.com", "tags": ["tomcat"]}`)
|
||||
_, tomcat, err := r.ParseLine(`{"target": "https://a.com", "tags": ["tomcat"]}`)
|
||||
require.NoError(t, err)
|
||||
php, tomcat := phpLine.Selection, tomcatLine.Selection
|
||||
|
||||
r.Bind(input("https://a.com"), php)
|
||||
r.Merge(input("https://a.com"), tomcat)
|
||||
r.Merge(input("https://a.com"), php)
|
||||
r.Bind(input("https://b.com"), php)
|
||||
r.Merge(input("https://b.com"), nil)
|
||||
r.Bind(input("https://c.com"), nil)
|
||||
r.Merge(input("https://c.com"), php)
|
||||
r.Bind("https://a.com", php)
|
||||
r.Merge("https://a.com", tomcat)
|
||||
r.Merge("https://a.com", php)
|
||||
r.Bind("https://b.com", php)
|
||||
r.Merge("https://b.com", nil)
|
||||
r.Bind("https://c.com", nil)
|
||||
r.Merge("https://c.com", php)
|
||||
r.Prepare(allTemplates)
|
||||
|
||||
require.Equal(t, []string{"thinkphp-rce", "tomcat-panel", "CVE-2020-1938"}, selected(t, r, "https://a.com"))
|
||||
require.Nil(t, r.For(&contextargs.MetaInput{Input: "https://b.com"}), "a plain duplicate lifts the restriction")
|
||||
require.Nil(t, r.For(&contextargs.MetaInput{Input: "https://c.com"}), "a plain target stays unrestricted")
|
||||
|
||||
again, err := r.ParseLine(`{"target": "https://z.com", "tags": ["thinkphp"]}`)
|
||||
_, again, err := r.ParseLine(`{"target": "https://z.com", "tags": ["thinkphp"]}`)
|
||||
require.NoError(t, err)
|
||||
require.Same(t, php, again.Selection, "identical selectors are interned")
|
||||
require.Same(t, php, again, "identical selectors are interned")
|
||||
}
|
||||
|
||||
func TestLoadFilter(t *testing.T) {
|
||||
@@ -236,14 +183,14 @@ func TestLoadFilter(t *testing.T) {
|
||||
t.Run("unrestricted target loads everything", func(t *testing.T) {
|
||||
r := NewRegistry(templatesDir)
|
||||
bindLine(t, r, `{"target": "https://a.com", "tags": ["thinkphp"]}`)
|
||||
r.Bind(input("https://b.com"), nil)
|
||||
r.Bind("https://b.com", nil)
|
||||
require.Nil(t, r.LoadFilter())
|
||||
})
|
||||
|
||||
t.Run("plain duplicate of a target line loads everything", func(t *testing.T) {
|
||||
r := NewRegistry(templatesDir)
|
||||
target := bindLine(t, r, `{"target": "https://a.com", "tags": ["thinkphp"]}`)
|
||||
r.Merge(input(target), nil)
|
||||
r.Merge(target, nil)
|
||||
require.Nil(t, r.LoadFilter())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,34 +2,27 @@
|
||||
// single scan can run a different profile on each target.
|
||||
//
|
||||
// A target line is a JSON object whose keys mirror template profiles and CLI
|
||||
// flags, and may pin the target to an ip:
|
||||
// flags:
|
||||
//
|
||||
// {"target": "https://a.example", "tags": ["thinkphp"], "tech": ["php"]}
|
||||
// {"target": "https://b.example", "profile": "wordpress", "severity": ["high"]}
|
||||
// {"target": "https://c.example", "ip": "192.0.2.10"}
|
||||
package targetprofile
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/mapcidr/asn"
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/configuration"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
|
||||
iputil "github.com/projectdiscovery/utils/ip"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
|
||||
const (
|
||||
keyTarget = "target"
|
||||
keyProfile = "profile"
|
||||
keyIP = "ip"
|
||||
|
||||
keyTemplates = "templates"
|
||||
keyTags = "tags"
|
||||
@@ -61,64 +54,36 @@ func IsLine(line string) bool {
|
||||
|
||||
// parseLine decodes a target line. Unknown keys are rejected so that a typo
|
||||
// cannot silently widen the scan to every template.
|
||||
func parseLine(line string) (target, profile, ip string, sel selector, err error) {
|
||||
func parseLine(line string) (target, profile string, sel selector, err error) {
|
||||
var raw map[string]any
|
||||
if err := json.Unmarshal([]byte(line), &raw); err != nil {
|
||||
return "", "", "", nil, fmt.Errorf("invalid target line %q: %w", line, err)
|
||||
return "", "", nil, fmt.Errorf("invalid target line %q: %w", line, err)
|
||||
}
|
||||
|
||||
target, err = stringValue(raw, keyTarget)
|
||||
if err != nil {
|
||||
return "", "", "", nil, err
|
||||
return "", "", nil, err
|
||||
}
|
||||
target = strings.TrimSpace(target)
|
||||
if target == "" {
|
||||
return "", "", "", nil, fmt.Errorf("target line %q has no %q", line, keyTarget)
|
||||
return "", "", nil, fmt.Errorf("target line %q has no %q", line, keyTarget)
|
||||
}
|
||||
|
||||
profile, err = stringValue(raw, keyProfile)
|
||||
if err != nil {
|
||||
return "", "", "", nil, err
|
||||
}
|
||||
ip, err = stringValue(raw, keyIP)
|
||||
if err != nil {
|
||||
return "", "", "", nil, err
|
||||
}
|
||||
if ip != "" {
|
||||
if ip, err = pinnedIP(target, ip); err != nil {
|
||||
return "", "", "", nil, fmt.Errorf("target %q: %w", target, err)
|
||||
}
|
||||
return "", "", nil, err
|
||||
}
|
||||
delete(raw, keyTarget)
|
||||
delete(raw, keyProfile)
|
||||
delete(raw, keyIP)
|
||||
|
||||
sel, ignored, err := toSelector(raw)
|
||||
if err != nil {
|
||||
return "", "", "", nil, fmt.Errorf("target %q: %w", target, err)
|
||||
return "", "", nil, fmt.Errorf("target %q: %w", target, err)
|
||||
}
|
||||
if len(ignored) > 0 {
|
||||
return "", "", "", nil, fmt.Errorf("target %q: unknown keys %s (supported: target, ip, profile, %s)", target, strings.Join(ignored, ", "), strings.Join(selectorKeys, ", "))
|
||||
return "", "", nil, fmt.Errorf("target %q: unknown keys %s (supported: target, profile, %s)", target, strings.Join(ignored, ", "), strings.Join(selectorKeys, ", "))
|
||||
}
|
||||
return target, profile, ip, sel, nil
|
||||
}
|
||||
|
||||
// pinnedIP validates the ip a target is pinned to and returns it in canonical
|
||||
// form. Only a target with a hostname can be pinned: the hostname is kept for
|
||||
// the Host header, SNI and template variables while connections go to the ip.
|
||||
func pinnedIP(target, value string) (string, error) {
|
||||
addr, err := netip.ParseAddr(strings.TrimSpace(value))
|
||||
if err != nil || addr.Zone() != "" || addr.IsUnspecified() {
|
||||
return "", fmt.Errorf("invalid ip %q", value)
|
||||
}
|
||||
if iputil.IsCIDR(target) || asn.IsASN(target) {
|
||||
return "", errors.New("a CIDR or ASN target cannot have an ip")
|
||||
}
|
||||
parsed, err := urlutil.Parse(target)
|
||||
if err != nil || parsed.Hostname() == "" || iputil.IsIP(parsed.Hostname()) {
|
||||
return "", errors.New("a target with an ip must have a hostname")
|
||||
}
|
||||
return addr.Unmap().String(), nil
|
||||
return target, profile, sel, nil
|
||||
}
|
||||
|
||||
// loadProfile reads the selection keys of a template profile. Keys that
|
||||
|
||||
@@ -16,14 +16,6 @@ import (
|
||||
// Helper is a structure for helping with input transformation
|
||||
type Helper struct {
|
||||
InputsHTTP *hybrid.HybridMap
|
||||
// InputsHTTPProbed is true when InputsHTTP was filled by httpx probing.
|
||||
// False means probing was skipped (e.g. MultiFormat) and map absence must
|
||||
// not be treated as "not HTTP".
|
||||
InputsHTTPProbed bool
|
||||
// StrictProbe, when set, makes web-template input resolution skip inputs that
|
||||
// were probed and found NOT to be HTTP services (instead of falling back to
|
||||
// a raw URL). Lossless: a non-HTTP port has no HTTP application to find.
|
||||
StrictProbe bool
|
||||
}
|
||||
|
||||
// NewHelper returns a new input helper instance
|
||||
@@ -61,32 +53,6 @@ func (h *Helper) Transform(input string, protocol templateTypes.ProtocolType) st
|
||||
return input
|
||||
}
|
||||
|
||||
// TransformPinned is Transform for an input pinned to ip, which takes the
|
||||
// HTTP probe result made through that ip and never the result of the input
|
||||
// without an ip, as that one may come from another server.
|
||||
func (h *Helper) TransformPinned(input, ip string, protocol templateTypes.ProtocolType) string {
|
||||
if ip == "" || (protocol != templateTypes.HTTPProtocol && protocol != templateTypes.HeadlessProtocol) {
|
||||
return h.Transform(input, protocol)
|
||||
}
|
||||
if h.InputsHTTP != nil {
|
||||
if probed, ok := h.InputsHTTP.Get(ProbeKey(input, ip)); ok {
|
||||
return string(probed)
|
||||
}
|
||||
}
|
||||
// no probe through this ip: convert it as an input that was not probed
|
||||
return (&Helper{}).Transform(input, protocol)
|
||||
}
|
||||
|
||||
// ProbeKey returns the key of the HTTP probe result of input in InputsHTTP.
|
||||
// A target pinned to an ip is probed through that ip, so each ip has its own.
|
||||
// The separator cannot occur in an input, which may hold an "@" of its own.
|
||||
func ProbeKey(input, ip string) string {
|
||||
if ip == "" {
|
||||
return input
|
||||
}
|
||||
return input + "\x00" + ip
|
||||
}
|
||||
|
||||
type inputType int
|
||||
|
||||
const (
|
||||
@@ -148,12 +114,6 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
|
||||
if probed, ok := h.InputsHTTP.Get(input); ok {
|
||||
return string(probed)
|
||||
}
|
||||
// Strict reachability: this input was probed and is not an HTTP
|
||||
// service, so skip it for web templates rather than falling back to
|
||||
// a raw URL. Lossless — a non-HTTP port has no HTTP app to match.
|
||||
if h.StrictProbe {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
// try to parse it as absolute url and return
|
||||
if absUrl, err := urlutil.ParseAbsoluteURL(input, false); err == nil {
|
||||
|
||||
@@ -4,28 +4,9 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/hmap/store/hybrid"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestTransformPinned(t *testing.T) {
|
||||
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
||||
require.NoError(t, err, "could not create hybrid map")
|
||||
defer func() {
|
||||
_ = hm.Close()
|
||||
}()
|
||||
helper := &Helper{InputsHTTP: hm}
|
||||
|
||||
_ = hm.Set(ProbeKey("app.test", ""), []byte("http://app.test"))
|
||||
_ = hm.Set(ProbeKey("app.test", "127.0.0.2"), []byte("https://app.test"))
|
||||
|
||||
require.Equal(t, "https://app.test", helper.TransformPinned("app.test", "127.0.0.2", templateTypes.HTTPProtocol), "the probe made through the pinned ip")
|
||||
require.Equal(t, "http://app.test", helper.TransformPinned("app.test", "", templateTypes.HTTPProtocol), "an unpinned input")
|
||||
require.Equal(t, "app.test", helper.TransformPinned("app.test", "127.0.0.3", templateTypes.HTTPProtocol), "an ip without a probe of its own does not take the unpinned result")
|
||||
require.Equal(t, "app.test", helper.TransformPinned("app.test", "127.0.0.2", templateTypes.DNSProtocol), "a protocol that is not probed")
|
||||
require.NotEqual(t, ProbeKey("[email protected]", ""), ProbeKey("u.example", "192.0.2.10"), "an input holding an @ does not collide with a pinned one")
|
||||
}
|
||||
|
||||
func TestConvertInputToType(t *testing.T) {
|
||||
helper := &Helper{}
|
||||
|
||||
@@ -91,32 +72,3 @@ func TestConvertInputToType(t *testing.T) {
|
||||
require.Equal(t, test.result, result, "could not get correct result %+v", test)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrictProbeSkipsUnconfirmedHTTPInput(t *testing.T) {
|
||||
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = hm.Close() })
|
||||
|
||||
_ = hm.Set("ok.example", []byte("https://ok.example"))
|
||||
|
||||
helper := &Helper{InputsHTTP: hm, StrictProbe: true}
|
||||
|
||||
// confirmed host -> resolved URL
|
||||
require.Equal(t, "https://ok.example", helper.convertInputToType("ok.example", typeURL, ""))
|
||||
// probed but non-HTTP hosts -> skipped (no raw fallback)
|
||||
require.Equal(t, "", helper.convertInputToType("127.0.0.1:3306", typeURL, ""))
|
||||
require.Equal(t, "", helper.convertInputToType("zombie.example:6379", typeURL, ""))
|
||||
// already-URL inputs are not gated by probe results
|
||||
require.Equal(t, "https://direct.example", helper.convertInputToType("https://direct.example", typeURL, ""))
|
||||
// non-HTTP protocols still transform normally
|
||||
require.Equal(t, "127.0.0.1:3306", helper.convertInputToType("127.0.0.1:3306", typeHostWithOptionalPort, ""))
|
||||
}
|
||||
|
||||
func TestStrictProbeDisabledKeepsRawFallback(t *testing.T) {
|
||||
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = hm.Close() })
|
||||
|
||||
helper := &Helper{InputsHTTP: hm, StrictProbe: false}
|
||||
require.Equal(t, "127.0.0.1:3306", helper.convertInputToType("127.0.0.1:3306", typeURL, ""))
|
||||
}
|
||||
+8
-19
@@ -112,27 +112,16 @@ func (rr *RequestResponse) ID() string {
|
||||
func (rr *RequestResponse) MarshalJSON() ([]byte, error) {
|
||||
m := make(map[string]interface{})
|
||||
m["url"] = rr.URL.String()
|
||||
// Request and Response are optional (UnmarshalJSON only reads the keys when
|
||||
// present), so nil pointers omit their keys instead of emitting "null".
|
||||
//
|
||||
// The pre-marshaled bytes must be wrapped as json.Message: encoding/json
|
||||
// (and sonic) encode a bare []byte as a base64 *string*, which this type's
|
||||
// own UnmarshalJSON — expecting a JSON object — cannot decode, breaking the
|
||||
// json.JSONCodec round-trip contract asserted above.
|
||||
if rr.Request != nil {
|
||||
reqBin, err := json.Marshal(rr.Request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m["request"] = json.Message(reqBin)
|
||||
reqBin, err := json.Marshal(rr.Request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rr.Response != nil {
|
||||
respBin, err := json.Marshal(rr.Response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m["response"] = json.Message(respBin)
|
||||
m["request"] = reqBin
|
||||
respBin, err := json.Marshal(rr.Response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m["response"] = respBin
|
||||
return json.Marshal(m)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestRequestResponseJSONRoundTrip pins the json.JSONCodec contract asserted
|
||||
// on RequestResponse: MarshalJSON output must be decodable by the type's own
|
||||
// UnmarshalJSON.
|
||||
//
|
||||
// It previously failed because MarshalJSON put the pre-marshaled request and
|
||||
// response into a map[string]interface{} as []byte, which encoding/json (and
|
||||
// sonic) encode as base64 *strings* - while UnmarshalJSON expects JSON
|
||||
// objects:
|
||||
//
|
||||
// {"request":"eyJtZXRob2QiOiJHRVQifQ==", ...} -> mismatched type string
|
||||
//
|
||||
// MetaInput serializes a captured request as the "raw-request" key with this
|
||||
// codec, so the broken round-trip surfaced through MetaInput.MarshalString ->
|
||||
// MetaInput.Unmarshal (pkg/input/provider/list/hmap.go) as a decode error.
|
||||
func TestRequestResponseJSONRoundTrip(t *testing.T) {
|
||||
rr, err := ParseRawRequest("GET /path?q=1 HTTP/1.1\r\nHost: example.com\r\nX-Test: hello\r\n\r\nbody")
|
||||
require.NoError(t, err)
|
||||
rr.Response = &HttpResponse{
|
||||
StatusCode: 200,
|
||||
Body: "ok",
|
||||
Raw: "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok",
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(rr)
|
||||
require.NoError(t, err)
|
||||
|
||||
var decoded RequestResponse
|
||||
require.NoError(t, json.Unmarshal(encoded, &decoded), "marshal/unmarshal round-trip must succeed, got %s", encoded)
|
||||
|
||||
require.Equal(t, rr.URL.String(), decoded.URL.String())
|
||||
require.NotNil(t, decoded.Request)
|
||||
require.Equal(t, rr.Request.Method, decoded.Request.Method)
|
||||
require.Equal(t, rr.Request.Raw, decoded.Request.Raw)
|
||||
require.Equal(t, rr.Request.Body, decoded.Request.Body)
|
||||
require.NotNil(t, decoded.Response)
|
||||
require.Equal(t, rr.Response.StatusCode, decoded.Response.StatusCode)
|
||||
require.Equal(t, rr.Response.Raw, decoded.Response.Raw)
|
||||
}
|
||||
|
||||
// TestRequestResponseMarshalOmitsNilParts pins that nil Request/Response stay
|
||||
// nil across the round-trip instead of turning into zero-valued structs (or a
|
||||
// decode error, which is what the previous base64 "null" encoding produced).
|
||||
func TestRequestResponseMarshalOmitsNilParts(t *testing.T) {
|
||||
rr, err := ParseRawRequest("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, rr.Response)
|
||||
|
||||
encoded, err := json.Marshal(rr)
|
||||
require.NoError(t, err)
|
||||
|
||||
var decoded RequestResponse
|
||||
require.NoError(t, json.Unmarshal(encoded, &decoded), "got %s", encoded)
|
||||
require.NotNil(t, decoded.Request)
|
||||
require.Nil(t, decoded.Response)
|
||||
}
|
||||
@@ -1,28 +1,10 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/google/go-github/v92/github"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
updateutils "github.com/projectdiscovery/utils/update"
|
||||
)
|
||||
|
||||
const (
|
||||
// TemplatesChannelStable updates nuclei-templates to the latest release.
|
||||
TemplatesChannelStable = "stable"
|
||||
// TemplatesChannelLatest updates nuclei-templates to the head of the
|
||||
// default branch, including templates merged since the latest release.
|
||||
TemplatesChannelLatest = "latest"
|
||||
|
||||
templatesDefaultBranch = "main"
|
||||
shortCommitLength = 7
|
||||
)
|
||||
|
||||
// templateRelease is a published release of the official nuclei-templates.
|
||||
type templateRelease interface {
|
||||
Version() string
|
||||
@@ -48,76 +30,3 @@ func latestGitHubTemplateRelease() (templateRelease, error) {
|
||||
}
|
||||
return githubTemplateRelease{downloader}, nil
|
||||
}
|
||||
|
||||
// githubTemplateHead is the head commit of the official nuclei-templates
|
||||
// default branch.
|
||||
type githubTemplateHead struct {
|
||||
release templateRelease // latest release, the commit is ahead of it
|
||||
commit string
|
||||
fetchArchive templateArchiveFetcher
|
||||
}
|
||||
|
||||
// Version returns the latest release with the short commit as semver build
|
||||
// metadata (e.g. v10.3.1+3de629d). Build metadata is ignored when comparing
|
||||
// versions, so it is only outdated once a newer release is published.
|
||||
func (h githubTemplateHead) Version() string {
|
||||
return h.release.Version() + "+" + h.commit[:min(len(h.commit), shortCommitLength)]
|
||||
}
|
||||
|
||||
// Changelog is empty since unreleased commits have no release notes.
|
||||
func (h githubTemplateHead) Changelog() string { return "" }
|
||||
|
||||
// DownloadSource downloads the archive of the head commit and calls callback
|
||||
// for every file in it.
|
||||
func (h githubTemplateHead) DownloadSource(_ bool, callback updateutils.AssetFileCallback) error {
|
||||
archive, err := h.fetchArchive(h.commit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return updateutils.UnpackAssetWithCallback(updateutils.Zip, archive, callback)
|
||||
}
|
||||
|
||||
// latestGitHubTemplateHead returns the current head of the nuclei-templates
|
||||
// default branch, based on the latest release.
|
||||
func latestGitHubTemplateHead() (templateRelease, error) {
|
||||
release, err := latestGitHubTemplateRelease()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client, err := newGitHubClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
commit, _, err := client.Repositories.GetCommitSHA1(context.Background(), updateutils.Organization, config.OfficialNucleiTemplatesRepoName, templatesDefaultBranch, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get nuclei-templates %s head: %w", templatesDefaultBranch, err)
|
||||
}
|
||||
|
||||
return githubTemplateHead{release: release, commit: commit, fetchArchive: fetchTemplateArchive}, nil
|
||||
}
|
||||
|
||||
// templateVersionRef returns the git ref of an installed templates version.
|
||||
// Versions installed from the latest channel carry their commit as build
|
||||
// metadata.
|
||||
func templateVersionRef(version string) string {
|
||||
if _, commit, ok := strings.Cut(version, "+"); ok && commit != "" {
|
||||
return commit
|
||||
}
|
||||
return version
|
||||
}
|
||||
|
||||
// newGitHubClient returns a GitHub client that uses GITHUB_TOKEN when set,
|
||||
// like the release downloader does.
|
||||
func newGitHubClient() (*github.Client, error) {
|
||||
opts := []github.ClientOptionsFunc{github.WithHTTPClient(&http.Client{Timeout: updateutils.DownloadUpdateTimeout})}
|
||||
if token := os.Getenv("GITHUB_TOKEN"); token != "" {
|
||||
opts = append(opts, github.WithAuthToken(token))
|
||||
}
|
||||
|
||||
client, err := github.NewClient(opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create github client: %w", err)
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
@@ -1,19 +1,14 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
updateutils "github.com/projectdiscovery/utils/update"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeTemplateRelease is an in-memory nuclei-templates release, mapping paths
|
||||
@@ -40,19 +35,6 @@ func (r fakeTemplateRelease) DownloadSource(_ bool, callback updateutils.AssetFi
|
||||
return nil
|
||||
}
|
||||
|
||||
// installedTemplateRelease is a release that is already installed, so
|
||||
// downloading it fails the test.
|
||||
type installedTemplateRelease struct {
|
||||
fakeTemplateRelease
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
// DownloadSource fails the test since the release is already installed.
|
||||
func (r installedTemplateRelease) DownloadSource(bool, updateutils.AssetFileCallback) error {
|
||||
r.t.Fatalf("downloaded already installed templates %s", r.version)
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeFileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
@@ -64,40 +46,3 @@ func (f fakeFileInfo) Mode() fs.FileMode { return 0o644 }
|
||||
func (f fakeFileInfo) ModTime() time.Time { return time.Time{} }
|
||||
func (f fakeFileInfo) IsDir() bool { return false }
|
||||
func (f fakeFileInfo) Sys() any { return nil }
|
||||
|
||||
// TestGithubTemplateHead checks the version of the head and that it downloads
|
||||
// the exact head commit.
|
||||
func TestGithubTemplateHead(t *testing.T) {
|
||||
commit := "3de629d823bb6eb00aa3769edc365b6525134fb1"
|
||||
entry := "projectdiscovery-nuclei-templates-3de629d/http/head.yaml"
|
||||
var fetchedRef string
|
||||
head := githubTemplateHead{
|
||||
release: fakeTemplateRelease{version: "v10.3.1"},
|
||||
commit: commit,
|
||||
fetchArchive: func(ref string) (*bytes.Reader, error) {
|
||||
fetchedRef = ref
|
||||
return newTemplateReleaseArchive(t, entry, []byte("id: head")), nil
|
||||
},
|
||||
}
|
||||
|
||||
require.Equal(t, "v10.3.1+3de629d", head.Version())
|
||||
// build metadata is ignored, so only a newer release outdates the head
|
||||
require.False(t, config.IsOutdatedVersion(head.Version(), "v10.3.1"))
|
||||
require.True(t, config.IsOutdatedVersion(head.Version(), "v10.3.2"))
|
||||
|
||||
var entries []string
|
||||
require.NoError(t, head.DownloadSource(false, func(uri string, _ fs.FileInfo, _ io.Reader) error {
|
||||
entries = append(entries, uri)
|
||||
return nil
|
||||
}))
|
||||
require.Equal(t, commit, fetchedRef, "should download the exact head commit")
|
||||
require.Equal(t, []string{entry}, entries)
|
||||
}
|
||||
|
||||
// TestTemplateVersionRef checks the git ref of release and latest channel
|
||||
// versions.
|
||||
func TestTemplateVersionRef(t *testing.T) {
|
||||
require.Equal(t, "v10.3.1", templateVersionRef("v10.3.1"))
|
||||
require.Equal(t, "3de629d", templateVersionRef("v10.3.1+3de629d"))
|
||||
require.Equal(t, "v10.3.1+", templateVersionRef("v10.3.1+"))
|
||||
}
|
||||
+15
-72
@@ -77,24 +77,15 @@ type TemplateManager struct {
|
||||
DisablePublicTemplates bool // if true,
|
||||
// public templates are not downloaded from the GitHub nuclei-templates repository
|
||||
|
||||
// Channel is the nuclei-templates update channel, TemplatesChannelStable
|
||||
// (default) or TemplatesChannelLatest
|
||||
Channel string
|
||||
|
||||
// fetchLatestRelease overrides where the official release comes from; nil
|
||||
// means GitHub
|
||||
fetchLatestRelease func() (templateRelease, error)
|
||||
}
|
||||
|
||||
// latestRelease returns the official templates to install for the configured
|
||||
// channel.
|
||||
func (t *TemplateManager) latestRelease() (templateRelease, error) {
|
||||
if t.fetchLatestRelease != nil {
|
||||
return t.fetchLatestRelease()
|
||||
}
|
||||
if t.Channel == TemplatesChannelLatest {
|
||||
return latestGitHubTemplateHead()
|
||||
}
|
||||
return latestGitHubTemplateRelease()
|
||||
}
|
||||
|
||||
@@ -122,8 +113,6 @@ func (t *TemplateManager) UpdateIfOutdated() error {
|
||||
return withTemplatesUpdateLock(config.DefaultConfig.TemplatesDirectory, t.updateIfOutdatedLocked)
|
||||
}
|
||||
|
||||
// updateIfOutdatedLocked installs or updates the templates when they are
|
||||
// outdated. It must be called while holding the templates update lock.
|
||||
func (t *TemplateManager) updateIfOutdatedLocked() error {
|
||||
// if the templates folder does not exist, it's a fresh installation and do not update
|
||||
if !fileutil.FolderExists(config.DefaultConfig.TemplatesDirectory) {
|
||||
@@ -140,17 +129,6 @@ func (t *TemplateManager) updateIfOutdatedLocked() error {
|
||||
gologger.Debug().Msgf("Could not reload templates version, using the one loaded at startup: %s", err)
|
||||
}
|
||||
|
||||
if t.Channel == TemplatesChannelLatest {
|
||||
needsUpdate, err := t.latestCommitNeedsUpdate()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if needsUpdate {
|
||||
return t.updateTemplatesAt(config.DefaultConfig.TemplatesDirectory)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
needsUpdate := config.DefaultConfig.NeedsTemplateUpdate()
|
||||
|
||||
// NOTE(dwisiswant0): if PDTM API data is not available
|
||||
@@ -173,22 +151,6 @@ func (t *TemplateManager) updateIfOutdatedLocked() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// latestCommitNeedsUpdate returns true if the head of the nuclei-templates
|
||||
// default branch differs from the installed templates. The version check only
|
||||
// knows about releases, so it can't tell when the head moves.
|
||||
func (t *TemplateManager) latestCommitNeedsUpdate() (bool, error) {
|
||||
if t.DisablePublicTemplates || !config.DefaultConfig.CanCheckForUpdates() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
release, err := t.latestRelease()
|
||||
if err != nil {
|
||||
return false, errkit.Wrap(err, "failed to get latest nuclei-templates commit")
|
||||
}
|
||||
|
||||
return release.Version() != config.DefaultConfig.TemplateVersion, nil
|
||||
}
|
||||
|
||||
// installTemplatesAt installs templates at given directory
|
||||
func (t *TemplateManager) installTemplatesAt(dir string) error {
|
||||
if !fileutil.FolderExists(dir) {
|
||||
@@ -272,9 +234,6 @@ func (t *TemplateManager) finalizeTemplateWrite(cfg *config.Config, writtenOutpu
|
||||
return t.finalizeTemplateRelease(cfg, writtenOutputs, version)
|
||||
}
|
||||
|
||||
// bootstrapTemplateOwnership builds the ownership manifest from the archive of
|
||||
// the installed version when it is missing, so an update can tell official
|
||||
// templates apart from local ones.
|
||||
func (t *TemplateManager) bootstrapTemplateOwnership(dir, version string, fetchArchive templateArchiveFetcher) error {
|
||||
if _, err := loadTemplateOwnership(dir); err == nil {
|
||||
return nil
|
||||
@@ -305,7 +264,7 @@ func (t *TemplateManager) bootstrapTemplateOwnership(dir, version string, fetchA
|
||||
return nil
|
||||
}
|
||||
|
||||
archive, err := fetchArchive(templateVersionRef(version))
|
||||
archive, err := fetchArchive(version)
|
||||
if err != nil {
|
||||
gologger.Verbose().Msgf("Continuing template update without prior ownership for %q: %s", version, err)
|
||||
return nil
|
||||
@@ -323,54 +282,40 @@ func (t *TemplateManager) bootstrapTemplateOwnership(dir, version string, fetchA
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetchTemplateArchive downloads the nuclei-templates zipball at ref, a release
|
||||
// tag or a commit.
|
||||
func fetchTemplateArchive(ref string) (*bytes.Reader, error) {
|
||||
func fetchTemplateReleaseArchive(version string) (*bytes.Reader, error) {
|
||||
ctx := context.Background()
|
||||
client, err := newGitHubClient()
|
||||
httpClient := &http.Client{Timeout: updateutils.DownloadUpdateTimeout}
|
||||
client, err := github.NewClient(github.WithHTTPClient(httpClient))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("create github client for prior template release %q: %w", version, err)
|
||||
}
|
||||
archiveURL, _, err := client.Repositories.GetArchiveLink(
|
||||
ctx,
|
||||
updateutils.Organization,
|
||||
config.OfficialNucleiTemplatesRepoName,
|
||||
github.Zipball,
|
||||
&github.RepositoryContentGetOptions{Ref: ref},
|
||||
&github.RepositoryContentGetOptions{Ref: version},
|
||||
1, // Follow one repository rename, as the previous client did.
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve nuclei-templates %q archive: %w", ref, err)
|
||||
}
|
||||
if archiveURL.Scheme != "https" {
|
||||
return nil, fmt.Errorf("resolve nuclei-templates %q archive: unexpected archive url %q", ref, archiveURL)
|
||||
return nil, fmt.Errorf("resolve prior template release %q archive: %w", version, err)
|
||||
}
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, archiveURL.String(), nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create nuclei-templates %q archive request: %w", ref, err)
|
||||
return nil, fmt.Errorf("create prior template release %q archive request: %w", version, err)
|
||||
}
|
||||
|
||||
// the archive is public, so it is downloaded without the token the github
|
||||
// client sends to every host
|
||||
httpClient := &http.Client{Timeout: updateutils.DownloadUpdateTimeout}
|
||||
response, err := httpClient.Do(request)
|
||||
var contents bytes.Buffer
|
||||
response, err := client.Do(request, &contents)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download nuclei-templates %q: %w", ref, err)
|
||||
return nil, fmt.Errorf("download prior template release %q: %w", version, err)
|
||||
}
|
||||
defer func() {
|
||||
_ = response.Body.Close()
|
||||
}()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("download nuclei-templates %q: unexpected HTTP status %s", ref, response.Status)
|
||||
return nil, fmt.Errorf("download prior template release %q: unexpected HTTP status %s", version, response.Status)
|
||||
}
|
||||
|
||||
contents, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read nuclei-templates %q archive: %w", ref, err)
|
||||
}
|
||||
|
||||
return bytes.NewReader(contents), nil
|
||||
return bytes.NewReader(contents.Bytes()), nil
|
||||
}
|
||||
|
||||
func (t *TemplateManager) bootstrapTemplateOwnershipFromArchive(dir string, archive *bytes.Reader) error {
|
||||
@@ -436,7 +381,7 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := t.bootstrapTemplateOwnership(dir, config.DefaultConfig.TemplateVersion, fetchTemplateArchive); err != nil {
|
||||
if err := t.bootstrapTemplateOwnership(dir, config.DefaultConfig.TemplateVersion, fetchTemplateReleaseArchive); err != nil {
|
||||
return errkit.Wrapf(err, "failed to migrate template ownership at %s", dir)
|
||||
}
|
||||
|
||||
@@ -455,9 +400,7 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error {
|
||||
latestVersion := release.Version()
|
||||
currentVersion := config.DefaultConfig.TemplateVersion
|
||||
|
||||
// versions on the latest channel only differ in build metadata, which
|
||||
// version comparison ignores
|
||||
if t.Channel == TemplatesChannelLatest || config.IsOutdatedVersion(currentVersion, latestVersion) {
|
||||
if config.IsOutdatedVersion(currentVersion, latestVersion) {
|
||||
gologger.Info().Msgf("Your current nuclei-templates %s are outdated. Latest is %s\n", currentVersion, latestVersion)
|
||||
} else {
|
||||
gologger.Debug().Msgf("Updating nuclei-templates from %s to %s (forced update)\n", currentVersion, latestVersion)
|
||||
|
||||
@@ -83,59 +83,6 @@ func TestUpdateIfOutdatedSkipsReleaseInstalledWhileWaiting(t *testing.T) {
|
||||
require.Equal(t, "v2.0.0", cfg.TemplateVersion)
|
||||
}
|
||||
|
||||
// TestUpdateIfOutdatedLatestChannel checks that the latest channel updates when
|
||||
// the head moves, even though the release stays the same.
|
||||
func TestUpdateIfOutdatedLatestChannel(t *testing.T) {
|
||||
HideProgressBar = true
|
||||
|
||||
cfg := &config.Config{Logger: gologger.DefaultLogger}
|
||||
cfg.SetConfigDir(t.TempDir())
|
||||
cfg.SetTemplatesDir(filepath.Join(t.TempDir(), "templates"))
|
||||
previousConfig := config.DefaultConfig
|
||||
config.DefaultConfig = cfg
|
||||
t.Cleanup(func() { config.DefaultConfig = previousConfig })
|
||||
|
||||
installed := fakeTemplateRelease{version: "v1.0.0+aaaaaaa", files: map[string]string{
|
||||
"http/old.yaml": "id: old",
|
||||
}}
|
||||
tm := &TemplateManager{Channel: TemplatesChannelLatest, fetchLatestRelease: installed.fetch}
|
||||
require.NoError(t, tm.FreshInstallIfNotExists())
|
||||
require.Equal(t, "v1.0.0+aaaaaaa", cfg.TemplateVersion)
|
||||
|
||||
// the version check reports the release the installed commit is based on,
|
||||
// so the stable check alone would never update
|
||||
cfg.LatestNucleiTemplatesVersion = "v1.0.0"
|
||||
require.False(t, cfg.NeedsTemplateUpdate())
|
||||
|
||||
t.Run("updates when the head moves", func(t *testing.T) {
|
||||
head := fakeTemplateRelease{version: "v1.0.0+bbbbbbb", files: map[string]string{
|
||||
"http/old.yaml": "id: old",
|
||||
"http/new.yaml": "id: new",
|
||||
}}
|
||||
tm := &TemplateManager{Channel: TemplatesChannelLatest, fetchLatestRelease: head.fetch}
|
||||
require.NoError(t, tm.UpdateIfOutdated())
|
||||
require.Equal(t, "v1.0.0+bbbbbbb", cfg.TemplateVersion)
|
||||
require.FileExists(t, filepath.Join(cfg.TemplatesDirectory, "http", "new.yaml"))
|
||||
})
|
||||
|
||||
t.Run("skips the download when already at the head", func(t *testing.T) {
|
||||
head := installedTemplateRelease{fakeTemplateRelease: fakeTemplateRelease{version: "v1.0.0+bbbbbbb"}, t: t}
|
||||
tm := &TemplateManager{Channel: TemplatesChannelLatest, fetchLatestRelease: func() (templateRelease, error) {
|
||||
return head, nil
|
||||
}}
|
||||
require.NoError(t, tm.UpdateIfOutdated())
|
||||
require.Equal(t, "v1.0.0+bbbbbbb", cfg.TemplateVersion)
|
||||
})
|
||||
|
||||
t.Run("returns the error when the head can't be fetched", func(t *testing.T) {
|
||||
tm := &TemplateManager{Channel: TemplatesChannelLatest, fetchLatestRelease: func() (templateRelease, error) {
|
||||
return nil, errors.New("rate limited")
|
||||
}}
|
||||
require.ErrorContains(t, tm.UpdateIfOutdated(), "rate limited")
|
||||
require.Equal(t, "v1.0.0+bbbbbbb", cfg.TemplateVersion)
|
||||
})
|
||||
}
|
||||
|
||||
func TestIsOutdatedVersion(t *testing.T) {
|
||||
testCases := []struct {
|
||||
current string
|
||||
|
||||
@@ -2,36 +2,22 @@ package installer
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
)
|
||||
|
||||
// Tunables for withTemplatesUpdateLock. Package vars (instead of constants) so
|
||||
// tests can shrink the 10m/15m windows without waiting them out.
|
||||
var (
|
||||
templatesUpdateLockTimeout = 10 * time.Minute
|
||||
templatesUpdateLockStaleAfter = 15 * time.Minute
|
||||
templatesUpdateLockPollInterval = 250 * time.Millisecond
|
||||
// removeTemplatesUpdateLock is os.Remove except in tests that simulate a
|
||||
// lock disappearing, or refusing to disappear, between Stat and Remove.
|
||||
removeTemplatesUpdateLock = os.Remove
|
||||
)
|
||||
|
||||
// withTemplatesUpdateLock serializes template install/update of templatesDir
|
||||
// across processes. Parallel `go test` packages each have their own sync.Once,
|
||||
// so without a cross-process lock they race on a shared templates directory
|
||||
// (ENOENT, partial trees, checksum mismatches).
|
||||
func withTemplatesUpdateLock(templatesDir string, fn func() error) error {
|
||||
lockPath := templatesUpdateLockPath(templatesDir)
|
||||
deadline := time.Now().Add(templatesUpdateLockTimeout)
|
||||
deadline := time.Now().Add(10 * time.Minute)
|
||||
|
||||
for {
|
||||
f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
@@ -44,110 +30,17 @@ func withTemplatesUpdateLock(templatesDir string, fn func() error) error {
|
||||
return fn()
|
||||
}
|
||||
|
||||
// Anything other than "already exists" (e.g. /tmp not writable,
|
||||
// read-only filesystem) will never succeed by retrying: fail fast
|
||||
// instead of spinning until the deadline.
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
return errkit.Wrapf(err, "could not create nuclei templates update lock at %s", lockPath)
|
||||
}
|
||||
|
||||
info, statErr := os.Stat(lockPath)
|
||||
if statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
// Lost a race with the holder releasing the lock; retry
|
||||
// immediately instead of sleeping through the window.
|
||||
continue
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return errkit.Wrapf(statErr, "timed out waiting for nuclei templates update lock at %s (could not stat lock); remove it manually if it is stale, or rerun with -duc/-disable-update-check to skip the automatic template update", lockPath)
|
||||
}
|
||||
time.Sleep(templatesUpdateLockPollInterval)
|
||||
continue
|
||||
}
|
||||
|
||||
if age := time.Since(info.ModTime()); age > templatesUpdateLockStaleAfter {
|
||||
if rmErr := removeTemplatesUpdateLock(lockPath); rmErr != nil && !os.IsNotExist(rmErr) {
|
||||
// The classic multi-user hang: the lock lives in the shared
|
||||
// os.TempDir(), which has the sticky bit, so a lock file
|
||||
// created by another user cannot be unlinked (EPERM) by this
|
||||
// process. Retrying is pointless; surface who owns it and
|
||||
// how to recover instead of looping until the deadline.
|
||||
return errkit.Wrapf(rmErr, "stale nuclei templates update lock at %s (%s) could not be removed; it may belong to another user and cannot be unlinked from the shared temp dir, remove it manually (e.g. sudo rm %s) or rerun with -duc/-disable-update-check to skip the automatic template update", lockPath, describeUpdateLock(info, lockPath), lockPath)
|
||||
}
|
||||
if info, statErr := os.Stat(lockPath); statErr == nil && time.Since(info.ModTime()) > 15*time.Minute {
|
||||
_ = os.Remove(lockPath)
|
||||
continue
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return errkit.Wrapf(err, "timed out waiting for nuclei templates update lock at %s (%s); another process may be updating templates, if no update is running remove the stale lock manually (e.g. sudo rm %s) or rerun with -duc/-disable-update-check to skip the automatic template update", lockPath, describeUpdateLock(info, lockPath), lockPath)
|
||||
return errkit.Wrap(err, "timed out waiting for nuclei templates update lock")
|
||||
}
|
||||
time.Sleep(templatesUpdateLockPollInterval)
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// describeUpdateLock renders age, ownership and recorded pid of an existing
|
||||
// lock for error messages. Everything is best-effort: the file is usually
|
||||
// 0600, so a lock owned by another user may be neither readable nor statable
|
||||
// beyond what os.Stat already returned.
|
||||
func describeUpdateLock(info os.FileInfo, lockPath string) string {
|
||||
var parts []string
|
||||
parts = append(parts, fmt.Sprintf("age %s", time.Since(info.ModTime()).Round(time.Second)))
|
||||
parts = append(parts, fmt.Sprintf("mode %s", info.Mode()))
|
||||
if uid, gid, ok := updateLockOwner(info); ok {
|
||||
extra := fmt.Sprintf("uid=%d gid=%d", uid, gid)
|
||||
if current := os.Getuid(); current >= 0 && uid != uint64(current) {
|
||||
extra += " (another user)"
|
||||
}
|
||||
parts = append(parts, extra)
|
||||
}
|
||||
if pid, err := os.ReadFile(lockPath); err == nil {
|
||||
if trimmed := strings.TrimSpace(string(pid)); trimmed != "" {
|
||||
parts = append(parts, fmt.Sprintf("pid %s", trimmed))
|
||||
}
|
||||
} else if os.IsPermission(err) {
|
||||
parts = append(parts, "pid unreadable (permission denied, likely another user)")
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// updateLockOwner extracts uid/gid from os.Stat without importing syscall
|
||||
// directly (syscall.Stat_t does not exist on windows). It uses reflection on
|
||||
// info.Sys(): on unix this is *syscall.Stat_t with Uid/Gid fields, elsewhere
|
||||
// the assertion fails and ok=false is reported.
|
||||
func updateLockOwner(info os.FileInfo) (uid, gid uint64, ok bool) {
|
||||
sys := info.Sys()
|
||||
if sys == nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
v := reflect.ValueOf(sys)
|
||||
if v.Kind() == reflect.Pointer && !v.IsNil() {
|
||||
v = v.Elem()
|
||||
}
|
||||
if v.Kind() != reflect.Struct {
|
||||
return 0, 0, false
|
||||
}
|
||||
uidField := v.FieldByName("Uid")
|
||||
gidField := v.FieldByName("Gid")
|
||||
if !uidField.IsValid() || !gidField.IsValid() {
|
||||
return 0, 0, false
|
||||
}
|
||||
toUint := func(rv reflect.Value) (uint64, bool) {
|
||||
switch rv.Kind() {
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
return rv.Uint(), true
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
if n := rv.Int(); n >= 0 {
|
||||
return uint64(n), true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
u, okU := toUint(uidField)
|
||||
g, okG := toUint(gidField)
|
||||
if !okU || !okG {
|
||||
return 0, 0, false
|
||||
}
|
||||
return u, g, true
|
||||
}
|
||||
|
||||
// templatesUpdateLockPath derives one lock per templates directory, so updates
|
||||
// of different directories do not wait on each other. Every alias of a
|
||||
// directory maps to the same lock, including a symlink whose target does not
|
||||
|
||||
@@ -3,24 +3,11 @@ package installer
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// errkit quotes the error chain, so a Windows path's backslashes are escaped
|
||||
// in Error() even though the message itself contains the real path.
|
||||
func requireErrorContainsPath(t *testing.T, err error, path string) {
|
||||
t.Helper()
|
||||
rendered := err.Error()
|
||||
if strings.Contains(rendered, path) || strings.Contains(rendered, strings.ReplaceAll(path, `\`, `\\`)) {
|
||||
return
|
||||
}
|
||||
require.Contains(t, rendered, path)
|
||||
}
|
||||
|
||||
func TestTemplatesUpdateLockPath(t *testing.T) {
|
||||
t.Run("same directory, same lock", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
@@ -71,95 +58,3 @@ func TestTemplatesUpdateLockPath(t *testing.T) {
|
||||
require.Equal(t, templatesUpdateLockPath(target), templatesUpdateLockPath(link))
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithTemplatesUpdateLockStaleLockIsCleared(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
lockPath := templatesUpdateLockPath(dir)
|
||||
require.NoError(t, os.WriteFile(lockPath, []byte("1234\n"), 0o600))
|
||||
stale := time.Now().Add(-1 * time.Hour)
|
||||
require.NoError(t, os.Chtimes(lockPath, stale, stale))
|
||||
t.Cleanup(func() { _ = os.Remove(lockPath) })
|
||||
|
||||
ran := false
|
||||
require.NoError(t, withTemplatesUpdateLock(dir, func() error {
|
||||
ran = true
|
||||
return nil
|
||||
}))
|
||||
require.True(t, ran, "fn should run after the stale lock is cleared")
|
||||
}
|
||||
|
||||
func TestWithTemplatesUpdateLockFreshLockTimesOutWithHelpfulMessage(t *testing.T) {
|
||||
oldTimeout, oldStale, oldPoll := templatesUpdateLockTimeout, templatesUpdateLockStaleAfter, templatesUpdateLockPollInterval
|
||||
templatesUpdateLockTimeout = 200 * time.Millisecond
|
||||
templatesUpdateLockStaleAfter = time.Hour
|
||||
templatesUpdateLockPollInterval = 10 * time.Millisecond
|
||||
defer func() {
|
||||
templatesUpdateLockTimeout, templatesUpdateLockStaleAfter, templatesUpdateLockPollInterval = oldTimeout, oldStale, oldPoll
|
||||
}()
|
||||
|
||||
dir := t.TempDir()
|
||||
lockPath := templatesUpdateLockPath(dir)
|
||||
require.NoError(t, os.WriteFile(lockPath, []byte("99999\n"), 0o600))
|
||||
t.Cleanup(func() { _ = os.Remove(lockPath) })
|
||||
|
||||
start := time.Now()
|
||||
err := withTemplatesUpdateLock(dir, func() error { return nil })
|
||||
elapsed := time.Since(start)
|
||||
require.Error(t, err)
|
||||
require.Less(t, elapsed, 10*time.Second, "must fail fast instead of hanging")
|
||||
require.Contains(t, err.Error(), "timed out waiting for nuclei templates update lock")
|
||||
requireErrorContainsPath(t, err, lockPath)
|
||||
require.Contains(t, err.Error(), "-duc")
|
||||
}
|
||||
|
||||
func TestWithTemplatesUpdateLockStaleLockAlreadyRemoved(t *testing.T) {
|
||||
previous := removeTemplatesUpdateLock
|
||||
t.Cleanup(func() { removeTemplatesUpdateLock = previous })
|
||||
removeTemplatesUpdateLock = func(path string) error {
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
// Another process removed the stale lock after we stat-ed it.
|
||||
return os.ErrNotExist
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
lockPath := templatesUpdateLockPath(dir)
|
||||
require.NoError(t, os.WriteFile(lockPath, []byte("1234\n"), 0o600))
|
||||
stale := time.Now().Add(-1 * time.Hour)
|
||||
require.NoError(t, os.Chtimes(lockPath, stale, stale))
|
||||
t.Cleanup(func() { _ = os.Remove(lockPath) })
|
||||
|
||||
ran := false
|
||||
require.NoError(t, withTemplatesUpdateLock(dir, func() error {
|
||||
ran = true
|
||||
return nil
|
||||
}))
|
||||
require.True(t, ran, "a lock another process already removed must be taken, not treated as a failure")
|
||||
}
|
||||
|
||||
func TestWithTemplatesUpdateLockUnremovableStaleLockFailsFast(t *testing.T) {
|
||||
previous := removeTemplatesUpdateLock
|
||||
t.Cleanup(func() { removeTemplatesUpdateLock = previous })
|
||||
removeTemplatesUpdateLock = func(string) error { return os.ErrPermission }
|
||||
|
||||
dir := t.TempDir()
|
||||
lockPath := templatesUpdateLockPath(dir)
|
||||
require.NoError(t, os.WriteFile(lockPath, []byte("1234\n"), 0o600))
|
||||
stale := time.Now().Add(-1 * time.Hour)
|
||||
require.NoError(t, os.Chtimes(lockPath, stale, stale))
|
||||
t.Cleanup(func() { _ = os.Remove(lockPath) })
|
||||
|
||||
ran := false
|
||||
start := time.Now()
|
||||
err := withTemplatesUpdateLock(dir, func() error {
|
||||
ran = true
|
||||
return nil
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.False(t, ran)
|
||||
require.Less(t, time.Since(start), 2*time.Second)
|
||||
require.ErrorIs(t, err, os.ErrPermission)
|
||||
require.Contains(t, err.Error(), "could not be removed")
|
||||
requireErrorContainsPath(t, err, lockPath)
|
||||
}
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
|
||||
jsnet "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/net"
|
||||
@@ -156,7 +156,7 @@ func initBuiltInFunc(runtime *goja.Runtime) {
|
||||
executionId := ctx.Value("executionId").(string)
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
return false, errkit.New("dialers with executionId %s not found", executionId)
|
||||
panic("dialers with executionId " + executionId + " not found")
|
||||
}
|
||||
|
||||
conn, err := dialer.Fastdialer.Dial(ctx, "udp", net.JoinHostPort(host, port))
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package ldap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
@@ -10,7 +9,6 @@ import (
|
||||
|
||||
"github.com/go-ldap/ldap/v3"
|
||||
"github.com/projectdiscovery/goja"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
)
|
||||
@@ -94,8 +92,7 @@ func NewClient(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
|
||||
executionId := c.nj.ExecutionId()
|
||||
dialers := protocolstate.GetDialersWithId(executionId)
|
||||
if dialers == nil {
|
||||
gologger.Error().Msgf("dialers with executionId %s not found", executionId)
|
||||
return nil
|
||||
panic("dialers with executionId " + executionId + " not found")
|
||||
}
|
||||
|
||||
dialCtx := c.nj.Context()
|
||||
@@ -146,10 +143,6 @@ func NewClient(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
|
||||
}
|
||||
c.nj.HandleError(err, "failed to connect to ldap server")
|
||||
}
|
||||
// go-ldap waits for a reply with no timeout, and an interrupt cannot stop the runtime
|
||||
// inside that read: a server that accepts and never answers would keep the goroutine,
|
||||
// and everything the abandoned runtime references, alive for the life of the process.
|
||||
context.AfterFunc(dialCtx, func() { _ = conn.Close() })
|
||||
c.conn = ldap.NewConn(conn, u.Scheme == "ldaps")
|
||||
if u.Scheme != "ldaps" && c.cfg.Upgrade {
|
||||
serverName := u.Hostname()
|
||||
|
||||
Loaded 100 of 240 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user