mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-08 02:08:33 -10:00
Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe24f8638b | ||
|
|
8b7cbf44e6 | ||
|
|
59ae4cac04 | ||
|
|
64965ba776 | ||
|
|
15271926ad | ||
|
|
46fbc92a76 | ||
|
|
36d0adba18 | ||
|
|
049e6b629e | ||
|
|
2c6b223bb4 | ||
|
|
0827b2779b | ||
|
|
0971fb83e2 | ||
|
|
62dae92b3d | ||
|
|
e44df6e947 | ||
|
|
faaeaf30b1 | ||
|
|
b7fe335285 | ||
|
|
9b04153a3b | ||
|
|
23f3407af7 | ||
|
|
64f94f3e66 | ||
|
|
ac52212d17 |
No files matched your search
@@ -166,7 +166,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -178,7 +177,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
|
||||
+1
-2
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -176,7 +175,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
|
||||
+1
-2
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -176,7 +175,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
|
||||
+1
-2
@@ -181,7 +181,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -193,7 +192,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
|
||||
+1
-2
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -176,7 +175,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
|
||||
+1
-2
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -176,7 +175,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
|
||||
+1
-2
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -176,7 +175,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
|
||||
+1
-2
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -176,7 +175,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
|
||||
+1
-2
@@ -164,7 +164,6 @@ TEMPLATES:
|
||||
-w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file)
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
@@ -176,7 +175,7 @@ TEMPLATES:
|
||||
-egm, -enable-global-matchers enable loading global matchers templates
|
||||
-file enable loading file templates
|
||||
-llm enable llm matchers and extractors (semantic matching)
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
|
||||
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
|
||||
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
|
||||
-llm-model string model used for semantic matching
|
||||
-llm-timeout int time in seconds to wait for a single llm call (default 30)
|
||||
|
||||
+1
-2
@@ -297,7 +297,6 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
|
||||
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
|
||||
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
|
||||
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
|
||||
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
|
||||
@@ -310,7 +309,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVarP(&options.EnableGlobalMatchersTemplates, "enable-global-matchers", "egm", false, "enable loading global matchers templates"),
|
||||
flagSet.BoolVar(&options.EnableFileTemplates, "file", false, "enable loading file templates"),
|
||||
flagSet.BoolVar(&options.EnableLLM, "llm", false, "enable llm matchers and extractors (semantic matching)"),
|
||||
flagSet.StringVar(&options.LLMProvider, "llm-provider", "openai", "llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter)"),
|
||||
flagSet.StringVar(&options.LLMProvider, "llm-provider", "openai", "llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev)"),
|
||||
flagSet.StringVar(&options.LLMBaseURL, "llm-base-url", "", "openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)"),
|
||||
flagSet.StringVar(&options.LLMModel, "llm-model", "", "model used for semantic matching"),
|
||||
flagSet.IntVar(&options.LLMTimeout, "llm-timeout", 30, "time in seconds to wait for a single llm call"),
|
||||
|
||||
@@ -105,7 +105,7 @@ require (
|
||||
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
|
||||
github.com/projectdiscovery/uncover v1.2.1
|
||||
github.com/projectdiscovery/useragent v0.0.109
|
||||
github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5
|
||||
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350
|
||||
github.com/projectdiscovery/wappalyzergo v0.3.6
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.7
|
||||
github.com/redis/go-redis/v9 v9.19.0
|
||||
|
||||
@@ -956,8 +956,6 @@ github.com/projectdiscovery/gologger v1.1.73 h1:WX2nzq29cL7HdvOHqIttacoid/5lMwap
|
||||
github.com/projectdiscovery/gologger v1.1.73/go.mod h1:mJwODZcFDg70ihINpOvZevmBtgvpP8H9/l8Y+OPhZPY=
|
||||
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
|
||||
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
|
||||
github.com/projectdiscovery/govaluate v0.0.0-20260729205521-4c61bf19382e h1:0vaevfTiQeRCkX2xbY/mR38422kHAxGzpVV8FQJVfoc=
|
||||
github.com/projectdiscovery/govaluate v0.0.0-20260729205521-4c61bf19382e/go.mod h1:Z3VjpUJV1pnXDwvAddA8aZ4e839SoeftB7CSdZntAVI=
|
||||
github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2 h1:aSLbDvrJQPlp4HvojU+bTU5XV8xdAVCjoc5gX1Wfbw4=
|
||||
github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2/go.mod h1:Z3VjpUJV1pnXDwvAddA8aZ4e839SoeftB7CSdZntAVI=
|
||||
github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76 h1:AN70bbi6BBs7KpIM9w0LxygUN7uzT/oH+owDIQ+Fz/k=
|
||||
@@ -1000,8 +998,8 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
|
||||
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
|
||||
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
|
||||
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
|
||||
github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5 h1:AOspf7ScbwZr8A2YnjCFd4ZJAMSFqzfEaIJIbJPuu10=
|
||||
github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
|
||||
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350 h1:uXqpYa6XKV9dCvoti9PbSgIWIsBKPu5YKMywB9G4q0Q=
|
||||
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
|
||||
github.com/projectdiscovery/wappalyzergo v0.3.6 h1:1FXCMeCa1xVu2zaW+eeBcbVxtlMStqGOlKcMLyh2pzI=
|
||||
github.com/projectdiscovery/wappalyzergo v0.3.6/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
|
||||
|
||||
+35
-1
@@ -2,8 +2,12 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
|
||||
llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
utilsllm "github.com/projectdiscovery/utils/llm"
|
||||
@@ -28,6 +32,32 @@ func (a *llmMatcherClient) Complete(ctx context.Context, prompt string, asJSON b
|
||||
})
|
||||
}
|
||||
|
||||
// llmJudgeClient is an llmMatcherClient whose provider classifies natively, so
|
||||
// llm matchers ask it through Judge instead of Complete.
|
||||
type llmJudgeClient struct {
|
||||
*llmMatcherClient
|
||||
warnComplete sync.Once
|
||||
}
|
||||
|
||||
func (a *llmJudgeClient) Judge(ctx context.Context, question string, labels, inputs []string) (llmclient.Verdict, error) {
|
||||
verdict, err := a.client.Judge(ctx, utilsllm.JudgeRequest{Question: question, Labels: labels, Inputs: inputs})
|
||||
if err != nil {
|
||||
return llmclient.Verdict{}, err
|
||||
}
|
||||
return llmclient.Verdict{Label: verdict.Label, Confidence: verdict.Confidence, Model: verdict.Model}, nil
|
||||
}
|
||||
|
||||
// Complete is only reached by llm extractors, which need free text a
|
||||
// classifier cannot produce. Returning here spends no call budget and
|
||||
// makes no provider request. Extractors turn the error into an empty
|
||||
// result, and the warning says why.
|
||||
func (a *llmJudgeClient) Complete(context.Context, string, bool) (string, error) {
|
||||
a.warnComplete.Do(func() {
|
||||
gologger.Warning().Msgf("llm provider %q cannot run llm extractors, they return nothing", utilsllm.ProviderJev)
|
||||
})
|
||||
return "", errors.New("llm provider cannot run extractors")
|
||||
}
|
||||
|
||||
// configureLLM builds the scan's llm client from options. It returns nil unless
|
||||
// -llm is set, so nothing reaches a model until the user opts in. The API key is
|
||||
// read from the environment by the provider layer, never from options.
|
||||
@@ -52,5 +82,9 @@ func configureLLM(options *types.Options) (llmclient.Client, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &llmMatcherClient{client: client, model: options.LLMModel}, nil
|
||||
matcherClient := &llmMatcherClient{client: client, model: options.LLMModel}
|
||||
if client.CanJudge() {
|
||||
return &llmJudgeClient{llmMatcherClient: matcherClient}, nil
|
||||
}
|
||||
return matcherClient, nil
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestConfigureLLMExposesJudgeOnlyForClassifiers(t *testing.T) {
|
||||
t.Setenv("LLM_API_KEY", "key")
|
||||
|
||||
client, err := configureLLM(&types.Options{EnableLLM: true, LLMProvider: "jev", LLMModel: "jev-latest"})
|
||||
require.NoError(t, err)
|
||||
require.Implements(t, (*llmclient.Judge)(nil), client)
|
||||
|
||||
client, err = configureLLM(&types.Options{EnableLLM: true, LLMProvider: "ollama", LLMModel: "llama3.2"})
|
||||
require.NoError(t, err)
|
||||
_, isJudge := client.(llmclient.Judge)
|
||||
require.False(t, isJudge, "chat providers keep the Complete path")
|
||||
}
|
||||
|
||||
func TestJevExtractorDoesNotSpendCallBudget(t *testing.T) {
|
||||
t.Setenv("LLM_API_KEY", "key")
|
||||
// ProxyFromEnvironment caches the first request in the process. Match the
|
||||
// proxy the interactsh client test expects, and keep the local server direct.
|
||||
for _, key := range []string{"HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "https_proxy", "no_proxy"} {
|
||||
t.Setenv(key, "")
|
||||
}
|
||||
t.Setenv("HTTPS_PROXY", "http://127.0.0.1:9")
|
||||
t.Setenv("NO_PROXY", "127.0.0.1,localhost")
|
||||
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
_, _ = w.Write([]byte(`{"model":"jev-1.14.0","answers":{"q":{"type":"noul","noul":0.9}}}`))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
client, err := configureLLM(&types.Options{
|
||||
EnableLLM: true,
|
||||
LLMProvider: "jev",
|
||||
LLMBaseURL: server.URL + "/v1",
|
||||
LLMModel: "jev-latest",
|
||||
LLMMaxCalls: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.Complete(context.Background(), "extract the title", true)
|
||||
require.ErrorContains(t, err, "cannot run extractors")
|
||||
_, err = client.Complete(context.Background(), "extract another", true)
|
||||
require.Error(t, err)
|
||||
require.Zero(t, calls.Load())
|
||||
|
||||
judge := client.(llmclient.Judge)
|
||||
verdict, err := judge.Judge(context.Background(), "login page?", nil, []string{"<form>"})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "yes", verdict.Label)
|
||||
require.EqualValues(t, 1, calls.Load())
|
||||
}
|
||||
@@ -119,14 +119,6 @@ func ParseOptions(options *types.Options) {
|
||||
options.Logger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n")
|
||||
options.StoreResponse = true
|
||||
}
|
||||
// Derived before validation: -uncover-query turns uncover on, and -plan
|
||||
// has to reject that before any target lookup starts.
|
||||
if options.UncoverQuery != nil {
|
||||
options.Uncover = true
|
||||
if len(options.UncoverEngine) == 0 {
|
||||
options.UncoverEngine = append(options.UncoverEngine, "shodan")
|
||||
}
|
||||
}
|
||||
// Validate the options passed by the user and if any
|
||||
// invalid options have been used, exit.
|
||||
if err := ValidateOptions(options); err != nil {
|
||||
@@ -146,6 +138,13 @@ func ParseOptions(options *types.Options) {
|
||||
_ = os.Setenv("GITHUB_TOKEN", options.GitHubToken)
|
||||
}
|
||||
|
||||
if options.UncoverQuery != nil {
|
||||
options.Uncover = true
|
||||
if len(options.UncoverEngine) == 0 {
|
||||
options.UncoverEngine = append(options.UncoverEngine, "shodan")
|
||||
}
|
||||
}
|
||||
|
||||
if options.OfflineHTTP {
|
||||
options.DisableHTTPProbe = true
|
||||
}
|
||||
@@ -163,9 +162,6 @@ func ValidateOptions(options *types.Options) error {
|
||||
}
|
||||
return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields")
|
||||
}
|
||||
if err := validatePlanOptions(options); err != nil {
|
||||
return err
|
||||
}
|
||||
if options.Verbose && options.Silent {
|
||||
return errors.New("both verbose and silent mode specified")
|
||||
}
|
||||
@@ -540,22 +536,3 @@ func getBoolEnvValue(key string) bool {
|
||||
value := os.Getenv(key)
|
||||
return strings.EqualFold(value, "true")
|
||||
}
|
||||
|
||||
// validatePlanOptions rejects options that change the templates or targets
|
||||
// only once scanning starts, which a plan written beforehand cannot show.
|
||||
func validatePlanOptions(options *types.Options) error {
|
||||
if options.PlanFile == "" {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case options.AutomaticScan:
|
||||
return errors.New("-plan cannot be used with -automatic-scan, which picks templates while scanning")
|
||||
case options.Uncover || len(options.UncoverQuery) > 0:
|
||||
return errors.New("-plan cannot be used with -uncover, which finds targets over the network")
|
||||
case options.PreflightPortScan:
|
||||
return errors.New("-plan cannot be used with -preflight-portscan, which drops targets after scanning their ports")
|
||||
case options.DASTServer || options.DASTProxy:
|
||||
return errors.New("-plan cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"slices"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
)
|
||||
|
||||
// httpRequestSeconds is the time of one http request the plan costs are
|
||||
// weighted against (see core.PlanEntry.Cost).
|
||||
const httpRequestSeconds = 0.015
|
||||
|
||||
// planSummary is the last line of a plan file, written once every entry is
|
||||
// known.
|
||||
type planSummary struct {
|
||||
Entries int `json:"entries"`
|
||||
Requests int `json:"requests"`
|
||||
Cost int `json:"cost"`
|
||||
// EstimatedSeconds is the largest of the time the rate limit allows, the
|
||||
// cost spread over every concurrent slot, and the longest entry, whose
|
||||
// requests run one after another. Hosts slower than the measured request
|
||||
// time make the scan take longer.
|
||||
EstimatedSeconds int `json:"estimated-seconds"`
|
||||
}
|
||||
|
||||
// writePlan writes one JSON line per unit of work the scan would run,
|
||||
// followed by the summary. Targets are listed as given: inputs that the scan
|
||||
// would probe to an http url first are not probed here, since probing sends
|
||||
// requests.
|
||||
func (r *Runner) writePlan(store *loader.Store, engine *core.Engine) error {
|
||||
if !r.options.DisableHTTPProbe && loader.IsHTTPBasedProtocolUsed(store) && r.isInputNonHTTP() {
|
||||
r.Logger.Info().Msgf("Targets without a scheme are planned as given; the scan probes them for http(s) first")
|
||||
}
|
||||
file, err := os.Create(r.options.PlanFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
writer := bufio.NewWriter(file)
|
||||
encoder := json.NewEncoder(writer)
|
||||
|
||||
var summary planSummary
|
||||
var headlessCost, longest int
|
||||
var encodeErr error
|
||||
engine.Plan(slices.Concat(store.Templates(), store.Workflows()), r.inputProvider, r.options.DisableClustering, func(entry core.PlanEntry) {
|
||||
if encodeErr != nil {
|
||||
return
|
||||
}
|
||||
encodeErr = encoder.Encode(entry)
|
||||
summary.Entries++
|
||||
summary.Requests += entry.Requests
|
||||
summary.Cost += entry.Cost
|
||||
longest = max(longest, entry.Cost)
|
||||
if entry.Protocol == types.HeadlessProtocol.String() {
|
||||
headlessCost += entry.Cost
|
||||
}
|
||||
})
|
||||
if encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
summary.EstimatedSeconds = r.estimateSeconds(summary.Requests, summary.Cost-headlessCost, headlessCost, longest)
|
||||
if err := encoder.Encode(map[string]planSummary{"summary": summary}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writer.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
r.Logger.Info().Msgf("Plan written to %s: %d template runs, up to %d requests, about %ds", r.options.PlanFile, summary.Entries, summary.Requests, summary.EstimatedSeconds)
|
||||
return nil
|
||||
}
|
||||
|
||||
// estimateSeconds estimates the scan duration. Headless templates run in
|
||||
// their own, smaller pool, so their cost is spread over it separately.
|
||||
func (r *Runner) estimateSeconds(requests, cost, headlessCost, longest int) int {
|
||||
seconds := float64(longest) * httpRequestSeconds
|
||||
if r.options.RateLimit > 0 && r.options.RateLimitDuration > 0 {
|
||||
seconds = max(seconds, float64(requests)/(float64(r.options.RateLimit)/r.options.RateLimitDuration.Seconds()))
|
||||
}
|
||||
if slots := r.options.TemplateThreads * r.options.BulkSize; slots > 0 {
|
||||
seconds = max(seconds, float64(cost)*httpRequestSeconds/float64(slots))
|
||||
}
|
||||
if slots := r.options.HeadlessTemplateThreads * r.options.HeadlessBulkSize; slots > 0 {
|
||||
seconds = max(seconds, float64(headlessCost)*httpRequestSeconds/float64(slots))
|
||||
}
|
||||
return int(seconds + 0.5)
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
|
||||
for name, options := range map[string]*types.Options{
|
||||
"automatic scan": {PlanFile: "p", AutomaticScan: true},
|
||||
"uncover": {PlanFile: "p", Uncover: true},
|
||||
"uncover query": {PlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
|
||||
"preflight": {PlanFile: "p", PreflightPortScan: true},
|
||||
"dast server": {PlanFile: "p", DASTServer: true},
|
||||
"dast proxy": {PlanFile: "p", DASTProxy: true},
|
||||
} {
|
||||
require.Error(t, validatePlanOptions(options), name)
|
||||
}
|
||||
require.NoError(t, validatePlanOptions(&types.Options{PlanFile: "p"}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
|
||||
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEstimateSecondsTakesTheSlowestBound(t *testing.T) {
|
||||
r := &Runner{options: &types.Options{
|
||||
RateLimit: 150, RateLimitDuration: time.Second,
|
||||
TemplateThreads: 25, BulkSize: 25,
|
||||
HeadlessTemplateThreads: 10, HeadlessBulkSize: 10,
|
||||
}}
|
||||
|
||||
require.Equal(t, 20, r.estimateSeconds(3000, 3000, 0, 12), "rate limit bound")
|
||||
require.Equal(t, 10, r.estimateSeconds(52, 41, 660, 660), "one long entry runs its requests in sequence")
|
||||
require.Equal(t, 9, r.estimateSeconds(100, 0, 60000, 60), "headless cost spreads over the headless pool")
|
||||
|
||||
r.options.RateLimit = 0
|
||||
require.Equal(t, 0, r.estimateSeconds(3000, 3000, 0, 1), "no rate limit, cost spread over 625 slots")
|
||||
}
|
||||
@@ -146,18 +146,10 @@ func New(options *types.Options) (*Runner, error) {
|
||||
|
||||
// Version check by default
|
||||
if config.DefaultConfig.CanCheckForUpdates() {
|
||||
// -plan sends no request it can avoid, so it skips the version and
|
||||
// update checks; it still installs templates on a fresh setup, which
|
||||
// would otherwise plan nothing
|
||||
checkUpdates := options.PlanFile == ""
|
||||
var latestIgnoreHash string
|
||||
if checkUpdates {
|
||||
var versionCheckErr error
|
||||
latestIgnoreHash, versionCheckErr = installer.NucleiVersionCheck()
|
||||
if versionCheckErr != nil {
|
||||
if options.Verbose || options.Debug {
|
||||
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
|
||||
}
|
||||
latestIgnoreHash, versionCheckErr := installer.NucleiVersionCheck()
|
||||
if versionCheckErr != nil {
|
||||
if options.Verbose || options.Debug {
|
||||
runner.Logger.Error().Msgf("nuclei version check failed got: %s\n", versionCheckErr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,19 +170,17 @@ func New(options *types.Options) (*Runner, error) {
|
||||
if err := tm.FreshInstallIfNotExists(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to install nuclei templates: %s\n", err)
|
||||
}
|
||||
if checkUpdates {
|
||||
if err := tm.UpdateIfOutdated(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
|
||||
}
|
||||
if err := tm.UpdateIfOutdated(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to update nuclei templates: %s\n", err)
|
||||
}
|
||||
|
||||
if checkUpdates && config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
|
||||
if config.DefaultConfig.IgnoreFileNeedsUpdate(latestIgnoreHash) {
|
||||
if err := installer.UpdateIgnoreFile(); err != nil {
|
||||
runner.Logger.Warning().Msgf("Failed to update .nuclei-ignore file: %s\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if checkUpdates && options.UpdateTemplates {
|
||||
if options.UpdateTemplates {
|
||||
// we automatically check for updates unless explicitly disabled
|
||||
// this print statement is only to inform the user that there are no updates
|
||||
noUpdates := !config.DefaultConfig.NeedsTemplateUpdate()
|
||||
@@ -601,13 +591,8 @@ func (r *Runner) RunEnumeration() error {
|
||||
httpclientpool.ResetConnectionStats()
|
||||
|
||||
// If the user has asked for DAST server mode, run the live
|
||||
// DAST fuzzing server. -plan never reaches this with those flags set:
|
||||
// validation rejects the combination, and this check covers a caller
|
||||
// that skipped it.
|
||||
// DAST fuzzing server.
|
||||
if r.options.DASTServer || r.options.DASTProxy {
|
||||
if err := validatePlanOptions(r.options); err != nil {
|
||||
return err
|
||||
}
|
||||
execurOpts := &server.NucleiExecutorOptions{
|
||||
Options: r.options,
|
||||
Output: r.output,
|
||||
@@ -830,11 +815,6 @@ func (r *Runner) RunEnumeration() error {
|
||||
profiles.Prepare(slices.Concat(store.Templates(), store.Workflows()))
|
||||
executorOpts.TargetScope = profiles
|
||||
}
|
||||
// before uncover, preflight, auth prefetch and probing below, all of which
|
||||
// reach the network
|
||||
if r.options.PlanFile != "" {
|
||||
return r.writePlan(store, executorEngine)
|
||||
}
|
||||
// TODO: remove below functions after v3 or update warning messages
|
||||
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
|
||||
|
||||
|
||||
+13
-15
@@ -38,7 +38,19 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
|
||||
|
||||
totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
|
||||
|
||||
finalTemplates, clusterCount := e.clusterTemplates(templatesList, noCluster)
|
||||
// attempt to cluster templates if noCluster is false
|
||||
var finalTemplates []*templates.Template
|
||||
clusterCount := 0
|
||||
if !noCluster {
|
||||
var clusterMappings map[string][]string
|
||||
finalTemplates, clusterCount, clusterMappings = templates.ClusterTemplates(templatesList, e.executerOpts)
|
||||
// Store cluster mappings in executerOpts for SDK access (thread-safe)
|
||||
if clusterMappings != nil {
|
||||
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
|
||||
}
|
||||
} else {
|
||||
finalTemplates = templatesList
|
||||
}
|
||||
|
||||
totalReqAfterClustering := getRequestCount(finalTemplates) * int(target.Count())
|
||||
|
||||
@@ -174,20 +186,6 @@ func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*template
|
||||
return results
|
||||
}
|
||||
|
||||
// clusterTemplates merges templates that send identical requests unless
|
||||
// noCluster is set, and returns the templates to run with the number merged.
|
||||
func (e *Engine) clusterTemplates(templatesList []*templates.Template, noCluster bool) ([]*templates.Template, int) {
|
||||
if noCluster {
|
||||
return templatesList, 0
|
||||
}
|
||||
finalTemplates, clusterCount, clusterMappings := templates.ClusterTemplates(templatesList, e.executerOpts)
|
||||
// Store cluster mappings in executerOpts for SDK access (thread-safe)
|
||||
if clusterMappings != nil {
|
||||
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
|
||||
}
|
||||
return finalTemplates, clusterCount
|
||||
}
|
||||
|
||||
// expectedWork returns the requests and the tasks the target-bound templates
|
||||
// will run. It counts only the template and target pairs that per-target
|
||||
// profiles select, so progress reflects the work that will be done. A task is
|
||||
|
||||
+6
-15
@@ -382,19 +382,6 @@ func newOriginAttempt() *originAttempt {
|
||||
return attempt
|
||||
}
|
||||
|
||||
// originScopeKey returns the key an origin-scoped template runs once per, and
|
||||
// false when the template runs on every target.
|
||||
func originScopeKey(template *templates.Template, value *contextargs.MetaInput) (string, bool) {
|
||||
if template == nil || value == nil || !template.IsOriginScoped() {
|
||||
return "", false
|
||||
}
|
||||
origin := normalize.Origin(value.Input)
|
||||
if origin == "" {
|
||||
return "", false
|
||||
}
|
||||
return template.ID + "\x00" + origin + "\x00" + value.CustomIP, true
|
||||
}
|
||||
|
||||
// claimOriginScoped reports whether this caller should execute an origin-scoped
|
||||
// template. finish(true) records a successful run. finish(false) releases the
|
||||
// claim so a later target, including one that waited, can try again. Templates
|
||||
@@ -402,10 +389,14 @@ func originScopeKey(template *templates.Template, value *contextargs.MetaInput)
|
||||
// key so each address selected for one URL is tested on its own.
|
||||
func (e *Engine) claimOriginScoped(template *templates.Template, value *contextargs.MetaInput) (finish func(bool), run bool) {
|
||||
noop := func(bool) {}
|
||||
key, scoped := originScopeKey(template, value)
|
||||
if !scoped {
|
||||
if template == nil || value == nil || !template.IsOriginScoped() {
|
||||
return noop, true
|
||||
}
|
||||
origin := normalize.Origin(value.Input)
|
||||
if origin == "" {
|
||||
return noop, true
|
||||
}
|
||||
key := template.ID + "\x00" + origin + "\x00" + value.CustomIP
|
||||
actual, _ := e.originScoped.LoadOrStore(key, newOriginAttempt())
|
||||
slot := actual.(*originAttempt)
|
||||
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
)
|
||||
|
||||
// PlanEntry is one unit of work a scan runs: a template, a cluster of
|
||||
// templates sharing one request, or a workflow, on one target.
|
||||
type PlanEntry struct {
|
||||
// ID identifies the entry across runs of the same plan.
|
||||
ID string `json:"id"`
|
||||
// Target is empty for self-contained templates, which run once.
|
||||
Target string `json:"target,omitempty"`
|
||||
// IP is the address the target is pinned to, when it is.
|
||||
IP string `json:"ip,omitempty"`
|
||||
Templates []string `json:"templates"`
|
||||
Protocol string `json:"protocol"`
|
||||
// Requests is what the templates declare, the count -stats starts from.
|
||||
// The scan sends fewer when a request needs a value the target lacks,
|
||||
// such as {{DN}} on an ip.
|
||||
Requests int `json:"requests"`
|
||||
// Cost is Requests weighted by how long one request of the protocol
|
||||
// takes, in http requests.
|
||||
Cost int `json:"cost"`
|
||||
// Dynamic marks entries whose requests depend on responses: workflows
|
||||
// (only the root is counted), flows and fuzzing.
|
||||
Dynamic bool `json:"dynamic,omitempty"`
|
||||
// Fallbacks are later targets of the same origin, in scan order. The
|
||||
// scan tries the next one only when this entry's target fails, so a plan
|
||||
// keeps them instead of dropping the retry.
|
||||
Fallbacks []string `json:"fallbacks,omitempty"`
|
||||
}
|
||||
|
||||
// Plan reports the work ExecuteScanWithOpts would do with the same arguments,
|
||||
// without sending anything. It applies the same clustering, per-target
|
||||
// profiles and once-per-origin rule, so a pair is listed only if the scan
|
||||
// would run it. Later targets of an origin-scoped template are kept on the
|
||||
// first entry as fallbacks: the scan runs the next one when the earlier
|
||||
// target fails.
|
||||
func (e *Engine) Plan(templatesList []*templates.Template, target provider.InputProvider, noCluster bool, emit func(PlanEntry)) {
|
||||
finalTemplates, _ := e.clusterTemplates(templatesList, noCluster)
|
||||
// clustering does not keep an order; a plan is diffed and edited, so it does
|
||||
finalTemplates = slices.SortedFunc(slices.Values(finalTemplates), func(a, b *templates.Template) int {
|
||||
return strings.Compare(a.ID, b.ID)
|
||||
})
|
||||
|
||||
var targetBound []*templates.Template
|
||||
for _, template := range finalTemplates {
|
||||
if template.SelfContained {
|
||||
emit(withPlanID(planEntry(template, &contextargs.MetaInput{}, nil)))
|
||||
} else {
|
||||
targetBound = append(targetBound, template)
|
||||
}
|
||||
}
|
||||
|
||||
// Buffered so a later target of the same origin can be recorded on the
|
||||
// entry before its id is derived.
|
||||
var planned []PlanEntry
|
||||
originIndex := make(map[string]int)
|
||||
target.Iterate(func(value *contextargs.MetaInput) bool {
|
||||
selection := e.selectionFor(value)
|
||||
for _, template := range targetBound {
|
||||
if selection != nil && !template.SelectedBy(selection) {
|
||||
continue
|
||||
}
|
||||
if key, scoped := originScopeKey(template, value); scoped {
|
||||
if idx, seen := originIndex[key]; seen {
|
||||
planned[idx].Fallbacks = append(planned[idx].Fallbacks, value.Input)
|
||||
continue
|
||||
}
|
||||
originIndex[key] = len(planned)
|
||||
}
|
||||
planned = append(planned, planEntry(template, value, selection))
|
||||
}
|
||||
return true
|
||||
})
|
||||
for _, entry := range planned {
|
||||
emit(withPlanID(entry))
|
||||
}
|
||||
}
|
||||
|
||||
func withPlanID(entry PlanEntry) PlanEntry {
|
||||
entry.ID = planEntryID(entry)
|
||||
return entry
|
||||
}
|
||||
|
||||
// protocolCost is how many http requests one request of a protocol is worth
|
||||
// in time. Measured per call against a CDN-fronted site, where an http
|
||||
// request took about 15ms: a javascript call adds about 150ms of runtime, a
|
||||
// headless page load with waitload took about 1s. Unlisted protocols cost
|
||||
// about one round trip, like http.
|
||||
var protocolCost = map[types.ProtocolType]int{
|
||||
types.JavascriptProtocol: 10,
|
||||
types.HeadlessProtocol: 60,
|
||||
}
|
||||
|
||||
func planEntry(template *templates.Template, value *contextargs.MetaInput, selection protocols.TemplateSelection) PlanEntry {
|
||||
entry := PlanEntry{
|
||||
Target: value.Input,
|
||||
IP: value.CustomIP,
|
||||
Templates: []string{template.ID},
|
||||
Protocol: template.Type().String(),
|
||||
Requests: template.TotalRequests,
|
||||
Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
|
||||
}
|
||||
weight := func(protocol types.ProtocolType) int { return max(protocolCost[protocol], 1) }
|
||||
|
||||
switch {
|
||||
case template.CompiledWorkflow != nil:
|
||||
// only the root of a workflow is certain to run; subtemplates depend
|
||||
// on what it matches
|
||||
entry.Requests = 0
|
||||
for _, workflow := range template.CompiledWorkflow.Workflows {
|
||||
for _, executer := range workflow.Executers {
|
||||
requests := executer.Executer.Requests()
|
||||
entry.Requests += requests
|
||||
entry.Cost += requests * weight(executer.TemplateType)
|
||||
}
|
||||
}
|
||||
entry.Dynamic = true
|
||||
case isCluster(template):
|
||||
cluster := template.Executer.(*templates.ClusterExecuter)
|
||||
entry.Templates = slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
|
||||
// TotalRequests of a cluster counts only its http and dns requests,
|
||||
// so an ssl cluster would show none
|
||||
entry.Requests = cluster.Requests()
|
||||
entry.Cost = entry.Requests * weight(template.Type())
|
||||
case len(template.RequestsQueue) > 0:
|
||||
// each block is weighted by its own protocol, so a template mixing
|
||||
// http with javascript is not priced as all http
|
||||
for _, request := range template.RequestsQueue {
|
||||
entry.Cost += request.Requests() * weight(request.Type())
|
||||
}
|
||||
default:
|
||||
entry.Cost = entry.Requests * weight(template.Type())
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func isCluster(template *templates.Template) bool {
|
||||
_, ok := template.Executer.(*templates.ClusterExecuter)
|
||||
return ok
|
||||
}
|
||||
|
||||
// planEntryID derives the id from what the entry runs, so planning the same
|
||||
// scan again yields the same ids.
|
||||
func planEntryID(entry PlanEntry) string {
|
||||
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00") + "\x00" + strings.Join(entry.Fallbacks, "\x00")))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
@@ -1,155 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
|
||||
tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanEntryCostWeightsByProtocol(t *testing.T) {
|
||||
headless := &templates.Template{ID: "h", RequestsHeadless: []*headless.Request{{}}, TotalRequests: 2}
|
||||
require.Equal(t, 120, planEntry(headless, &contextargs.MetaInput{Input: "t"}, nil).Cost)
|
||||
|
||||
plain := &templates.Template{ID: "p", TotalRequests: 2}
|
||||
require.Equal(t, 2, planEntry(plain, &contextargs.MetaInput{Input: "t"}, nil).Cost, "unlisted protocols cost one request each")
|
||||
}
|
||||
|
||||
// countedRequest is a request block with a fixed protocol and count.
|
||||
type countedRequest struct {
|
||||
protocols.Request
|
||||
protocol tmpltypes.ProtocolType
|
||||
requests int
|
||||
}
|
||||
|
||||
func (r countedRequest) Type() tmpltypes.ProtocolType { return r.protocol }
|
||||
func (r countedRequest) Requests() int { return r.requests }
|
||||
|
||||
func TestPlanEntryWeightsEachProtocolBlock(t *testing.T) {
|
||||
// one http request and ten javascript calls: Type() reports http for the
|
||||
// whole template, which would price it as 11
|
||||
mixed := &templates.Template{
|
||||
ID: "mixed",
|
||||
RequestsHTTP: []*http.Request{{}},
|
||||
TotalRequests: 11,
|
||||
RequestsQueue: []protocols.Request{
|
||||
countedRequest{protocol: tmpltypes.HTTPProtocol, requests: 1},
|
||||
countedRequest{protocol: tmpltypes.JavascriptProtocol, requests: 10},
|
||||
},
|
||||
}
|
||||
entry := planEntry(mixed, &contextargs.MetaInput{Input: "t"}, nil)
|
||||
require.Equal(t, 11, entry.Requests)
|
||||
require.Equal(t, 101, entry.Cost)
|
||||
}
|
||||
|
||||
func TestPlanListsThePairsTheScanRuns(t *testing.T) {
|
||||
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
|
||||
engine := New(options)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{
|
||||
Logger: engine.Logger,
|
||||
Options: options,
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: &progressRecorder{},
|
||||
TargetScope: scopeByInput{
|
||||
"php": {paths: []string{"thinkphp.yaml"}},
|
||||
"none": {},
|
||||
},
|
||||
})
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
runs []string
|
||||
)
|
||||
var templatesList []*templates.Template
|
||||
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
|
||||
templatesList = append(templatesList, &templates.Template{
|
||||
ID: path,
|
||||
Path: path,
|
||||
TotalRequests: 3,
|
||||
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
|
||||
})
|
||||
}
|
||||
selfContained := &templates.Template{ID: "self.yaml", Path: "self.yaml", SelfContained: true, TotalRequests: 1}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "none"}, {Input: "plain"}}}
|
||||
|
||||
var plan []PlanEntry
|
||||
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
|
||||
|
||||
require.Empty(t, plan[0].Target, "a self-contained template runs once, with no target")
|
||||
require.Equal(t, []string{"self.yaml"}, plan[0].Templates)
|
||||
var planned []string
|
||||
ids := make(map[string]struct{})
|
||||
for _, entry := range plan[1:] {
|
||||
require.Equal(t, 3, entry.Requests)
|
||||
planned = append(planned, entry.Templates[0]+"@"+entry.Target)
|
||||
ids[entry.ID] = struct{}{}
|
||||
}
|
||||
require.Len(t, ids, len(plan)-1, "every entry has its own id")
|
||||
|
||||
var again []PlanEntry
|
||||
engine.Plan(append(templatesList, selfContained), targets, true, func(entry PlanEntry) { again = append(again, entry) })
|
||||
require.Equal(t, plan, again, "planning the same scan again yields the same entries and ids")
|
||||
|
||||
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
|
||||
require.ElementsMatch(t, runs, planned, "the plan lists exactly the pairs the scan runs")
|
||||
require.ElementsMatch(t, []string{"thinkphp.yaml@php", "thinkphp.yaml@plain", "generic.yaml@plain"}, planned)
|
||||
}
|
||||
|
||||
func TestPlanKeepsOriginFallbacks(t *testing.T) {
|
||||
options := &types.Options{BulkSize: 1, TemplateThreads: 1, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
|
||||
engine := New(options)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{
|
||||
Logger: engine.Logger,
|
||||
Options: options,
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: &progressRecorder{},
|
||||
})
|
||||
|
||||
root := &templates.Template{
|
||||
ID: "root.yaml",
|
||||
RequestsHTTP: []*http.Request{{Path: []string{"{{RootURL}}/health"}}},
|
||||
TotalRequests: 1,
|
||||
}
|
||||
require.True(t, root.IsOriginScoped())
|
||||
generic := &templates.Template{ID: "generic.yaml", TotalRequests: 1}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{
|
||||
{Input: "https://example.com/a"},
|
||||
{Input: "https://example.com/b"},
|
||||
{Input: "https://other.example/a"},
|
||||
{Input: "https://example.com/a", CustomIP: "203.0.113.5"},
|
||||
}}
|
||||
|
||||
var plan []PlanEntry
|
||||
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
|
||||
plan = append(plan, entry)
|
||||
})
|
||||
|
||||
var roots []PlanEntry
|
||||
for _, entry := range plan {
|
||||
if entry.Templates[0] == "root.yaml" {
|
||||
roots = append(roots, entry)
|
||||
}
|
||||
}
|
||||
require.Equal(t, []PlanEntry{
|
||||
{ID: roots[0].ID, Target: "https://example.com/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1, Fallbacks: []string{"https://example.com/b"}},
|
||||
{ID: roots[1].ID, Target: "https://other.example/a", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
|
||||
{ID: roots[2].ID, Target: "https://example.com/a", IP: "203.0.113.5", Templates: []string{"root.yaml"}, Protocol: "http", Requests: 1, Cost: 1},
|
||||
}, roots)
|
||||
require.NotEqual(t, roots[0].ID, roots[2].ID, "a pinned address is its own attempt")
|
||||
|
||||
var again []PlanEntry
|
||||
engine.Plan([]*templates.Template{generic, root}, targets, true, func(entry PlanEntry) {
|
||||
again = append(again, entry)
|
||||
})
|
||||
require.Equal(t, plan, again)
|
||||
}
|
||||
@@ -21,6 +21,22 @@ type Client interface {
|
||||
Complete(ctx context.Context, prompt string, asJSON bool) (string, error)
|
||||
}
|
||||
|
||||
// Judge is implemented by clients whose provider classifies natively. Its
|
||||
// confidence is a calibrated probability rather than a number the model
|
||||
// reports about itself, and the inputs travel apart from the question, so
|
||||
// matchers use it over Complete when it is available.
|
||||
type Judge interface {
|
||||
Judge(ctx context.Context, question string, labels, inputs []string) (Verdict, error)
|
||||
}
|
||||
|
||||
// Verdict is the label a Judge chose. Model is the exact model version that
|
||||
// answered, when the provider reports one.
|
||||
type Verdict struct {
|
||||
Label string
|
||||
Confidence float64
|
||||
Model string
|
||||
}
|
||||
|
||||
// TruncateApproxTokens trims input to roughly maxTokens using the common
|
||||
// 4-chars-per-token approximation. Exact counting is provider-specific; this
|
||||
// only needs to keep a large body from blowing the context window.
|
||||
|
||||
@@ -65,26 +65,21 @@ func (matcher *Matcher) MatchLLMWithAudit(corpus string, inputs []string, values
|
||||
return false, nil, nil
|
||||
}
|
||||
|
||||
prompt := matcher.buildLLMPrompt(matcher.truncate(corpus, inputs), values)
|
||||
answer, err := client.Complete(context.Background(), prompt, true)
|
||||
if err != nil {
|
||||
responses := matcher.truncate(corpus, inputs)
|
||||
var (
|
||||
verdict llmVerdict
|
||||
audit *LLMAudit
|
||||
ok bool
|
||||
)
|
||||
if judge, isJudge := client.(llmclient.Judge); isJudge {
|
||||
verdict, audit, ok = matcher.judgeLLM(judge, responses, values)
|
||||
} else {
|
||||
verdict, audit, ok = matcher.completeLLM(client, responses, values)
|
||||
}
|
||||
if !ok {
|
||||
return false, nil, nil
|
||||
}
|
||||
|
||||
var verdict llmVerdict
|
||||
if err := json.Unmarshal([]byte(answer), &verdict); err != nil {
|
||||
return false, nil, nil
|
||||
}
|
||||
|
||||
audit := &LLMAudit{
|
||||
PromptHash: hashPrompt(prompt),
|
||||
Verdict: strings.TrimSpace(verdict.Verdict),
|
||||
Confidence: verdict.Confidence,
|
||||
}
|
||||
if namer, ok := client.(modelNamer); ok {
|
||||
audit.Model = namer.Model()
|
||||
}
|
||||
|
||||
expect := matcher.Expect
|
||||
if expect == "" {
|
||||
expect = "yes"
|
||||
@@ -107,9 +102,55 @@ func (matcher *Matcher) MatchLLMWithAudit(corpus string, inputs []string, values
|
||||
return false, nil, audit
|
||||
}
|
||||
|
||||
if verdict.Evidence == "" {
|
||||
return true, nil, audit
|
||||
}
|
||||
return true, []string{verdict.Evidence}, audit
|
||||
}
|
||||
|
||||
// completeLLM asks a chat model for the verdict as JSON.
|
||||
func (matcher *Matcher) completeLLM(client LLMClient, inputs []string, values map[string]interface{}) (llmVerdict, *LLMAudit, bool) {
|
||||
prompt := matcher.buildLLMPrompt(inputs, values)
|
||||
answer, err := client.Complete(context.Background(), prompt, true)
|
||||
if err != nil {
|
||||
return llmVerdict{}, nil, false
|
||||
}
|
||||
|
||||
var verdict llmVerdict
|
||||
if err := json.Unmarshal([]byte(answer), &verdict); err != nil {
|
||||
return llmVerdict{}, nil, false
|
||||
}
|
||||
|
||||
audit := &LLMAudit{
|
||||
PromptHash: hashPrompt(prompt),
|
||||
Verdict: strings.TrimSpace(verdict.Verdict),
|
||||
Confidence: verdict.Confidence,
|
||||
}
|
||||
if namer, ok := client.(modelNamer); ok {
|
||||
audit.Model = namer.Model()
|
||||
}
|
||||
return verdict, audit, true
|
||||
}
|
||||
|
||||
// judgeLLM asks a native classifier. It needs no prompt framing: the question
|
||||
// and the responses are separate fields of the request, and the answer is a
|
||||
// probability over the allowed verdicts, not text. It quotes no evidence.
|
||||
func (matcher *Matcher) judgeLLM(judge llmclient.Judge, inputs []string, values map[string]interface{}) (llmVerdict, *LLMAudit, bool) {
|
||||
question := interpolate(matcher.Prompt, values)
|
||||
answer, err := judge.Judge(context.Background(), question, matcher.Options, inputs)
|
||||
if err != nil {
|
||||
return llmVerdict{}, nil, false
|
||||
}
|
||||
|
||||
audit := &LLMAudit{
|
||||
Model: answer.Model,
|
||||
PromptHash: hashPrompt(strings.Join(append([]string{question, strings.Join(matcher.Options, ",")}, inputs...), "\x00")),
|
||||
Verdict: answer.Label,
|
||||
Confidence: answer.Confidence,
|
||||
}
|
||||
return llmVerdict{Verdict: answer.Label, Confidence: answer.Confidence}, audit, true
|
||||
}
|
||||
|
||||
// buildLLMPrompt wraps the author's question with the output contract and the
|
||||
// response under a random boundary (see FrameResponse).
|
||||
//
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -277,3 +278,61 @@ func TestMatchLLMWithoutInputsUsesPart(t *testing.T) {
|
||||
require.Contains(t, stub.lastGot, "the body")
|
||||
require.NotContains(t, stub.lastGot, "Response 1")
|
||||
}
|
||||
|
||||
type stubJudge struct {
|
||||
stubLLM
|
||||
verdict llmclient.Verdict
|
||||
err error
|
||||
question string
|
||||
labels []string
|
||||
inputs []string
|
||||
}
|
||||
|
||||
func (s *stubJudge) Judge(_ context.Context, question string, labels, inputs []string) (llmclient.Verdict, error) {
|
||||
s.question, s.labels, s.inputs = question, labels, inputs
|
||||
return s.verdict, s.err
|
||||
}
|
||||
|
||||
func TestMatchLLMPrefersJudgeAndKeepsResponseOutOfQuestion(t *testing.T) {
|
||||
m := llmMatcher("login page for {{Hostname}}?")
|
||||
judge := &stubJudge{verdict: llmclient.Verdict{Label: "yes", Confidence: 0.97, Model: "jev-1.14.0"}}
|
||||
m.SetLLMClient(judge)
|
||||
|
||||
ok, snips, audit := m.MatchLLMWithAudit("<form>", nil, map[string]interface{}{"Hostname": "example.com"})
|
||||
require.True(t, ok)
|
||||
require.Empty(t, snips, "a classifier quotes no evidence")
|
||||
require.Zero(t, judge.calls, "Complete must not be used when the client can judge")
|
||||
require.Equal(t, "login page for example.com?", judge.question)
|
||||
require.Nil(t, judge.labels, "no options means a yes or no question")
|
||||
require.Equal(t, []string{"<form>"}, judge.inputs)
|
||||
require.Equal(t, "jev-1.14.0", audit.Model)
|
||||
require.InDelta(t, 0.97, audit.Confidence, 1e-9)
|
||||
}
|
||||
|
||||
func TestMatchLLMJudgeAppliesExpectOptionsAndConfidence(t *testing.T) {
|
||||
m := llmMatcher("same access?")
|
||||
m.Options = []string{"same", "different"}
|
||||
m.Expect = "different"
|
||||
m.MinConfidence = 0.9
|
||||
judge := &stubJudge{verdict: llmclient.Verdict{Label: "different", Confidence: 0.8}}
|
||||
m.SetLLMClient(judge)
|
||||
|
||||
ok, _, audit := m.MatchLLMWithAudit("", []string{"a", "b"}, nil)
|
||||
require.False(t, ok, "below min-confidence")
|
||||
require.NotNil(t, audit)
|
||||
require.Equal(t, []string{"same", "different"}, judge.labels)
|
||||
require.Equal(t, []string{"a", "b"}, judge.inputs)
|
||||
|
||||
judge.verdict.Confidence = 0.95
|
||||
ok, _, _ = m.MatchLLMWithAudit("", []string{"a", "b"}, nil)
|
||||
require.True(t, ok)
|
||||
}
|
||||
|
||||
func TestMatchLLMJudgeFailsClosed(t *testing.T) {
|
||||
m := llmMatcher("q")
|
||||
m.SetLLMClient(&stubJudge{err: errors.New("boom")})
|
||||
|
||||
ok, _, audit := m.MatchLLMWithAudit("body", nil, nil)
|
||||
require.False(t, ok)
|
||||
require.Nil(t, audit)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package protocolstate
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/networkpolicy"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIsHostAllowedRacesExcludeUpdate(t *testing.T) {
|
||||
np, err := networkpolicy.New(networkpolicy.Options{DenyList: []string{"127.0.0.0/8"}})
|
||||
require.NoError(t, err)
|
||||
|
||||
const executionID = "exclude-race"
|
||||
entry := &Dialers{NetworkPolicy: np, ExcludeTargets: []string{"192.0.2.1"}}
|
||||
err = dialers.Set(executionID, entry)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { dialers.Delete(executionID) })
|
||||
|
||||
stop := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
lists := [][]string{{"192.0.2.1"}, {"192.0.2.2", "192.0.2.3"}}
|
||||
i := 0
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
entry.Lock()
|
||||
entry.ExcludeTargets = lists[i%2]
|
||||
entry.Unlock()
|
||||
i++
|
||||
}
|
||||
}()
|
||||
|
||||
for i := 0; i < 2000; i++ {
|
||||
_ = IsHostAllowed(executionID, "http://192.0.2.1")
|
||||
}
|
||||
close(stop)
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -138,11 +138,14 @@ func CanonicalizeHost(host string) string {
|
||||
// IsHostAllowed checks if the host is allowed by network policy
|
||||
func IsHostAllowed(executionId string, targetUrl string) bool {
|
||||
dialers, ok := dialers.Get(executionId)
|
||||
if !ok {
|
||||
if !ok || dialers == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
dialers.Lock()
|
||||
np := dialers.NetworkPolicy
|
||||
excludedTargets := append([]string(nil), dialers.ExcludeTargets...)
|
||||
dialers.Unlock()
|
||||
if np == nil {
|
||||
// no policy present: deny rather than allow.
|
||||
return false
|
||||
@@ -174,7 +177,7 @@ func IsHostAllowed(executionId string, targetUrl string) bool {
|
||||
canonHost := CanonicalizeHost(host)
|
||||
|
||||
// Check explicit ExcludeTargets against raw, host, and canonical host
|
||||
for _, excluded := range dialers.ExcludeTargets {
|
||||
for _, excluded := range excludedTargets {
|
||||
ex := CanonicalizeHost(strings.TrimSpace(excluded))
|
||||
if ex != "" {
|
||||
if strings.EqualFold(canonHost, ex) || strings.EqualFold(host, ex) || strings.EqualFold(rawTarget, ex) {
|
||||
|
||||
@@ -115,7 +115,10 @@ func initDialers(options *types.Options) error {
|
||||
if options.RestrictLocalNetworkAccess {
|
||||
expandedDenyList = append(expandedDenyList, networkpolicy.DefaultIPv4DenylistRanges...)
|
||||
expandedDenyList = append(expandedDenyList, networkpolicy.DefaultIPv6DenylistRanges...)
|
||||
// the unspecified address connects to the local host
|
||||
expandedDenyList = append(expandedDenyList, "::/128")
|
||||
}
|
||||
expandedDenyList = append(expandedDenyList, mappedDenyRanges(expandedDenyList)...)
|
||||
npOptions := &networkpolicy.Options{
|
||||
DenyList: expandedDenyList,
|
||||
}
|
||||
@@ -342,6 +345,27 @@ func Close(executionId string) {
|
||||
}
|
||||
}
|
||||
|
||||
// mappedDenyRanges returns the IPv4-mapped IPv6 form (::ffff:a.b.c.d) of each
|
||||
// IPv4 address and range in denyList. The network policy checks a mapped
|
||||
// address such as [::ffff:127.0.0.1] only against IPv6 ranges, but the dialer
|
||||
// connects to the IPv4 host, so without these forms it passes -eh and -lna.
|
||||
func mappedDenyRanges(denyList []string) []string {
|
||||
var mapped []string
|
||||
for _, entry := range denyList {
|
||||
cidr := entry
|
||||
if !strings.Contains(cidr, "/") {
|
||||
cidr += "/32"
|
||||
}
|
||||
_, ipNet, err := net.ParseCIDR(cidr)
|
||||
if err != nil || ipNet.IP.To4() == nil || strings.Contains(entry, ":") {
|
||||
continue
|
||||
}
|
||||
ones, _ := ipNet.Mask.Size()
|
||||
mapped = append(mapped, fmt.Sprintf("::ffff:%s/%d", ipNet.IP.To4(), 96+ones))
|
||||
}
|
||||
return mapped
|
||||
}
|
||||
|
||||
// excludeHostRules turns an exclude entry into network policy deny rules.
|
||||
// networkpolicy treats non-IP entries as unanchored regexes, so a plain
|
||||
// "test.com" would also deny "latest.com". Plain hostnames match the host and
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
package protocolstate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -27,3 +32,47 @@ func TestExcludeTargetsMatchExactHost(t *testing.T) {
|
||||
require.True(t, np.Validate("testXcom"))
|
||||
require.True(t, np.Validate("sub.regex.example"))
|
||||
}
|
||||
|
||||
func TestMappedFormsDeniedByDialer(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer server.Close()
|
||||
_, port, err := net.SplitHostPort(server.Listener.Addr().String())
|
||||
require.NoError(t, err)
|
||||
|
||||
for name, set := range map[string]func(*types.Options){
|
||||
"lna": func(o *types.Options) { o.RestrictLocalNetworkAccess = true },
|
||||
"exclude-hosts": func(o *types.Options) { o.ExcludeTargets = []string{"127.0.0.1"} },
|
||||
"exclude-cidr": func(o *types.Options) { o.ExcludeTargets = []string{"127.0.0.0/8"} },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
opts := types.DefaultOptions()
|
||||
opts.ExecutionId = "test-mapped-" + name
|
||||
set(opts)
|
||||
require.NoError(t, Init(opts))
|
||||
defer Close(opts.ExecutionId)
|
||||
|
||||
d := GetDialersWithId(opts.ExecutionId)
|
||||
hosts := []string{"127.0.0.1", "::ffff:127.0.0.1", "::ffff:7f00:1", "0:0:0:0:0:ffff:127.0.0.1"}
|
||||
if opts.RestrictLocalNetworkAccess {
|
||||
hosts = append(hosts, "::")
|
||||
}
|
||||
for _, host := range hosts {
|
||||
require.False(t, d.NetworkPolicy.Validate(host), host)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
conn, err := d.Fastdialer.Dial(ctx, "tcp", net.JoinHostPort(host, port))
|
||||
cancel()
|
||||
if conn != nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
require.Error(t, err, host)
|
||||
}
|
||||
require.True(t, d.NetworkPolicy.Validate("::ffff:8.8.8.8"))
|
||||
require.True(t, d.NetworkPolicy.Validate("8.8.8.8"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappedDenyRanges(t *testing.T) {
|
||||
require.Equal(t, []string{"::ffff:127.0.0.0/104", "::ffff:192.0.2.10/128", "::ffff:0.0.0.0/104"},
|
||||
mappedDenyRanges([]string{"127.0.0.0/8", "192.0.2.10", "::1/128", "fc00::/7", `(?i)(^|\.)test\.com\.?$`, "0.0.0.0/8"}))
|
||||
}
|
||||
@@ -26,33 +26,42 @@ var defaultResolvers = []string{
|
||||
"8.8.4.4:53", // Google
|
||||
}
|
||||
|
||||
// Init initializes the client pool implementation
|
||||
// Init initializes the client pool implementation.
|
||||
// Each resolver list gets its own client, so a later scan is not stuck with the first scan's resolvers.
|
||||
func Init(options *types.Options) error {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
// Don't create clients if already created in the past.
|
||||
if normalClient != nil {
|
||||
return nil
|
||||
}
|
||||
clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]()
|
||||
|
||||
resolvers := defaultResolvers
|
||||
if len(options.InternalResolversList) > 0 {
|
||||
resolvers = options.InternalResolversList
|
||||
}
|
||||
var err error
|
||||
normalClient, err = retryabledns.New(resolvers, 1)
|
||||
client, err := cachedClient(resolversFor(options))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not create dns client")
|
||||
}
|
||||
m.Lock()
|
||||
normalClient = client
|
||||
m.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func getNormalClient() *retryabledns.Client {
|
||||
func resolversFor(options *types.Options) []string {
|
||||
if options != nil && len(options.InternalResolversList) > 0 {
|
||||
return options.InternalResolversList
|
||||
}
|
||||
return defaultResolvers
|
||||
}
|
||||
|
||||
func cachedClient(resolvers []string) (*retryabledns.Client, error) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
return normalClient
|
||||
if clientPool == nil {
|
||||
clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]()
|
||||
}
|
||||
key := strings.Join(resolvers, ",")
|
||||
if client, ok := clientPool.Get(key); ok {
|
||||
return client, nil
|
||||
}
|
||||
client, err := retryabledns.New(resolvers, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = clientPool.Set(key, client)
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// Configuration contains the custom configuration options for a client
|
||||
@@ -81,7 +90,7 @@ func (c *Configuration) Hash() string {
|
||||
// Get creates or gets a client for the protocol based on custom configuration
|
||||
func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) {
|
||||
if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
|
||||
return getNormalClient(), nil
|
||||
return cachedClient(resolversFor(options))
|
||||
}
|
||||
hash := configuration.Hash()
|
||||
if client, ok := clientPool.Get(hash); ok {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package dnsclientpool
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestClientsFollowResolverList(t *testing.T) {
|
||||
previousClient := normalClient
|
||||
previousPool := clientPool
|
||||
t.Cleanup(func() {
|
||||
normalClient = previousClient
|
||||
clientPool = previousPool
|
||||
})
|
||||
normalClient = nil
|
||||
clientPool = nil
|
||||
|
||||
firstOptions := &types.Options{InternalResolversList: []string{"192.0.2.1:53"}}
|
||||
secondOptions := &types.Options{InternalResolversList: []string{"192.0.2.2:53"}}
|
||||
|
||||
require.NoError(t, Init(firstOptions))
|
||||
require.NoError(t, Init(secondOptions))
|
||||
|
||||
first, err := Get(firstOptions, &Configuration{})
|
||||
require.NoError(t, err)
|
||||
second, err := Get(secondOptions, &Configuration{})
|
||||
require.NoError(t, err)
|
||||
require.NotSame(t, first, second)
|
||||
|
||||
again, err := Get(firstOptions, &Configuration{})
|
||||
require.NoError(t, err)
|
||||
require.Same(t, first, again)
|
||||
}
|
||||
@@ -94,10 +94,21 @@ func NewTransportWrapper() func(http.RoundTripper) http.RoundTripper {
|
||||
},
|
||||
DisableWarningHeader: true,
|
||||
}
|
||||
return transport
|
||||
return &unwrappingTransport{Transport: transport}
|
||||
}
|
||||
}
|
||||
|
||||
// unwrappingTransport lets code that must not use the cache, such as reporting
|
||||
// clients that need TLS verification, reach the wrapped transport.
|
||||
type unwrappingTransport struct {
|
||||
*httpcache.Transport
|
||||
}
|
||||
|
||||
// Unwrap returns the transport the cache wraps.
|
||||
func (t *unwrappingTransport) Unwrap() http.RoundTripper {
|
||||
return t.Transport.Transport
|
||||
}
|
||||
|
||||
func openDiskCache(dir string, maxBytes int64) (httpcache.Cache, *leveldb.DB, error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, nil, fmt.Errorf("create cache dir: %w", err)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpcache
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
|
||||
libcache "github.com/sandrolain/httpcache"
|
||||
)
|
||||
|
||||
@@ -178,3 +180,29 @@ func TestDiskCacheRespectsSizeCap(t *testing.T) {
|
||||
t.Fatal("stored a response over a 1-byte directory cap")
|
||||
}
|
||||
}
|
||||
|
||||
// Reporting clients send tokens, so tlsverify must reach the scan transport
|
||||
// under the cache wrapper and turn certificate verification on.
|
||||
func TestTransportWrapperUnwrapsForTLSVerification(t *testing.T) {
|
||||
cacheMu.Lock()
|
||||
saved := cache
|
||||
cache = libcache.NewMemoryCache()
|
||||
cacheMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
cacheMu.Lock()
|
||||
cache = saved
|
||||
cacheMu.Unlock()
|
||||
})
|
||||
|
||||
base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
wrapped := NewTransportWrapper()(base)
|
||||
u, ok := wrapped.(interface{ Unwrap() http.RoundTripper })
|
||||
if !ok || u.Unwrap() != base {
|
||||
t.Fatal("the cache wrapper does not unwrap to the scan transport")
|
||||
}
|
||||
|
||||
got, ok := tlsverify.Client(&http.Client{Transport: wrapped}).Transport.(*http.Transport)
|
||||
if !ok || got.TLSClientConfig.InsecureSkipVerify {
|
||||
t.Fatal("tlsverify did not turn verification on under the cache wrapper")
|
||||
}
|
||||
}
|
||||
@@ -166,6 +166,11 @@ func (t *connTrackingTransport) RoundTrip(req *http.Request) (*http.Response, er
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// Unwrap returns the underlying transport.
|
||||
func (t *connTrackingTransport) Unwrap() http.RoundTripper {
|
||||
return t.base
|
||||
}
|
||||
|
||||
func (t *connTrackingTransport) CloseIdleConnections() {
|
||||
type closeIdler interface{ CloseIdleConnections() }
|
||||
if ci, ok := t.base.(closeIdler); ok {
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/pdf"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
"go.uber.org/multierr"
|
||||
@@ -76,6 +78,17 @@ type IssueTrackerStats struct {
|
||||
func New(options *Options, db string, doNotDedupe bool) (Client, error) {
|
||||
client := &ReportingClient{options: options}
|
||||
|
||||
// The caller can pass the scan client, which skips TLS verification;
|
||||
// trackers and exporters send tokens and passwords, so they get a copy
|
||||
// that verifies certificates.
|
||||
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
|
||||
if hc := tlsverify.Client(options.HttpClient.HTTPClient); hc != options.HttpClient.HTTPClient {
|
||||
opts := retryablehttp.DefaultOptionsSingle
|
||||
opts.Timeout = hc.Timeout // keep the caller's timeout, also when it is 0
|
||||
options.HttpClient = retryablehttp.NewWithHTTPClient(hc, opts)
|
||||
}
|
||||
}
|
||||
|
||||
if options.GitHub != nil {
|
||||
options.GitHub.HttpClient = options.HttpClient
|
||||
options.GitHub.OmitRaw = options.OmitRaw
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
package reporting_test
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// connectProxy tunnels every CONNECT to target, whatever host the client asked
|
||||
// for, so no name is resolved and nothing leaves 127.0.0.1.
|
||||
func connectProxy(t *testing.T, target string) *httptest.Server {
|
||||
p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodConnect {
|
||||
http.Error(w, "connect only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
up, err := net.Dial("tcp", target)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
c, _, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
_ = up.Close()
|
||||
return
|
||||
}
|
||||
_, _ = c.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n"))
|
||||
go func() { _, _ = io.Copy(up, c); _ = up.Close() }()
|
||||
go func() { _, _ = io.Copy(c, up); _ = c.Close() }()
|
||||
}))
|
||||
t.Cleanup(p.Close)
|
||||
return p
|
||||
}
|
||||
|
||||
// With -pi and a proxy, the runner gives reporting the scan client, which skips
|
||||
// TLS verification. No tracker or exporter may send its secret to a server with
|
||||
// an untrusted certificate.
|
||||
func TestNewVerifiesTLSWithScanClient(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
proxy := connectProxy(t, srv.Listener.Addr().String())
|
||||
|
||||
opts := types.DefaultOptions()
|
||||
opts.SetExecutionID("reporting-tls-test")
|
||||
opts.ProxyInternal = true
|
||||
opts.AliveHttpProxy = proxy.URL
|
||||
require.NoError(t, protocolstate.Init(opts))
|
||||
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
|
||||
scanClient, err := httpclientpool.Get(opts, &httpclientpool.Configuration{}, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "test-template",
|
||||
Host: "https://target.example",
|
||||
Matched: "https://target.example/",
|
||||
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
|
||||
}
|
||||
configs := map[string]*reporting.Options{
|
||||
"github": {GitHub: &github.Options{BaseURL: "https://github.victim.example/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token"}},
|
||||
"gitlab": {GitLab: &gitlab.Options{BaseURL: "https://gitlab.victim.example/", Username: "u", Token: "test-token", ProjectName: "1"}},
|
||||
"gitea": {Gitea: &gitea.Options{BaseURL: "https://gitea.victim.example/", Token: "test-token", ProjectOwner: "o", ProjectName: "r"}},
|
||||
"jira": {Jira: &jira.Options{Cloud: true, URL: "https://jira.victim.example", AccountID: "a", Email: "[email protected]", Token: "test-token", ProjectName: "P", IssueType: "Bug"}},
|
||||
"elasticsearch": {ElasticsearchExporter: &es.Options{Host: "es.victim.example", Port: 443, SSL: true, Username: "u", Password: "test-password", IndexName: "n"}},
|
||||
"splunk": {SplunkExporter: &splunk.Options{Host: "splunk.victim.example", Port: 443, SSL: true, Token: "test-token", IndexName: "n"}},
|
||||
}
|
||||
for name, ro := range configs {
|
||||
ro.HttpClient = scanClient
|
||||
ro.ExecutionId = opts.ExecutionId
|
||||
client, err := reporting.New(ro, "", true)
|
||||
if err == nil {
|
||||
_ = client.CreateIssue(event)
|
||||
client.Close()
|
||||
}
|
||||
require.Zero(t, hits.Load(), "%s sent a request to a server with an untrusted certificate", name)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Package tlsverify gives trackers and exporters an HTTP client that verifies
|
||||
// TLS certificates, since they send tokens and passwords.
|
||||
package tlsverify
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// errTLSDialerOnly is returned when a transport dials HTTPS only through a
|
||||
// custom TLS dialer: dropping that dialer would also drop its route or address
|
||||
// policy, so the copy refuses requests instead.
|
||||
var errTLSDialerOnly = errors.New("tlsverify: the transport has a custom TLS dialer and no DialContext, so certificate verification cannot be enforced")
|
||||
|
||||
type refusingTransport struct{}
|
||||
|
||||
func (refusingTransport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||
return nil, errTLSDialerOnly
|
||||
}
|
||||
|
||||
// Client returns c, or a copy of c whose transport verifies TLS certificates
|
||||
// when c can skip verification (the scan client can). The copy keeps the proxy
|
||||
// and the dialer, so the address policy still applies, but it drops a custom
|
||||
// TLS dialer, because that dialer can use the scan TLS config. So the token
|
||||
// never goes over an unverified connection.
|
||||
//
|
||||
// A transport with an Unwrap() http.RoundTripper method declares that its
|
||||
// RoundTrip only observes or caches requests and enforces no policy; the copy
|
||||
// uses the unwrapped transport. Other transports are returned unchanged.
|
||||
func Client(c *http.Client) *http.Client {
|
||||
rt := c.Transport
|
||||
for {
|
||||
u, ok := rt.(interface{ Unwrap() http.RoundTripper })
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
rt = u.Unwrap()
|
||||
}
|
||||
t, ok := rt.(*http.Transport)
|
||||
if !ok {
|
||||
return c
|
||||
}
|
||||
insecure := t.TLSClientConfig != nil && t.TLSClientConfig.InsecureSkipVerify
|
||||
tlsDialer := t.DialTLSContext != nil || t.DialTLS != nil //nolint:staticcheck // DialTLS is deprecated but still used
|
||||
if !insecure && !tlsDialer {
|
||||
return c
|
||||
}
|
||||
copied := *c
|
||||
if tlsDialer && t.DialContext == nil && t.Dial == nil { //nolint:staticcheck // Dial is deprecated but still used
|
||||
copied.Transport = refusingTransport{}
|
||||
return &copied
|
||||
}
|
||||
verifying := t.Clone()
|
||||
verifying.DialTLSContext = nil
|
||||
verifying.DialTLS = nil //nolint:staticcheck // DialTLS is deprecated but still used
|
||||
if insecure {
|
||||
// Keep the caller's roots, client certificates and checks; the server
|
||||
// name of the scan config belongs to a scan target, not to this host.
|
||||
verifying.TLSClientConfig.InsecureSkipVerify = false
|
||||
verifying.TLSClientConfig.ServerName = ""
|
||||
if verifying.TLSClientConfig.MinVersion < tls.VersionTLS12 {
|
||||
verifying.TLSClientConfig.MinVersion = tls.VersionTLS12
|
||||
}
|
||||
}
|
||||
copied.Transport = verifying
|
||||
return &copied
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package tlsverify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The scan client can skip TLS verification; the tracker must not send the
|
||||
// token over such a connection, and must not change the caller's client.
|
||||
func TestClientTurnsOnTLSVerification(t *testing.T) {
|
||||
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
caller := &http.Client{Transport: insecure}
|
||||
|
||||
got := Client(caller)
|
||||
gotTransport, ok := got.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
require.False(t, gotTransport.TLSClientConfig.InsecureSkipVerify)
|
||||
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's transport changed")
|
||||
|
||||
secure := &http.Client{Transport: &http.Transport{}}
|
||||
require.Same(t, secure, Client(secure))
|
||||
}
|
||||
|
||||
func TestClientKeepsCallerTLSSettings(t *testing.T) {
|
||||
pinned := func(tls.ConnectionState) error { return nil }
|
||||
insecure := &http.Transport{
|
||||
DialTLSContext: func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("scan dialer") },
|
||||
DialContext: (&net.Dialer{}).DialContext,
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
ServerName: "scan-target.example",
|
||||
MinVersion: tls.VersionTLS10,
|
||||
RootCAs: x509.NewCertPool(),
|
||||
VerifyConnection: pinned,
|
||||
},
|
||||
}
|
||||
got := Client(&http.Client{Transport: insecure}).Transport.(*http.Transport)
|
||||
require.Nil(t, got.DialTLSContext)
|
||||
require.NotNil(t, got.DialContext)
|
||||
cfg := got.TLSClientConfig
|
||||
require.False(t, cfg.InsecureSkipVerify)
|
||||
require.Empty(t, cfg.ServerName)
|
||||
require.Equal(t, uint16(tls.VersionTLS12), cfg.MinVersion)
|
||||
require.Same(t, insecure.TLSClientConfig.RootCAs, cfg.RootCAs)
|
||||
require.NotNil(t, cfg.VerifyConnection)
|
||||
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's config changed")
|
||||
}
|
||||
|
||||
// A custom TLS dialer may carry the only route or address policy, so a
|
||||
// transport without DialContext cannot be rebuilt and must refuse requests.
|
||||
func TestClientRefusesTLSDialerWithoutDialContext(t *testing.T) {
|
||||
var dialed bool
|
||||
tlsOnly := &http.Transport{DialTLSContext: func(context.Context, string, string) (net.Conn, error) {
|
||||
dialed = true
|
||||
return nil, errors.New("policy dialer")
|
||||
}}
|
||||
_, err := Client(&http.Client{Transport: tlsOnly}).Get("https://tracker.example/")
|
||||
require.ErrorIs(t, err, errTLSDialerOnly)
|
||||
require.False(t, dialed)
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/google/go-github/v92/github"
|
||||
@@ -12,6 +11,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"golang.org/x/oauth2"
|
||||
@@ -55,17 +55,16 @@ type Options struct {
|
||||
// New creates a new issue tracker integration client based on options.
|
||||
func New(options *Options) (*Integration, error) {
|
||||
ctx := context.Background()
|
||||
// Build the oauth2 client on top of the caller's client, so its transport
|
||||
// (proxy, address policy) and redirect rules apply to every request.
|
||||
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
|
||||
ctx = context.WithValue(ctx, oauth2.HTTPClient, tlsverify.Client(options.HttpClient.HTTPClient))
|
||||
}
|
||||
ts := oauth2.StaticTokenSource(
|
||||
&oauth2.Token{AccessToken: options.Token},
|
||||
)
|
||||
tc := oauth2.NewClient(ctx, ts)
|
||||
|
||||
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
|
||||
if tcTransport, ok := tc.Transport.(*http.Transport); ok {
|
||||
tcTransport.Proxy = options.HttpClient.HTTPClient.Transport.(*http.Transport).Proxy
|
||||
}
|
||||
}
|
||||
|
||||
clientOptions := []github.ClientOptionsFunc{github.WithHTTPClient(tc)}
|
||||
if options.BaseURL != "" {
|
||||
clientOptions = append(clientOptions, github.WithURLs(&options.BaseURL, nil))
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package github
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func testEvent() *output.ResultEvent {
|
||||
return &output.ResultEvent{
|
||||
TemplateID: "test-template",
|
||||
Host: "https://target.example",
|
||||
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
|
||||
}
|
||||
}
|
||||
|
||||
func clientWithTransport(rt http.RoundTripper) *retryablehttp.Client {
|
||||
c := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
|
||||
c.HTTPClient = &http.Client{Transport: rt}
|
||||
return c
|
||||
}
|
||||
|
||||
// The tracker must send every request through the caller's client, so a
|
||||
// caller's address policy also covers the GitHub API base URL.
|
||||
func TestNewUsesCallerHTTPClient(t *testing.T) {
|
||||
var serverHits atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
serverHits.Add(1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"number":1,"html_url":"https://github.example/o/r/issues/1"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
t.Run("requests go through the caller's transport", func(t *testing.T) {
|
||||
var seen atomic.Int32
|
||||
rt := roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
seen.Add(1)
|
||||
require.Equal(t, "Bearer test-token", r.Header.Get("Authorization"))
|
||||
return http.DefaultTransport.RoundTrip(r)
|
||||
})
|
||||
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
|
||||
require.NoError(t, err)
|
||||
_, err = tracker.CreateIssue(testEvent())
|
||||
require.NoError(t, err)
|
||||
require.Positive(t, seen.Load())
|
||||
})
|
||||
|
||||
t.Run("a refusing transport stops the request", func(t *testing.T) {
|
||||
before := serverHits.Load()
|
||||
denied := errors.New("denied by address policy")
|
||||
rt := roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, denied })
|
||||
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
|
||||
require.NoError(t, err)
|
||||
_, err = tracker.CreateIssue(testEvent())
|
||||
require.ErrorContains(t, err, "denied by address policy")
|
||||
require.Equal(t, before, serverHits.Load(), "the request reached the server")
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewRefusesUntrustedCertificateFromInsecureCallerClient(t *testing.T) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"number":1}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(insecure)})
|
||||
require.NoError(t, err)
|
||||
_, err = tracker.CreateIssue(testEvent())
|
||||
require.ErrorContains(t, err, "certificate")
|
||||
}
|
||||
|
||||
type wrappingTransport struct{ base http.RoundTripper }
|
||||
|
||||
func (w *wrappingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return w.base.RoundTrip(r)
|
||||
}
|
||||
func (w *wrappingTransport) Unwrap() http.RoundTripper { return w.base }
|
||||
|
||||
// The scan client wraps its transport and, with a SOCKS proxy, does TLS in its
|
||||
// own dialer with the scan TLS config, which skips verification.
|
||||
func TestNewRefusesUntrustedCertificateFromInsecureTLSDialer(t *testing.T) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"number":1}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
var dialed atomic.Int32
|
||||
var dialer net.Dialer
|
||||
base := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dialed.Add(1)
|
||||
return dialer.DialContext(ctx, network, addr)
|
||||
},
|
||||
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
conn, err := dialer.DialContext(ctx, network, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tls.Client(conn, &tls.Config{InsecureSkipVerify: true}), nil
|
||||
},
|
||||
}
|
||||
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(&wrappingTransport{base: base})})
|
||||
require.NoError(t, err)
|
||||
_, err = tracker.CreateIssue(testEvent())
|
||||
require.ErrorContains(t, err, "certificate")
|
||||
require.Positive(t, dialed.Load(), "the caller's dialer was not used")
|
||||
}
|
||||
@@ -289,18 +289,6 @@ func (template *Template) SelectedBy(selection protocols.TemplateSelection) bool
|
||||
return selection.Allows(template.Path)
|
||||
}
|
||||
|
||||
// TemplateIDs returns the IDs of the members selection selects; a nil
|
||||
// selection selects every member.
|
||||
func (e *ClusterExecuter) TemplateIDs(selection protocols.TemplateSelection) []string {
|
||||
ids := make([]string, 0, len(e.operators))
|
||||
for _, operator := range e.operators {
|
||||
if selection == nil || selection.Allows(operator.templatePath) {
|
||||
ids = append(ids, operator.templateID)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// operatorsFor returns the members selected for input by per-target profiles.
|
||||
func (e *ClusterExecuter) operatorsFor(input *contextargs.MetaInput) []*clusteredOperator {
|
||||
if e.options.TargetScope == nil {
|
||||
|
||||
@@ -251,9 +251,6 @@ type Options struct {
|
||||
Silent bool
|
||||
// Validate validates the templates passed to nuclei.
|
||||
Validate bool
|
||||
// PlanFile is where the work a scan would do is written as JSON lines;
|
||||
// nuclei exits after writing it, without sending requests.
|
||||
PlanFile string
|
||||
// NoStrictSyntax disables strict syntax check on nuclei templates (allows custom key-value pairs).
|
||||
NoStrictSyntax bool
|
||||
// Verbose flag indicates whether to show verbose output or not
|
||||
@@ -710,7 +707,6 @@ func (options *Options) Copy() *Options {
|
||||
AutomaticScan: options.AutomaticScan,
|
||||
Silent: options.Silent,
|
||||
Validate: options.Validate,
|
||||
PlanFile: options.PlanFile,
|
||||
NoStrictSyntax: options.NoStrictSyntax,
|
||||
Verbose: options.Verbose,
|
||||
VerboseVerbose: options.VerboseVerbose,
|
||||
|
||||
@@ -17,13 +17,6 @@ func TestCopyPreservesPayloadRandomOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyPreservesPlanFile(t *testing.T) {
|
||||
options := &Options{PlanFile: "plan.jsonl"}
|
||||
if options.Copy().PlanFile != "plan.jsonl" {
|
||||
t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) {
|
||||
metadata := map[string]interface{}{
|
||||
"labels": []interface{}{map[interface{}]interface{}{1: "int", 1.0: "float64"}},
|
||||
|
||||
Reference in new issue
Block a user