Compare commits

...
Author SHA1 Message Date
Mzack9999 fe24f8638b review fixes 2026-11-04 05:09:15 +01:00
Mzack9999 8b7cbf44e6 all trackers 2026-11-04 03:26:07 +01:00
Mzack9999 59ae4cac04 tls dialer 2026-11-04 03:00:58 +01:00
Mzack9999 64965ba776 tls verify 2026-11-04 00:54:48 +01:00
Mzack9999 15271926ad tracker client 2026-11-03 23:28:48 +01:00
Mzack9999 46fbc92a76 Merge pull request #8023 from projectdiscovery/fix/dns-resolver-client
Give each scan its own DNS resolvers
2026-11-03 21:42:45 +01:00
Mzack9999 36d0adba18 Merge pull request #8024 from projectdiscovery/fix/headless-exclude-lock
Read headless excludes under the dialer lock
2026-11-03 20:47:14 +01:00
Mzack9999 049e6b629e Merge pull request #8018 from projectdiscovery/feat/jev-llm-provider
use jev for llm matchers when -llm-provider jev
2026-11-03 19:21:59 +01:00
Dogan Can Bakir 2c6b223bb4 Merge pull request #7983 from projectdiscovery/headless-host-canonical
Deny IPv4-mapped forms of denied hosts in the dialer
2026-11-03 20:45:05 +03:00
Mzack9999 0827b2779b preserve proxy 2026-11-03 18:34:18 +01:00
Mzack9999 0971fb83e2 skip extractors 2026-11-03 18:07:38 +01:00
Mzack9999 62dae92b3d refresh resolvers 2026-11-03 17:30:04 +01:00
Mzack9999 e44df6e947 lock excludes 2026-11-03 17:30:04 +01:00
Mzack9999 faaeaf30b1 mapped deny 2026-11-03 17:07:35 +01:00
Mzack9999 b7fe335285 merge dev 2026-11-03 17:02:03 +01:00
Mzack9999 9b04153a3b merge dev 2026-11-03 12:45:38 +01:00
Dogan Can Bakir 23f3407af7 bump utils for the jev provider 2026-11-03 12:50:46 +03:00
Dogan Can Bakir 64f94f3e66 use jev for llm matchers when -llm-provider jev 2026-11-03 12:04:47 +03:00
Mzack9999 ac52212d17 canonicalize headless 2026-10-29 21:25:10 +01:00
32 changed files with 856 additions and 62 deletions

No files matched your search

+1 -1
View File
@@ -177,7 +177,7 @@ TEMPLATES:
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
+1 -1
View File
@@ -175,7 +175,7 @@ TEMPLATES:
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
+1 -1
View File
@@ -175,7 +175,7 @@ TEMPLATES:
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
+1 -1
View File
@@ -192,7 +192,7 @@ TEMPLATES:
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
+1 -1
View File
@@ -175,7 +175,7 @@ TEMPLATES:
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
+1 -1
View File
@@ -175,7 +175,7 @@ TEMPLATES:
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
+1 -1
View File
@@ -175,7 +175,7 @@ TEMPLATES:
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
+1 -1
View File
@@ -175,7 +175,7 @@ TEMPLATES:
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
+1 -1
View File
@@ -175,7 +175,7 @@ TEMPLATES:
-egm, -enable-global-matchers enable loading global matchers templates
-file enable loading file templates
-llm enable llm matchers and extractors (semantic matching)
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
-llm-provider string llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
-llm-base-url string openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
-llm-model string model used for semantic matching
-llm-timeout int time in seconds to wait for a single llm call (default 30)
+1 -1
View File
@@ -309,7 +309,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVarP(&options.EnableGlobalMatchersTemplates, "enable-global-matchers", "egm", false, "enable loading global matchers templates"),
flagSet.BoolVar(&options.EnableFileTemplates, "file", false, "enable loading file templates"),
flagSet.BoolVar(&options.EnableLLM, "llm", false, "enable llm matchers and extractors (semantic matching)"),
flagSet.StringVar(&options.LLMProvider, "llm-provider", "openai", "llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter)"),
flagSet.StringVar(&options.LLMProvider, "llm-provider", "openai", "llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev)"),
flagSet.StringVar(&options.LLMBaseURL, "llm-base-url", "", "openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)"),
flagSet.StringVar(&options.LLMModel, "llm-model", "", "model used for semantic matching"),
flagSet.IntVar(&options.LLMTimeout, "llm-timeout", 30, "time in seconds to wait for a single llm call"),
+1 -1
View File
@@ -105,7 +105,7 @@ require (
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
github.com/projectdiscovery/uncover v1.2.1
github.com/projectdiscovery/useragent v0.0.109
github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350
github.com/projectdiscovery/wappalyzergo v0.3.6
github.com/projectdiscovery/yamldoc-go v1.0.7
github.com/redis/go-redis/v9 v9.19.0
+2 -4
View File
@@ -956,8 +956,6 @@ github.com/projectdiscovery/gologger v1.1.73 h1:WX2nzq29cL7HdvOHqIttacoid/5lMwap
github.com/projectdiscovery/gologger v1.1.73/go.mod h1:mJwODZcFDg70ihINpOvZevmBtgvpP8H9/l8Y+OPhZPY=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/govaluate v0.0.0-20260729205521-4c61bf19382e h1:0vaevfTiQeRCkX2xbY/mR38422kHAxGzpVV8FQJVfoc=
github.com/projectdiscovery/govaluate v0.0.0-20260729205521-4c61bf19382e/go.mod h1:Z3VjpUJV1pnXDwvAddA8aZ4e839SoeftB7CSdZntAVI=
github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2 h1:aSLbDvrJQPlp4HvojU+bTU5XV8xdAVCjoc5gX1Wfbw4=
github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2/go.mod h1:Z3VjpUJV1pnXDwvAddA8aZ4e839SoeftB7CSdZntAVI=
github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76 h1:AN70bbi6BBs7KpIM9w0LxygUN7uzT/oH+owDIQ+Fz/k=
@@ -1000,8 +998,8 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5 h1:AOspf7ScbwZr8A2YnjCFd4ZJAMSFqzfEaIJIbJPuu10=
github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350 h1:uXqpYa6XKV9dCvoti9PbSgIWIsBKPu5YKMywB9G4q0Q=
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
github.com/projectdiscovery/wappalyzergo v0.3.6 h1:1FXCMeCa1xVu2zaW+eeBcbVxtlMStqGOlKcMLyh2pzI=
github.com/projectdiscovery/wappalyzergo v0.3.6/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
+35 -1
View File
@@ -2,8 +2,12 @@ package runner
import (
"context"
"errors"
"sync"
"time"
"github.com/projectdiscovery/gologger"
llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
utilsllm "github.com/projectdiscovery/utils/llm"
@@ -28,6 +32,32 @@ func (a *llmMatcherClient) Complete(ctx context.Context, prompt string, asJSON b
})
}
// llmJudgeClient is an llmMatcherClient whose provider classifies natively, so
// llm matchers ask it through Judge instead of Complete.
type llmJudgeClient struct {
*llmMatcherClient
warnComplete sync.Once
}
func (a *llmJudgeClient) Judge(ctx context.Context, question string, labels, inputs []string) (llmclient.Verdict, error) {
verdict, err := a.client.Judge(ctx, utilsllm.JudgeRequest{Question: question, Labels: labels, Inputs: inputs})
if err != nil {
return llmclient.Verdict{}, err
}
return llmclient.Verdict{Label: verdict.Label, Confidence: verdict.Confidence, Model: verdict.Model}, nil
}
// Complete is only reached by llm extractors, which need free text a
// classifier cannot produce. Returning here spends no call budget and
// makes no provider request. Extractors turn the error into an empty
// result, and the warning says why.
func (a *llmJudgeClient) Complete(context.Context, string, bool) (string, error) {
a.warnComplete.Do(func() {
gologger.Warning().Msgf("llm provider %q cannot run llm extractors, they return nothing", utilsllm.ProviderJev)
})
return "", errors.New("llm provider cannot run extractors")
}
// configureLLM builds the scan's llm client from options. It returns nil unless
// -llm is set, so nothing reaches a model until the user opts in. The API key is
// read from the environment by the provider layer, never from options.
@@ -52,5 +82,9 @@ func configureLLM(options *types.Options) (llmclient.Client, error) {
return nil, err
}
return &llmMatcherClient{client: client, model: options.LLMModel}, nil
matcherClient := &llmMatcherClient{client: client, model: options.LLMModel}
if client.CanJudge() {
return &llmJudgeClient{llmMatcherClient: matcherClient}, nil
}
return matcherClient, nil
}
+65
View File
@@ -0,0 +1,65 @@
package runner
import (
"context"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestConfigureLLMExposesJudgeOnlyForClassifiers(t *testing.T) {
t.Setenv("LLM_API_KEY", "key")
client, err := configureLLM(&types.Options{EnableLLM: true, LLMProvider: "jev", LLMModel: "jev-latest"})
require.NoError(t, err)
require.Implements(t, (*llmclient.Judge)(nil), client)
client, err = configureLLM(&types.Options{EnableLLM: true, LLMProvider: "ollama", LLMModel: "llama3.2"})
require.NoError(t, err)
_, isJudge := client.(llmclient.Judge)
require.False(t, isJudge, "chat providers keep the Complete path")
}
func TestJevExtractorDoesNotSpendCallBudget(t *testing.T) {
t.Setenv("LLM_API_KEY", "key")
// ProxyFromEnvironment caches the first request in the process. Match the
// proxy the interactsh client test expects, and keep the local server direct.
for _, key := range []string{"HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "https_proxy", "no_proxy"} {
t.Setenv(key, "")
}
t.Setenv("HTTPS_PROXY", "http://127.0.0.1:9")
t.Setenv("NO_PROXY", "127.0.0.1,localhost")
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
_, _ = w.Write([]byte(`{"model":"jev-1.14.0","answers":{"q":{"type":"noul","noul":0.9}}}`))
}))
t.Cleanup(server.Close)
client, err := configureLLM(&types.Options{
EnableLLM: true,
LLMProvider: "jev",
LLMBaseURL: server.URL + "/v1",
LLMModel: "jev-latest",
LLMMaxCalls: 1,
})
require.NoError(t, err)
_, err = client.Complete(context.Background(), "extract the title", true)
require.ErrorContains(t, err, "cannot run extractors")
_, err = client.Complete(context.Background(), "extract another", true)
require.Error(t, err)
require.Zero(t, calls.Load())
judge := client.(llmclient.Judge)
verdict, err := judge.Judge(context.Background(), "login page?", nil, []string{"<form>"})
require.NoError(t, err)
require.Equal(t, "yes", verdict.Label)
require.EqualValues(t, 1, calls.Load())
}
+16
View File
@@ -21,6 +21,22 @@ type Client interface {
Complete(ctx context.Context, prompt string, asJSON bool) (string, error)
}
// Judge is implemented by clients whose provider classifies natively. Its
// confidence is a calibrated probability rather than a number the model
// reports about itself, and the inputs travel apart from the question, so
// matchers use it over Complete when it is available.
type Judge interface {
Judge(ctx context.Context, question string, labels, inputs []string) (Verdict, error)
}
// Verdict is the label a Judge chose. Model is the exact model version that
// answered, when the provider reports one.
type Verdict struct {
Label string
Confidence float64
Model string
}
// TruncateApproxTokens trims input to roughly maxTokens using the common
// 4-chars-per-token approximation. Exact counting is provider-specific; this
// only needs to keep a large body from blowing the context window.
+58 -17
View File
@@ -65,26 +65,21 @@ func (matcher *Matcher) MatchLLMWithAudit(corpus string, inputs []string, values
return false, nil, nil
}
prompt := matcher.buildLLMPrompt(matcher.truncate(corpus, inputs), values)
answer, err := client.Complete(context.Background(), prompt, true)
if err != nil {
responses := matcher.truncate(corpus, inputs)
var (
verdict llmVerdict
audit *LLMAudit
ok bool
)
if judge, isJudge := client.(llmclient.Judge); isJudge {
verdict, audit, ok = matcher.judgeLLM(judge, responses, values)
} else {
verdict, audit, ok = matcher.completeLLM(client, responses, values)
}
if !ok {
return false, nil, nil
}
var verdict llmVerdict
if err := json.Unmarshal([]byte(answer), &verdict); err != nil {
return false, nil, nil
}
audit := &LLMAudit{
PromptHash: hashPrompt(prompt),
Verdict: strings.TrimSpace(verdict.Verdict),
Confidence: verdict.Confidence,
}
if namer, ok := client.(modelNamer); ok {
audit.Model = namer.Model()
}
expect := matcher.Expect
if expect == "" {
expect = "yes"
@@ -107,9 +102,55 @@ func (matcher *Matcher) MatchLLMWithAudit(corpus string, inputs []string, values
return false, nil, audit
}
if verdict.Evidence == "" {
return true, nil, audit
}
return true, []string{verdict.Evidence}, audit
}
// completeLLM asks a chat model for the verdict as JSON.
func (matcher *Matcher) completeLLM(client LLMClient, inputs []string, values map[string]interface{}) (llmVerdict, *LLMAudit, bool) {
prompt := matcher.buildLLMPrompt(inputs, values)
answer, err := client.Complete(context.Background(), prompt, true)
if err != nil {
return llmVerdict{}, nil, false
}
var verdict llmVerdict
if err := json.Unmarshal([]byte(answer), &verdict); err != nil {
return llmVerdict{}, nil, false
}
audit := &LLMAudit{
PromptHash: hashPrompt(prompt),
Verdict: strings.TrimSpace(verdict.Verdict),
Confidence: verdict.Confidence,
}
if namer, ok := client.(modelNamer); ok {
audit.Model = namer.Model()
}
return verdict, audit, true
}
// judgeLLM asks a native classifier. It needs no prompt framing: the question
// and the responses are separate fields of the request, and the answer is a
// probability over the allowed verdicts, not text. It quotes no evidence.
func (matcher *Matcher) judgeLLM(judge llmclient.Judge, inputs []string, values map[string]interface{}) (llmVerdict, *LLMAudit, bool) {
question := interpolate(matcher.Prompt, values)
answer, err := judge.Judge(context.Background(), question, matcher.Options, inputs)
if err != nil {
return llmVerdict{}, nil, false
}
audit := &LLMAudit{
Model: answer.Model,
PromptHash: hashPrompt(strings.Join(append([]string{question, strings.Join(matcher.Options, ",")}, inputs...), "\x00")),
Verdict: answer.Label,
Confidence: answer.Confidence,
}
return llmVerdict{Verdict: answer.Label, Confidence: answer.Confidence}, audit, true
}
// buildLLMPrompt wraps the author's question with the output contract and the
// response under a random boundary (see FrameResponse).
//
+59
View File
@@ -6,6 +6,7 @@ import (
"regexp"
"testing"
llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
"github.com/stretchr/testify/require"
)
@@ -277,3 +278,61 @@ func TestMatchLLMWithoutInputsUsesPart(t *testing.T) {
require.Contains(t, stub.lastGot, "the body")
require.NotContains(t, stub.lastGot, "Response 1")
}
type stubJudge struct {
stubLLM
verdict llmclient.Verdict
err error
question string
labels []string
inputs []string
}
func (s *stubJudge) Judge(_ context.Context, question string, labels, inputs []string) (llmclient.Verdict, error) {
s.question, s.labels, s.inputs = question, labels, inputs
return s.verdict, s.err
}
func TestMatchLLMPrefersJudgeAndKeepsResponseOutOfQuestion(t *testing.T) {
m := llmMatcher("login page for {{Hostname}}?")
judge := &stubJudge{verdict: llmclient.Verdict{Label: "yes", Confidence: 0.97, Model: "jev-1.14.0"}}
m.SetLLMClient(judge)
ok, snips, audit := m.MatchLLMWithAudit("<form>", nil, map[string]interface{}{"Hostname": "example.com"})
require.True(t, ok)
require.Empty(t, snips, "a classifier quotes no evidence")
require.Zero(t, judge.calls, "Complete must not be used when the client can judge")
require.Equal(t, "login page for example.com?", judge.question)
require.Nil(t, judge.labels, "no options means a yes or no question")
require.Equal(t, []string{"<form>"}, judge.inputs)
require.Equal(t, "jev-1.14.0", audit.Model)
require.InDelta(t, 0.97, audit.Confidence, 1e-9)
}
func TestMatchLLMJudgeAppliesExpectOptionsAndConfidence(t *testing.T) {
m := llmMatcher("same access?")
m.Options = []string{"same", "different"}
m.Expect = "different"
m.MinConfidence = 0.9
judge := &stubJudge{verdict: llmclient.Verdict{Label: "different", Confidence: 0.8}}
m.SetLLMClient(judge)
ok, _, audit := m.MatchLLMWithAudit("", []string{"a", "b"}, nil)
require.False(t, ok, "below min-confidence")
require.NotNil(t, audit)
require.Equal(t, []string{"same", "different"}, judge.labels)
require.Equal(t, []string{"a", "b"}, judge.inputs)
judge.verdict.Confidence = 0.95
ok, _, _ = m.MatchLLMWithAudit("", []string{"a", "b"}, nil)
require.True(t, ok)
}
func TestMatchLLMJudgeFailsClosed(t *testing.T) {
m := llmMatcher("q")
m.SetLLMClient(&stubJudge{err: errors.New("boom")})
ok, _, audit := m.MatchLLMWithAudit("body", nil, nil)
require.False(t, ok)
require.Nil(t, audit)
}
@@ -0,0 +1,46 @@
package protocolstate
import (
"sync"
"testing"
"github.com/projectdiscovery/networkpolicy"
"github.com/stretchr/testify/require"
)
func TestIsHostAllowedRacesExcludeUpdate(t *testing.T) {
np, err := networkpolicy.New(networkpolicy.Options{DenyList: []string{"127.0.0.0/8"}})
require.NoError(t, err)
const executionID = "exclude-race"
entry := &Dialers{NetworkPolicy: np, ExcludeTargets: []string{"192.0.2.1"}}
err = dialers.Set(executionID, entry)
require.NoError(t, err)
t.Cleanup(func() { dialers.Delete(executionID) })
stop := make(chan struct{})
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
lists := [][]string{{"192.0.2.1"}, {"192.0.2.2", "192.0.2.3"}}
i := 0
for {
select {
case <-stop:
return
default:
}
entry.Lock()
entry.ExcludeTargets = lists[i%2]
entry.Unlock()
i++
}
}()
for i := 0; i < 2000; i++ {
_ = IsHostAllowed(executionID, "http://192.0.2.1")
}
close(stop)
wg.Wait()
}
@@ -138,11 +138,14 @@ func CanonicalizeHost(host string) string {
// IsHostAllowed checks if the host is allowed by network policy
func IsHostAllowed(executionId string, targetUrl string) bool {
dialers, ok := dialers.Get(executionId)
if !ok {
if !ok || dialers == nil {
return true
}
dialers.Lock()
np := dialers.NetworkPolicy
excludedTargets := append([]string(nil), dialers.ExcludeTargets...)
dialers.Unlock()
if np == nil {
// no policy present: deny rather than allow.
return false
@@ -174,7 +177,7 @@ func IsHostAllowed(executionId string, targetUrl string) bool {
canonHost := CanonicalizeHost(host)
// Check explicit ExcludeTargets against raw, host, and canonical host
for _, excluded := range dialers.ExcludeTargets {
for _, excluded := range excludedTargets {
ex := CanonicalizeHost(strings.TrimSpace(excluded))
if ex != "" {
if strings.EqualFold(canonHost, ex) || strings.EqualFold(host, ex) || strings.EqualFold(rawTarget, ex) {
@@ -115,7 +115,10 @@ func initDialers(options *types.Options) error {
if options.RestrictLocalNetworkAccess {
expandedDenyList = append(expandedDenyList, networkpolicy.DefaultIPv4DenylistRanges...)
expandedDenyList = append(expandedDenyList, networkpolicy.DefaultIPv6DenylistRanges...)
// the unspecified address connects to the local host
expandedDenyList = append(expandedDenyList, "::/128")
}
expandedDenyList = append(expandedDenyList, mappedDenyRanges(expandedDenyList)...)
npOptions := &networkpolicy.Options{
DenyList: expandedDenyList,
}
@@ -342,6 +345,27 @@ func Close(executionId string) {
}
}
// mappedDenyRanges returns the IPv4-mapped IPv6 form (::ffff:a.b.c.d) of each
// IPv4 address and range in denyList. The network policy checks a mapped
// address such as [::ffff:127.0.0.1] only against IPv6 ranges, but the dialer
// connects to the IPv4 host, so without these forms it passes -eh and -lna.
func mappedDenyRanges(denyList []string) []string {
var mapped []string
for _, entry := range denyList {
cidr := entry
if !strings.Contains(cidr, "/") {
cidr += "/32"
}
_, ipNet, err := net.ParseCIDR(cidr)
if err != nil || ipNet.IP.To4() == nil || strings.Contains(entry, ":") {
continue
}
ones, _ := ipNet.Mask.Size()
mapped = append(mapped, fmt.Sprintf("::ffff:%s/%d", ipNet.IP.To4(), 96+ones))
}
return mapped
}
// excludeHostRules turns an exclude entry into network policy deny rules.
// networkpolicy treats non-IP entries as unanchored regexes, so a plain
// "test.com" would also deny "latest.com". Plain hostnames match the host and
@@ -1,7 +1,12 @@
package protocolstate
import (
"context"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
@@ -27,3 +32,47 @@ func TestExcludeTargetsMatchExactHost(t *testing.T) {
require.True(t, np.Validate("testXcom"))
require.True(t, np.Validate("sub.regex.example"))
}
func TestMappedFormsDeniedByDialer(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
defer server.Close()
_, port, err := net.SplitHostPort(server.Listener.Addr().String())
require.NoError(t, err)
for name, set := range map[string]func(*types.Options){
"lna": func(o *types.Options) { o.RestrictLocalNetworkAccess = true },
"exclude-hosts": func(o *types.Options) { o.ExcludeTargets = []string{"127.0.0.1"} },
"exclude-cidr": func(o *types.Options) { o.ExcludeTargets = []string{"127.0.0.0/8"} },
} {
t.Run(name, func(t *testing.T) {
opts := types.DefaultOptions()
opts.ExecutionId = "test-mapped-" + name
set(opts)
require.NoError(t, Init(opts))
defer Close(opts.ExecutionId)
d := GetDialersWithId(opts.ExecutionId)
hosts := []string{"127.0.0.1", "::ffff:127.0.0.1", "::ffff:7f00:1", "0:0:0:0:0:ffff:127.0.0.1"}
if opts.RestrictLocalNetworkAccess {
hosts = append(hosts, "::")
}
for _, host := range hosts {
require.False(t, d.NetworkPolicy.Validate(host), host)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
conn, err := d.Fastdialer.Dial(ctx, "tcp", net.JoinHostPort(host, port))
cancel()
if conn != nil {
_ = conn.Close()
}
require.Error(t, err, host)
}
require.True(t, d.NetworkPolicy.Validate("::ffff:8.8.8.8"))
require.True(t, d.NetworkPolicy.Validate("8.8.8.8"))
})
}
}
func TestMappedDenyRanges(t *testing.T) {
require.Equal(t, []string{"::ffff:127.0.0.0/104", "::ffff:192.0.2.10/128", "::ffff:0.0.0.0/104"},
mappedDenyRanges([]string{"127.0.0.0/8", "192.0.2.10", "::1/128", "fc00::/7", `(?i)(^|\.)test\.com\.?$`, "0.0.0.0/8"}))
}
+28 -19
View File
@@ -26,33 +26,42 @@ var defaultResolvers = []string{
"8.8.4.4:53", // Google
}
// Init initializes the client pool implementation
// Init initializes the client pool implementation.
// Each resolver list gets its own client, so a later scan is not stuck with the first scan's resolvers.
func Init(options *types.Options) error {
m.Lock()
defer m.Unlock()
// Don't create clients if already created in the past.
if normalClient != nil {
return nil
}
clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]()
resolvers := defaultResolvers
if len(options.InternalResolversList) > 0 {
resolvers = options.InternalResolversList
}
var err error
normalClient, err = retryabledns.New(resolvers, 1)
client, err := cachedClient(resolversFor(options))
if err != nil {
return errors.Wrap(err, "could not create dns client")
}
m.Lock()
normalClient = client
m.Unlock()
return nil
}
func getNormalClient() *retryabledns.Client {
func resolversFor(options *types.Options) []string {
if options != nil && len(options.InternalResolversList) > 0 {
return options.InternalResolversList
}
return defaultResolvers
}
func cachedClient(resolvers []string) (*retryabledns.Client, error) {
m.Lock()
defer m.Unlock()
return normalClient
if clientPool == nil {
clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]()
}
key := strings.Join(resolvers, ",")
if client, ok := clientPool.Get(key); ok {
return client, nil
}
client, err := retryabledns.New(resolvers, 1)
if err != nil {
return nil, err
}
_ = clientPool.Set(key, client)
return client, nil
}
// Configuration contains the custom configuration options for a client
@@ -81,7 +90,7 @@ func (c *Configuration) Hash() string {
// Get creates or gets a client for the protocol based on custom configuration
func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) {
if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
return getNormalClient(), nil
return cachedClient(resolversFor(options))
}
hash := configuration.Hash()
if client, ok := clientPool.Get(hash); ok {
@@ -0,0 +1,35 @@
package dnsclientpool
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestClientsFollowResolverList(t *testing.T) {
previousClient := normalClient
previousPool := clientPool
t.Cleanup(func() {
normalClient = previousClient
clientPool = previousPool
})
normalClient = nil
clientPool = nil
firstOptions := &types.Options{InternalResolversList: []string{"192.0.2.1:53"}}
secondOptions := &types.Options{InternalResolversList: []string{"192.0.2.2:53"}}
require.NoError(t, Init(firstOptions))
require.NoError(t, Init(secondOptions))
first, err := Get(firstOptions, &Configuration{})
require.NoError(t, err)
second, err := Get(secondOptions, &Configuration{})
require.NoError(t, err)
require.NotSame(t, first, second)
again, err := Get(firstOptions, &Configuration{})
require.NoError(t, err)
require.Same(t, first, again)
}
+12 -1
View File
@@ -94,10 +94,21 @@ func NewTransportWrapper() func(http.RoundTripper) http.RoundTripper {
},
DisableWarningHeader: true,
}
return transport
return &unwrappingTransport{Transport: transport}
}
}
// unwrappingTransport lets code that must not use the cache, such as reporting
// clients that need TLS verification, reach the wrapped transport.
type unwrappingTransport struct {
*httpcache.Transport
}
// Unwrap returns the transport the cache wraps.
func (t *unwrappingTransport) Unwrap() http.RoundTripper {
return t.Transport.Transport
}
func openDiskCache(dir string, maxBytes int64) (httpcache.Cache, *leveldb.DB, error) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, nil, fmt.Errorf("create cache dir: %w", err)
@@ -1,6 +1,7 @@
package httpcache
import (
"crypto/tls"
"fmt"
"io"
"net/http"
@@ -10,6 +11,7 @@ import (
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
libcache "github.com/sandrolain/httpcache"
)
@@ -178,3 +180,29 @@ func TestDiskCacheRespectsSizeCap(t *testing.T) {
t.Fatal("stored a response over a 1-byte directory cap")
}
}
// Reporting clients send tokens, so tlsverify must reach the scan transport
// under the cache wrapper and turn certificate verification on.
func TestTransportWrapperUnwrapsForTLSVerification(t *testing.T) {
cacheMu.Lock()
saved := cache
cache = libcache.NewMemoryCache()
cacheMu.Unlock()
t.Cleanup(func() {
cacheMu.Lock()
cache = saved
cacheMu.Unlock()
})
base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
wrapped := NewTransportWrapper()(base)
u, ok := wrapped.(interface{ Unwrap() http.RoundTripper })
if !ok || u.Unwrap() != base {
t.Fatal("the cache wrapper does not unwrap to the scan transport")
}
got, ok := tlsverify.Client(&http.Client{Transport: wrapped}).Transport.(*http.Transport)
if !ok || got.TLSClientConfig.InsecureSkipVerify {
t.Fatal("tlsverify did not turn verification on under the cache wrapper")
}
}
@@ -166,6 +166,11 @@ func (t *connTrackingTransport) RoundTrip(req *http.Request) (*http.Response, er
return resp, err
}
// Unwrap returns the underlying transport.
func (t *connTrackingTransport) Unwrap() http.RoundTripper {
return t.base
}
func (t *connTrackingTransport) CloseIdleConnections() {
type closeIdler interface{ CloseIdleConnections() }
if ci, ok := t.base.(closeIdler); ok {
+13
View File
@@ -20,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/pdf"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
@@ -27,6 +28,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/errkit"
fileutil "github.com/projectdiscovery/utils/file"
"go.uber.org/multierr"
@@ -76,6 +78,17 @@ type IssueTrackerStats struct {
func New(options *Options, db string, doNotDedupe bool) (Client, error) {
client := &ReportingClient{options: options}
// The caller can pass the scan client, which skips TLS verification;
// trackers and exporters send tokens and passwords, so they get a copy
// that verifies certificates.
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
if hc := tlsverify.Client(options.HttpClient.HTTPClient); hc != options.HttpClient.HTTPClient {
opts := retryablehttp.DefaultOptionsSingle
opts.Timeout = hc.Timeout // keep the caller's timeout, also when it is 0
options.HttpClient = retryablehttp.NewWithHTTPClient(hc, opts)
}
}
if options.GitHub != nil {
options.GitHub.HttpClient = options.HttpClient
options.GitHub.OmitRaw = options.OmitRaw
+98
View File
@@ -0,0 +1,98 @@
package reporting_test
import (
"io"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
// connectProxy tunnels every CONNECT to target, whatever host the client asked
// for, so no name is resolved and nothing leaves 127.0.0.1.
func connectProxy(t *testing.T, target string) *httptest.Server {
p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodConnect {
http.Error(w, "connect only", http.StatusMethodNotAllowed)
return
}
up, err := net.Dial("tcp", target)
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
c, _, err := w.(http.Hijacker).Hijack()
if err != nil {
_ = up.Close()
return
}
_, _ = c.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n"))
go func() { _, _ = io.Copy(up, c); _ = up.Close() }()
go func() { _, _ = io.Copy(c, up); _ = c.Close() }()
}))
t.Cleanup(p.Close)
return p
}
// With -pi and a proxy, the runner gives reporting the scan client, which skips
// TLS verification. No tracker or exporter may send its secret to a server with
// an untrusted certificate.
func TestNewVerifiesTLSWithScanClient(t *testing.T) {
var hits atomic.Int32
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.WriteHeader(http.StatusUnauthorized)
}))
t.Cleanup(srv.Close)
proxy := connectProxy(t, srv.Listener.Addr().String())
opts := types.DefaultOptions()
opts.SetExecutionID("reporting-tls-test")
opts.ProxyInternal = true
opts.AliveHttpProxy = proxy.URL
require.NoError(t, protocolstate.Init(opts))
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
scanClient, err := httpclientpool.Get(opts, &httpclientpool.Configuration{}, "")
require.NoError(t, err)
event := &output.ResultEvent{
TemplateID: "test-template",
Host: "https://target.example",
Matched: "https://target.example/",
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
}
configs := map[string]*reporting.Options{
"github": {GitHub: &github.Options{BaseURL: "https://github.victim.example/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token"}},
"gitlab": {GitLab: &gitlab.Options{BaseURL: "https://gitlab.victim.example/", Username: "u", Token: "test-token", ProjectName: "1"}},
"gitea": {Gitea: &gitea.Options{BaseURL: "https://gitea.victim.example/", Token: "test-token", ProjectOwner: "o", ProjectName: "r"}},
"jira": {Jira: &jira.Options{Cloud: true, URL: "https://jira.victim.example", AccountID: "a", Email: "[email protected]", Token: "test-token", ProjectName: "P", IssueType: "Bug"}},
"elasticsearch": {ElasticsearchExporter: &es.Options{Host: "es.victim.example", Port: 443, SSL: true, Username: "u", Password: "test-password", IndexName: "n"}},
"splunk": {SplunkExporter: &splunk.Options{Host: "splunk.victim.example", Port: 443, SSL: true, Token: "test-token", IndexName: "n"}},
}
for name, ro := range configs {
ro.HttpClient = scanClient
ro.ExecutionId = opts.ExecutionId
client, err := reporting.New(ro, "", true)
if err == nil {
_ = client.CreateIssue(event)
client.Close()
}
require.Zero(t, hits.Load(), "%s sent a request to a server with an untrusted certificate", name)
}
}
+68
View File
@@ -0,0 +1,68 @@
// Package tlsverify gives trackers and exporters an HTTP client that verifies
// TLS certificates, since they send tokens and passwords.
package tlsverify
import (
"crypto/tls"
"errors"
"net/http"
)
// errTLSDialerOnly is returned when a transport dials HTTPS only through a
// custom TLS dialer: dropping that dialer would also drop its route or address
// policy, so the copy refuses requests instead.
var errTLSDialerOnly = errors.New("tlsverify: the transport has a custom TLS dialer and no DialContext, so certificate verification cannot be enforced")
type refusingTransport struct{}
func (refusingTransport) RoundTrip(*http.Request) (*http.Response, error) {
return nil, errTLSDialerOnly
}
// Client returns c, or a copy of c whose transport verifies TLS certificates
// when c can skip verification (the scan client can). The copy keeps the proxy
// and the dialer, so the address policy still applies, but it drops a custom
// TLS dialer, because that dialer can use the scan TLS config. So the token
// never goes over an unverified connection.
//
// A transport with an Unwrap() http.RoundTripper method declares that its
// RoundTrip only observes or caches requests and enforces no policy; the copy
// uses the unwrapped transport. Other transports are returned unchanged.
func Client(c *http.Client) *http.Client {
rt := c.Transport
for {
u, ok := rt.(interface{ Unwrap() http.RoundTripper })
if !ok {
break
}
rt = u.Unwrap()
}
t, ok := rt.(*http.Transport)
if !ok {
return c
}
insecure := t.TLSClientConfig != nil && t.TLSClientConfig.InsecureSkipVerify
tlsDialer := t.DialTLSContext != nil || t.DialTLS != nil //nolint:staticcheck // DialTLS is deprecated but still used
if !insecure && !tlsDialer {
return c
}
copied := *c
if tlsDialer && t.DialContext == nil && t.Dial == nil { //nolint:staticcheck // Dial is deprecated but still used
copied.Transport = refusingTransport{}
return &copied
}
verifying := t.Clone()
verifying.DialTLSContext = nil
verifying.DialTLS = nil //nolint:staticcheck // DialTLS is deprecated but still used
if insecure {
// Keep the caller's roots, client certificates and checks; the server
// name of the scan config belongs to a scan target, not to this host.
verifying.TLSClientConfig.InsecureSkipVerify = false
verifying.TLSClientConfig.ServerName = ""
if verifying.TLSClientConfig.MinVersion < tls.VersionTLS12 {
verifying.TLSClientConfig.MinVersion = tls.VersionTLS12
}
}
copied.Transport = verifying
return &copied
}
+67
View File
@@ -0,0 +1,67 @@
package tlsverify
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"net"
"net/http"
"testing"
"github.com/stretchr/testify/require"
)
// The scan client can skip TLS verification; the tracker must not send the
// token over such a connection, and must not change the caller's client.
func TestClientTurnsOnTLSVerification(t *testing.T) {
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
caller := &http.Client{Transport: insecure}
got := Client(caller)
gotTransport, ok := got.Transport.(*http.Transport)
require.True(t, ok)
require.False(t, gotTransport.TLSClientConfig.InsecureSkipVerify)
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's transport changed")
secure := &http.Client{Transport: &http.Transport{}}
require.Same(t, secure, Client(secure))
}
func TestClientKeepsCallerTLSSettings(t *testing.T) {
pinned := func(tls.ConnectionState) error { return nil }
insecure := &http.Transport{
DialTLSContext: func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("scan dialer") },
DialContext: (&net.Dialer{}).DialContext,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
ServerName: "scan-target.example",
MinVersion: tls.VersionTLS10,
RootCAs: x509.NewCertPool(),
VerifyConnection: pinned,
},
}
got := Client(&http.Client{Transport: insecure}).Transport.(*http.Transport)
require.Nil(t, got.DialTLSContext)
require.NotNil(t, got.DialContext)
cfg := got.TLSClientConfig
require.False(t, cfg.InsecureSkipVerify)
require.Empty(t, cfg.ServerName)
require.Equal(t, uint16(tls.VersionTLS12), cfg.MinVersion)
require.Same(t, insecure.TLSClientConfig.RootCAs, cfg.RootCAs)
require.NotNil(t, cfg.VerifyConnection)
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's config changed")
}
// A custom TLS dialer may carry the only route or address policy, so a
// transport without DialContext cannot be rebuilt and must refuse requests.
func TestClientRefusesTLSDialerWithoutDialContext(t *testing.T) {
var dialed bool
tlsOnly := &http.Transport{DialTLSContext: func(context.Context, string, string) (net.Conn, error) {
dialed = true
return nil, errors.New("policy dialer")
}}
_, err := Client(&http.Client{Transport: tlsOnly}).Get("https://tracker.example/")
require.ErrorIs(t, err, errTLSDialerOnly)
require.False(t, dialed)
}
+6 -7
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
"net/http"
"strconv"
"github.com/google/go-github/v92/github"
@@ -12,6 +11,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
"github.com/projectdiscovery/retryablehttp-go"
"golang.org/x/oauth2"
@@ -55,17 +55,16 @@ type Options struct {
// New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) {
ctx := context.Background()
// Build the oauth2 client on top of the caller's client, so its transport
// (proxy, address policy) and redirect rules apply to every request.
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
ctx = context.WithValue(ctx, oauth2.HTTPClient, tlsverify.Client(options.HttpClient.HTTPClient))
}
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: options.Token},
)
tc := oauth2.NewClient(ctx, ts)
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
if tcTransport, ok := tc.Transport.(*http.Transport); ok {
tcTransport.Proxy = options.HttpClient.HTTPClient.Transport.(*http.Transport).Proxy
}
}
clientOptions := []github.ClientOptionsFunc{github.WithHTTPClient(tc)}
if options.BaseURL != "" {
clientOptions = append(clientOptions, github.WithURLs(&options.BaseURL, nil))
@@ -0,0 +1,126 @@
package github
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/stretchr/testify/require"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func testEvent() *output.ResultEvent {
return &output.ResultEvent{
TemplateID: "test-template",
Host: "https://target.example",
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
}
}
func clientWithTransport(rt http.RoundTripper) *retryablehttp.Client {
c := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
c.HTTPClient = &http.Client{Transport: rt}
return c
}
// The tracker must send every request through the caller's client, so a
// caller's address policy also covers the GitHub API base URL.
func TestNewUsesCallerHTTPClient(t *testing.T) {
var serverHits atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
serverHits.Add(1)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"number":1,"html_url":"https://github.example/o/r/issues/1"}`))
}))
defer srv.Close()
t.Run("requests go through the caller's transport", func(t *testing.T) {
var seen atomic.Int32
rt := roundTripFunc(func(r *http.Request) (*http.Response, error) {
seen.Add(1)
require.Equal(t, "Bearer test-token", r.Header.Get("Authorization"))
return http.DefaultTransport.RoundTrip(r)
})
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.NoError(t, err)
require.Positive(t, seen.Load())
})
t.Run("a refusing transport stops the request", func(t *testing.T) {
before := serverHits.Load()
denied := errors.New("denied by address policy")
rt := roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, denied })
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.ErrorContains(t, err, "denied by address policy")
require.Equal(t, before, serverHits.Load(), "the request reached the server")
})
}
func TestNewRefusesUntrustedCertificateFromInsecureCallerClient(t *testing.T) {
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"number":1}`))
}))
defer srv.Close()
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(insecure)})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.ErrorContains(t, err, "certificate")
}
type wrappingTransport struct{ base http.RoundTripper }
func (w *wrappingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
return w.base.RoundTrip(r)
}
func (w *wrappingTransport) Unwrap() http.RoundTripper { return w.base }
// The scan client wraps its transport and, with a SOCKS proxy, does TLS in its
// own dialer with the scan TLS config, which skips verification.
func TestNewRefusesUntrustedCertificateFromInsecureTLSDialer(t *testing.T) {
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"number":1}`))
}))
defer srv.Close()
var dialed atomic.Int32
var dialer net.Dialer
base := &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
dialed.Add(1)
return dialer.DialContext(ctx, network, addr)
},
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
conn, err := dialer.DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
return tls.Client(conn, &tls.Config{InsecureSkipVerify: true}), nil
},
}
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(&wrappingTransport{base: base})})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.ErrorContains(t, err, "certificate")
require.Positive(t, dialed.Load(), "the caller's dialer was not used")
}