mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-13 05:41:25 -10:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
582adc6028 | ||
|
|
005fc41c97 | ||
|
|
b2ea1489fd | ||
|
|
e6ce143015 | ||
|
|
894f90dd11 | ||
|
|
a351263c86 |
No files matched your search
@@ -167,6 +167,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -182,6 +182,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -165,6 +165,7 @@ TEMPLATES:
|
||||
-wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file)
|
||||
-validate validate the passed templates to nuclei
|
||||
-plan string write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target
|
||||
-run-plan string run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes
|
||||
-nss, -no-strict-syntax disable strict syntax check on templates
|
||||
-td, -template-display displays the templates content
|
||||
-tl list all templates matching current filters
|
||||
|
||||
@@ -314,6 +314,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
|
||||
flagSet.StringVar(&options.PlanFile, "plan", "", "write the work a scan would do with its estimated cost to a jsonl file, then exit without sending requests to any target"),
|
||||
flagSet.StringVar(&options.RunPlanFile, "run-plan", "", "run only the entries of a -plan file, logging finished ones to <file>.done so running it again resumes"),
|
||||
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
|
||||
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
|
||||
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all templates matching current filters"),
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
|
||||
@@ -23,20 +26,55 @@ type Concurrency struct {
|
||||
type Server struct {
|
||||
addr string
|
||||
config *types.Options
|
||||
// token is a random Bearer token that must be presented by callers.
|
||||
token string
|
||||
}
|
||||
|
||||
// New creates a new instance of Server.
|
||||
// New creates a new instance of Server. A random 32-byte hex Bearer token is
|
||||
// generated and stored; the caller should log it so operators can authenticate.
|
||||
func New(addr string, config *types.Options) *Server {
|
||||
return &Server{
|
||||
addr: addr,
|
||||
config: config,
|
||||
token: randomToken(),
|
||||
}
|
||||
}
|
||||
|
||||
// Token returns the Bearer token that callers must supply in an
|
||||
// "Authorization: Bearer <token>" header.
|
||||
func (s *Server) Token() string {
|
||||
return s.token
|
||||
}
|
||||
|
||||
// randomToken generates a 32-byte (64 hex char) cryptographically random token.
|
||||
func randomToken() string {
|
||||
b := make([]byte, 32)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
panic("httpapi: could not generate random token: " + err.Error())
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// authenticate rejects requests whose Authorization header does not carry the
|
||||
// server's Bearer token. Returns true when the request is authorized.
|
||||
func (s *Server) authenticate(w http.ResponseWriter, r *http.Request) bool {
|
||||
if s.token == "" {
|
||||
return true // token-less mode: explicitly disabled
|
||||
}
|
||||
auth := r.Header.Get("Authorization")
|
||||
want := "Bearer " + s.token
|
||||
if !strings.EqualFold(auth, want) {
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Start initializes the server and its routes, then starts listening on the specified address.
|
||||
func (s *Server) Start() error {
|
||||
http.HandleFunc("/api/concurrency", s.handleConcurrency)
|
||||
if err := http.ListenAndServe(s.addr, nil); err != nil {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/api/concurrency", s.handleConcurrency)
|
||||
if err := http.ListenAndServe(s.addr, mux); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@@ -44,6 +82,9 @@ func (s *Server) Start() error {
|
||||
|
||||
// handleConcurrency routes the request based on its method to the appropriate handler.
|
||||
func (s *Server) handleConcurrency(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.authenticate(w, r) {
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
s.getSettings(w, r)
|
||||
@@ -54,7 +95,7 @@ func (s *Server) handleConcurrency(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// GetSettings handles GET requests and returns the current concurrency settings
|
||||
// getSettings handles GET requests and returns the current concurrency settings.
|
||||
func (s *Server) getSettings(w http.ResponseWriter, _ *http.Request) {
|
||||
concurrencySettings := Concurrency{
|
||||
BulkSize: s.config.BulkSize,
|
||||
@@ -72,7 +113,7 @@ func (s *Server) getSettings(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateSettings handles PUT requests to update the concurrency settings
|
||||
// updateSettings handles PUT requests to update the concurrency settings.
|
||||
func (s *Server) updateSettings(w http.ResponseWriter, r *http.Request) {
|
||||
var newSettings Concurrency
|
||||
if err := json.NewDecoder(r.Body).Decode(&newSettings); err != nil {
|
||||
@@ -105,7 +146,7 @@ func (s *Server) updateSettings(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
if newSettings.JavascriptConcurrency > 0 {
|
||||
compiler.PoolingJsVmConcurrency = newSettings.JavascriptConcurrency
|
||||
s.config.JsConcurrency = newSettings.JavascriptConcurrency // no-op on speed change
|
||||
s.config.JsConcurrency = newSettings.JavascriptConcurrency
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package httpapi
|
||||
|
||||
// Tests that the /api/concurrency endpoint requires a valid Bearer token and
|
||||
// rejects unauthenticated requests with 401 Unauthorized.
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newTestServer() *Server {
|
||||
return New("127.0.0.1:0", &types.Options{
|
||||
BulkSize: 10,
|
||||
TemplateThreads: 5,
|
||||
RateLimit: 150,
|
||||
RateLimitDuration: time.Second,
|
||||
})
|
||||
}
|
||||
|
||||
// TestGetWithoutTokenReturns401 verifies that an unauthenticated GET is rejected.
|
||||
func TestGetWithoutTokenReturns401(t *testing.T) {
|
||||
s := newTestServer()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/concurrency", nil)
|
||||
w := httptest.NewRecorder()
|
||||
s.handleConcurrency(w, req)
|
||||
require.Equal(t, http.StatusUnauthorized, w.Code,
|
||||
"GET /api/concurrency without token must return 401")
|
||||
}
|
||||
|
||||
// TestPutWithoutTokenReturns401 verifies that an unauthenticated PUT is rejected.
|
||||
func TestPutWithoutTokenReturns401(t *testing.T) {
|
||||
s := newTestServer()
|
||||
req := httptest.NewRequest(http.MethodPut, "/api/concurrency",
|
||||
strings.NewReader(`{"bulk_size":999}`))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleConcurrency(w, req)
|
||||
require.Equal(t, http.StatusUnauthorized, w.Code,
|
||||
"PUT /api/concurrency without token must return 401")
|
||||
}
|
||||
|
||||
// TestGetWithWrongTokenReturns401 verifies that a wrong token is rejected.
|
||||
func TestGetWithWrongTokenReturns401(t *testing.T) {
|
||||
s := newTestServer()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/concurrency", nil)
|
||||
req.Header.Set("Authorization", "Bearer wrong-token")
|
||||
w := httptest.NewRecorder()
|
||||
s.handleConcurrency(w, req)
|
||||
require.Equal(t, http.StatusUnauthorized, w.Code,
|
||||
"GET /api/concurrency with wrong token must return 401")
|
||||
}
|
||||
|
||||
// TestGetWithCorrectTokenReturns200 verifies that a correct Bearer token allows access.
|
||||
func TestGetWithCorrectTokenReturns200(t *testing.T) {
|
||||
s := newTestServer()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/concurrency", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+s.Token())
|
||||
w := httptest.NewRecorder()
|
||||
s.handleConcurrency(w, req)
|
||||
require.Equal(t, http.StatusOK, w.Code,
|
||||
"GET /api/concurrency with valid token must return 200")
|
||||
require.Contains(t, w.Header().Get("Content-Type"), "application/json")
|
||||
}
|
||||
|
||||
// TestPutWithCorrectTokenModifiesSettings verifies a valid PUT is applied.
|
||||
func TestPutWithCorrectTokenModifiesSettings(t *testing.T) {
|
||||
s := newTestServer()
|
||||
req := httptest.NewRequest(http.MethodPut, "/api/concurrency",
|
||||
strings.NewReader(`{"bulk_size":42}`))
|
||||
req.Header.Set("Authorization", "Bearer "+s.Token())
|
||||
w := httptest.NewRecorder()
|
||||
s.handleConcurrency(w, req)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
require.Equal(t, 42, s.config.BulkSize)
|
||||
}
|
||||
|
||||
// TestTokenIsNotEmpty verifies that every Server is initialised with a non-empty token.
|
||||
func TestTokenIsNotEmpty(t *testing.T) {
|
||||
s := newTestServer()
|
||||
require.NotEmpty(t, s.Token(), "API server token must not be empty")
|
||||
}
|
||||
|
||||
// TestTokensAreUnique verifies that two servers get different tokens.
|
||||
func TestTokensAreUnique(t *testing.T) {
|
||||
s1 := newTestServer()
|
||||
s2 := newTestServer()
|
||||
require.NotEqual(t, s1.Token(), s2.Token(),
|
||||
"Each server instance must have a unique token")
|
||||
}
|
||||
@@ -558,18 +558,31 @@ func getBoolEnvValue(key string) bool {
|
||||
// validatePlanOptions rejects options that change the templates or targets
|
||||
// only once scanning starts, which a plan written beforehand cannot show.
|
||||
func validatePlanOptions(options *types.Options) error {
|
||||
if options.PlanFile == "" {
|
||||
if options.RunPlanFile != "" && options.PlanFile != "" {
|
||||
return errors.New("-plan cannot be used with -run-plan")
|
||||
}
|
||||
if options.RunPlanFile != "" && options.Resume != "" {
|
||||
return errors.New("-run-plan cannot be used with -resume, the plan's done log resumes it")
|
||||
}
|
||||
if options.PlanFile == "" && options.RunPlanFile == "" {
|
||||
return nil
|
||||
}
|
||||
// -automatic-scan builds its own engine, so a plan set on the main engine
|
||||
// would never be consulted. The other flags change the targets only after
|
||||
// a plan has been written.
|
||||
flag := "-plan"
|
||||
if options.PlanFile == "" {
|
||||
flag = "-run-plan"
|
||||
}
|
||||
switch {
|
||||
case options.AutomaticScan:
|
||||
return errors.New("-plan cannot be used with -automatic-scan, which picks templates while scanning")
|
||||
return fmt.Errorf("%s cannot be used with -automatic-scan, which picks templates while scanning", flag)
|
||||
case options.Uncover || len(options.UncoverQuery) > 0:
|
||||
return errors.New("-plan cannot be used with -uncover, which finds targets over the network")
|
||||
return fmt.Errorf("%s cannot be used with -uncover, which finds targets over the network", flag)
|
||||
case options.PreflightPortScan:
|
||||
return errors.New("-plan cannot be used with -preflight-portscan, which drops targets after scanning their ports")
|
||||
return fmt.Errorf("%s cannot be used with -preflight-portscan, which drops targets after scanning their ports", flag)
|
||||
case options.DASTServer || options.DASTProxy:
|
||||
return errors.New("-plan cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan")
|
||||
return fmt.Errorf("%s cannot be used with -dast-server or -dast-proxy, which start a server instead of writing a plan", flag)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -23,4 +23,19 @@ func TestPlanRejectsOptionsThatChangeTargetsLater(t *testing.T) {
|
||||
require.NoError(t, validatePlanOptions(&types.Options{Uncover: true}), "only -plan restricts them")
|
||||
require.NoError(t, validatePlanOptions(&types.Options{UncoverQuery: goflags.StringSlice{"example"}}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{DASTServer: true}))
|
||||
|
||||
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", PlanFile: "q"}))
|
||||
require.Error(t, validatePlanOptions(&types.Options{RunPlanFile: "p", Resume: "r"}))
|
||||
require.NoError(t, validatePlanOptions(&types.Options{RunPlanFile: "p"}))
|
||||
|
||||
for name, options := range map[string]*types.Options{
|
||||
"run automatic scan": {RunPlanFile: "p", AutomaticScan: true},
|
||||
"run uncover": {RunPlanFile: "p", Uncover: true},
|
||||
"run uncover query": {RunPlanFile: "p", UncoverQuery: goflags.StringSlice{"example"}},
|
||||
"run preflight": {RunPlanFile: "p", PreflightPortScan: true},
|
||||
"run dast server": {RunPlanFile: "p", DASTServer: true},
|
||||
"run dast proxy": {RunPlanFile: "p", DASTProxy: true},
|
||||
} {
|
||||
require.Error(t, validatePlanOptions(options), name)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
|
||||
)
|
||||
|
||||
// planRun limits a scan to the entries of a plan file and appends the id of
|
||||
// each finished entry to the plan's done log, so running the plan again skips
|
||||
// them. Ids are derived from what an entry runs, not from its position, so the
|
||||
// log still applies after the plan is written again for changed targets or
|
||||
// templates.
|
||||
type planRun struct {
|
||||
mu sync.Mutex
|
||||
entries map[uint64]plannedEntry
|
||||
// done holds finished origin groups by the id of their first entry; an
|
||||
// entry outside a group is its own group
|
||||
done map[uint64]struct{}
|
||||
log *os.File
|
||||
logErr error
|
||||
// resumed counts the entries finished before this run
|
||||
resumed int
|
||||
}
|
||||
|
||||
type plannedEntry struct {
|
||||
group uint64
|
||||
seen bool
|
||||
}
|
||||
|
||||
func planDoneLog(planFile string) string {
|
||||
return planFile + ".done"
|
||||
}
|
||||
|
||||
func openPlanRun(planFile string) (*planRun, error) {
|
||||
run := &planRun{entries: make(map[uint64]plannedEntry), done: make(map[uint64]struct{})}
|
||||
if err := run.readPlan(planFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := run.readDoneLog(planDoneLog(planFile)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log, err := os.OpenFile(planDoneLog(planFile), os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// an interrupted write leaves a partial line, which the next id must not
|
||||
// extend
|
||||
if info, err := log.Stat(); err == nil && info.Size() > 0 {
|
||||
last := make([]byte, 1)
|
||||
if _, err := log.ReadAt(last, info.Size()-1); err == nil && last[0] != '\n' {
|
||||
_, _ = log.WriteString("\n")
|
||||
}
|
||||
}
|
||||
run.log = log
|
||||
return run, nil
|
||||
}
|
||||
|
||||
func (run *planRun) readPlan(planFile string) error {
|
||||
file, err := os.Open(planFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
reader := bufio.NewReader(file)
|
||||
for number := 1; ; number++ {
|
||||
line, readErr := reader.ReadBytes('\n')
|
||||
if line = bytes.TrimSpace(line); len(line) > 0 {
|
||||
var entry struct {
|
||||
ID string `json:"id"`
|
||||
AlternativeTo string `json:"alternative-to"`
|
||||
}
|
||||
if err := json.Unmarshal(line, &entry); err != nil {
|
||||
return fmt.Errorf("%s:%d: %w", planFile, number, err)
|
||||
}
|
||||
// the summary line has no id
|
||||
if entry.ID != "" {
|
||||
id, ok := parsePlanID(entry.ID)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s:%d: invalid id %q", planFile, number, entry.ID)
|
||||
}
|
||||
group := id
|
||||
if entry.AlternativeTo != "" {
|
||||
if group, ok = parsePlanID(entry.AlternativeTo); !ok {
|
||||
return fmt.Errorf("%s:%d: invalid alternative-to %q", planFile, number, entry.AlternativeTo)
|
||||
}
|
||||
}
|
||||
run.entries[id] = plannedEntry{group: group}
|
||||
}
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (run *planRun) readDoneLog(path string) error {
|
||||
file, err := os.Open(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
// a partial line from an interrupted write does not parse and is
|
||||
// run again
|
||||
id, ok := parsePlanID(scanner.Text())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if entry, planned := run.entries[id]; planned {
|
||||
run.done[entry.group] = struct{}{}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range run.entries {
|
||||
if _, done := run.done[entry.group]; done {
|
||||
run.resumed++
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parsePlanID keeps ids as integers, which a plan of millions of entries
|
||||
// holds in a fraction of the memory of strings.
|
||||
func parsePlanID(id string) (uint64, bool) {
|
||||
if len(id) != 16 {
|
||||
return 0, false
|
||||
}
|
||||
value, err := strconv.ParseUint(id, 16, 64)
|
||||
return value, err == nil
|
||||
}
|
||||
|
||||
func (run *planRun) Skip(id string) bool {
|
||||
key, ok := parsePlanID(id)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
run.mu.Lock()
|
||||
defer run.mu.Unlock()
|
||||
entry, planned := run.entries[key]
|
||||
if !planned {
|
||||
return true
|
||||
}
|
||||
if !entry.seen {
|
||||
entry.seen = true
|
||||
run.entries[key] = entry
|
||||
}
|
||||
_, done := run.done[entry.group]
|
||||
return done
|
||||
}
|
||||
|
||||
func (run *planRun) Finished(id string) {
|
||||
key, ok := parsePlanID(id)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
run.mu.Lock()
|
||||
defer run.mu.Unlock()
|
||||
entry, planned := run.entries[key]
|
||||
if !planned {
|
||||
return
|
||||
}
|
||||
run.done[entry.group] = struct{}{}
|
||||
// written straight to the file: ctrl+c exits without running defers
|
||||
if _, err := run.log.WriteString(id + "\n"); err != nil && run.logErr == nil {
|
||||
run.logErr = err
|
||||
}
|
||||
}
|
||||
|
||||
// unreached counts unfinished entries the scan did not get to: those of a
|
||||
// host it found unresponsive, and those it does not have because it runs
|
||||
// with other templates, targets or options than the plan.
|
||||
func (run *planRun) unreached() int {
|
||||
run.mu.Lock()
|
||||
defer run.mu.Unlock()
|
||||
count := 0
|
||||
for _, entry := range run.entries {
|
||||
if _, done := run.done[entry.group]; !done && !entry.seen {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (run *planRun) close() error {
|
||||
if err := run.log.Close(); err != nil && run.logErr == nil {
|
||||
run.logErr = err
|
||||
}
|
||||
return run.logErr
|
||||
}
|
||||
|
||||
// finishPlanRun closes the done log and reports the entries left to run.
|
||||
func (r *Runner) finishPlanRun(run *planRun) {
|
||||
if err := run.close(); err != nil {
|
||||
r.Logger.Error().Msgf("Could not write %s: %s", planDoneLog(r.options.RunPlanFile), err)
|
||||
}
|
||||
if unreached := run.unreached(); unreached > 0 {
|
||||
r.Logger.Info().Msgf("%d plan entries did not run; running the plan again retries them. Entries run only when the scan has the templates, targets and options the plan was written with", unreached)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanRunResumesFromItsDoneLog(t *testing.T) {
|
||||
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
|
||||
require.NoError(t, os.WriteFile(planFile, []byte(`{"id":"00000000000000a1","target":"https://a.example/x","templates":["root"]}
|
||||
{"id":"00000000000000a2","target":"https://a.example/y","templates":["root"],"alternative-to":"00000000000000a1"}
|
||||
{"id":"00000000000000b1","target":"https://b.example","templates":["generic"]}
|
||||
{"id":"00000000000000c1","target":"https://c.example","templates":["generic"]}
|
||||
{"summary":{"entries":3,"alternatives":1}}
|
||||
`), 0o644))
|
||||
// the last line was cut off by an interrupted write
|
||||
require.NoError(t, os.WriteFile(planDoneLog(planFile), []byte("00000000000000b1\n00000000"), 0o644))
|
||||
|
||||
run, err := openPlanRun(planFile)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, run.entries, 4)
|
||||
require.Equal(t, 1, run.resumed)
|
||||
|
||||
require.True(t, run.Skip("00000000000000b1"), "finished in the previous run")
|
||||
require.True(t, run.Skip("00000000000000ff"), "not in the plan")
|
||||
require.False(t, run.Skip("00000000000000a1"))
|
||||
run.Finished("00000000000000a1")
|
||||
require.True(t, run.Skip("00000000000000a2"), "its origin is covered")
|
||||
require.Equal(t, 1, run.unreached(), "c1 never ran")
|
||||
require.NoError(t, run.close())
|
||||
|
||||
log, err := os.ReadFile(planDoneLog(planFile))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "00000000000000b1\n00000000\n00000000000000a1\n", string(log), "the partial line is not extended")
|
||||
|
||||
again, err := openPlanRun(planFile)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 3, again.resumed, "b1, and a1 with its alternative")
|
||||
require.True(t, again.Skip("00000000000000a2"), "a finished origin stays covered")
|
||||
require.False(t, again.Skip("00000000000000c1"))
|
||||
require.NoError(t, again.close())
|
||||
}
|
||||
|
||||
func TestPlanRunRejectsAMalformedPlan(t *testing.T) {
|
||||
planFile := filepath.Join(t.TempDir(), "plan.jsonl")
|
||||
require.NoError(t, os.WriteFile(planFile, []byte("{\"id\":\"00000000000000a1\"}\nnot json\n"), 0o644))
|
||||
_, err := openPlanRun(planFile)
|
||||
require.ErrorContains(t, err, "plan.jsonl:2")
|
||||
}
|
||||
@@ -855,6 +855,15 @@ func (r *Runner) RunEnumeration() error {
|
||||
}
|
||||
}()
|
||||
}
|
||||
if r.options.RunPlanFile != "" {
|
||||
run, err := openPlanRun(r.options.RunPlanFile)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not read plan")
|
||||
}
|
||||
r.Logger.Info().Msgf("Running %d plan entries from %s, %d already finished", len(run.entries), r.options.RunPlanFile, run.resumed)
|
||||
executorEngine.SetPlanRun(run)
|
||||
defer r.finishPlanRun(run)
|
||||
}
|
||||
// TODO: remove below functions after v3 or update warning messages
|
||||
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ type Engine struct {
|
||||
executerOpts *protocols.ExecutorOptions
|
||||
Callback func(*output.ResultEvent) // Executed on results
|
||||
templateExecutionCallback TemplateExecutionCallback
|
||||
planRun PlanRun
|
||||
Logger *gologger.Logger
|
||||
}
|
||||
|
||||
|
||||
+17
-1
@@ -36,9 +36,13 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
|
||||
if sharedBudget {
|
||||
defer e.options.ReleaseTemplateThread()
|
||||
}
|
||||
defer e.tasksDone(1)
|
||||
planned, skip := e.claimPlanned(template, &contextargs.MetaInput{})
|
||||
if skip {
|
||||
return
|
||||
}
|
||||
finished := e.templateExecutionStarted(template, "")
|
||||
defer func() { finished(ctx.Err()) }()
|
||||
defer e.tasksDone(1)
|
||||
var err error
|
||||
var match bool
|
||||
ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
|
||||
@@ -55,6 +59,8 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
|
||||
}
|
||||
if err != nil {
|
||||
e.options.Logger.Warning().Msgf("[%s] Could not execute step (self-contained): %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
|
||||
} else if ctx.Context().Err() == nil {
|
||||
planned()
|
||||
}
|
||||
results.CompareAndSwap(false, match)
|
||||
}(v, usesSharedTemplateBudget)
|
||||
@@ -330,6 +336,16 @@ func (e *Engine) tasksDone(count int64) {
|
||||
func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (matched bool, err error) {
|
||||
// a pair skipped as a repeat of its origin is done as well
|
||||
defer e.tasksDone(1)
|
||||
planned, skip := e.claimPlanned(template, value)
|
||||
if skip {
|
||||
return false, nil
|
||||
}
|
||||
// an origin a sibling already covered is settled too, so it is recorded
|
||||
defer func() {
|
||||
if err == nil && ctx.Err() == nil {
|
||||
planned()
|
||||
}
|
||||
}()
|
||||
// A template fixed to the target's origin makes the same requests for every
|
||||
// target that shares one, which a crawled list produces by the hundred.
|
||||
// Running it once per origin sends exactly the same traffic, minus the
|
||||
|
||||
+43
-2
@@ -104,7 +104,7 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
|
||||
entry := PlanEntry{
|
||||
Target: value.Input,
|
||||
IP: value.CustomIP,
|
||||
Templates: []string{template.ID},
|
||||
Templates: planTemplates(template, selection),
|
||||
Protocol: template.Type().String(),
|
||||
Requests: template.TotalRequests,
|
||||
Dynamic: template.Flow != "" || template.IsFuzzableRequest(),
|
||||
@@ -126,7 +126,6 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
|
||||
entry.Dynamic = true
|
||||
case isCluster(template):
|
||||
cluster := template.Executer.(*templates.ClusterExecuter)
|
||||
entry.Templates = slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
|
||||
// TotalRequests of a cluster counts only its http and dns requests,
|
||||
// so an ssl cluster would show none
|
||||
entry.Requests = cluster.Requests()
|
||||
@@ -143,6 +142,15 @@ func planEntry(template *templates.Template, value *contextargs.MetaInput, selec
|
||||
return entry
|
||||
}
|
||||
|
||||
// planTemplates lists the templates an entry runs, sorted so the entry id does
|
||||
// not depend on clustering order.
|
||||
func planTemplates(template *templates.Template, selection protocols.TemplateSelection) []string {
|
||||
if cluster, ok := template.Executer.(*templates.ClusterExecuter); ok {
|
||||
return slices.Sorted(slices.Values(cluster.TemplateIDs(selection)))
|
||||
}
|
||||
return []string{template.ID}
|
||||
}
|
||||
|
||||
func isCluster(template *templates.Template) bool {
|
||||
_, ok := template.Executer.(*templates.ClusterExecuter)
|
||||
return ok
|
||||
@@ -154,3 +162,36 @@ func planEntryID(entry PlanEntry) string {
|
||||
sum := sha256.Sum256([]byte(entry.Target + "\x00" + entry.IP + "\x00" + strings.Join(entry.Templates, "\x00")))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// PlanRun limits a scan to the entries of a plan and records the ones that
|
||||
// finish, so running the plan again resumes it.
|
||||
type PlanRun interface {
|
||||
// Skip reports whether the entry is not in the plan or already finished.
|
||||
Skip(id string) bool
|
||||
// Finished records an entry that ran without error.
|
||||
Finished(id string)
|
||||
}
|
||||
|
||||
// SetPlanRun makes the scan run only the entries run allows.
|
||||
func (e *Engine) SetPlanRun(run PlanRun) {
|
||||
e.planRun = run
|
||||
}
|
||||
|
||||
// claimPlanned reports whether the pair is skipped under a plan run, and
|
||||
// otherwise returns the func that records it finished. Without a plan run
|
||||
// every pair runs.
|
||||
func (e *Engine) claimPlanned(template *templates.Template, value *contextargs.MetaInput) (finished func(), skip bool) {
|
||||
if e.planRun == nil {
|
||||
return func() {}, false
|
||||
}
|
||||
// as in Plan, a self-contained template is not narrowed by any target
|
||||
var selection protocols.TemplateSelection
|
||||
if !template.SelfContained {
|
||||
selection = e.selectionFor(value)
|
||||
}
|
||||
id := planEntryID(PlanEntry{Target: value.Input, IP: value.CustomIP, Templates: planTemplates(template, selection)})
|
||||
if e.planRun.Skip(id) {
|
||||
return nil, true
|
||||
}
|
||||
return func() { e.planRun.Finished(id) }, false
|
||||
}
|
||||
@@ -154,3 +154,62 @@ func TestPlanListsOriginAlternatives(t *testing.T) {
|
||||
})
|
||||
require.Equal(t, plan, again)
|
||||
}
|
||||
|
||||
// fixedPlanRun is a plan run over fixed ids that records the finished ones.
|
||||
type fixedPlanRun struct {
|
||||
mu sync.Mutex
|
||||
allowed map[string]bool
|
||||
finished []string
|
||||
}
|
||||
|
||||
func (a *fixedPlanRun) Skip(id string) bool { return !a.allowed[id] }
|
||||
|
||||
func (a *fixedPlanRun) Finished(id string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.finished = append(a.finished, id)
|
||||
}
|
||||
|
||||
func TestPlanRunRunsOnlyThePlannedPairs(t *testing.T) {
|
||||
options := &types.Options{BulkSize: 2, TemplateThreads: 2, HeadlessBulkSize: 1, HeadlessTemplateThreads: 1}
|
||||
engine := New(options)
|
||||
engine.SetExecuterOptions(&protocols.ExecutorOptions{
|
||||
Logger: engine.Logger,
|
||||
Options: options,
|
||||
ResumeCfg: types.NewResumeCfg(),
|
||||
ProtocolType: tmpltypes.HTTPProtocol,
|
||||
Progress: &progressRecorder{},
|
||||
TargetScope: scopeByInput{"php": {paths: []string{"thinkphp.yaml"}}},
|
||||
})
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
runs []string
|
||||
)
|
||||
var templatesList []*templates.Template
|
||||
for _, path := range []string{"thinkphp.yaml", "generic.yaml"} {
|
||||
templatesList = append(templatesList, &templates.Template{
|
||||
ID: path,
|
||||
Path: path,
|
||||
TotalRequests: 1,
|
||||
Executer: &recordingExecuter{path: path, mu: &mu, runs: &runs},
|
||||
})
|
||||
}
|
||||
targets := &fakeTargetProvider{values: []*contextargs.MetaInput{{Input: "php"}, {Input: "plain"}}}
|
||||
|
||||
var plan []PlanEntry
|
||||
engine.Plan(templatesList, targets, true, func(entry PlanEntry) { plan = append(plan, entry) })
|
||||
require.Len(t, plan, 3)
|
||||
|
||||
// the first entry is left out of the plan, as if an agent removed it
|
||||
run := &fixedPlanRun{allowed: map[string]bool{plan[1].ID: true, plan[2].ID: true}}
|
||||
engine.SetPlanRun(run)
|
||||
engine.ExecuteScanWithOpts(context.Background(), templatesList, targets, true)
|
||||
|
||||
var want []string
|
||||
for _, entry := range plan[1:] {
|
||||
want = append(want, entry.Templates[0]+"@"+entry.Target)
|
||||
}
|
||||
require.ElementsMatch(t, want, runs, "only the planned pairs run")
|
||||
require.ElementsMatch(t, []string{plan[1].ID, plan[2].ID}, run.finished, "the scan derives the ids the plan wrote")
|
||||
}
|
||||
@@ -43,6 +43,10 @@ func IsPOP3(ctx context.Context, host string, port int) (IsPOP3Response, error)
|
||||
func isPoP3(ctx context.Context, executionId string, host string, port int) (IsPOP3Response, error) {
|
||||
resp := IsPOP3Response{}
|
||||
|
||||
if !protocolstate.IsHostAllowed(executionId, host) {
|
||||
return resp, protocolstate.ErrHostDenied.Msgf(host)
|
||||
}
|
||||
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
return IsPOP3Response{}, fmt.Errorf("dialers not initialized for %s", executionId)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package pop3
|
||||
|
||||
// Tests that the network policy (IsHostAllowed + fastdialer deny) is enforced
|
||||
// by isPoP3 before any successful connection is established to a denied host.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestIsPOP3DeniesExcludedHost verifies that isPoP3 returns an error when the
|
||||
// target host is in the --exclude-hosts list. The network-policy enforcement
|
||||
// happens either via IsHostAllowed (hostname form) or via the fastdialer's own
|
||||
// IP-level deny check; either way the call must not succeed.
|
||||
func TestIsPOP3DeniesExcludedHost(t *testing.T) {
|
||||
execID := t.Name()
|
||||
require.NoError(t, protocolstate.Init(&types.Options{
|
||||
ExecutionId: execID,
|
||||
ExcludeTargets: []string{"203.0.113.50"},
|
||||
}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
|
||||
_, err := isPoP3(ctx, execID, "203.0.113.50", 110)
|
||||
require.Error(t, err, "isPoP3 must reject a host excluded by network policy")
|
||||
// The error may come from IsHostAllowed (hostname form) or from the
|
||||
// fastdialer IP-level deny; either contains "denied" or "dropped".
|
||||
errStr := err.Error()
|
||||
require.True(t,
|
||||
containsAny(errStr, "denied", "dropped", "not allowed", "excluded"),
|
||||
"error %q should indicate a network-policy denial", errStr)
|
||||
}
|
||||
|
||||
func containsAny(s string, subs ...string) bool {
|
||||
for _, sub := range subs {
|
||||
if len(s) >= len(sub) {
|
||||
for i := 0; i <= len(s)-len(sub); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -46,6 +46,10 @@ func IsRDP(ctx context.Context, host string, port int) (IsRDPResponse, error) {
|
||||
func isRDP(ctx context.Context, executionId string, host string, port int) (IsRDPResponse, error) {
|
||||
resp := IsRDPResponse{}
|
||||
|
||||
if !protocolstate.IsHostAllowed(executionId, host) {
|
||||
return resp, protocolstate.ErrHostDenied.Msgf(host)
|
||||
}
|
||||
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
return IsRDPResponse{}, fmt.Errorf("dialers not initialized for %s", executionId)
|
||||
@@ -105,6 +109,10 @@ func CheckRDPAuth(ctx context.Context, host string, port int) (CheckRDPAuthRespo
|
||||
func checkRDPAuth(ctx context.Context, executionId string, host string, port int) (CheckRDPAuthResponse, error) {
|
||||
resp := CheckRDPAuthResponse{}
|
||||
|
||||
if !protocolstate.IsHostAllowed(executionId, host) {
|
||||
return resp, protocolstate.ErrHostDenied.Msgf(host)
|
||||
}
|
||||
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
return CheckRDPAuthResponse{}, fmt.Errorf("dialers not initialized for %s", executionId)
|
||||
@@ -193,6 +201,10 @@ func CheckRDPEncryption(ctx context.Context, host string, port int) (RDPEncrypti
|
||||
|
||||
// @memo
|
||||
func checkRDPEncryption(ctx context.Context, executionId string, host string, port int) (RDPEncryptionResponse, error) {
|
||||
if !protocolstate.IsHostAllowed(executionId, host) {
|
||||
return RDPEncryptionResponse{}, protocolstate.ErrHostDenied.Msgf(host)
|
||||
}
|
||||
|
||||
dialer := protocolstate.GetDialersWithId(executionId)
|
||||
if dialer == nil {
|
||||
return RDPEncryptionResponse{}, fmt.Errorf("dialers not initialized for %s", executionId)
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package rdp
|
||||
|
||||
// Tests that IsHostAllowed is enforced by IsRDP, CheckRDPAuth and
|
||||
// CheckRDPEncryption before any network dial is attempted.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func setupPolicy(t *testing.T, excludedHosts ...string) string {
|
||||
t.Helper()
|
||||
execID := t.Name()
|
||||
require.NoError(t, protocolstate.Init(&types.Options{
|
||||
ExecutionId: execID,
|
||||
ExcludeTargets: excludedHosts,
|
||||
}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
return execID
|
||||
}
|
||||
|
||||
func ctxWithExec(execID string) context.Context {
|
||||
return context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
|
||||
}
|
||||
|
||||
// TestIsRDPDeniesExcludedHost verifies that isRDP returns an error when the
|
||||
// target host is in the --exclude-hosts list.
|
||||
func TestIsRDPDeniesExcludedHost(t *testing.T) {
|
||||
execID := setupPolicy(t, "203.0.113.50")
|
||||
_, err := isRDP(ctxWithExec(execID), execID, "203.0.113.50", 3389)
|
||||
require.Error(t, err, "isRDP must reject a host that is excluded by policy")
|
||||
require.Contains(t, err.Error(), "203.0.113.50")
|
||||
}
|
||||
|
||||
// TestCheckRDPAuthDeniesExcludedHost verifies that checkRDPAuth respects policy.
|
||||
func TestCheckRDPAuthDeniesExcludedHost(t *testing.T) {
|
||||
execID := setupPolicy(t, "203.0.113.50")
|
||||
_, err := checkRDPAuth(ctxWithExec(execID), execID, "203.0.113.50", 3389)
|
||||
require.Error(t, err, "checkRDPAuth must reject a host that is excluded by policy")
|
||||
require.Contains(t, err.Error(), "203.0.113.50")
|
||||
}
|
||||
|
||||
// TestCheckRDPEncryptionDeniesExcludedHost verifies that checkRDPEncryption
|
||||
// respects policy before attempting any network connection.
|
||||
func TestCheckRDPEncryptionDeniesExcludedHost(t *testing.T) {
|
||||
execID := setupPolicy(t, "203.0.113.50")
|
||||
_, err := checkRDPEncryption(ctxWithExec(execID), execID, "203.0.113.50", 3389)
|
||||
require.Error(t, err, "checkRDPEncryption must reject a host that is excluded by policy")
|
||||
require.Contains(t, err.Error(), "203.0.113.50")
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package vnc
|
||||
|
||||
// Tests that the network policy is enforced by isVNC before any connection.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestIsVNCDeniesExcludedHost verifies that isVNC rejects a host that appears
|
||||
// in the --exclude-hosts list.
|
||||
func TestIsVNCDeniesExcludedHost(t *testing.T) {
|
||||
execID := t.Name()
|
||||
require.NoError(t, protocolstate.Init(&types.Options{
|
||||
ExecutionId: execID,
|
||||
ExcludeTargets: []string{"203.0.113.50"},
|
||||
}))
|
||||
t.Cleanup(func() { protocolstate.Close(execID) })
|
||||
|
||||
ctx := context.WithValue(context.Background(), "executionId", execID) //nolint:staticcheck
|
||||
_, err := isVNC(ctx, execID, "203.0.113.50", 5900)
|
||||
require.Error(t, err, "isVNC must reject a host excluded by network policy")
|
||||
errStr := err.Error()
|
||||
require.True(t,
|
||||
containsAny(errStr, "denied", "dropped", "not allowed", "excluded"),
|
||||
"error %q should indicate a network-policy denial", errStr)
|
||||
}
|
||||
|
||||
func containsAny(s string, subs ...string) bool {
|
||||
for _, sub := range subs {
|
||||
for i := 0; i <= len(s)-len(sub); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -37,6 +37,8 @@ import (
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
osutils "github.com/projectdiscovery/utils/os"
|
||||
|
||||
filepathutil "github.com/projectdiscovery/nuclei/v3/pkg/utils/filepath"
|
||||
unitutils "github.com/projectdiscovery/utils/unit"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
@@ -734,10 +736,24 @@ func (w *StandardWriter) WriteStoreDebugData(host, templateID, eventType string,
|
||||
|
||||
filename := sanitizeFileName(fmt.Sprintf("%s_%s", host, templateID))
|
||||
subFolder := filepath.Join(w.storeResponseDir, sanitizeFileName(eventType))
|
||||
// Defense-in-depth: verify the constructed sub-folder is still inside
|
||||
// the configured store-response directory before creating it. The
|
||||
// sanitizeFileName call above should prevent any traversal sequence from
|
||||
// surviving, but an explicit containment check is a cheap second gate
|
||||
// that will catch any future regression in the sanitizer.
|
||||
if !filepathutil.IsPathWithinDirectory(subFolder, w.storeResponseDir) {
|
||||
gologger.Error().Msgf("Computed debug sub-folder %s is outside store directory %s; skipping", subFolder, w.storeResponseDir)
|
||||
return
|
||||
}
|
||||
if !fileutil.FolderExists(subFolder) {
|
||||
_ = fileutil.CreateFolder(subFolder)
|
||||
}
|
||||
filename = filepath.Join(subFolder, fmt.Sprintf("%s.txt", filename))
|
||||
// Second containment check for the final file path.
|
||||
if !filepathutil.IsPathWithinDirectory(filename, w.storeResponseDir) {
|
||||
gologger.Error().Msgf("Computed debug file path %s is outside store directory %s; skipping", filename, w.storeResponseDir)
|
||||
return
|
||||
}
|
||||
f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not open debug output file: %s", err)
|
||||
|
||||
@@ -2,6 +2,8 @@ package output
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -239,3 +241,105 @@ func TestFormatJSONSanitizesNestedMetadataMaps(t *testing.T) {
|
||||
require.NoError(t, err, "formatJSON should not fail on nested map[interface{}]interface{} (#7971)")
|
||||
require.Contains(t, string(data), `"labels":{"en":"Hello","nested_list":[{"inner_key":"inner_val"}]}`)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Path-traversal containment tests for sanitizeFileName and WriteStoreDebugData
|
||||
// (CWE-22, defense-in-depth – mirrors the markdown exporter's regression suite)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestSanitizeFileNameContainment verifies that sanitizeFileName strips every
|
||||
// character that could allow a path to escape its intended directory. The
|
||||
// markdown exporter has an equivalent fuzz-driven suite; this test covers the
|
||||
// same invariants for the output writer's sanitizer.
|
||||
func TestSanitizeFileNameContainment(t *testing.T) {
|
||||
reportDir := t.TempDir()
|
||||
|
||||
hostileInputs := []string{
|
||||
"../../etc/passwd",
|
||||
"..\\..\\Windows\\System32",
|
||||
"..",
|
||||
"http://evil.com/../../../etc/shadow",
|
||||
"https://attacker.example/../../../../root/.ssh/authorized_keys",
|
||||
"some/host/../../escape",
|
||||
strings.Repeat("../", 200) + "etc/passwd",
|
||||
strings.Repeat("..\\", 200) + "Windows",
|
||||
"...",
|
||||
"....",
|
||||
".....",
|
||||
"\x00/etc/passwd", // null byte (not a real traversal on Linux but should be safe)
|
||||
"/../../foo",
|
||||
"\\..\\..\\foo",
|
||||
}
|
||||
|
||||
for _, in := range hostileInputs {
|
||||
in := in // capture
|
||||
t.Run(in, func(t *testing.T) {
|
||||
out := sanitizeFileName(in)
|
||||
// After sanitisation no path separator must survive.
|
||||
require.False(t, strings.ContainsAny(out, "/\\"),
|
||||
"sanitizeFileName leaked a separator: in=%q out=%q", in, out)
|
||||
// No ".." parent-ref must survive either.
|
||||
require.False(t, strings.Contains(out, ".."),
|
||||
"sanitizeFileName leaked '..': in=%q out=%q", in, out)
|
||||
|
||||
// Joining the sanitised value with an arbitrary base must stay
|
||||
// inside that base – the same invariant the exporter relies on.
|
||||
joined := filepath.Clean(filepath.Join(reportDir, out))
|
||||
rel, err := filepath.Rel(reportDir, joined)
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, "..", rel)
|
||||
require.False(t, strings.HasPrefix(rel, ".."+string(filepath.Separator)),
|
||||
"sanitizeFileName: joined path escaped reportDir: in=%q out=%q rel=%q", in, out, rel)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteStoreDebugDataContainment is an integration test that calls
|
||||
// WriteStoreDebugData with attacker-shaped host / templateID values and verifies
|
||||
// that no file is ever created outside the configured storeResponseDir.
|
||||
// Without the containment guard added in this fix, a future regression in
|
||||
// sanitizeFileName could silently allow a file to escape; this test would catch
|
||||
// such a regression immediately.
|
||||
func TestWriteStoreDebugDataContainment(t *testing.T) {
|
||||
storeDir := t.TempDir()
|
||||
|
||||
w := &StandardWriter{
|
||||
storeResponse: true,
|
||||
storeResponseDir: storeDir,
|
||||
}
|
||||
|
||||
hostilePairs := []struct{ host, templateID string }{
|
||||
{"../../etc/passwd", "cve-2023-1234"},
|
||||
{"..\\..\\Windows\\system32", "cve-2023-1234"},
|
||||
{"http://evil.com/../../../etc/shadow", "poc"},
|
||||
{"https://attacker.example/" + strings.Repeat("../", 50) + "root/.ssh", "recon"},
|
||||
{"..", ".."},
|
||||
{"/etc/passwd", "test"},
|
||||
{"\\\\unc\\path\\..\\evil", "smb-test"},
|
||||
}
|
||||
|
||||
for _, tc := range hostilePairs {
|
||||
tc := tc
|
||||
t.Run(tc.host+"|"+tc.templateID, func(t *testing.T) {
|
||||
w.WriteStoreDebugData(tc.host, tc.templateID, "http", "debug-data")
|
||||
})
|
||||
}
|
||||
|
||||
// Walk storeDir and assert every file created is still inside it.
|
||||
err := filepath.Walk(storeDir, func(path string, info os.FileInfo, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
rel, relErr := filepath.Rel(storeDir, path)
|
||||
if relErr != nil {
|
||||
return relErr
|
||||
}
|
||||
if rel == "." {
|
||||
return nil // the root itself
|
||||
}
|
||||
require.False(t, strings.HasPrefix(rel, ".."),
|
||||
"WriteStoreDebugData created a file outside store dir: %s", path)
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -94,21 +94,10 @@ func NewTransportWrapper() func(http.RoundTripper) http.RoundTripper {
|
||||
},
|
||||
DisableWarningHeader: true,
|
||||
}
|
||||
return &unwrappingTransport{Transport: transport}
|
||||
return transport
|
||||
}
|
||||
}
|
||||
|
||||
// unwrappingTransport lets code that must not use the cache, such as reporting
|
||||
// clients that need TLS verification, reach the wrapped transport.
|
||||
type unwrappingTransport struct {
|
||||
*httpcache.Transport
|
||||
}
|
||||
|
||||
// Unwrap returns the transport the cache wraps.
|
||||
func (t *unwrappingTransport) Unwrap() http.RoundTripper {
|
||||
return t.Transport.Transport
|
||||
}
|
||||
|
||||
func openDiskCache(dir string, maxBytes int64) (httpcache.Cache, *leveldb.DB, error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, nil, fmt.Errorf("create cache dir: %w", err)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package httpcache
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -11,7 +10,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
|
||||
libcache "github.com/sandrolain/httpcache"
|
||||
)
|
||||
|
||||
@@ -180,29 +178,3 @@ func TestDiskCacheRespectsSizeCap(t *testing.T) {
|
||||
t.Fatal("stored a response over a 1-byte directory cap")
|
||||
}
|
||||
}
|
||||
|
||||
// Reporting clients send tokens, so tlsverify must reach the scan transport
|
||||
// under the cache wrapper and turn certificate verification on.
|
||||
func TestTransportWrapperUnwrapsForTLSVerification(t *testing.T) {
|
||||
cacheMu.Lock()
|
||||
saved := cache
|
||||
cache = libcache.NewMemoryCache()
|
||||
cacheMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
cacheMu.Lock()
|
||||
cache = saved
|
||||
cacheMu.Unlock()
|
||||
})
|
||||
|
||||
base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
wrapped := NewTransportWrapper()(base)
|
||||
u, ok := wrapped.(interface{ Unwrap() http.RoundTripper })
|
||||
if !ok || u.Unwrap() != base {
|
||||
t.Fatal("the cache wrapper does not unwrap to the scan transport")
|
||||
}
|
||||
|
||||
got, ok := tlsverify.Client(&http.Client{Transport: wrapped}).Transport.(*http.Transport)
|
||||
if !ok || got.TLSClientConfig.InsecureSkipVerify {
|
||||
t.Fatal("tlsverify did not turn verification on under the cache wrapper")
|
||||
}
|
||||
}
|
||||
@@ -166,11 +166,6 @@ func (t *connTrackingTransport) RoundTrip(req *http.Request) (*http.Response, er
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// Unwrap returns the underlying transport.
|
||||
func (t *connTrackingTransport) Unwrap() http.RoundTripper {
|
||||
return t.base
|
||||
}
|
||||
|
||||
func (t *connTrackingTransport) CloseIdleConnections() {
|
||||
type closeIdler interface{ CloseIdleConnections() }
|
||||
if ci, ok := t.base.(closeIdler); ok {
|
||||
|
||||
@@ -124,12 +124,6 @@ func (t *sharingTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
||||
return entry.response.copyFor(req), nil
|
||||
}
|
||||
|
||||
// Unwrap returns the underlying transport, so reporting clients that need TLS
|
||||
// verification reach the scan transport and never share responses.
|
||||
func (t *sharingTransport) Unwrap() http.RoundTripper {
|
||||
return t.base
|
||||
}
|
||||
|
||||
func (t *sharingTransport) CloseIdleConnections() {
|
||||
type closeIdler interface{ CloseIdleConnections() }
|
||||
if ci, ok := t.base.(closeIdler); ok {
|
||||
|
||||
@@ -3,7 +3,6 @@ package httpclientpool
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -15,7 +14,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/useragent"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -308,14 +306,3 @@ func TestSharedResponsesDoNotKeepShortReads(t *testing.T) {
|
||||
_ = second.Body.Close()
|
||||
require.EqualValues(t, 2, base.hits.Load(), "a short read is not reused")
|
||||
}
|
||||
|
||||
// Reporting clients send tokens, so tlsverify must reach the scan transport
|
||||
// under the sharing wrapper and turn certificate verification on.
|
||||
func TestSharingTransportUnwrapsForTLSVerification(t *testing.T) {
|
||||
base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
wrapped := newSharingTransport(&connTrackingTransport{base: base})
|
||||
|
||||
got, ok := tlsverify.Client(&http.Client{Transport: wrapped}).Transport.(*http.Transport)
|
||||
require.True(t, ok, "tlsverify did not reach the scan transport under the sharing wrapper")
|
||||
require.False(t, got.TLSClientConfig.InsecureSkipVerify)
|
||||
}
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/pdf"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
|
||||
@@ -29,7 +28,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/utils/errkit"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
"go.uber.org/multierr"
|
||||
@@ -79,17 +77,6 @@ type IssueTrackerStats struct {
|
||||
func New(options *Options, db string, doNotDedupe bool) (Client, error) {
|
||||
client := &ReportingClient{options: options}
|
||||
|
||||
// The caller can pass the scan client, which skips TLS verification;
|
||||
// trackers and exporters send tokens and passwords, so they get a copy
|
||||
// that verifies certificates.
|
||||
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
|
||||
if hc := tlsverify.Client(options.HttpClient.HTTPClient); hc != options.HttpClient.HTTPClient {
|
||||
opts := retryablehttp.DefaultOptionsSingle
|
||||
opts.Timeout = hc.Timeout // keep the caller's timeout, also when it is 0
|
||||
options.HttpClient = retryablehttp.NewWithHTTPClient(hc, opts)
|
||||
}
|
||||
}
|
||||
|
||||
if options.GitHub != nil {
|
||||
options.GitHub.HttpClient = options.HttpClient
|
||||
options.GitHub.OmitRaw = options.OmitRaw
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
package reporting_test
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// connectProxy tunnels every CONNECT to target, whatever host the client asked
|
||||
// for, so no name is resolved and nothing leaves 127.0.0.1.
|
||||
func connectProxy(t *testing.T, target string) *httptest.Server {
|
||||
p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodConnect {
|
||||
http.Error(w, "connect only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
up, err := net.Dial("tcp", target)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
c, _, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
_ = up.Close()
|
||||
return
|
||||
}
|
||||
_, _ = c.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n"))
|
||||
go func() { _, _ = io.Copy(up, c); _ = up.Close() }()
|
||||
go func() { _, _ = io.Copy(c, up); _ = c.Close() }()
|
||||
}))
|
||||
t.Cleanup(p.Close)
|
||||
return p
|
||||
}
|
||||
|
||||
// With -pi and a proxy, the runner gives reporting the scan client, which skips
|
||||
// TLS verification. No tracker or exporter may send its secret to a server with
|
||||
// an untrusted certificate.
|
||||
func TestNewVerifiesTLSWithScanClient(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
proxy := connectProxy(t, srv.Listener.Addr().String())
|
||||
|
||||
opts := types.DefaultOptions()
|
||||
opts.SetExecutionID("reporting-tls-test")
|
||||
opts.ProxyInternal = true
|
||||
opts.AliveHttpProxy = proxy.URL
|
||||
require.NoError(t, protocolstate.Init(opts))
|
||||
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
|
||||
scanClient, err := httpclientpool.Get(opts, &httpclientpool.Configuration{}, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
event := &output.ResultEvent{
|
||||
TemplateID: "test-template",
|
||||
Host: "https://target.example",
|
||||
Matched: "https://target.example/",
|
||||
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
|
||||
}
|
||||
configs := map[string]*reporting.Options{
|
||||
"github": {GitHub: &github.Options{BaseURL: "https://github.victim.example/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token"}},
|
||||
"gitlab": {GitLab: &gitlab.Options{BaseURL: "https://gitlab.victim.example/", Username: "u", Token: "test-token", ProjectName: "1"}},
|
||||
"gitea": {Gitea: &gitea.Options{BaseURL: "https://gitea.victim.example/", Token: "test-token", ProjectOwner: "o", ProjectName: "r"}},
|
||||
"jira": {Jira: &jira.Options{Cloud: true, URL: "https://jira.victim.example", AccountID: "a", Email: "[email protected]", Token: "test-token", ProjectName: "P", IssueType: "Bug"}},
|
||||
"elasticsearch": {ElasticsearchExporter: &es.Options{Host: "es.victim.example", Port: 443, SSL: true, Username: "u", Password: "test-password", IndexName: "n"}},
|
||||
"splunk": {SplunkExporter: &splunk.Options{Host: "splunk.victim.example", Port: 443, SSL: true, Token: "test-token", IndexName: "n"}},
|
||||
}
|
||||
for name, ro := range configs {
|
||||
ro.HttpClient = scanClient
|
||||
ro.ExecutionId = opts.ExecutionId
|
||||
client, err := reporting.New(ro, "", true)
|
||||
if err == nil {
|
||||
_ = client.CreateIssue(event)
|
||||
client.Close()
|
||||
}
|
||||
require.Zero(t, hits.Load(), "%s sent a request to a server with an untrusted certificate", name)
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
// Package tlsverify gives trackers and exporters an HTTP client that verifies
|
||||
// TLS certificates, since they send tokens and passwords.
|
||||
package tlsverify
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// errTLSDialerOnly is returned when a transport dials HTTPS only through a
|
||||
// custom TLS dialer: dropping that dialer would also drop its route or address
|
||||
// policy, so the copy refuses requests instead.
|
||||
var errTLSDialerOnly = errors.New("tlsverify: the transport has a custom TLS dialer and no DialContext, so certificate verification cannot be enforced")
|
||||
|
||||
type refusingTransport struct{}
|
||||
|
||||
func (refusingTransport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||
return nil, errTLSDialerOnly
|
||||
}
|
||||
|
||||
// Client returns c, or a copy of c whose transport verifies TLS certificates
|
||||
// when c can skip verification (the scan client can). The copy keeps the proxy
|
||||
// and the dialer, so the address policy still applies, but it drops a custom
|
||||
// TLS dialer, because that dialer can use the scan TLS config. So the token
|
||||
// never goes over an unverified connection.
|
||||
//
|
||||
// A transport with an Unwrap() http.RoundTripper method declares that its
|
||||
// RoundTrip only observes or caches requests and enforces no policy; the copy
|
||||
// uses the unwrapped transport. Other transports are returned unchanged.
|
||||
func Client(c *http.Client) *http.Client {
|
||||
rt := c.Transport
|
||||
for {
|
||||
u, ok := rt.(interface{ Unwrap() http.RoundTripper })
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
rt = u.Unwrap()
|
||||
}
|
||||
t, ok := rt.(*http.Transport)
|
||||
if !ok {
|
||||
return c
|
||||
}
|
||||
insecure := t.TLSClientConfig != nil && t.TLSClientConfig.InsecureSkipVerify
|
||||
tlsDialer := t.DialTLSContext != nil || t.DialTLS != nil //nolint:staticcheck // DialTLS is deprecated but still used
|
||||
if !insecure && !tlsDialer {
|
||||
return c
|
||||
}
|
||||
copied := *c
|
||||
if tlsDialer && t.DialContext == nil && t.Dial == nil { //nolint:staticcheck // Dial is deprecated but still used
|
||||
copied.Transport = refusingTransport{}
|
||||
return &copied
|
||||
}
|
||||
verifying := t.Clone()
|
||||
verifying.DialTLSContext = nil
|
||||
verifying.DialTLS = nil //nolint:staticcheck // DialTLS is deprecated but still used
|
||||
if insecure {
|
||||
// Keep the caller's roots, client certificates and checks; the server
|
||||
// name of the scan config belongs to a scan target, not to this host.
|
||||
verifying.TLSClientConfig.InsecureSkipVerify = false
|
||||
verifying.TLSClientConfig.ServerName = ""
|
||||
if verifying.TLSClientConfig.MinVersion < tls.VersionTLS12 {
|
||||
verifying.TLSClientConfig.MinVersion = tls.VersionTLS12
|
||||
}
|
||||
}
|
||||
copied.Transport = verifying
|
||||
return &copied
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package tlsverify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The scan client can skip TLS verification; the tracker must not send the
|
||||
// token over such a connection, and must not change the caller's client.
|
||||
func TestClientTurnsOnTLSVerification(t *testing.T) {
|
||||
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
caller := &http.Client{Transport: insecure}
|
||||
|
||||
got := Client(caller)
|
||||
gotTransport, ok := got.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
require.False(t, gotTransport.TLSClientConfig.InsecureSkipVerify)
|
||||
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's transport changed")
|
||||
|
||||
secure := &http.Client{Transport: &http.Transport{}}
|
||||
require.Same(t, secure, Client(secure))
|
||||
}
|
||||
|
||||
func TestClientKeepsCallerTLSSettings(t *testing.T) {
|
||||
pinned := func(tls.ConnectionState) error { return nil }
|
||||
insecure := &http.Transport{
|
||||
DialTLSContext: func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("scan dialer") },
|
||||
DialContext: (&net.Dialer{}).DialContext,
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
ServerName: "scan-target.example",
|
||||
MinVersion: tls.VersionTLS10,
|
||||
RootCAs: x509.NewCertPool(),
|
||||
VerifyConnection: pinned,
|
||||
},
|
||||
}
|
||||
got := Client(&http.Client{Transport: insecure}).Transport.(*http.Transport)
|
||||
require.Nil(t, got.DialTLSContext)
|
||||
require.NotNil(t, got.DialContext)
|
||||
cfg := got.TLSClientConfig
|
||||
require.False(t, cfg.InsecureSkipVerify)
|
||||
require.Empty(t, cfg.ServerName)
|
||||
require.Equal(t, uint16(tls.VersionTLS12), cfg.MinVersion)
|
||||
require.Same(t, insecure.TLSClientConfig.RootCAs, cfg.RootCAs)
|
||||
require.NotNil(t, cfg.VerifyConnection)
|
||||
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's config changed")
|
||||
}
|
||||
|
||||
// A custom TLS dialer may carry the only route or address policy, so a
|
||||
// transport without DialContext cannot be rebuilt and must refuse requests.
|
||||
func TestClientRefusesTLSDialerWithoutDialContext(t *testing.T) {
|
||||
var dialed bool
|
||||
tlsOnly := &http.Transport{DialTLSContext: func(context.Context, string, string) (net.Conn, error) {
|
||||
dialed = true
|
||||
return nil, errors.New("policy dialer")
|
||||
}}
|
||||
_, err := Client(&http.Client{Transport: tlsOnly}).Get("https://tracker.example/")
|
||||
require.ErrorIs(t, err, errTLSDialerOnly)
|
||||
require.False(t, dialed)
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/google/go-github/v92/github"
|
||||
@@ -11,7 +12,6 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"golang.org/x/oauth2"
|
||||
@@ -55,16 +55,17 @@ type Options struct {
|
||||
// New creates a new issue tracker integration client based on options.
|
||||
func New(options *Options) (*Integration, error) {
|
||||
ctx := context.Background()
|
||||
// Build the oauth2 client on top of the caller's client, so its transport
|
||||
// (proxy, address policy) and redirect rules apply to every request.
|
||||
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
|
||||
ctx = context.WithValue(ctx, oauth2.HTTPClient, tlsverify.Client(options.HttpClient.HTTPClient))
|
||||
}
|
||||
ts := oauth2.StaticTokenSource(
|
||||
&oauth2.Token{AccessToken: options.Token},
|
||||
)
|
||||
tc := oauth2.NewClient(ctx, ts)
|
||||
|
||||
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
|
||||
if tcTransport, ok := tc.Transport.(*http.Transport); ok {
|
||||
tcTransport.Proxy = options.HttpClient.HTTPClient.Transport.(*http.Transport).Proxy
|
||||
}
|
||||
}
|
||||
|
||||
clientOptions := []github.ClientOptionsFunc{github.WithHTTPClient(tc)}
|
||||
if options.BaseURL != "" {
|
||||
clientOptions = append(clientOptions, github.WithURLs(&options.BaseURL, nil))
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
package github
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func testEvent() *output.ResultEvent {
|
||||
return &output.ResultEvent{
|
||||
TemplateID: "test-template",
|
||||
Host: "https://target.example",
|
||||
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
|
||||
}
|
||||
}
|
||||
|
||||
func clientWithTransport(rt http.RoundTripper) *retryablehttp.Client {
|
||||
c := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
|
||||
c.HTTPClient = &http.Client{Transport: rt}
|
||||
return c
|
||||
}
|
||||
|
||||
// The tracker must send every request through the caller's client, so a
|
||||
// caller's address policy also covers the GitHub API base URL.
|
||||
func TestNewUsesCallerHTTPClient(t *testing.T) {
|
||||
var serverHits atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
serverHits.Add(1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"number":1,"html_url":"https://github.example/o/r/issues/1"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
t.Run("requests go through the caller's transport", func(t *testing.T) {
|
||||
var seen atomic.Int32
|
||||
rt := roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
seen.Add(1)
|
||||
require.Equal(t, "Bearer test-token", r.Header.Get("Authorization"))
|
||||
return http.DefaultTransport.RoundTrip(r)
|
||||
})
|
||||
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
|
||||
require.NoError(t, err)
|
||||
_, err = tracker.CreateIssue(testEvent())
|
||||
require.NoError(t, err)
|
||||
require.Positive(t, seen.Load())
|
||||
})
|
||||
|
||||
t.Run("a refusing transport stops the request", func(t *testing.T) {
|
||||
before := serverHits.Load()
|
||||
denied := errors.New("denied by address policy")
|
||||
rt := roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, denied })
|
||||
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
|
||||
require.NoError(t, err)
|
||||
_, err = tracker.CreateIssue(testEvent())
|
||||
require.ErrorContains(t, err, "denied by address policy")
|
||||
require.Equal(t, before, serverHits.Load(), "the request reached the server")
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewRefusesUntrustedCertificateFromInsecureCallerClient(t *testing.T) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"number":1}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(insecure)})
|
||||
require.NoError(t, err)
|
||||
_, err = tracker.CreateIssue(testEvent())
|
||||
require.ErrorContains(t, err, "certificate")
|
||||
}
|
||||
|
||||
type wrappingTransport struct{ base http.RoundTripper }
|
||||
|
||||
func (w *wrappingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return w.base.RoundTrip(r)
|
||||
}
|
||||
func (w *wrappingTransport) Unwrap() http.RoundTripper { return w.base }
|
||||
|
||||
// The scan client wraps its transport and, with a SOCKS proxy, does TLS in its
|
||||
// own dialer with the scan TLS config, which skips verification.
|
||||
func TestNewRefusesUntrustedCertificateFromInsecureTLSDialer(t *testing.T) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"number":1}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
var dialed atomic.Int32
|
||||
var dialer net.Dialer
|
||||
base := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dialed.Add(1)
|
||||
return dialer.DialContext(ctx, network, addr)
|
||||
},
|
||||
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
conn, err := dialer.DialContext(ctx, network, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tls.Client(conn, &tls.Config{InsecureSkipVerify: true}), nil
|
||||
},
|
||||
}
|
||||
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(&wrappingTransport{base: base})})
|
||||
require.NoError(t, err)
|
||||
_, err = tracker.CreateIssue(testEvent())
|
||||
require.ErrorContains(t, err, "certificate")
|
||||
require.Positive(t, dialed.Load(), "the caller's dialer was not used")
|
||||
}
|
||||
@@ -254,6 +254,9 @@ type Options struct {
|
||||
// PlanFile is where the work a scan would do is written as JSON lines;
|
||||
// nuclei exits after writing it, without sending requests.
|
||||
PlanFile string
|
||||
// RunPlanFile is a plan whose entries the scan runs, skipping those its
|
||||
// done log lists as finished.
|
||||
RunPlanFile string
|
||||
// StoreEvents is where the scan writes the event of every request, as
|
||||
// JSON lines, for -replay.
|
||||
StoreEvents string
|
||||
@@ -721,6 +724,7 @@ func (options *Options) Copy() *Options {
|
||||
Silent: options.Silent,
|
||||
Validate: options.Validate,
|
||||
PlanFile: options.PlanFile,
|
||||
RunPlanFile: options.RunPlanFile,
|
||||
StoreEvents: options.StoreEvents,
|
||||
ReplayFile: options.ReplayFile,
|
||||
NoStrictSyntax: options.NoStrictSyntax,
|
||||
|
||||
@@ -18,10 +18,13 @@ func TestCopyPreservesPayloadRandomOrder(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCopyPreservesPlanFile(t *testing.T) {
|
||||
options := &Options{PlanFile: "plan.jsonl"}
|
||||
options := &Options{PlanFile: "plan.jsonl", RunPlanFile: "run.jsonl"}
|
||||
if options.Copy().PlanFile != "plan.jsonl" {
|
||||
t.Fatal("Copy must preserve PlanFile, or an embedded caller loses plan mode")
|
||||
}
|
||||
if options.Copy().RunPlanFile != "run.jsonl" {
|
||||
t.Fatal("Copy must preserve RunPlanFile, or an embedded caller runs the whole scan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestToJSONMapRejectsKeysSharingAStringForm(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user