Honor interval

go.mod

@@ -102,7 +102,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
 	github.com/projectdiscovery/uncover v1.2.1
 	github.com/projectdiscovery/useragent v0.0.109
-	github.com/projectdiscovery/utils v0.11.12-0.20261026114542-c746c2a9b8ec
+	github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5
 	github.com/projectdiscovery/wappalyzergo v0.3.5
 	github.com/projectdiscovery/yamldoc-go v1.0.7
 	github.com/redis/go-redis/v9 v9.19.0

go.sum

@@ -999,8 +999,8 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
 github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
 github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
 github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
-github.com/projectdiscovery/utils v0.11.12-0.20261026114542-c746c2a9b8ec h1:yoAPKM61ofYFGaUUSqTCW4J0B4mHJoTg5FDv0ZrCDb4=
-github.com/projectdiscovery/utils v0.11.12-0.20261026114542-c746c2a9b8ec/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
+github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5 h1:AOspf7ScbwZr8A2YnjCFd4ZJAMSFqzfEaIJIbJPuu10=
+github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
 github.com/projectdiscovery/wappalyzergo v0.3.5 h1:UZradYF1GIcruSIV5wwo/Coph9/VnM17oajLD9eXR2U=
 github.com/projectdiscovery/wappalyzergo v0.3.5/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
 github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=

internal/runner/inputs.go

@@ -18,6 +18,10 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
+// Minimum number of hosts that need httpx probing before we emit periodic
+// progress lines. Keeps small scans unchanged.
+const httpxProbeProgressMinTargets = 1000
+
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
@@ -56,7 +60,24 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
-	var count atomic.Int32
+	var alive atomic.Int32
+	var probed atomic.Int32
+
+	// swg.Add blocks once the pool is full, so the probe loop below only
+	// returns when almost every probe is done. Count the hosts first so the
+	// progress ticker runs while the probes run.
+	var stop func()
+	if r.options.EnableProgressBar {
+		var probeTotal int
+		r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
+			if !stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
+				probeTotal++
+			}
+			return true
+		})
+		stop = r.startHTTPXProbeProgress(probeTotal, &probed, &alive)
+	}
+
 	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
 		if stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
 			return true
@@ -71,19 +92,63 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 		swg.Add()
 		go func(target *contextargs.MetaInput) {
 			defer swg.Done()
+			defer probed.Add(1)
 
 			// a pinned target is probed through its ip, and each ip keeps its
 			// own result, as servers of one host may serve different schemes
 			ctx := contextargs.WithCustomIP(context.Background(), target)
 			if result := utils.ProbeURLWithContext(ctx, target.Input, httpxClient); result != "" {
-				count.Add(1)
+				alive.Add(1)
 				_ = hm.Set(input.ProbeKey(target.Input, target.CustomIP), []byte(result))
 			}
 		}(value)
 		return true
 	})
 	swg.Wait()
+	if stop != nil {
+		stop()
+	}
 
-	r.Logger.Info().Msgf("Found %d URL from httpx", count.Load())
+	r.Logger.Info().Msgf("Found %d URL from httpx", alive.Load())
 	return hm, nil
 }
+
+// startHTTPXProbeProgress optionally logs probe progress for large input sets
+// when -stats is enabled. Returns a stop func that waits for the ticker goroutine
+// to exit, or nil when progress is inactive.
+func (r *Runner) startHTTPXProbeProgress(probeTotal int, probed, alive *atomic.Int32) func() {
+	if probeTotal < httpxProbeProgressMinTargets || !r.options.EnableProgressBar {
+		return nil
+	}
+
+	// like the stats ticker, an interval of zero or less disables periodic output
+	interval := time.Duration(r.options.StatsInterval) * time.Second
+	if interval <= 0 {
+		return nil
+	}
+
+	r.Logger.Info().Msgf("httpx probe progress enabled for %d hosts (interval %s)", probeTotal, interval)
+
+	done := make(chan struct{})
+	exited := make(chan struct{})
+	ticker := time.NewTicker(interval)
+	go func() {
+		defer close(exited)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-done:
+				return
+			case <-ticker.C:
+				r.Logger.Info().Msgf(
+					"[httpx] probe progress: %d/%d (alive: %d)",
+					probed.Load(), probeTotal, alive.Load(),
+				)
+			}
+		}
+	}()
+	return func() {
+		close(done)
+		<-exited
+	}
+}

internal/runner/inputs_progress_test.go

@@ -0,0 +1,29 @@
+package runner
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTTPXProbeProgressRespectsDisabledInterval(t *testing.T) {
+	var probed, alive atomic.Int32
+	for _, interval := range []int{0, -1} {
+		options := types.DefaultOptions()
+		options.EnableProgressBar = true
+		options.StatsInterval = interval
+		r := &Runner{options: options, Logger: gologger.DefaultLogger}
+		require.Nil(t, r.startHTTPXProbeProgress(httpxProbeProgressMinTargets, &probed, &alive), "interval %d", interval)
+	}
+
+	options := types.DefaultOptions()
+	options.EnableProgressBar = true
+	options.StatsInterval = 5
+	r := &Runner{options: options, Logger: gologger.DefaultLogger}
+	stop := r.startHTTPXProbeProgress(httpxProbeProgressMinTargets, &probed, &alive)
+	require.NotNil(t, stop)
+	stop()
+}

internal/tests/integration/testdata/protocols/headless/file-upload.yaml

@@ -23,11 +23,6 @@ headless:
       args:
         by: x
         xpath: /html/body/form/input[2]
-    # the upload response echoes the file, rendered as text in a pre element
-    - action: waitvisible
-      args:
-        by: xpath
-        xpath: //pre[contains(., "Basic File Upload")]
     matchers:
       - type: word
         words:

pkg/protocols/headless/engine/engine.go

@@ -1,7 +1,6 @@
 package engine
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 	"os"
@@ -13,9 +12,7 @@ import (
 	"github.com/go-rod/rod/lib/launcher/flags"
 	"github.com/pkg/errors"
 
-	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/projectdiscovery/utils/chromeshell"
 	fileutil "github.com/projectdiscovery/utils/file"
 	osutils "github.com/projectdiscovery/utils/os"
 )
@@ -41,7 +38,7 @@ func New(options *types.Options) (*Browser, error) {
 	var proxyServer *headlessProxy
 	var err error
 
-	var chromeLauncher *launcher.Launcher
+	chromeLauncher := launcher.New()
 
 	if options.CDPEndpoint == "" {
 		dataStore, err = os.MkdirTemp("", "nuclei-*")
@@ -49,51 +46,58 @@ func New(options *types.Options) (*Browser, error) {
 			return nil, errors.Wrap(err, "could not create temporary directory")
 		}
 
-		newChromeLauncher := func(browserPath string) *launcher.Launcher {
-			configured := launcher.New().
-				Leakless(false).
-				Set("disable-crash-reporter").
-				Set("disable-gpu").
-				Set("disable-notifications").
-				Set("hide-scrollbars").
-				Set("ignore-certificate-errors").
-				Set("ignore-ssl-errors").
-				Set("incognito").
-				Set("mute-audio").
-				Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)).
-				Delete("use-mock-keychain").
-				UserDataDir(dataStore)
+		chromeLauncher = chromeLauncher.
+			Leakless(false).
+			Set("disable-crash-reporter").
+			Set("disable-gpu").
+			Set("disable-notifications").
+			Set("hide-scrollbars").
+			Set("ignore-certificate-errors").
+			Set("ignore-ssl-errors").
+			Set("incognito").
+			Set("mute-audio").
+			Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)).
+			Delete("use-mock-keychain").
+			UserDataDir(dataStore)
 
-			if browserPath != "" {
-				configured.Bin(browserPath)
-			}
-			if MustDisableSandbox() {
-				configured.NoSandbox(true)
-			}
-			configured.Headless(!options.ShowBrowser)
-			configured.
-				Proxy(proxyServer.Addr()).
-				Set("proxy-bypass-list", "<-loopback>")
-			for k, v := range options.ParseHeadlessOptionalArguments() {
-				configured.Set(flags.Flag(k), v)
-			}
-			return configured
+		if MustDisableSandbox() {
+			chromeLauncher = chromeLauncher.NoSandbox(true)
 		}
 
-		browserPath, isChromeShell, err := browserPath(options)
+		executablePath, err := os.Executable()
 		if err != nil {
 			return nil, err
 		}
+
+		// if musl is used, most likely we are on alpine linux which is not supported by go-rod, so we fallback to default chrome
+		useMusl, _ := fileutil.UseMusl(executablePath)
+		if options.UseInstalledChrome || useMusl {
+			if chromePath, hasChrome := launcher.LookPath(); hasChrome {
+				chromeLauncher.Bin(chromePath)
+			} else {
+				return nil, errors.New("the chrome browser is not installed")
+			}
+		}
+
+		if options.ShowBrowser {
+			chromeLauncher = chromeLauncher.Headless(false)
+		} else {
+			chromeLauncher = chromeLauncher.Headless(true)
+		}
 		proxyServer, err = newHeadlessProxy(options)
 		if err != nil {
 			return nil, err
 		}
-		launcherURL, err = launchBrowser(browserPath, isChromeShell, func(path string) (string, error) {
-			chromeLauncher = newChromeLauncher(path)
-			return chromeLauncher.Launch()
-		})
+		chromeLauncher = chromeLauncher.
+			Proxy(proxyServer.Addr()).
+			Set("proxy-bypass-list", "<-loopback>")
+
+		for k, v := range options.ParseHeadlessOptionalArguments() {
+			chromeLauncher.Set(flags.Flag(k), v)
+		}
+
+		launcherURL, err = chromeLauncher.Launch()
 		if err != nil {
-			_ = proxyServer.Close()
 			return nil, err
 		}
 	} else {
@@ -139,60 +143,6 @@ func New(options *types.Options) (*Browser, error) {
 	return engine, nil
 }
 
-// browserPath returns the browser binary to launch, empty for the rod default,
-// and whether that binary is chrome-headless-shell.
-func browserPath(options *types.Options) (string, bool, error) {
-	executablePath, err := os.Executable()
-	if err != nil {
-		return "", false, err
-	}
-
-	// if musl is used, most likely we are on alpine linux which is not supported by go-rod, so we fallback to default chrome
-	useMusl, _ := fileutil.UseMusl(executablePath)
-	if options.UseInstalledChrome || useMusl {
-		if chromePath, hasChrome := launcher.LookPath(); hasChrome {
-			return chromePath, false, nil
-		}
-		return "", false, errors.New("the chrome browser is not installed")
-	}
-	if options.ShowBrowser || !chromeshell.Supported() {
-		return "", false, nil
-	}
-
-	// Prefer chrome-headless-shell for local headless templates; skip it when
-	// headed since the shell binary cannot show a UI.
-	ctx, cancel := context.WithTimeout(context.Background(), chromeshell.DefaultDownloadTimeout)
-	defer cancel()
-	shellPath, err := chromeshell.EnsureWithOptions(ctx, chromeshell.Options{Proxy: downloadProxy(options)})
-	if err != nil {
-		gologger.Warning().Msgf("Could not prepare chrome-headless-shell, using the default browser: %s\n", err)
-		return "", false, nil
-	}
-	return shellPath, true, nil
-}
-
-// downloadProxy returns the scan proxy for the chrome-headless-shell download,
-// or empty to use the environment proxy.
-func downloadProxy(options *types.Options) string {
-	if options.AliveHttpProxy != "" {
-		return options.AliveHttpProxy
-	}
-	return options.AliveSocksProxy
-}
-
-// launchBrowser launches browserPath and, only when it is chrome-headless-shell,
-// retries once with the default browser. An installed Chrome that fails to
-// launch is returned as an error, as before.
-func launchBrowser(browserPath string, isChromeShell bool, launch func(string) (string, error)) (string, error) {
-	launcherURL, err := launch(browserPath)
-	if err == nil || !isChromeShell {
-		return launcherURL, err
-	}
-
-	gologger.Warning().Msgf("Could not launch chrome-headless-shell, using the default browser: %s\n", err)
-	return launch("")
-}
-
 // MustDisableSandbox determines if the current os and user needs sandbox mode disabled
 func MustDisableSandbox() bool {
 	// linux with root user needs "--no-sandbox" option

pkg/protocols/headless/engine/engine_test.go

@@ -1,67 +0,0 @@
-package engine
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLaunchBrowserFallsBackFromChromeShell(t *testing.T) {
-	var paths []string
-	got, err := launchBrowser("/cache/chrome-headless-shell", true, func(path string) (string, error) {
-		paths = append(paths, path)
-		if path != "" {
-			return "", errors.New("shell launch failed")
-		}
-		return "ws://default-browser", nil
-	})
-
-	require.NoError(t, err)
-	require.Equal(t, "ws://default-browser", got)
-	require.Equal(t, []string{"/cache/chrome-headless-shell", ""}, paths)
-}
-
-func TestLaunchBrowserDoesNotRetryDefaultBrowser(t *testing.T) {
-	var calls int
-	expected := errors.New("default launch failed")
-	_, err := launchBrowser("", false, func(string) (string, error) {
-		calls++
-		return "", expected
-	})
-
-	require.ErrorIs(t, err, expected)
-	require.Equal(t, 1, calls)
-}
-
-func TestLaunchBrowserDoesNotRetryInstalledChrome(t *testing.T) {
-	var paths []string
-	expected := errors.New("installed chrome launch failed")
-	_, err := launchBrowser("/usr/bin/google-chrome", false, func(path string) (string, error) {
-		paths = append(paths, path)
-		return "", expected
-	})
-
-	require.ErrorIs(t, err, expected)
-	require.Equal(t, []string{"/usr/bin/google-chrome"}, paths)
-}
-
-// The browser download goes through the scan proxy, or the environment proxy
-// when there is none.
-func TestDownloadProxy(t *testing.T) {
-	cases := []struct {
-		name string
-		opts *types.Options
-		want string
-	}{
-		{"no proxy uses the environment", &types.Options{}, ""},
-		{"http proxy", &types.Options{AliveHttpProxy: "http://127.0.0.1:8080"}, "http://127.0.0.1:8080"},
-		{"socks proxy", &types.Options{AliveSocksProxy: "socks5://127.0.0.1:1080"}, "socks5://127.0.0.1:1080"},
-	}
-	for _, c := range cases {
-		if got := downloadProxy(c.opts); got != c.want {
-			t.Errorf("%s: downloadProxy()=%q want %q", c.name, got, c.want)
-		}
-	}
-}

pkg/protocols/headless/engine/page_test.go

@@ -117,10 +117,8 @@ func TestRunReportsNavigationResponseWhenActionsFail(t *testing.T) {
 
 	var got *http.Response
 	actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}}
-	// Full Chromium fails a navigation to an empty error page, while
-	// chrome-headless-shell renders it, so only the reported response is
-	// checked: it must arrive whether or not the actions fail.
-	_, _, _ = instance.Run(input, actions, nil, &Options{Timeout: 20 * time.Second, Options: opts, OnNavigationResponse: func(resp *http.Response) { got = resp }})
+	_, _, err = instance.Run(input, actions, nil, &Options{Timeout: 20 * time.Second, Options: opts, OnNavigationResponse: func(resp *http.Response) { got = resp }})
+	require.Error(t, err, "the browser fails a navigation to an empty error page")
 	require.NotNil(t, got)
 	require.Equal(t, http.StatusTooManyRequests, got.StatusCode)
 	require.Equal(t, "7", got.Header.Get("Retry-After"))