Honor interval

internal/runner/inputs.go

@@ -18,6 +18,10 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
+// Minimum number of hosts that need httpx probing before we emit periodic
+// progress lines. Keeps small scans unchanged.
+const httpxProbeProgressMinTargets = 1000
+
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
@@ -56,7 +60,24 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
-	var count atomic.Int32
+	var alive atomic.Int32
+	var probed atomic.Int32
+
+	// swg.Add blocks once the pool is full, so the probe loop below only
+	// returns when almost every probe is done. Count the hosts first so the
+	// progress ticker runs while the probes run.
+	var stop func()
+	if r.options.EnableProgressBar {
+		var probeTotal int
+		r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
+			if !stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
+				probeTotal++
+			}
+			return true
+		})
+		stop = r.startHTTPXProbeProgress(probeTotal, &probed, &alive)
+	}
+
 	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
 		if stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
 			return true
@@ -71,19 +92,63 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 		swg.Add()
 		go func(target *contextargs.MetaInput) {
 			defer swg.Done()
+			defer probed.Add(1)
 
 			// a pinned target is probed through its ip, and each ip keeps its
 			// own result, as servers of one host may serve different schemes
 			ctx := contextargs.WithCustomIP(context.Background(), target)
 			if result := utils.ProbeURLWithContext(ctx, target.Input, httpxClient); result != "" {
-				count.Add(1)
+				alive.Add(1)
 				_ = hm.Set(input.ProbeKey(target.Input, target.CustomIP), []byte(result))
 			}
 		}(value)
 		return true
 	})
 	swg.Wait()
+	if stop != nil {
+		stop()
+	}
 
-	r.Logger.Info().Msgf("Found %d URL from httpx", count.Load())
+	r.Logger.Info().Msgf("Found %d URL from httpx", alive.Load())
 	return hm, nil
 }
+
+// startHTTPXProbeProgress optionally logs probe progress for large input sets
+// when -stats is enabled. Returns a stop func that waits for the ticker goroutine
+// to exit, or nil when progress is inactive.
+func (r *Runner) startHTTPXProbeProgress(probeTotal int, probed, alive *atomic.Int32) func() {
+	if probeTotal < httpxProbeProgressMinTargets || !r.options.EnableProgressBar {
+		return nil
+	}
+
+	// like the stats ticker, an interval of zero or less disables periodic output
+	interval := time.Duration(r.options.StatsInterval) * time.Second
+	if interval <= 0 {
+		return nil
+	}
+
+	r.Logger.Info().Msgf("httpx probe progress enabled for %d hosts (interval %s)", probeTotal, interval)
+
+	done := make(chan struct{})
+	exited := make(chan struct{})
+	ticker := time.NewTicker(interval)
+	go func() {
+		defer close(exited)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-done:
+				return
+			case <-ticker.C:
+				r.Logger.Info().Msgf(
+					"[httpx] probe progress: %d/%d (alive: %d)",
+					probed.Load(), probeTotal, alive.Load(),
+				)
+			}
+		}
+	}()
+	return func() {
+		close(done)
+		<-exited
+	}
+}

internal/runner/inputs_progress_test.go

@@ -0,0 +1,29 @@
+package runner
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTTPXProbeProgressRespectsDisabledInterval(t *testing.T) {
+	var probed, alive atomic.Int32
+	for _, interval := range []int{0, -1} {
+		options := types.DefaultOptions()
+		options.EnableProgressBar = true
+		options.StatsInterval = interval
+		r := &Runner{options: options, Logger: gologger.DefaultLogger}
+		require.Nil(t, r.startHTTPXProbeProgress(httpxProbeProgressMinTargets, &probed, &alive), "interval %d", interval)
+	}
+
+	options := types.DefaultOptions()
+	options.EnableProgressBar = true
+	options.StatsInterval = 5
+	r := &Runner{options: options, Logger: gologger.DefaultLogger}
+	stop := r.startHTTPXProbeProgress(httpxProbeProgressMinTargets, &probed, &alive)
+	require.NotNil(t, stop)
+	stop()
+}