Drain conns

internal/server/dedupe.go

@@ -1,122 +0,0 @@
-package server
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"net/url"
-	"sort"
-	"strings"
-	"sync"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
-	mapsutil "github.com/projectdiscovery/utils/maps"
-)
-
-var dynamicHeaders = map[string]bool{
-	"date":                true,
-	"if-modified-since":   true,
-	"if-unmodified-since": true,
-	"cache-control":       true,
-	"if-none-match":       true,
-	"if-match":            true,
-	"authorization":       true,
-	"cookie":              true,
-	"x-csrf-token":        true,
-	"content-length":      true,
-	"content-md5":         true,
-	"host":                true,
-	"x-request-id":        true,
-	"x-correlation-id":    true,
-	"user-agent":          true,
-	"referer":             true,
-}
-
-type requestDeduplicator struct {
-	hashes map[string]struct{}
-	lock   *sync.RWMutex
-}
-
-func newRequestDeduplicator() *requestDeduplicator {
-	return &requestDeduplicator{
-		hashes: make(map[string]struct{}),
-		lock:   &sync.RWMutex{},
-	}
-}
-
-func (r *requestDeduplicator) isDuplicate(req *types.RequestResponse) bool {
-	hash, err := hashRequest(req)
-	if err != nil {
-		return false
-	}
-
-	r.lock.RLock()
-	_, ok := r.hashes[hash]
-	r.lock.RUnlock()
-	if ok {
-		return true
-	}
-
-	r.lock.Lock()
-	r.hashes[hash] = struct{}{}
-	r.lock.Unlock()
-	return false
-}
-
-func hashRequest(req *types.RequestResponse) (string, error) {
-	normalizedURL, err := normalizeURL(req.URL.URL)
-	if err != nil {
-		return "", err
-	}
-
-	var hashContent strings.Builder
-	hashContent.WriteString(req.Request.Method)
-	hashContent.WriteString(normalizedURL)
-
-	headers := sortedNonDynamicHeaders(req.Request.Headers)
-	for _, header := range headers {
-		hashContent.WriteString(header.Key)
-		hashContent.WriteString(header.Value)
-	}
-
-	if len(req.Request.Body) > 0 {
-		hashContent.Write([]byte(req.Request.Body))
-	}
-
-	// Calculate the SHA256 hash
-	hash := sha256.Sum256([]byte(hashContent.String()))
-	return hex.EncodeToString(hash[:]), nil
-}
-
-func normalizeURL(u *url.URL) (string, error) {
-	query := u.Query()
-	sortedQuery := make(url.Values)
-	for k, v := range query {
-		sort.Strings(v)
-		sortedQuery[k] = v
-	}
-	u.RawQuery = sortedQuery.Encode()
-
-	if u.Path == "" {
-		u.Path = "/"
-	}
-	return u.String(), nil
-}
-
-type header struct {
-	Key   string
-	Value string
-}
-
-func sortedNonDynamicHeaders(headers mapsutil.OrderedMap[string, string]) []header {
-	var result []header
-	headers.Iterate(func(k, v string) bool {
-		if !dynamicHeaders[strings.ToLower(k)] {
-			result = append(result, header{Key: k, Value: v})
-		}
-		return true
-	})
-	sort.Slice(result, func(i, j int) bool {
-		return result[i].Key < result[j].Key
-	})
-	return result
-}

internal/server/requests_worker.go

@@ -39,7 +39,7 @@ func (s *DASTServer) consumeTaskRequest(req PostRequestsHandlerRequest) {
 		return
 	}
 
-	if s.deduplicator.isDuplicate(parsedReq) {
+	if s.deduplicator.IsDuplicate(parsedReq) {
 		gologger.Warning().Msgf("Duplicate request detected: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String())
 		return
 	}

internal/server/server.go

@@ -19,6 +19,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/internal/server/scope"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/dedupe"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/utils/env"
@@ -30,7 +31,7 @@ type DASTServer struct {
 	httpServer   *http.Server
 	options      *Options
 	tasksPool    *pond.WorkerPool
-	deduplicator *requestDeduplicator
+	deduplicator *dedupe.RequestDeduplicator
 	scopeManager *scope.Manager
 	proxy        *proxy.Proxy
 	startTime    time.Time
@@ -96,7 +97,7 @@ func New(options *Options) (*DASTServer, error) {
 	server := &DASTServer{
 		options:      options,
 		tasksPool:    pond.New(maxWorkers, bufferSize),
-		deduplicator: newRequestDeduplicator(),
+		deduplicator: dedupe.NewRequestDeduplicator(),
 		startTime:    time.Now(),
 	}
 	server.setupHandlers(false)

internal/tests/integration/network_test.go

@@ -19,7 +19,7 @@ func TestNetwork(t *testing.T) {
 	t.Run("Basic", func(t *testing.T) {
 		errState := &capturedError{}
 		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
 			if err != nil {
@@ -48,7 +48,7 @@ func TestNetwork(t *testing.T) {
 
 	t.Run("PinnedTargetLine", func(t *testing.T) {
 		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
 			if err == nil && string(data) == "PING" {
@@ -81,7 +81,7 @@ func TestNetwork(t *testing.T) {
 	t.Run("Hex", func(t *testing.T) {
 		errState := &capturedError{}
 		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
 			if err != nil {
@@ -111,7 +111,7 @@ func TestNetwork(t *testing.T) {
 	t.Run("MultiStep", func(t *testing.T) {
 		errState := &capturedError{}
 		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 5, 5*time.Second)
 			if err != nil {
@@ -153,7 +153,7 @@ func TestNetwork(t *testing.T) {
 	t.Run("Duration", func(t *testing.T) {
 		errState := &capturedError{}
 		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 5, 5*time.Second)
 			if err != nil {
@@ -192,7 +192,7 @@ func TestNetwork(t *testing.T) {
 
 	t.Run("SelfContained", func(t *testing.T) {
 		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 			_, _ = conn.Write([]byte("Authentication successful"))
 		})
 		defer server.Close()
@@ -212,7 +212,7 @@ func TestNetwork(t *testing.T) {
 	t.Run("Variables", func(t *testing.T) {
 		errState := &capturedError{}
 		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
 			if err != nil {
@@ -240,7 +240,7 @@ func TestNetwork(t *testing.T) {
 	t.Run("SameAddress", func(t *testing.T) {
 		errState := &capturedError{}
 		server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
 			if err != nil {
@@ -269,7 +269,7 @@ func TestNetwork(t *testing.T) {
 
 	t.Run("NetworkPort", func(t *testing.T) {
 		server := testutils.NewTCPServer(nil, 23846, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
 			if err == nil && string(data) == "PING" {
@@ -292,7 +292,7 @@ func TestNetwork(t *testing.T) {
 		// listener binds without root. The server is started on 8081 and the
 		// template (port 23846) must dial the operator-specified 8081 to match.
 		serverReserved := testutils.NewTCPServer(nil, 8081, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
 			if err == nil && string(data) == "PING" {
@@ -310,7 +310,7 @@ func TestNetwork(t *testing.T) {
 		}
 
 		serverOverride := testutils.NewTCPServer(nil, 34567, func(conn net.Conn) {
-			defer func() { _ = conn.Close() }()
+			defer testutils.CloseConn(conn)
 
 			data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
 			if err == nil && string(data) == "PING" {

internal/tests/testutils/integration.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -497,6 +498,18 @@ func (s *TCPServer) Close() {
 	_ = s.listener.Close()
 }
 
+// CloseConn closes a server connection after it drains the unread input.
+// A close with unread input sends a RST, and Windows then drops the reply
+// that the client did not read yet, so the client reads nothing.
+func CloseConn(conn net.Conn) {
+	if tcpConn, ok := conn.(*net.TCPConn); ok {
+		_ = tcpConn.CloseWrite()
+	}
+	_ = conn.SetReadDeadline(time.Now().Add(time.Second))
+	_, _ = io.Copy(io.Discard, conn)
+	_ = conn.Close()
+}
+
 // NewWebsocketServer creates a new websocket server from a handler.
 func NewWebsocketServer(path string, handler func(conn net.Conn), originValidate func(origin string) bool, port ...int) *httptest.Server {
 	handlerFunc := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

internal/tests/testutils/integration_test.go

@@ -0,0 +1,49 @@
+package testutils
+
+import (
+	"io"
+	"net"
+	"testing"
+	"time"
+)
+
+// TestCloseConnUnreadInput checks that the client reads the reply and then a
+// clean EOF when the server did not read all of the request. A plain close
+// sends a RST here, and on Windows the RST drops the reply.
+func TestCloseConnUnreadInput(t *testing.T) {
+	handled := make(chan struct{})
+	server := NewTCPServer(nil, 0, func(conn net.Conn) {
+		defer close(handled)
+		defer CloseConn(conn)
+
+		buf := make([]byte, 4)
+		if _, err := io.ReadFull(conn, buf); err != nil {
+			return
+		}
+		_, _ = conn.Write([]byte("PONG"))
+	})
+	defer server.Close()
+
+	conn, err := net.Dial("tcp", server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = conn.Close() }()
+
+	if _, err := conn.Write([]byte("PING\r\n")); err != nil {
+		t.Fatal(err)
+	}
+	// let the server close before the client reads, as in the Windows failure
+	time.Sleep(200 * time.Millisecond)
+
+	_ = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
+	data, err := io.ReadAll(conn)
+	if err != nil {
+		t.Fatalf("read after server close: %v", err)
+	}
+	if string(data) != "PONG" {
+		t.Fatalf("got %q, want %q", data, "PONG")
+	}
+	_ = conn.Close()
+	<-handled
+}

pkg/input/dedupe/dedupe.go

@@ -0,0 +1,218 @@
+// Package dedupe provides request-level deduplication for HTTP inputs used by
+// fuzzing / DAST (burp, openapi, proxify dumps, live dast-server, etc.).
+package dedupe
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"net/url"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
+	mapsutil "github.com/projectdiscovery/utils/maps"
+)
+
+// Headers that commonly change between equivalent requests and should not
+// affect the fingerprint used for deduplication. Cookie and Authorization are
+// not here: requests that differ only in credentials are distinct (e.g. for
+// authorization or IDOR testing), so their normalized values are part of the
+// fingerprint.
+var dynamicHeaders = map[string]bool{
+	"date":                true,
+	"if-modified-since":   true,
+	"if-unmodified-since": true,
+	"cache-control":       true,
+	"if-none-match":       true,
+	"if-match":            true,
+	"x-csrf-token":        true,
+	"content-length":      true,
+	"content-md5":         true,
+	"host":                true,
+	"x-request-id":        true,
+	"x-correlation-id":    true,
+	"user-agent":          true,
+	"referer":             true,
+}
+
+// RequestDeduplicator tracks seen HTTP requests by a stable fingerprint.
+type RequestDeduplicator struct {
+	hashes map[string]struct{}
+	lock   sync.RWMutex
+}
+
+// NewRequestDeduplicator creates an empty request deduplicator.
+func NewRequestDeduplicator() *RequestDeduplicator {
+	return &RequestDeduplicator{
+		hashes: make(map[string]struct{}),
+	}
+}
+
+// IsDuplicate returns true if an equivalent request was already seen.
+// The first occurrence of a fingerprint is recorded and returns false.
+func (r *RequestDeduplicator) IsDuplicate(req *types.RequestResponse) bool {
+	if req == nil {
+		return false
+	}
+	hash, err := HashRequest(req)
+	if err != nil || hash == "" {
+		return false
+	}
+
+	r.lock.RLock()
+	_, ok := r.hashes[hash]
+	r.lock.RUnlock()
+	if ok {
+		return true
+	}
+
+	r.lock.Lock()
+	defer r.lock.Unlock()
+	if _, ok := r.hashes[hash]; ok {
+		return true
+	}
+	r.hashes[hash] = struct{}{}
+	return false
+}
+
+// Len returns the number of unique fingerprints recorded.
+func (r *RequestDeduplicator) Len() int {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+	return len(r.hashes)
+}
+
+// HashRequest builds a stable fingerprint for a request:
+// method + normalized URL + non-dynamic headers + body.
+func HashRequest(req *types.RequestResponse) (string, error) {
+	if req == nil || req.URL.URL == nil {
+		return "", nil
+	}
+
+	normalizedURL, err := NormalizeURL(req.URL.URL)
+	if err != nil {
+		return "", err
+	}
+
+	var hashContent strings.Builder
+	method := "GET"
+	var body string
+	var headers mapsutil.OrderedMap[string, string]
+	if req.Request != nil {
+		if req.Request.Method != "" {
+			method = strings.ToUpper(req.Request.Method)
+		}
+		body = req.Request.Body
+		headers = req.Request.Headers
+	}
+	// NUL separators keep field boundaries unambiguous, so "ab"+"c" and
+	// "a"+"bc" do not hash the same.
+	hashContent.WriteString(method)
+	hashContent.WriteByte(0)
+	hashContent.WriteString(normalizedURL)
+
+	for _, header := range sortedNonDynamicHeaders(headers) {
+		hashContent.WriteByte(0)
+		hashContent.WriteString(header.Key)
+		hashContent.WriteByte(0)
+		hashContent.WriteString(header.Value)
+	}
+
+	if len(body) > 0 {
+		hashContent.WriteByte(0)
+		hashContent.WriteString(body)
+	}
+
+	hash := sha256.Sum256([]byte(hashContent.String()))
+	return hex.EncodeToString(hash[:]), nil
+}
+
+// NormalizeURL returns a stable string form of u without mutating the original.
+// Scheme and host are lowercased, query keys are sorted and the fragment is
+// dropped since it is never sent to the server. The values of a repeated key
+// keep their order, because servers can use the first or the last value. A
+// query that does not parse is kept as sent, so no malformed pair is dropped.
+func NormalizeURL(u *url.URL) (string, error) {
+	if u == nil {
+		return "", nil
+	}
+	cloned := *u
+	cloned.Scheme = strings.ToLower(cloned.Scheme)
+	cloned.Host = strings.ToLower(cloned.Host)
+	cloned.Fragment = ""
+	cloned.RawFragment = ""
+	if query, err := url.ParseQuery(cloned.RawQuery); err == nil {
+		// Encode sorts the keys and keeps the order of the values of each key.
+		cloned.RawQuery = query.Encode()
+	}
+	if cloned.Path == "" {
+		cloned.Path = "/"
+	}
+	return cloned.String(), nil
+}
+
+type header struct {
+	Key   string
+	Value string
+}
+
+func sortedNonDynamicHeaders(headers mapsutil.OrderedMap[string, string]) []header {
+	var result []header
+	headers.Iterate(func(k, v string) bool {
+		key := strings.ToLower(k)
+		if dynamicHeaders[key] {
+			return true
+		}
+		switch key {
+		case "cookie":
+			v = normalizeCookie(v)
+		case "authorization":
+			v = normalizeAuthorization(v)
+		}
+		result = append(result, header{Key: key, Value: v})
+		return true
+	})
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Key < result[j].Key
+	})
+	return result
+}
+
+// normalizeCookie sorts the cookie pairs so that only their order does not make
+// two requests different. Empty pairs are dropped; names and values are kept
+// as sent, since both are case sensitive. When a name occurs more than once,
+// the pairs keep their order, because servers can use the first or the last
+// value.
+func normalizeCookie(value string) string {
+	parts := strings.Split(value, ";")
+	pairs := parts[:0]
+	names := make(map[string]struct{}, len(parts))
+	unique := true
+	for _, part := range parts {
+		if part = strings.TrimSpace(part); part != "" {
+			pairs = append(pairs, part)
+			name, _, _ := strings.Cut(part, "=")
+			name = strings.TrimSpace(name)
+			if _, seen := names[name]; seen {
+				unique = false
+			}
+			names[name] = struct{}{}
+		}
+	}
+	if unique {
+		sort.Strings(pairs)
+	}
+	return strings.Join(pairs, "; ")
+}
+
+// normalizeAuthorization lowercases the case insensitive auth scheme and trims
+// the surrounding space. The credentials are kept as sent.
+func normalizeAuthorization(value string) string {
+	value = strings.TrimSpace(value)
+	scheme, credentials, found := strings.Cut(value, " ")
+	if !found {
+		return value
+	}
+	return strings.ToLower(scheme) + " " + strings.TrimSpace(credentials)
+}

pkg/input/dedupe/dedupe_test.go

@@ -0,0 +1,220 @@
+package dedupe
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
+	mapsutil "github.com/projectdiscovery/utils/maps"
+	urlutil "github.com/projectdiscovery/utils/url"
+	"github.com/stretchr/testify/require"
+)
+
+func mustURL(t *testing.T, raw string) urlutil.URL {
+	t.Helper()
+	parsed, err := urlutil.ParseAbsoluteURL(raw, false)
+	require.NoError(t, err)
+	return *parsed
+}
+
+func newReq(t *testing.T, method, rawURL, body string, headers map[string]string) *types.RequestResponse {
+	t.Helper()
+	om := mapsutil.NewOrderedMap[string, string]()
+	for k, v := range headers {
+		om.Set(k, v)
+	}
+	return &types.RequestResponse{
+		URL: mustURL(t, rawURL),
+		Request: &types.HttpRequest{
+			Method:  method,
+			Body:    body,
+			Headers: om,
+		},
+	}
+}
+
+func TestRequestDeduplicatorDuplicates(t *testing.T) {
+	t.Parallel()
+
+	d := NewRequestDeduplicator()
+	a := newReq(t, "GET", "https://example.com/a?b=2&a=1", "", nil)
+	b := newReq(t, "GET", "https://example.com/a?a=1&b=2", "", nil) // query order differs
+	c := newReq(t, "POST", "https://example.com/a?a=1&b=2", "", nil)
+
+	require.False(t, d.IsDuplicate(a))
+	require.True(t, d.IsDuplicate(b), "normalized query order should collide")
+	require.False(t, d.IsDuplicate(c), "different method should not collide")
+	require.Equal(t, 2, d.Len())
+}
+
+func TestRequestDeduplicatorIgnoresDynamicHeaders(t *testing.T) {
+	t.Parallel()
+
+	d := NewRequestDeduplicator()
+	a := newReq(t, "GET", "https://example.com/x", "", map[string]string{
+		"Cookie":     "a=1; b=2",
+		"User-Agent": "ua-a",
+		"X-Custom":   "keep",
+	})
+	b := newReq(t, "GET", "https://example.com/x", "", map[string]string{
+		"Cookie":     "b=2;a=1",
+		"User-Agent": "ua-b",
+		"X-Custom":   "keep",
+	})
+	c := newReq(t, "GET", "https://example.com/x", "", map[string]string{
+		"X-Custom": "other",
+	})
+
+	require.False(t, d.IsDuplicate(a))
+	require.True(t, d.IsDuplicate(b), "dynamic headers and cookie order should be ignored")
+	require.False(t, d.IsDuplicate(c), "stable header change should not collide")
+}
+
+func TestRequestDeduplicatorBodyMatters(t *testing.T) {
+	t.Parallel()
+
+	d := NewRequestDeduplicator()
+	a := newReq(t, "POST", "https://example.com/x", `{"a":1}`, nil)
+	b := newReq(t, "POST", "https://example.com/x", `{"a":2}`, nil)
+
+	require.False(t, d.IsDuplicate(a))
+	require.False(t, d.IsDuplicate(b))
+}
+
+func TestNormalizeURLDoesNotMutateInput(t *testing.T) {
+	t.Parallel()
+
+	u, err := url.Parse("https://example.com/path?b=2&a=1")
+	require.NoError(t, err)
+	original := u.RawQuery
+
+	got, err := NormalizeURL(u)
+	require.NoError(t, err)
+	require.Equal(t, original, u.RawQuery)
+	require.Contains(t, got, "a=1")
+	require.Contains(t, got, "b=2")
+}
+
+func TestHashRequestEmptyPath(t *testing.T) {
+	t.Parallel()
+
+	a := newReq(t, "GET", "https://example.com", "", nil)
+	b := newReq(t, "GET", "https://example.com/", "", nil)
+
+	ha, err := HashRequest(a)
+	require.NoError(t, err)
+	hb, err := HashRequest(b)
+	require.NoError(t, err)
+	require.Equal(t, ha, hb)
+}
+
+func TestHashRequestFieldBoundaries(t *testing.T) {
+	t.Parallel()
+
+	a := newReq(t, "GET", "https://example.com/x", "", map[string]string{"X-Ab": "c"})
+	b := newReq(t, "GET", "https://example.com/x", "", map[string]string{"X-A": "bc"})
+
+	ha, err := HashRequest(a)
+	require.NoError(t, err)
+	hb, err := HashRequest(b)
+	require.NoError(t, err)
+	require.NotEqual(t, ha, hb)
+}
+
+func TestHashRequestHostCaseAndFragment(t *testing.T) {
+	t.Parallel()
+
+	a := newReq(t, "GET", "https://Example.COM/x?a=1#top", "", nil)
+	b := newReq(t, "GET", "https://example.com/x?a=1", "", nil)
+	c := newReq(t, "GET", "https://example.com/X?a=1", "", nil)
+
+	ha, err := HashRequest(a)
+	require.NoError(t, err)
+	hb, err := HashRequest(b)
+	require.NoError(t, err)
+	hc, err := HashRequest(c)
+	require.NoError(t, err)
+	require.Equal(t, ha, hb, "host case and fragment should not matter")
+	require.NotEqual(t, hb, hc, "path case should matter")
+}
+
+func TestRequestDeduplicatorAuthHeadersMatter(t *testing.T) {
+	t.Parallel()
+
+	d := NewRequestDeduplicator()
+	alice := newReq(t, "GET", "https://example.com/orders/1", "", map[string]string{"Authorization": "Bearer alice"})
+	bob := newReq(t, "GET", "https://example.com/orders/1", "", map[string]string{"Authorization": "Bearer bob"})
+	aliceAgain := newReq(t, "GET", "https://example.com/orders/1", "", map[string]string{"authorization": "bearer  alice "})
+	anonymous := newReq(t, "GET", "https://example.com/orders/1", "", nil)
+
+	require.False(t, d.IsDuplicate(alice))
+	require.False(t, d.IsDuplicate(bob), "different authorization must not collide")
+	require.True(t, d.IsDuplicate(aliceAgain), "scheme case and spacing should not matter")
+	require.False(t, d.IsDuplicate(anonymous), "request without credentials must not collide")
+	require.Equal(t, 3, d.Len())
+}
+
+func TestRequestDeduplicatorCookies(t *testing.T) {
+	t.Parallel()
+
+	d := NewRequestDeduplicator()
+	a := newReq(t, "GET", "https://example.com/x", "", map[string]string{"Cookie": "session=alice; theme=dark"})
+	b := newReq(t, "GET", "https://example.com/x", "", map[string]string{"Cookie": "theme=dark;session=alice;"})
+	c := newReq(t, "GET", "https://example.com/x", "", map[string]string{"Cookie": "session=bob; theme=dark"})
+
+	require.False(t, d.IsDuplicate(a))
+	require.True(t, d.IsDuplicate(b), "cookie pair order should not matter")
+	require.False(t, d.IsDuplicate(c), "different cookie value must not collide")
+	require.Equal(t, 2, d.Len())
+}
+
+func TestRequestDeduplicatorWithoutAuthHeaders(t *testing.T) {
+	t.Parallel()
+
+	d := NewRequestDeduplicator()
+	a := newReq(t, "GET", "https://example.com/x?b=2&a=1", "", map[string]string{"User-Agent": "a", "X-Request-Id": "1"})
+	b := newReq(t, "GET", "https://example.com/x?a=1&b=2", "", map[string]string{"User-Agent": "b", "X-Request-Id": "2"})
+
+	require.False(t, d.IsDuplicate(a))
+	require.True(t, d.IsDuplicate(b))
+	require.Equal(t, 1, d.Len())
+}
+
+func TestNormalizeCookie(t *testing.T) {
+	t.Parallel()
+
+	require.Equal(t, "a=1; b=2", normalizeCookie(" b=2 ;; a=1 ;"))
+	require.Equal(t, "", normalizeCookie(""))
+	require.Equal(t, "a=2; a=1", normalizeCookie("a=2; a=1"), "repeated names keep their order")
+	require.Equal(t, "b=3; a=2; a=1", normalizeCookie("b=3; a=2; a=1"))
+}
+
+func TestHashRequestRepeatedCookieOrder(t *testing.T) {
+	t.Parallel()
+
+	a := newReq(t, "GET", "https://example.com/x", "", map[string]string{"Cookie": "session=alice; session=bob"})
+	b := newReq(t, "GET", "https://example.com/x", "", map[string]string{"Cookie": "session=bob; session=alice"})
+
+	ha, err := HashRequest(a)
+	require.NoError(t, err)
+	hb, err := HashRequest(b)
+	require.NoError(t, err)
+	require.NotEqual(t, ha, hb)
+}
+
+func TestNormalizeURLQuery(t *testing.T) {
+	t.Parallel()
+
+	normalize := func(raw string) string {
+		u, err := url.Parse(raw)
+		require.NoError(t, err)
+		got, err := NormalizeURL(u)
+		require.NoError(t, err)
+		return got
+	}
+
+	require.Equal(t, normalize("https://example.com/x?b=2&a=1"), normalize("https://example.com/x?a=1&b=2"), "key order should not matter")
+	require.NotEqual(t, normalize("https://example.com/x?a=1&a=2"), normalize("https://example.com/x?a=2&a=1"), "order of repeated values should matter")
+	require.NotEqual(t, normalize("https://example.com/x?a=1&probe=%ZZ"), normalize("https://example.com/x?a=1"), "malformed pairs should not be dropped")
+	require.Equal(t, "https://example.com/x?a=1&probe=%ZZ", normalize("https://example.com/x?a=1&probe=%ZZ"))
+}

pkg/input/provider/http/multiformat.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/dedupe"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/burp"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/json"
@@ -39,6 +40,7 @@ type HttpInputProvider struct {
 	inputData []byte
 	inputFile string
 	count     int64
+	dupeCount int64
 }
 
 // NewHttpInputProvider creates a new input provider for nuclei from a file
@@ -48,9 +50,12 @@ type HttpInputProvider struct {
 // otherwise it will use the input string
 func NewHttpInputProvider(opts *HttpMultiFormatOptions) (*HttpInputProvider, error) {
 	var format formats.Format
-	for _, provider := range providersList {
-		if provider.Name() == opts.InputMode {
+	for _, newProvider := range providersList {
+		// Use a fresh instance per provider so concurrent callers do not
+		// share format options.
+		if provider := newProvider(); provider.Name() == opts.InputMode {
 			format = provider
+			break
 		}
 	}
 	if format == nil {
@@ -60,6 +65,8 @@ func NewHttpInputProvider(opts *HttpMultiFormatOptions) (*HttpInputProvider, err
 	// Do a first pass over the input to identify any errors
 	// and get the count of the input file as well
 	count := int64(0)
+	dupeCount := int64(0)
+	deduplicator := dedupe.NewRequestDeduplicator()
 	var inputFile *os.File
 	var inputReader io.Reader
 	if opts.InputFile != "" {
@@ -87,23 +94,34 @@ func NewHttpInputProvider(opts *HttpMultiFormatOptions) (*HttpInputProvider, err
 	}
 
 	parseErr := format.Parse(bytes.NewReader(data), func(request *types.RequestResponse) bool {
+		if deduplicator.IsDuplicate(request) {
+			dupeCount++
+			return false
+		}
 		count++
 		return false
 	}, opts.InputFile)
 	if parseErr != nil {
 		return nil, errors.Wrap(parseErr, "could not parse input file")
 	}
-	return &HttpInputProvider{format: format, inputData: data, inputFile: opts.InputFile, count: count}, nil
+	if dupeCount > 0 {
+		gologger.Info().Msgf("Supplied %s input was automatically deduplicated (%d removed).", opts.InputMode, dupeCount)
+	}
+	return &HttpInputProvider{format: format, inputData: data, inputFile: opts.InputFile, count: count, dupeCount: dupeCount}, nil
 }
 
-// Count returns the number of items for input provider
+// Count returns the number of unique items for input provider
 func (i *HttpInputProvider) Count() int64 {
 	return i.count
 }
 
-// Iterate over all inputs in order
+// Iterate over all unique inputs in order
 func (i *HttpInputProvider) Iterate(callback func(value *contextargs.MetaInput) bool) {
+	deduplicator := dedupe.NewRequestDeduplicator()
 	err := i.format.Parse(bytes.NewReader(i.inputData), func(request *types.RequestResponse) bool {
+		if deduplicator.IsDuplicate(request) {
+			return false
+		}
 		metaInput := contextargs.NewMetaInput()
 		metaInput.ReqResp = request
 		metaInput.Input = request.URL.String()
@@ -140,21 +158,21 @@ func (i *HttpInputProvider) InputType() string {
 func (i *HttpInputProvider) Close() {}
 
 // Supported Providers
-var providersList = []formats.Format{
-	burp.New(),
-	json.New(),
-	yaml.New(),
-	openapi.New(),
-	swagger.New(),
-	raw.New(),
+var providersList = []func() formats.Format{
+	func() formats.Format { return burp.New() },
+	func() formats.Format { return json.New() },
+	func() formats.Format { return yaml.New() },
+	func() formats.Format { return openapi.New() },
+	func() formats.Format { return swagger.New() },
+	func() formats.Format { return raw.New() },
 }
 
 // SupportedFormats returns the list of supported formats in comma-separated
 // manner
 func SupportedFormats() string {
 	var formats []string
-	for _, provider := range providersList {
-		formats = append(formats, provider.Name())
+	for _, newProvider := range providersList {
+		formats = append(formats, newProvider().Name())
 	}
 	return strings.Join(formats, ", ")
 }

pkg/input/provider/http/multiformat_test.go

@@ -0,0 +1,77 @@
+package http
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHttpInputProviderDeduplicatesJSONL(t *testing.T) {
+	t.Parallel()
+
+	// Three lines: two equivalent GETs (different cookie and query order) and one unique POST.
+	input := "" +
+		`{"url":"https://example.com/search?q=1&lang=en","request":{"raw":"GET /search?q=1&lang=en HTTP/1.1\r\nHost: example.com\r\nCookie: a=1; b=2\r\n\r\n"}}` + "\n" +
+		`{"url":"https://example.com/search?lang=en&q=1","request":{"raw":"GET /search?lang=en&q=1 HTTP/1.1\r\nHost: example.com\r\nCookie: b=2; a=1\r\n\r\n"}}` + "\n" +
+		`{"url":"https://example.com/search","request":{"raw":"POST /search HTTP/1.1\r\nHost: example.com\r\nContent-Type: application/json\r\nContent-Length: 7\r\n\r\n{\"x\":1}"}}` + "\n"
+
+	provider, err := NewHttpInputProvider(&HttpMultiFormatOptions{
+		InputMode:     "jsonl",
+		InputContents: input,
+	})
+	require.NoError(t, err)
+	require.Equal(t, int64(2), provider.Count())
+	require.Equal(t, int64(1), provider.dupeCount)
+
+	var methods []string
+	provider.Iterate(func(value *contextargs.MetaInput) bool {
+		require.NotNil(t, value.ReqResp)
+		require.NotNil(t, value.ReqResp.Request)
+		methods = append(methods, value.ReqResp.Request.Method+" "+value.Input)
+		return true
+	})
+	require.Len(t, methods, 2)
+	require.Contains(t, methods[0], "GET")
+	require.Contains(t, methods[1], "POST")
+}
+
+func TestHttpInputProviderIterateSkipsDuplicates(t *testing.T) {
+	t.Parallel()
+
+	input := "" +
+		`{"url":"https://example.com/a","request":{"raw":"GET /a HTTP/1.1\r\nHost: example.com\r\n\r\n"}}` + "\n" +
+		`{"url":"https://example.com/a","request":{"raw":"GET /a HTTP/1.1\r\nHost: example.com\r\nUser-Agent: other\r\n\r\n"}}` + "\n" +
+		`{"url":"https://example.com/b","request":{"raw":"GET /b HTTP/1.1\r\nHost: example.com\r\n\r\n"}}` + "\n"
+
+	provider, err := NewHttpInputProvider(&HttpMultiFormatOptions{
+		InputMode:     "jsonl",
+		InputContents: input,
+	})
+	require.NoError(t, err)
+	require.Equal(t, int64(2), provider.Count())
+	require.Equal(t, int64(1), provider.dupeCount)
+
+	var urls []string
+	provider.Iterate(func(value *contextargs.MetaInput) bool {
+		urls = append(urls, value.Input)
+		return true
+	})
+	require.Equal(t, []string{"https://example.com/a", "https://example.com/b"}, urls)
+}
+
+func TestHttpInputProviderKeepsAuthOnlyDifferences(t *testing.T) {
+	t.Parallel()
+
+	input := "" +
+		`{"url":"https://example.com/orders/1","request":{"raw":"GET /orders/1 HTTP/1.1\r\nHost: example.com\r\nAuthorization: Bearer alice\r\n\r\n"}}` + "\n" +
+		`{"url":"https://example.com/orders/1","request":{"raw":"GET /orders/1 HTTP/1.1\r\nHost: example.com\r\nAuthorization: Bearer bob\r\n\r\n"}}` + "\n"
+
+	provider, err := NewHttpInputProvider(&HttpMultiFormatOptions{
+		InputMode:     "jsonl",
+		InputContents: input,
+	})
+	require.NoError(t, err)
+	require.Equal(t, int64(2), provider.Count())
+	require.Equal(t, int64(0), provider.dupeCount)
+}