Compare commits

...
Author SHA1 Message Date
Mzack9999 806085bf05 align impersonate 2026-07-17 12:34:53 +04:00
Mzack9999 76a77432e7 integrate reqx 2026-07-17 11:42:35 +04:00
14 changed files with 553 additions and 124 deletions

No files matched your search

+1 -1
View File
@@ -376,7 +376,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"),
flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable browser TLS and HTTP/2 fingerprint impersonation"),
flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
)
+7 -3
View File
@@ -1,6 +1,6 @@
module github.com/projectdiscovery/nuclei/v3
go 1.25.7
go 1.26.1
require (
github.com/andygrunwald/go-jira v1.16.1
@@ -22,7 +22,7 @@ require (
github.com/projectdiscovery/fastdialer v0.5.10
github.com/projectdiscovery/hmap v0.0.101
github.com/projectdiscovery/interactsh v1.3.1
github.com/projectdiscovery/rawhttp v0.1.90
github.com/projectdiscovery/rawhttp v0.1.90 // indirect
github.com/projectdiscovery/retryabledns v1.0.115
github.com/projectdiscovery/retryablehttp-go v1.3.15
github.com/projectdiscovery/yamldoc-go v1.0.6
@@ -112,6 +112,7 @@ require (
github.com/projectdiscovery/networkpolicy v0.1.40
github.com/projectdiscovery/ratelimit v0.0.88
github.com/projectdiscovery/rdap v0.9.0
github.com/projectdiscovery/reqx v0.0.0-20260717073230-60a4df85eb4c
github.com/projectdiscovery/sarif v0.1.0
github.com/projectdiscovery/tlsx v1.2.2
github.com/projectdiscovery/uncover v1.2.1
@@ -340,6 +341,8 @@ require (
github.com/projectdiscovery/freeport v0.0.7 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/quic-go/quic-go v0.59.0 // indirect
github.com/refraction-networking/utls v1.8.2 // indirect
github.com/sashabaranov/go-openai v1.37.0 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
@@ -366,6 +369,7 @@ require (
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.69.0 // indirect
github.com/vmihailenco/bufpool v0.1.11 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
github.com/vmihailenco/tagparser v0.1.2 // indirect
@@ -423,7 +427,7 @@ require (
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
golang.org/x/mod v0.36.0 // indirect
golang.org/x/sys v0.45.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.45.0
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
+12 -2
View File
@@ -940,6 +940,10 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs=
github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo=
@@ -1074,6 +1078,8 @@ github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI=
github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
@@ -1199,6 +1205,8 @@ go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
@@ -1463,8 +1471,8 @@ golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -1646,3 +1654,5 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
software.sslmate.com/src/go-pkcs12 v0.7.0 h1:Db8W44cB54TWD7stUFFSWxdfpdn6fZVcDl0w3R4RVM0=
software.sslmate.com/src/go-pkcs12 v0.7.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI=
github.com/projectdiscovery/reqx v0.0.0-20260717073230-60a4df85eb4c h1:/1JsMDRIdeRi3lOEjGWJUjTtW2wxCTuVYp6v7M6/4ps=
github.com/projectdiscovery/reqx v0.0.0-20260717073230-60a4df85eb4c/go.mod h1:hK9i+LySFhIu8tgt1j8BON9pJr/46avEJ5En7MgTRN0=
@@ -5,13 +5,12 @@ import (
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/networkpolicy"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
)
type Dialers struct {
Fastdialer *fastdialer.Dialer
RawHTTPClient *rawhttp.Client
RawHTTPClient *RawHTTPClient
DefaultHTTPClient *retryablehttp.Client
HTTPClientPool *HTTPPool
NetworkPolicy *networkpolicy.NetworkPolicy
@@ -0,0 +1,372 @@
package protocolstate
import (
"crypto/tls"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/reqx"
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
urlutil "github.com/projectdiscovery/utils/url"
)
// rawNewLine is the wire-level line terminator used for raw request serialization.
const rawNewLine = "\r\n"
// RawHeader is a single ordered raw HTTP header that preserves the exact key
// text and ordering. It replaces github.com/projectdiscovery/rawhttp/client.Header.
type RawHeader struct {
Key string
Value string
}
// RawHeaders is an ordered list of raw headers, replacing rawhttp/client.Headers.
type RawHeaders []RawHeader
// RawHTTPOptions configures a single raw (unsafe) HTTP request. It mirrors the
// subset of rawhttp.Options that nuclei relies on.
type RawHTTPOptions struct {
Timeout time.Duration
FollowRedirects bool
MaxRedirects int
AutomaticHostHeader bool
AutomaticContentLength bool
CustomHeaders RawHeaders
ForceReadAllBody bool
CustomRawBytes []byte
Proxy string
SNI string
FastDialer *fastdialer.Dialer
}
// DefaultRawHTTPOptions returns the default raw client options, matching the
// historical rawhttp.DefaultOptions.
func DefaultRawHTTPOptions() *RawHTTPOptions {
return &RawHTTPOptions{
Timeout: 30 * time.Second,
FollowRedirects: true,
MaxRedirects: 10,
AutomaticHostHeader: true,
AutomaticContentLength: true,
}
}
// Clone returns a shallow copy of the options so callers can mutate per request.
func (o *RawHTTPOptions) Clone() *RawHTTPOptions {
if o == nil {
return DefaultRawHTTPOptions()
}
clone := *o
return &clone
}
// RawHTTPClient is a reqx-backed replacement for rawhttp.Client. It sends
// wire-level (unsafe) HTTP/1.1 requests by serializing them to exact bytes and
// handing them to reqx's raw engine via SetRawRequest, which writes them
// verbatim on the wire.
type RawHTTPClient struct {
client *reqx.Client
Options *RawHTTPOptions
}
// newRawReqxClient builds the underlying reqx client used by the raw paths.
// It runs in raw mode (no automatic header/path mangling), forces HTTP/1.1,
// keeps fastdialer as the dialer, and disables reqx's own redirect following so
// per-request redirect policy can be honored on a shared client.
func newRawReqxClient(timeout time.Duration, dialer *fastdialer.Dialer, sni, proxy string, pipelining bool, pipelineDepth int) *reqx.Client {
opts := []reqx.Option{
reqx.WithInsecureSkipVerify(),
reqx.WithTLSMinVersion(tls.VersionTLS10),
reqx.WithForceHTTP1(),
reqx.WithKeepAlive(!pipelining),
reqx.WithRawMode(),
}
if timeout > 0 {
opts = append(opts, reqx.WithTimeout(timeout))
}
if dialer != nil {
opts = append(opts, reqx.WithDialer(reqx.DialerFunc(dialer.Dial)))
}
if sni != "" {
opts = append(opts, reqx.WithSNI(sni))
}
if proxy != "" {
if u, err := url.Parse(proxy); err == nil {
opts = append(opts, reqx.WithProxyURL(u))
}
}
if pipelining {
opts = append(opts, reqx.WithPipelining(true))
if pipelineDepth > 0 {
opts = append(opts, reqx.WithPipelineDepth(pipelineDepth))
}
}
client := reqx.New(opts...)
// never auto-follow: redirects are followed manually so that per-request
// FollowRedirects / MaxRedirects are honored on a shared client.
client.SetRedirectPolicy(func(_ *http.Request, _ []*http.Request) error {
return http.ErrUseLastResponse
})
client.SetMaxRedirects(1 << 30)
return client
}
// NewRawHTTPClient creates a new reqx-backed raw client with the provided options.
func NewRawHTTPClient(options *RawHTTPOptions) *RawHTTPClient {
if options == nil {
options = DefaultRawHTTPOptions()
}
client := newRawReqxClient(options.Timeout, options.FastDialer, options.SNI, options.Proxy, false, 0)
return &RawHTTPClient{client: client, Options: options}
}
// DoRaw performs a raw request using the client's default options.
func (c *RawHTTPClient) DoRaw(method, reqURL, uripath string, headers map[string][]string, body io.Reader) (*http.Response, error) {
return c.DoRawWithOptions(method, reqURL, uripath, headers, body, c.Options)
}
// DoRawWithOptions performs a raw request with per-request options.
func (c *RawHTTPClient) DoRawWithOptions(method, reqURL, uripath string, headers map[string][]string, body io.Reader, options *RawHTTPOptions) (*http.Response, error) {
if options == nil {
options = c.Options
}
bodyBytes, bodyProvided, err := readBody(body)
if err != nil {
return nil, err
}
return c.do(method, reqURL, uripath, headers, bodyBytes, bodyProvided, options, 0)
}
func (c *RawHTTPClient) do(method, reqURL, uripath string, headers map[string][]string, body []byte, bodyProvided bool, options *RawHTTPOptions, depth int) (*http.Response, error) {
rawBytes, err := serializeRawRequest(method, reqURL, uripath, headers, body, bodyProvided, options)
if err != nil {
return nil, err
}
resp, err := c.client.R().SetRawRequest(rawBytes).Send(method, dialURLFromRaw(reqURL))
if err != nil {
return nil, err
}
httpResp := resp.Response
if httpResp == nil {
return nil, fmt.Errorf("rawhttp: nil response")
}
maxRedirects := options.MaxRedirects
if maxRedirects <= 0 {
maxRedirects = 10
}
if options.FollowRedirects && isRedirectStatus(httpResp.StatusCode) && depth < maxRedirects {
if loc := httpResp.Header.Get("Location"); loc != "" {
_, _ = io.Copy(io.Discard, httpResp.Body)
_ = httpResp.Body.Close()
next := resolveLocation(reqURL, loc)
// after a redirect we rebuild the request for the new URL, so any
// custom raw bytes / explicit uri path no longer apply.
nextOptions := options.Clone()
nextOptions.CustomRawBytes = nil
return c.do(method, next, "", headers, body, bodyProvided, nextOptions, depth+1)
}
}
return httpResp, nil
}
// RawPipelineOptions configures the turbo (HTTP/1.1 pipelining) client.
type RawPipelineOptions struct {
Host string
MaxConnections int
MaxPendingRequests int
Timeout time.Duration
FastDialer *fastdialer.Dialer
SNI string
Proxy string
}
// RawPipelineClient is a reqx-backed replacement for rawhttp.PipelineClient,
// used by nuclei's turbo/pipeline mode.
type RawPipelineClient struct {
client *reqx.Client
options *RawHTTPOptions
}
// NewRawPipelineClient creates a new reqx-backed pipelining client.
func NewRawPipelineClient(opts *RawPipelineOptions) *RawPipelineClient {
if opts == nil {
opts = &RawPipelineOptions{}
}
client := newRawReqxClient(opts.Timeout, opts.FastDialer, opts.SNI, opts.Proxy, true, opts.MaxPendingRequests)
ro := DefaultRawHTTPOptions()
ro.FollowRedirects = false
return &RawPipelineClient{client: client, options: ro}
}
// DoRaw sends a raw pipelined request.
func (c *RawPipelineClient) DoRaw(method, reqURL, uripath string, headers map[string][]string, body io.Reader) (*http.Response, error) {
bodyBytes, bodyProvided, err := readBody(body)
if err != nil {
return nil, err
}
rawBytes, err := serializeRawRequest(method, reqURL, uripath, headers, bodyBytes, bodyProvided, c.options)
if err != nil {
return nil, err
}
resp, err := c.client.R().SetRawRequest(rawBytes).Send(method, dialURLFromRaw(reqURL))
if err != nil {
return nil, err
}
if resp.Response == nil {
return nil, fmt.Errorf("rawhttp: nil response")
}
return resp.Response, nil
}
// Dor sends a retryablehttp request over the pipelined connection.
func (c *RawPipelineClient) Dor(req *retryablehttp.Request) (*http.Response, error) {
var body io.Reader
if req.Body != nil {
body = req.Body
}
return c.DoRaw(req.Method, req.URL.String(), "", map[string][]string(req.Header), body)
}
// DumpRequestRaw renders the wire-level representation of a raw request,
// replacing rawhttp.DumpRequestRaw. It shares the serializer used to send raw
// requests, so the dumped bytes match what is sent.
func DumpRequestRaw(method, reqURL, uripath string, headers map[string][]string, body io.Reader, options *RawHTTPOptions) ([]byte, error) {
if options == nil {
options = DefaultRawHTTPOptions()
}
bodyBytes, bodyProvided, err := readBody(body)
if err != nil {
return nil, err
}
return serializeRawRequest(method, reqURL, uripath, headers, bodyBytes, bodyProvided, options)
}
// serializeRawRequest builds the exact wire bytes for a raw HTTP/1.1 request.
// When CustomRawBytes is set, it is returned verbatim. Otherwise the request
// line, headers (ordered CustomHeaders take precedence over the map), optional
// Host and Content-Length, and body are assembled with CRLF terminators.
func serializeRawRequest(method, reqURL, uripath string, headers map[string][]string, body []byte, bodyProvided bool, options *RawHTTPOptions) ([]byte, error) {
if len(options.CustomRawBytes) > 0 {
return options.CustomRawBytes, nil
}
if method == "" {
method = http.MethodGet
}
u, err := urlutil.ParseURL(reqURL, true)
if err != nil {
return nil, err
}
path := uripath
if path == "" {
path = u.Path
if path == "" {
path = "/"
}
if !u.Params.IsEmpty() {
path += "?" + u.Params.Encode()
}
}
var b strings.Builder
b.WriteString(method + " " + path + " HTTP/1.1" + rawNewLine)
if len(options.CustomHeaders) > 0 {
for _, h := range options.CustomHeaders {
if h.Value != "" {
b.WriteString(h.Key + ": " + h.Value + rawNewLine)
} else {
b.WriteString(h.Key + rawNewLine)
}
}
} else {
if _, ok := headers["Host"]; !ok && options.AutomaticHostHeader {
b.WriteString("Host: " + u.Host + rawNewLine)
}
for key, values := range headers {
for _, value := range values {
if value != "" {
b.WriteString(key + ": " + value + rawNewLine)
} else {
b.WriteString(key + rawNewLine)
}
}
}
}
if options.AutomaticContentLength && bodyProvided {
b.WriteString("Content-Length: " + strconv.Itoa(len(body)) + rawNewLine)
}
b.WriteString(rawNewLine)
if len(body) > 0 {
b.Write(body)
}
return []byte(b.String()), nil
}
// dialURLFromRaw returns a scheme://host URL suitable for dialing. The actual
// request line/path lives in the raw bytes, so the dial URL only needs the
// scheme and host.
func dialURLFromRaw(reqURL string) string {
u, err := urlutil.ParseURL(reqURL, true)
if err != nil {
return reqURL
}
scheme := u.Scheme
if scheme == "" {
scheme = "http"
}
return scheme + "://" + u.Host
}
func isRedirectStatus(code int) bool {
switch code {
case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
return true
}
return false
}
// resolveLocation mirrors rawhttp's redirect handling: absolute URLs are used
// as-is, root-relative locations are joined to scheme://host, anything else is
// passed through verbatim.
func resolveLocation(baseURL, location string) string {
if strings.HasPrefix(location, "http://") || strings.HasPrefix(location, "https://") {
return location
}
u, err := urlutil.ParseURL(baseURL, true)
if err != nil {
return location
}
scheme := u.Scheme
if scheme == "" {
scheme = "http"
}
if strings.HasPrefix(location, "/") {
return fmt.Sprintf("%s://%s%s", scheme, u.Host, location)
}
return location
}
func readBody(body io.Reader) ([]byte, bool, error) {
if body == nil {
return nil, false, nil
}
b, err := io.ReadAll(body)
if err != nil {
return nil, false, err
}
return b, true, nil
}
+8 -4
View File
@@ -18,13 +18,13 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/race"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/raw"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/errkit"
stringsutil "github.com/projectdiscovery/utils/strings"
@@ -76,7 +76,7 @@ type generatedRequest struct {
original *Request
rawRequest *raw.Request
meta map[string]interface{}
pipelinedClient *rawhttp.PipelineClient
pipelinedClient *protocolstate.RawPipelineClient
request *retryablehttp.Request
dynamicValues map[string]interface{}
interactshURLs []string
@@ -505,13 +505,17 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st
return nil, errors.Wrap(err, "failed to set request body")
}
}
if !r.request.Unsafe {
// Under -tls-impersonate, leave User-Agent / Accept* unset so the transport
// browser profile can supply matching defaults. Template and -H headers
// still win via SetHeader / setCustomHeaders.
impersonate := r.options.Options != nil && r.options.Options.TlsImpersonate
if !r.request.Unsafe && !impersonate {
userAgent := useragent.PickRandom()
httputil.SetHeader(req, "User-Agent", userAgent.Raw)
}
// Only set these headers on non-raw requests
if len(r.request.Raw) == 0 && !r.request.Unsafe {
if len(r.request.Raw) == 0 && !r.request.Unsafe && !impersonate {
httputil.SetHeader(req, "Accept", "*/*")
httputil.SetHeader(req, "Accept-Language", "en")
}
+1 -2
View File
@@ -26,7 +26,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
"github.com/projectdiscovery/rawhttp"
fileutil "github.com/projectdiscovery/utils/file"
)
@@ -143,7 +142,7 @@ type Request struct {
totalRequests int
customHeaders map[string]string
generator *generators.PayloadGenerator // optional, only enabled when using payloads
rawhttpClient *rawhttp.Client
rawhttpClient *protocolstate.RawHTTPClient
dialer *fastdialer.Dialer
// description: |
+22 -92
View File
@@ -1,8 +1,6 @@
package httpclientpool
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
@@ -16,15 +14,11 @@ import (
"time"
"github.com/pkg/errors"
"golang.org/x/net/proxy"
"golang.org/x/net/publicsuffix"
"github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -188,8 +182,8 @@ func (c *Configuration) HasStandardOptions() bool {
return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0
}
// GetRawHTTP returns the rawhttp request client
func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client {
// GetRawHTTP returns the reqx-backed raw request client
func GetRawHTTP(options *protocols.ExecutorOptions) *protocolstate.RawHTTPClient {
dialers := protocolstate.GetDialersWithId(options.Options.ExecutionId)
if dialers == nil {
panic("dialers not initialized for execution id: " + options.Options.ExecutionId)
@@ -203,16 +197,17 @@ func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client {
return dialers.RawHTTPClient
}
rawHttpOptionsCopy := *rawhttp.DefaultOptions
rawOptions := protocolstate.DefaultRawHTTPOptions()
if options.Options.AliveHttpProxy != "" {
rawHttpOptionsCopy.Proxy = options.Options.AliveHttpProxy
rawOptions.Proxy = options.Options.AliveHttpProxy
} else if options.Options.AliveSocksProxy != "" {
rawHttpOptionsCopy.Proxy = options.Options.AliveSocksProxy
rawOptions.Proxy = options.Options.AliveSocksProxy
} else if dialers.Fastdialer != nil {
rawHttpOptionsCopy.FastDialer = dialers.Fastdialer
rawOptions.FastDialer = dialers.Fastdialer
}
rawHttpOptionsCopy.Timeout = options.Options.GetTimeouts().HttpTimeout
dialers.RawHTTPClient = rawhttp.NewClient(&rawHttpOptionsCopy)
rawOptions.Timeout = options.Options.GetTimeouts().HttpTimeout
rawOptions.SNI = options.Options.SNI
dialers.RawHTTPClient = protocolstate.NewRawHTTPClient(rawOptions)
return dialers.RawHTTPClient
}
@@ -288,80 +283,20 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
transportKey := transportHash(host, disableKeepAlives, maxIdleConns, maxIdleConnsPerHost, maxConnsPerHost, responseHeaderTimeout)
createTransport := func() (http.RoundTripper, error) {
// Set the base TLS configuration definition
tlsConfig := &tls.Config{
Renegotiation: tls.RenegotiateOnceAsClient,
InsecureSkipVerify: true,
MinVersion: tls.VersionTLS10,
ClientSessionCache: sharedTLSSessionCache,
}
if options.SNI != "" {
tlsConfig.ServerName = options.SNI
}
tlsConfig, err := utils.AddConfiguredClientCertToRequest(tlsConfig, options)
// reqx replaces the net/http.Transport as nuclei's HTTP engine: fastdialer
// still performs the TCP dial (DNS cache, dialed-IP tracking, network
// policy) while reqx performs TLS and the HTTP exchange. The pool tuning,
// client certificates, SNI, impersonation and proxy settings are mapped
// onto reqx options in newReqxTransport.
transport, err := newReqxTransport(dialers.Fastdialer, options, reqxTransportParams{
disableKeepAlives: disableKeepAlives,
maxIdleConns: maxIdleConns,
maxIdleConnsPerHost: maxIdleConnsPerHost,
idleConnTimeout: 30 * time.Second,
responseHeaderTimeout: responseHeaderTimeout,
})
if err != nil {
return nil, errors.Wrap(err, "could not create client certificate")
}
transport := &http.Transport{
ForceAttemptHTTP2: options.ForceAttemptHTTP2,
DialContext: dialers.Fastdialer.Dial,
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
if options.TlsImpersonate {
return dialers.Fastdialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil)
}
if options.HasClientCertificates() || options.ForceAttemptHTTP2 {
return dialers.Fastdialer.DialTLSWithConfig(ctx, network, addr, tlsConfig)
}
return dialers.Fastdialer.DialTLS(ctx, network, addr)
},
MaxIdleConns: maxIdleConns,
MaxIdleConnsPerHost: maxIdleConnsPerHost,
MaxConnsPerHost: maxConnsPerHost,
TLSClientConfig: tlsConfig,
DisableKeepAlives: disableKeepAlives,
IdleConnTimeout: 30 * time.Second,
ResponseHeaderTimeout: responseHeaderTimeout,
}
if options.AliveHttpProxy != "" {
if proxyURL, err := url.Parse(options.AliveHttpProxy); err == nil {
transport.Proxy = http.ProxyURL(proxyURL)
}
} else if options.AliveSocksProxy != "" {
socksURL, proxyErr := url.Parse(options.AliveSocksProxy)
if proxyErr != nil {
return nil, proxyErr
}
dialer, err := proxy.FromURL(socksURL, proxy.Direct)
if err != nil {
return nil, err
}
dc := dialer.(interface {
DialContext(ctx context.Context, network, addr string) (net.Conn, error)
})
transport.DialContext = dc.DialContext
transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
// upgrade proxy connection to tls
conn, err := dc.DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
if tlsConfig.ServerName == "" {
// addr should be in form of host:port already set from canonicalAddr
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
tlsConfig.ServerName = host
}
return tls.Client(conn, tlsConfig), nil
}
return nil, err
}
return &connTrackingTransport{base: transport}, nil
@@ -434,11 +369,6 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
return pool.GetOrCreateClient(clientKey, transportKey, createTransport, createClient)
}
// sharedTLSSessionCache is shared by all pooled transports so TLS session
// resumption survives transport eviction/re-creation, and a single bounded
// LRU replaces a 128-entry cache per host client.
var sharedTLSSessionCache = tls.NewLRUClientSessionCache(2048)
// transportHash identifies a shareable transport. Only parameters that live
// on http.Transport participate; everything else (redirects, cookie jars,
// client timeouts) is layered on top by the per-configuration client.
+6 -5
View File
@@ -6,7 +6,7 @@ import (
"strings"
"time"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -29,16 +29,17 @@ type RawHttpRequestOpts struct {
// Body is the body to use
Body io.Reader
// Options is more client related options
Options *rawhttp.Options
Options *protocolstate.RawHTTPOptions
}
// SendRawRequest sends a raw http request with the provided options and returns http response
func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Response, error) {
func SendRawRequest(client *protocolstate.RawHTTPClient, opts *RawHttpRequestOpts) (*http.Response, error) {
resp, err := client.DoRawWithOptions(opts.Method, opts.URL, opts.Path, opts.Headers, opts.Body, opts.Options)
if err != nil {
cause := err.Error()
if stringsutil.ContainsAll(cause, "ReadStatusLine: ", "read: connection reset by peer") {
// this error is caused when rawhttp client sends a corrupted or malformed request packet to server
if stringsutil.ContainsAll(cause, "ReadStatusLine: ", "read: connection reset by peer") ||
stringsutil.ContainsAny(cause, "connection reset by peer") {
// this error is caused when raw client sends a corrupted or malformed request packet to server
// some servers may attempt gracefully shutdown but most will just abruptly close the connection which results
// in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow
req, reqErr := http.NewRequest(opts.Method, opts.URL, opts.Body)
+99
View File
@@ -0,0 +1,99 @@
package httpclientpool
import (
"crypto/tls"
"net/url"
"time"
"github.com/pkg/errors"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/reqx"
)
// reqxTransportParams carries the pooled-transport tuning resolved by Get() so
// the reqx transport can mirror the connection-pool semantics of the historical
// net/http.Transport.
type reqxTransportParams struct {
disableKeepAlives bool
maxIdleConns int
maxIdleConnsPerHost int
idleConnTimeout time.Duration
responseHeaderTimeout time.Duration
}
// newReqxTransport builds the reqx.Transport that backs nuclei's safe HTTP path,
// replacing the net/http.Transport previously constructed inline. fastdialer is
// preserved as the dialer so DNS caching, dialed-IP tracking and network policy
// keep working, while reqx performs TLS and the HTTP exchange. In standard mode
// reqx delegates to an internal net/http.Transport, so connection pooling,
// httptrace (connection-reuse stats), response.TLS and HTTP/2 negotiation all
// keep working as before.
//
// HTTP/2 is only attempted when the user explicitly opts in (-force-attempt-http2),
// matching the historical ForceAttemptHTTP2 behavior; impersonation applies a
// pinned browser profile (TLS + HTTP/2 + matching header values) instead.
func newReqxTransport(dialer *fastdialer.Dialer, options *types.Options, params reqxTransportParams) (*reqx.Transport, error) {
opts := []reqx.Option{
reqx.WithDialer(reqx.DialerFunc(dialer.Dial)),
reqx.WithInsecureSkipVerify(),
reqx.WithTLSMinVersion(tls.VersionTLS10),
reqx.WithKeepAlive(!params.disableKeepAlives),
reqx.WithMaxIdleConns(params.maxIdleConns),
// reqx couples MaxConnsPerHost and MaxIdleConnsPerHost to a single value;
// we use the idle-per-host count (threads) so connection reuse matches the
// historical transport. Per-host concurrency is governed by the scan
// concurrency manager rather than the transport.
reqx.WithMaxConnsPerHost(params.maxIdleConnsPerHost),
reqx.WithIdleConnTimeout(params.idleConnTimeout),
reqx.WithResponseHeaderTimeout(params.responseHeaderTimeout),
}
if options.SNI != "" {
opts = append(opts, reqx.WithSNI(options.SNI))
}
switch {
case options.TlsImpersonate:
// Keep-alive and a pinned profile keep TLS, HTTP/2, and default request
// headers aligned with the impersonated client.
opts = append(opts,
reqx.WithKeepAlive(true),
reqx.WithBrowserProfile(reqx.ProfileChrome131),
)
case options.ForceAttemptHTTP2:
// leave reqx in auto mode so HTTP/2 is negotiated via ALPN
default:
opts = append(opts, reqx.WithForceHTTP1())
}
// client certificate / custom CA support, mirroring the tls.Config that
// AddConfiguredClientCertToRequest used to populate on the net/http transport.
if options.HasClientCertificates() {
certConfig, err := utils.AddConfiguredClientCertToRequest(&tls.Config{}, options)
if err != nil {
return nil, errors.Wrap(err, "could not create client certificate")
}
for _, cert := range certConfig.Certificates {
opts = append(opts, reqx.WithClientCert(cert))
}
if certConfig.RootCAs != nil {
opts = append(opts, reqx.WithRootCAs(certConfig.RootCAs))
}
}
if options.AliveHttpProxy != "" {
if proxyURL, err := url.Parse(options.AliveHttpProxy); err == nil {
opts = append(opts, reqx.WithProxyURL(proxyURL))
}
} else if options.AliveSocksProxy != "" {
proxyURL, err := url.Parse(options.AliveSocksProxy)
if err != nil {
return nil, err
}
opts = append(opts, reqx.WithProxyURL(proxyURL))
}
return reqx.NewTransport(opts...), nil
}
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
"github.com/projectdiscovery/rawhttp/client"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/utils/errkit"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
@@ -27,7 +27,7 @@ type Request struct {
Path string
Data string
Headers map[string]string
UnsafeHeaders client.Headers
UnsafeHeaders protocolstate.RawHeaders
UnsafeRawBytes []byte
}
@@ -239,7 +239,7 @@ read_line:
// therefore use the full line as key
_, found := rawRequest.Headers[key]
if unsafe {
rawRequest.UnsafeHeaders = append(rawRequest.UnsafeHeaders, client.Header{Key: line})
rawRequest.UnsafeHeaders = append(rawRequest.UnsafeHeaders, protocolstate.RawHeader{Key: line})
}
if unsafe && found {
+10 -5
View File
@@ -39,7 +39,6 @@ import (
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
convUtil "github.com/projectdiscovery/utils/conversion"
"github.com/projectdiscovery/utils/errkit"
@@ -378,16 +377,22 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
return err
}
pipeOptions := rawhttp.DefaultPipelineOptions
pipeOptions.Host = URL.Host
pipeOptions.MaxConnections = 1
pipeOptions := &protocolstate.RawPipelineOptions{
Host: URL.Host,
MaxConnections: 1,
MaxPendingRequests: 100,
Timeout: request.options.Options.GetTimeouts().HttpTimeout,
}
if dialers := protocolstate.GetDialersWithId(request.options.Options.ExecutionId); dialers != nil {
pipeOptions.FastDialer = dialers.Fastdialer
}
if request.PipelineConcurrentConnections > 0 {
pipeOptions.MaxConnections = request.PipelineConcurrentConnections
}
if request.PipelineRequestsPerConnection > 0 {
pipeOptions.MaxPendingRequests = request.PipelineRequestsPerConnection
}
pipeClient := rawhttp.NewPipelineClient(pipeOptions)
pipeClient := protocolstate.NewRawPipelineClient(pipeOptions)
// defaultMaxWorkers should be a sufficient value to keep queues always full
// in case the queue is bigger increase the workers
+4 -2
View File
@@ -93,8 +93,10 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
if err != nil {
return errors.Wrap(err, "fuzz: could not build request from url")
}
userAgent := useragent.PickRandom()
baseRequest.Header.Set("User-Agent", userAgent.Raw)
if request.options.Options == nil || !request.options.Options.TlsImpersonate {
userAgent := useragent.PickRandom()
baseRequest.Header.Set("User-Agent", userAgent.Raw)
}
request.addHeadersToRequest(baseRequest)
// execute with one value first to checks its applicability
+7 -3
View File
@@ -6,7 +6,7 @@ import (
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/utils/errkit"
)
@@ -20,8 +20,12 @@ func dump(req *generatedRequest, reqURL string) ([]byte, error) {
}
return bin, nil
}
rawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes}
bin, err := rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions)
rawHttpOptions := &protocolstate.RawHTTPOptions{
CustomHeaders: req.rawRequest.UnsafeHeaders,
CustomRawBytes: req.rawRequest.UnsafeRawBytes,
AutomaticHostHeader: true,
}
bin, err := protocolstate.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions)
if err != nil {
return nil, errkit.Wrapf(err, "could not dump request: %v", reqURL)
}