mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-07 09:46:03 -10:00
feat(http): add RFC 9111 cache w/ per-request disable (#6746)
* feat(http): add RFC-9111 cache w/ per-request disable * wrap HTTP transports with sandrolain/httpcache and expose global `--no-http-cache` flag. * allow templates to set `disable-http-cache`. * bump Go to 1.25.7. Signed-off-by: Dwi Siswanto <[email protected]> * ci: pin go version by go.mod Signed-off-by: Dwi Siswanto <[email protected]> * build: set GODEBUG=tlssha1=1 Signed-off-by: Dwi Siswanto <[email protected]> * build(make): add more `GO*` envs for `go-build` Signed-off-by: Dwi Siswanto <[email protected]> * chore(cmd): fix typo Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * fix(types): add `DisableHTTPCache` to `Copy` method Signed-off-by: Dwi Siswanto <[email protected]> * test: add HTTP cache integration tests Signed-off-by: Dwi Siswanto <[email protected]> * refactor: inverse HTTP cache (RFC 9111) logic (opt-in) Signed-off-by: Dwi Siswanto <[email protected]> * chore(deps): go mod tidy Signed-off-by: Dwi Siswanto <[email protected]> * test(integration): add -http-cache to both HTTP cache test cases Signed-off-by: Dwi Siswanto <[email protected]> * filter cache * bound cache --------- Signed-off-by: Dwi Siswanto <[email protected]> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Mzack9999 <[email protected]>
This commit is contained in:
19 files changed
+646
-96
No files matched your search
@@ -16,5 +16,4 @@ jobs:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: projectdiscovery/actions/setup/go/compat-checks@v1
|
||||
with:
|
||||
go-version: "stable"
|
||||
release-test: true
|
||||
@@ -206,8 +206,6 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: projectdiscovery/actions/setup/go@v1
|
||||
with:
|
||||
go-version: "stable"
|
||||
- uses: projectdiscovery/actions/goreleaser@v1
|
||||
|
||||
flamegraph:
|
||||
|
||||
@@ -12,6 +12,7 @@ builds:
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
- GOEXPERIMENT=greenteagc
|
||||
- GODEBUG=tlssha1=1
|
||||
goos: [windows, linux, darwin]
|
||||
goarch: [amd64, '386', arm, arm64]
|
||||
ignore:
|
||||
|
||||
@@ -33,8 +33,9 @@ clean:
|
||||
|
||||
go-build: clean
|
||||
go-build:
|
||||
CGO_ENABLED=0 $(GOBUILD) -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
|
||||
-o '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' $(GOBUILD_PACKAGES)
|
||||
CGO_ENABLED=0 GOEXPERIMENT=greenteagc GODEBUG=tlssha1=1 $(GOBUILD) \
|
||||
-trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
|
||||
-o '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' $(GOBUILD_PACKAGES)
|
||||
|
||||
build: GOFLAGS = -pgo=auto
|
||||
build: GOBUILD_OUTPUT = ./bin/nuclei
|
||||
@@ -297,4 +298,4 @@ template-validate:
|
||||
./bin/nuclei -validate \
|
||||
-w workflows \
|
||||
-et http/technologies \
|
||||
-ept code
|
||||
-ept code
|
||||
@@ -1477,6 +1477,19 @@ DisableCookie is an optional setting that disables cookie reuse
|
||||
|
||||
<div class="dd">
|
||||
|
||||
<code>disable-http-cache</code> <i>bool</i>
|
||||
|
||||
</div>
|
||||
<div class="dt">
|
||||
|
||||
DisableHTTPCache turns off HTTP caching for this request. It cannot turn caching on when -http-cache is unset.
|
||||
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
<div class="dd">
|
||||
|
||||
<code>read-all</code> <i>bool</i>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -450,6 +450,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.BoolVarP(&options.DisableHTTPProbe, "no-httpx", "nh", false, "disable httpx probing for non-url input"),
|
||||
flagSet.BoolVar(&options.PreflightPortScan, "preflight-portscan", false, "run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)"),
|
||||
flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "disable stdin processing"),
|
||||
flagSet.BoolVar(&options.EnableHTTPCache, "http-cache", false, "enable HTTP cache (RFC 9111) for HTTP requests"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("headless", "Headless",
|
||||
|
||||
@@ -56,9 +56,9 @@ require (
|
||||
github.com/antchfx/xmlquery v1.4.4
|
||||
github.com/antchfx/xpath v1.3.6
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.17
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.70
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.16
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
|
||||
github.com/bytedance/sonic v1.15.0
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
@@ -118,6 +118,7 @@ require (
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.95
|
||||
github.com/redis/go-redis/v9 v9.11.0
|
||||
github.com/rs/zerolog v1.34.0
|
||||
github.com/sandrolain/httpcache v1.4.0
|
||||
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
|
||||
github.com/sijms/go-ora/v2 v2.9.0
|
||||
github.com/stretchr/testify v1.12.1
|
||||
@@ -174,19 +175,19 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/kms v1.37.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
|
||||
github.com/aws/smithy-go v1.24.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
@@ -317,7 +318,7 @@ require (
|
||||
github.com/libdns/libdns v1.1.1 // indirect
|
||||
github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20250827001030-24949be3fa54 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/mackerelio/go-osstat v0.2.6 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
|
||||
@@ -195,20 +195,20 @@ github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82 h1:EO13QJTCD1Ig2IrQnoHTRrn981H9mB7afXsZ89WptI4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82/go.mod h1:AGh1NCg0SH+uyJamiJA5tTQcql4MMRDXGRdMmCxCXzY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.16 h1:E4Tz+tJiPc7kGnXwIfCyUj6xHJNpENlY11oKpRTgsjc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.16/go.mod h1:2S9hBElpCyGMifv14WxQ7EfPumgoeCPZUpuPX8VtW34=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2 h1:9/HxDeIgA7DcKK6e6ZaP5PQiXugYbNERx3Z5u30mN+k=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2/go.mod h1:3N1RoxKNcVHmbOKVMMw8pvMs5TUhGYPQP/aq1zmAWqo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
@@ -223,12 +223,12 @@ github.com/aws/aws-sdk-go-v2/service/kms v1.37.7 h1:dZmNIRtPUvtvUIIDVNpvtnJQ8N8I
|
||||
github.com/aws/aws-sdk-go-v2/service/kms v1.37.7/go.mod h1:vj8PlfJH9mnGeIzd6uMLPi5VgiqzGG7AZoe1kf1uTXM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
@@ -783,8 +783,8 @@ github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8 h1:z9RDOBcFcf3f2hSfK
|
||||
github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8/go.mod h1:37YR9jabpiIxsb8X9VCIx8qFOjTDIIrIHHODa8C4gz0=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20250827001030-24949be3fa54 h1:mFWunSatvkQQDhpdyuFAYwyAan3hzCuma+Pz8sqvOfg=
|
||||
github.com/lufia/plan9stats v0.0.0-20250827001030-24949be3fa54/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/mackerelio/go-osstat v0.2.6 h1:gs4U8BZeS1tjrL08tt5VUliVvSWP26Ai2Ob8Lr7f2i0=
|
||||
github.com/mackerelio/go-osstat v0.2.6/go.mod h1:lRy8V9ZuHpuRVZh+vyTkODeDPl3/d5MgXHtLSaqG8bA=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
@@ -1037,6 +1037,8 @@ github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkB
|
||||
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/sandrolain/httpcache v1.4.0 h1:Jf4Vx62X2ybvNPSpPvI1kT3xvMdDG1AsApQjOQKO9E0=
|
||||
github.com/sandrolain/httpcache v1.4.0/go.mod h1:kHBuXveitSn39SNPBhdf/ybG272X706HJ2RJqOQ+Em0=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uRPSpfNZkY=
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
@@ -96,6 +97,8 @@ var httpTestcases = []integrationCase{
|
||||
{Path: "protocols/http/response-data-literal-reuse.yaml", TestCase: &httpResponseDataLiteralReuse{}},
|
||||
{Path: "protocols/http/raw-path-single-slash.yaml", TestCase: &httpRawPathSingleSlash{}},
|
||||
{Path: "protocols/http/raw-unsafe-path-single-slash.yaml", TestCase: &httpRawUnsafePathSingleSlash{}},
|
||||
{Path: "protocols/http/disable-http-cache.yaml", TestCase: &httpDisableCache{}},
|
||||
{Path: "protocols/http/http-cache.yaml", TestCase: &httpCache{}},
|
||||
}
|
||||
|
||||
type httpMultiVarSharing struct{}
|
||||
@@ -1849,3 +1852,67 @@ func (h *httpRawUnsafePathSingleSlash) Execute(filepath string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type httpCache struct{}
|
||||
|
||||
func (h *httpCache) Execute(filePath string) error {
|
||||
router := httprouter.New()
|
||||
var requestCount int32
|
||||
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
atomic.AddInt32(&requestCount, 1)
|
||||
w.Header().Set("Cache-Control", "max-age=2")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = fmt.Fprint(w, requestCount)
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-http-cache")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We expect 2 results because we made 2 requests and both should match
|
||||
if err := expectResultsCount(results, 2); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We expect only 1 actual request to the server because of caching
|
||||
if count := atomic.LoadInt32(&requestCount); count != 1 {
|
||||
return fmt.Errorf("expected 1 request to server, got %d", count)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type httpDisableCache struct{}
|
||||
|
||||
func (h *httpDisableCache) Execute(filePath string) error {
|
||||
var requestCount int32
|
||||
router := httprouter.New()
|
||||
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
atomic.AddInt32(&requestCount, 1)
|
||||
w.Header().Set("Cache-Control", "max-age=2")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = fmt.Fprint(w, requestCount)
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-http-cache")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We expect 2 results because we made 2 requests and both should match
|
||||
if err := expectResultsCount(results, 2); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We expect 2 actual requests to the server because cache is disabled
|
||||
if count := atomic.LoadInt32(&requestCount); count != 2 {
|
||||
return fmt.Errorf("expected 2 requests to server, got %d", count)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
id: disable-http-cache
|
||||
|
||||
info:
|
||||
name: Disable HTTP Cache Test
|
||||
author: dwisiswant0
|
||||
severity: info
|
||||
description: Tests if disable-http-cache works as expected
|
||||
tags: test
|
||||
|
||||
requests:
|
||||
- raw:
|
||||
- |
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
- |
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
disable-http-cache: true
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- body_1 == "1"
|
||||
- body_2 == "2"
|
||||
@@ -0,0 +1,23 @@
|
||||
id: http-cache
|
||||
|
||||
info:
|
||||
name: HTTP Cache Test
|
||||
author: dwisiswant0
|
||||
severity: info
|
||||
description: Tests if HTTP cache (RFC 9111) works as expected
|
||||
tags: test
|
||||
|
||||
requests:
|
||||
- raw:
|
||||
- |
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
- |
|
||||
GET / HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
disable-http-cache: false
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- body_1 == "1"
|
||||
- body_2 == "1"
|
||||
@@ -880,6 +880,11 @@
|
||||
"title": "optional disable cookie reuse",
|
||||
"description": "Optional setting that disables cookie reuse"
|
||||
},
|
||||
"disable-http-cache": {
|
||||
"type": "boolean",
|
||||
"title": "disable HTTP cache",
|
||||
"description": "Turns off HTTP caching for this request; cannot enable cache when -http-cache is unset"
|
||||
},
|
||||
"read-all": {
|
||||
"type": "boolean",
|
||||
"title": "force read all body",
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
"github.com/projectdiscovery/mapcidr/asn"
|
||||
"github.com/projectdiscovery/networkpolicy"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
|
||||
mapsutil "github.com/projectdiscovery/utils/maps"
|
||||
@@ -328,6 +329,7 @@ func Close(executionId string) {
|
||||
dialers.Delete(executionId)
|
||||
|
||||
if dialers.IsEmpty() {
|
||||
httpcache.Close()
|
||||
StopActiveMemGuardian()
|
||||
}
|
||||
}
|
||||
@@ -174,6 +174,10 @@ type Request struct {
|
||||
// DisableCookie is an optional setting that disables cookie reuse
|
||||
DisableCookie bool `yaml:"disable-cookie,omitempty" json:"disable-cookie,omitempty" jsonschema:"title=optional disable cookie reuse,description=Optional setting that disables cookie reuse"`
|
||||
|
||||
// description: |
|
||||
// DisableHTTPCache turns off HTTP caching for this request. It cannot turn caching on when -http-cache is unset.
|
||||
DisableHTTPCache bool `yaml:"disable-http-cache,omitempty" json:"disable-http-cache,omitempty" jsonschema:"title=disable HTTP cache,description=Turns off HTTP caching for this request; cannot enable cache when -http-cache is unset"`
|
||||
|
||||
// description: |
|
||||
// Enables force reading of the entire raw unsafe request body ignoring
|
||||
// any specified content length headers.
|
||||
@@ -354,10 +358,11 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
}
|
||||
|
||||
connectionConfiguration := &httpclientpool.Configuration{
|
||||
Threads: request.Threads,
|
||||
MaxRedirects: request.MaxRedirects,
|
||||
NoTimeout: false,
|
||||
DisableCookie: request.DisableCookie,
|
||||
Threads: request.Threads,
|
||||
MaxRedirects: request.MaxRedirects,
|
||||
NoTimeout: false,
|
||||
DisableCookie: request.DisableCookie,
|
||||
DisableHTTPCache: request.DisableHTTPCache,
|
||||
Connection: &httpclientpool.ConnectionConfiguration{
|
||||
DisableKeepAlive: disableKeepAlive,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
// Package httpcache provides RFC 9111 compliant HTTP response caching for nuclei.
|
||||
// It wraps http.RoundTripper to cache responses based on standard HTTP caching headers.
|
||||
package httpcache
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/sandrolain/httpcache"
|
||||
"github.com/sandrolain/httpcache/leveldbcache"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
)
|
||||
|
||||
const defaultMaxCacheBytes = 256 << 20 // 256 MiB
|
||||
|
||||
var (
|
||||
cacheMu sync.Mutex
|
||||
cache httpcache.Cache
|
||||
diskDB *leveldb.DB
|
||||
maxCacheBytes int64 = defaultMaxCacheBytes
|
||||
)
|
||||
|
||||
// GetCache returns the shared cache, opening LevelDB on first use.
|
||||
// After Close, the next call opens a new handle.
|
||||
func GetCache() httpcache.Cache {
|
||||
cacheMu.Lock()
|
||||
defer cacheMu.Unlock()
|
||||
if cache != nil {
|
||||
return cache
|
||||
}
|
||||
|
||||
cacheDir := filepath.Join(config.DefaultConfig.GetCacheDir(), "httpcache")
|
||||
opened, db, err := openDiskCache(cacheDir, maxCacheBytes)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("HTTP cache: could not open %q, using memory: %s", cacheDir, err)
|
||||
cache = newBoundedCache(httpcache.NewMemoryCache(), "", maxCacheBytes)
|
||||
return cache
|
||||
}
|
||||
gologger.Verbose().Msgf("HTTP cache initialized at %q (max %d bytes)", cacheDir, maxCacheBytes)
|
||||
diskDB = db
|
||||
cache = opened
|
||||
return cache
|
||||
}
|
||||
|
||||
// Close releases the LevelDB handle. Safe if the cache was never opened or
|
||||
// is using the memory fallback. The next GetCache call reopens it.
|
||||
func Close() {
|
||||
cacheMu.Lock()
|
||||
defer cacheMu.Unlock()
|
||||
if diskDB != nil {
|
||||
if err := diskDB.Close(); err != nil {
|
||||
gologger.Warning().Msgf("HTTP cache: close: %s", err)
|
||||
}
|
||||
diskDB = nil
|
||||
}
|
||||
cache = nil
|
||||
}
|
||||
|
||||
// NewTransportWrapper returns a function that wraps an [http.RoundTripper] with
|
||||
// caching.
|
||||
//
|
||||
// This is designed to be used with retryablehttp's WrapTransport option.
|
||||
func NewTransportWrapper() func(http.RoundTripper) http.RoundTripper {
|
||||
c := GetCache()
|
||||
|
||||
return func(rt http.RoundTripper) http.RoundTripper {
|
||||
transport := &httpcache.Transport{
|
||||
Transport: rt,
|
||||
Cache: c,
|
||||
MarkCachedResponses: false,
|
||||
SkipServerErrorsFromCache: true,
|
||||
AsyncRevalidateTimeout: 10 * time.Second,
|
||||
IsPublicCache: false,
|
||||
EnableVarySeparation: true,
|
||||
ShouldCache: shouldCache,
|
||||
CacheKeyHeaders: []string{
|
||||
"Authorization",
|
||||
"Cookie",
|
||||
"Accept-Encoding",
|
||||
"Accept-Language",
|
||||
"Accept",
|
||||
"Origin",
|
||||
},
|
||||
DisableWarningHeader: true,
|
||||
}
|
||||
return transport
|
||||
}
|
||||
}
|
||||
|
||||
func openDiskCache(dir string, maxBytes int64) (httpcache.Cache, *leveldb.DB, error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, nil, fmt.Errorf("create cache dir: %w", err)
|
||||
}
|
||||
|
||||
db, err := leveldb.OpenFile(dir, &opt.Options{
|
||||
WriteBuffer: 1 << 20,
|
||||
CompactionTableSize: 2 << 20,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return newBoundedCache(leveldbcache.NewWithDB(db), dir, maxBytes), db, nil
|
||||
}
|
||||
|
||||
func shouldCache(resp *http.Response) bool {
|
||||
return isCacheableStatus(resp.StatusCode)
|
||||
}
|
||||
|
||||
func isCacheableStatus(code int) bool {
|
||||
return code >= 200 && code < 300
|
||||
}
|
||||
|
||||
func newBoundedCache(inner httpcache.Cache, dir string, maxBytes int64) *statusFilterCache {
|
||||
return &statusFilterCache{
|
||||
inner: inner,
|
||||
dir: dir,
|
||||
maxBytes: maxBytes,
|
||||
}
|
||||
}
|
||||
|
||||
// statusFilterCache drops non-2xx entries. sandrolain/httpcache v1.4.0 stores
|
||||
// 301/404/405/410/414/501 without calling ShouldCache.
|
||||
// When dir is set, Set is skipped once the LevelDB directory reaches maxBytes.
|
||||
type statusFilterCache struct {
|
||||
inner httpcache.Cache
|
||||
dir string
|
||||
maxBytes int64
|
||||
setCount atomic.Uint32
|
||||
fullWarned atomic.Bool
|
||||
}
|
||||
|
||||
func (c *statusFilterCache) Get(key string) ([]byte, bool) {
|
||||
data, ok := c.inner.Get(key)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
if !storedResponseIs2xx(data) {
|
||||
c.inner.Delete(key)
|
||||
return nil, false
|
||||
}
|
||||
return data, true
|
||||
}
|
||||
|
||||
func (c *statusFilterCache) Set(key string, responseBytes []byte) {
|
||||
if !storedResponseIs2xx(responseBytes) {
|
||||
return
|
||||
}
|
||||
if c.maxBytes > 0 && c.overBudget(len(responseBytes)) {
|
||||
if c.fullWarned.CompareAndSwap(false, true) {
|
||||
gologger.Warning().Msgf("HTTP cache: at %d byte cap, not storing more responses", c.maxBytes)
|
||||
}
|
||||
return
|
||||
}
|
||||
c.inner.Set(key, responseBytes)
|
||||
}
|
||||
|
||||
func (c *statusFilterCache) Delete(key string) {
|
||||
c.inner.Delete(key)
|
||||
}
|
||||
|
||||
func (c *statusFilterCache) overBudget(incoming int) bool {
|
||||
if c.dir == "" {
|
||||
return false
|
||||
}
|
||||
// Directory walk is not free; sample every 32 writes after the first.
|
||||
n := c.setCount.Add(1)
|
||||
if n > 1 && n%32 != 0 {
|
||||
return false
|
||||
}
|
||||
return dirSize(c.dir)+int64(incoming) >= c.maxBytes
|
||||
}
|
||||
|
||||
func dirSize(root string) int64 {
|
||||
var total int64
|
||||
_ = filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
total += info.Size()
|
||||
return nil
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
func storedResponseIs2xx(data []byte) bool {
|
||||
resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(data)), nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return isCacheableStatus(resp.StatusCode)
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package httpcache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
libcache "github.com/sandrolain/httpcache"
|
||||
)
|
||||
|
||||
func dumpedResponse(t *testing.T, status int) []byte {
|
||||
t.Helper()
|
||||
resp := &http.Response{
|
||||
StatusCode: status,
|
||||
Status: http.StatusText(status),
|
||||
Proto: "HTTP/1.1",
|
||||
ProtoMajor: 1,
|
||||
ProtoMinor: 1,
|
||||
Header: make(http.Header),
|
||||
Body: http.NoBody,
|
||||
}
|
||||
resp.Header.Set("Cache-Control", "max-age=60")
|
||||
data, err := httputil.DumpResponse(resp, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func TestStatusFilterCacheRejectsNon2xx(t *testing.T) {
|
||||
inner := libcache.NewMemoryCache()
|
||||
c := &statusFilterCache{inner: inner}
|
||||
|
||||
okBody := dumpedResponse(t, http.StatusOK)
|
||||
c.Set("ok", okBody)
|
||||
got, ok := c.Get("ok")
|
||||
if !ok {
|
||||
t.Fatal("2xx response was not stored")
|
||||
}
|
||||
if string(got) != string(okBody) {
|
||||
t.Fatal("2xx cache payload changed")
|
||||
}
|
||||
|
||||
c.Set("missing", dumpedResponse(t, http.StatusNotFound))
|
||||
if _, ok := c.Get("missing"); ok {
|
||||
t.Fatal("404 response was stored")
|
||||
}
|
||||
|
||||
inner.Set("stale-404", dumpedResponse(t, http.StatusMovedPermanently))
|
||||
if _, ok := c.Get("stale-404"); ok {
|
||||
t.Fatal("existing 301 was served")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportDoesNotCache404(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Cache-Control", "max-age=60")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = io.WriteString(w, "gone")
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
rt := &libcache.Transport{
|
||||
Transport: ts.Client().Transport,
|
||||
Cache: &statusFilterCache{inner: libcache.NewMemoryCache()},
|
||||
SkipServerErrorsFromCache: true,
|
||||
AsyncRevalidateTimeout: time.Second,
|
||||
ShouldCache: shouldCache,
|
||||
DisableWarningHeader: true,
|
||||
}
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("status = %d", resp.StatusCode)
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
|
||||
if got := hits.Load(); got != 2 {
|
||||
t.Fatalf("server hits = %d, want 2 (404 must not be reused from cache)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportCaches200(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := hits.Add(1)
|
||||
w.Header().Set("Cache-Control", "max-age=60")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = fmt.Fprintf(w, "%d", n)
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
rt := &libcache.Transport{
|
||||
Transport: ts.Client().Transport,
|
||||
Cache: &statusFilterCache{inner: libcache.NewMemoryCache()},
|
||||
SkipServerErrorsFromCache: true,
|
||||
AsyncRevalidateTimeout: time.Second,
|
||||
ShouldCache: shouldCache,
|
||||
DisableWarningHeader: true,
|
||||
}
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != "1" {
|
||||
t.Fatalf("body = %q, want cached first response", body)
|
||||
}
|
||||
}
|
||||
|
||||
if got := hits.Load(); got != 1 {
|
||||
t.Fatalf("server hits = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheCloses(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
c, db, err := openDiskCache(dir, defaultMaxCacheBytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := dumpedResponse(t, http.StatusOK)
|
||||
c.Set("a", body)
|
||||
if _, ok := c.Get("a"); !ok {
|
||||
t.Fatal("value missing before close")
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c2, db2, err := openDiskCache(dir, defaultMaxCacheBytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db2.Close() })
|
||||
if _, ok := c2.Get("a"); !ok {
|
||||
t.Fatal("value missing after reopen")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheRespectsSizeCap(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
c, db, err := openDiskCache(dir, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
c.Set("a", dumpedResponse(t, http.StatusOK))
|
||||
if _, ok := c.Get("a"); ok {
|
||||
t.Fatal("stored a response over a 1-byte directory cap")
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
@@ -187,6 +188,8 @@ type Configuration struct {
|
||||
NoTimeout bool
|
||||
// DisableCookie disables cookie reuse for the http client (cookiejar impl)
|
||||
DisableCookie bool
|
||||
// DisableHTTPCache disables HTTP caching for this client instance
|
||||
DisableHTTPCache bool
|
||||
// FollowRedirects specifies the redirects flow
|
||||
RedirectFlow RedirectFlow
|
||||
// Connection defines custom connection configuration
|
||||
@@ -226,6 +229,8 @@ func (c *Configuration) Hash() string {
|
||||
builder.WriteString(strconv.Itoa(int(c.RedirectFlow)))
|
||||
builder.WriteString("r")
|
||||
builder.WriteString(strconv.FormatBool(c.DisableCookie))
|
||||
builder.WriteString("h")
|
||||
builder.WriteString(strconv.FormatBool(c.DisableHTTPCache))
|
||||
builder.WriteString("c")
|
||||
builder.WriteString(strconv.FormatBool(c.Connection != nil))
|
||||
if c.Connection != nil {
|
||||
@@ -247,7 +252,7 @@ func (c *Configuration) Hash() string {
|
||||
|
||||
// HasStandardOptions checks whether the configuration requires custom settings
|
||||
func (c *Configuration) HasStandardOptions() bool {
|
||||
return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0
|
||||
return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0 && !c.DisableHTTPCache
|
||||
}
|
||||
|
||||
// GetRawHTTP returns the rawhttp request client
|
||||
@@ -360,6 +365,9 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
|
||||
if configuration.ResponseHeaderTimeout > 0 && configuration.ResponseHeaderTimeout > retryableHttpOptions.Timeout {
|
||||
retryableHttpOptions.Timeout = configuration.ResponseHeaderTimeout
|
||||
}
|
||||
if options.EnableHTTPCache && !configuration.DisableHTTPCache {
|
||||
retryableHttpOptions.WrapTransport = httpcache.NewTransportWrapper()
|
||||
}
|
||||
|
||||
maxIdleConnsPerHost := idleConnPoolSize(options.TemplateThreads, configuration.Threads)
|
||||
maxIdleConns := maxIdleConnsPerHost
|
||||
@@ -463,7 +471,6 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
|
||||
|
||||
return &connTrackingTransport{base: transport}, nil
|
||||
}
|
||||
|
||||
redirectFlow := configuration.RedirectFlow
|
||||
maxRedirects := configuration.MaxRedirects
|
||||
|
||||
|
||||
@@ -460,7 +460,7 @@ func init() {
|
||||
Value: "HTTP response headers in name:value format",
|
||||
},
|
||||
}
|
||||
HTTPRequestDoc.Fields = make([]encoder.Doc, 39)
|
||||
HTTPRequestDoc.Fields = make([]encoder.Doc, 40)
|
||||
HTTPRequestDoc.Fields[0].Name = "path"
|
||||
HTTPRequestDoc.Fields[0].Type = "[]string"
|
||||
HTTPRequestDoc.Fields[0].Note = ""
|
||||
@@ -599,91 +599,96 @@ func init() {
|
||||
HTTPRequestDoc.Fields[21].Note = ""
|
||||
HTTPRequestDoc.Fields[21].Description = "DisableCookie is an optional setting that disables cookie reuse"
|
||||
HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "DisableCookie is an optional setting that disables cookie reuse"
|
||||
HTTPRequestDoc.Fields[22].Name = "read-all"
|
||||
HTTPRequestDoc.Fields[22].Name = "disable-http-cache"
|
||||
HTTPRequestDoc.Fields[22].Type = "bool"
|
||||
HTTPRequestDoc.Fields[22].Note = ""
|
||||
HTTPRequestDoc.Fields[22].Description = "Enables force reading of the entire raw unsafe request body ignoring\nany specified content length headers."
|
||||
HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Enables force reading of the entire raw unsafe request body ignoring"
|
||||
HTTPRequestDoc.Fields[23].Name = "redirects"
|
||||
HTTPRequestDoc.Fields[22].Description = "DisableHTTPCache turns off HTTP caching for this request. It cannot turn caching on when -http-cache is unset."
|
||||
HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "DisableHTTPCache turns off HTTP caching for this request. It cannot turn caching on when -http-cache is unset."
|
||||
HTTPRequestDoc.Fields[23].Name = "read-all"
|
||||
HTTPRequestDoc.Fields[23].Type = "bool"
|
||||
HTTPRequestDoc.Fields[23].Note = ""
|
||||
HTTPRequestDoc.Fields[23].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
|
||||
HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
|
||||
HTTPRequestDoc.Fields[24].Name = "host-redirects"
|
||||
HTTPRequestDoc.Fields[23].Description = "Enables force reading of the entire raw unsafe request body ignoring\nany specified content length headers."
|
||||
HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Enables force reading of the entire raw unsafe request body ignoring"
|
||||
HTTPRequestDoc.Fields[24].Name = "redirects"
|
||||
HTTPRequestDoc.Fields[24].Type = "bool"
|
||||
HTTPRequestDoc.Fields[24].Note = ""
|
||||
HTTPRequestDoc.Fields[24].Description = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
|
||||
HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client."
|
||||
HTTPRequestDoc.Fields[25].Name = "protocol-redirects"
|
||||
HTTPRequestDoc.Fields[24].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
|
||||
HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
|
||||
HTTPRequestDoc.Fields[25].Name = "host-redirects"
|
||||
HTTPRequestDoc.Fields[25].Type = "bool"
|
||||
HTTPRequestDoc.Fields[25].Note = ""
|
||||
HTTPRequestDoc.Fields[25].Description = "ProtocolRedirects specifies whether only redirects within the same protocol should be followed.\n\nWhen set to true with redirects enabled, cross-protocol redirects (e.g. HTTP to HTTPS) will be blocked."
|
||||
HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "ProtocolRedirects specifies whether only redirects within the same protocol should be followed."
|
||||
HTTPRequestDoc.Fields[26].Name = "pipeline"
|
||||
HTTPRequestDoc.Fields[25].Description = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
|
||||
HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client."
|
||||
HTTPRequestDoc.Fields[26].Name = "protocol-redirects"
|
||||
HTTPRequestDoc.Fields[26].Type = "bool"
|
||||
HTTPRequestDoc.Fields[26].Note = ""
|
||||
HTTPRequestDoc.Fields[26].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
|
||||
HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
|
||||
HTTPRequestDoc.Fields[27].Name = "unsafe"
|
||||
HTTPRequestDoc.Fields[26].Description = "ProtocolRedirects specifies whether only redirects within the same protocol should be followed.\n\nWhen set to true with redirects enabled, cross-protocol redirects (e.g. HTTP to HTTPS) will be blocked."
|
||||
HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "ProtocolRedirects specifies whether only redirects within the same protocol should be followed."
|
||||
HTTPRequestDoc.Fields[27].Name = "pipeline"
|
||||
HTTPRequestDoc.Fields[27].Type = "bool"
|
||||
HTTPRequestDoc.Fields[27].Note = ""
|
||||
HTTPRequestDoc.Fields[27].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
|
||||
HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
|
||||
HTTPRequestDoc.Fields[28].Name = "race"
|
||||
HTTPRequestDoc.Fields[27].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
|
||||
HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
|
||||
HTTPRequestDoc.Fields[28].Name = "unsafe"
|
||||
HTTPRequestDoc.Fields[28].Type = "bool"
|
||||
HTTPRequestDoc.Fields[28].Note = ""
|
||||
HTTPRequestDoc.Fields[28].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
|
||||
HTTPRequestDoc.Fields[28].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
|
||||
HTTPRequestDoc.Fields[29].Name = "req-condition"
|
||||
HTTPRequestDoc.Fields[28].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
|
||||
HTTPRequestDoc.Fields[28].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
|
||||
HTTPRequestDoc.Fields[29].Name = "race"
|
||||
HTTPRequestDoc.Fields[29].Type = "bool"
|
||||
HTTPRequestDoc.Fields[29].Note = ""
|
||||
HTTPRequestDoc.Fields[29].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
|
||||
HTTPRequestDoc.Fields[29].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
|
||||
HTTPRequestDoc.Fields[30].Name = "stop-at-first-match"
|
||||
HTTPRequestDoc.Fields[29].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
|
||||
HTTPRequestDoc.Fields[29].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
|
||||
HTTPRequestDoc.Fields[30].Name = "req-condition"
|
||||
HTTPRequestDoc.Fields[30].Type = "bool"
|
||||
HTTPRequestDoc.Fields[30].Note = ""
|
||||
HTTPRequestDoc.Fields[30].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HTTPRequestDoc.Fields[30].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HTTPRequestDoc.Fields[31].Name = "skip-variables-check"
|
||||
HTTPRequestDoc.Fields[30].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
|
||||
HTTPRequestDoc.Fields[30].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
|
||||
HTTPRequestDoc.Fields[31].Name = "stop-at-first-match"
|
||||
HTTPRequestDoc.Fields[31].Type = "bool"
|
||||
HTTPRequestDoc.Fields[31].Note = ""
|
||||
HTTPRequestDoc.Fields[31].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
|
||||
HTTPRequestDoc.Fields[31].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
|
||||
HTTPRequestDoc.Fields[32].Name = "iterate-all"
|
||||
HTTPRequestDoc.Fields[31].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HTTPRequestDoc.Fields[31].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HTTPRequestDoc.Fields[32].Name = "skip-variables-check"
|
||||
HTTPRequestDoc.Fields[32].Type = "bool"
|
||||
HTTPRequestDoc.Fields[32].Note = ""
|
||||
HTTPRequestDoc.Fields[32].Description = "IterateAll iterates all the values extracted from internal extractors"
|
||||
HTTPRequestDoc.Fields[32].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
|
||||
HTTPRequestDoc.Fields[33].Name = "digest-username"
|
||||
HTTPRequestDoc.Fields[33].Type = "string"
|
||||
HTTPRequestDoc.Fields[32].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
|
||||
HTTPRequestDoc.Fields[32].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
|
||||
HTTPRequestDoc.Fields[33].Name = "iterate-all"
|
||||
HTTPRequestDoc.Fields[33].Type = "bool"
|
||||
HTTPRequestDoc.Fields[33].Note = ""
|
||||
HTTPRequestDoc.Fields[33].Description = "DigestAuthUsername specifies the username for digest authentication"
|
||||
HTTPRequestDoc.Fields[33].Comments[encoder.LineComment] = "DigestAuthUsername specifies the username for digest authentication"
|
||||
HTTPRequestDoc.Fields[34].Name = "digest-password"
|
||||
HTTPRequestDoc.Fields[33].Description = "IterateAll iterates all the values extracted from internal extractors"
|
||||
HTTPRequestDoc.Fields[33].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
|
||||
HTTPRequestDoc.Fields[34].Name = "digest-username"
|
||||
HTTPRequestDoc.Fields[34].Type = "string"
|
||||
HTTPRequestDoc.Fields[34].Note = ""
|
||||
HTTPRequestDoc.Fields[34].Description = "DigestAuthPassword specifies the password for digest authentication"
|
||||
HTTPRequestDoc.Fields[34].Comments[encoder.LineComment] = "DigestAuthPassword specifies the password for digest authentication"
|
||||
HTTPRequestDoc.Fields[35].Name = "disable-path-automerge"
|
||||
HTTPRequestDoc.Fields[35].Type = "bool"
|
||||
HTTPRequestDoc.Fields[34].Description = "DigestAuthUsername specifies the username for digest authentication"
|
||||
HTTPRequestDoc.Fields[34].Comments[encoder.LineComment] = "DigestAuthUsername specifies the username for digest authentication"
|
||||
HTTPRequestDoc.Fields[35].Name = "digest-password"
|
||||
HTTPRequestDoc.Fields[35].Type = "string"
|
||||
HTTPRequestDoc.Fields[35].Note = ""
|
||||
HTTPRequestDoc.Fields[35].Description = "DisablePathAutomerge disables merging target url path with raw request path"
|
||||
HTTPRequestDoc.Fields[35].Comments[encoder.LineComment] = "DisablePathAutomerge disables merging target url path with raw request path"
|
||||
HTTPRequestDoc.Fields[36].Name = "pre-condition"
|
||||
HTTPRequestDoc.Fields[36].Type = "[]matchers.Matcher"
|
||||
HTTPRequestDoc.Fields[35].Description = "DigestAuthPassword specifies the password for digest authentication"
|
||||
HTTPRequestDoc.Fields[35].Comments[encoder.LineComment] = "DigestAuthPassword specifies the password for digest authentication"
|
||||
HTTPRequestDoc.Fields[36].Name = "disable-path-automerge"
|
||||
HTTPRequestDoc.Fields[36].Type = "bool"
|
||||
HTTPRequestDoc.Fields[36].Note = ""
|
||||
HTTPRequestDoc.Fields[36].Description = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
|
||||
HTTPRequestDoc.Fields[36].Comments[encoder.LineComment] = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
|
||||
HTTPRequestDoc.Fields[37].Name = "pre-condition-operator"
|
||||
HTTPRequestDoc.Fields[37].Type = "string"
|
||||
HTTPRequestDoc.Fields[36].Description = "DisablePathAutomerge disables merging target url path with raw request path"
|
||||
HTTPRequestDoc.Fields[36].Comments[encoder.LineComment] = "DisablePathAutomerge disables merging target url path with raw request path"
|
||||
HTTPRequestDoc.Fields[37].Name = "pre-condition"
|
||||
HTTPRequestDoc.Fields[37].Type = "[]matchers.Matcher"
|
||||
HTTPRequestDoc.Fields[37].Note = ""
|
||||
HTTPRequestDoc.Fields[37].Description = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
|
||||
HTTPRequestDoc.Fields[37].Comments[encoder.LineComment] = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
|
||||
HTTPRequestDoc.Fields[38].Name = "global-matchers"
|
||||
HTTPRequestDoc.Fields[38].Type = "bool"
|
||||
HTTPRequestDoc.Fields[37].Description = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
|
||||
HTTPRequestDoc.Fields[37].Comments[encoder.LineComment] = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
|
||||
HTTPRequestDoc.Fields[38].Name = "pre-condition-operator"
|
||||
HTTPRequestDoc.Fields[38].Type = "string"
|
||||
HTTPRequestDoc.Fields[38].Note = ""
|
||||
HTTPRequestDoc.Fields[38].Description = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
|
||||
HTTPRequestDoc.Fields[38].Comments[encoder.LineComment] = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
|
||||
HTTPRequestDoc.Fields[38].Description = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
|
||||
HTTPRequestDoc.Fields[38].Comments[encoder.LineComment] = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
|
||||
HTTPRequestDoc.Fields[39].Name = "global-matchers"
|
||||
HTTPRequestDoc.Fields[39].Type = "bool"
|
||||
HTTPRequestDoc.Fields[39].Note = ""
|
||||
HTTPRequestDoc.Fields[39].Description = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
|
||||
HTTPRequestDoc.Fields[39].Comments[encoder.LineComment] = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
|
||||
|
||||
GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder"
|
||||
GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol"
|
||||
|
||||
@@ -212,6 +212,11 @@ type Options struct {
|
||||
// When enabled, each host gets its own rate limiter and global rate limit becomes unlimited.
|
||||
// Disabled by default.
|
||||
PerHostRateLimit bool
|
||||
// EnableHTTPCache enables HTTP caching (RFC 9111) for requests
|
||||
//
|
||||
// NOTE(dwisiswant0): this is experimental and might be enabled by default
|
||||
// in the future releases.
|
||||
EnableHTTPCache bool
|
||||
// LeaveDefaultPorts skips normalization of default ports
|
||||
LeaveDefaultPorts bool
|
||||
// AutomaticScan enables automatic tech based template execution
|
||||
@@ -578,6 +583,7 @@ func (options *Options) Copy() *Options {
|
||||
DisableHTTPProbe: options.DisableHTTPProbe,
|
||||
PreflightPortScan: options.PreflightPortScan,
|
||||
PerHostRateLimit: options.PerHostRateLimit,
|
||||
EnableHTTPCache: options.EnableHTTPCache,
|
||||
LeaveDefaultPorts: options.LeaveDefaultPorts,
|
||||
AutomaticScan: options.AutomaticScan,
|
||||
Silent: options.Silent,
|
||||
|
||||
Reference in new issue
Block a user