From 1f5e9c8dea40083de8068448f2ed1c1111600c91 Mon Sep 17 00:00:00 2001 From: blotus Date: Wed, 1 Jul 2026 17:30:50 +0200 Subject: [PATCH] Waf challenge mode (#4268) --- .github/workflows/bats.yml | 3 - .github/workflows/go-tests-windows.yml | 3 - .github/workflows/go-tests.yml | 172 +- .gitignore | 4 + .gitmodules | 3 + .golangci.yml | 12 + Makefile | 17 + cmd/crowdsec-cli/climetrics/show.go | 4 +- .../climetrics/statappsecchallenge.go | 189 ++ .../climetrics/statappsecchallengeinfra.go | 62 + .../climetrics/statappsecengine.go | 4 +- cmd/crowdsec-cli/climetrics/store.go | 235 +- cmd/crowdsec/crowdsec.go | 28 + go.mod | 4 +- go.sum | 3 + .../modules/appsec/appsec_hooks_test.go | 361 ++ .../modules/appsec/appsec_runner.go | 97 +- pkg/acquisition/modules/appsec/appsec_test.go | 42 +- pkg/acquisition/modules/appsec/config.go | 98 +- pkg/acquisition/modules/appsec/config_test.go | 157 + pkg/acquisition/modules/appsec/init.go | 3 +- pkg/acquisition/modules/appsec/metrics.go | 20 +- pkg/acquisition/modules/appsec/run.go | 8 + .../modules/appsec/testhelpers_test.go | 30 + pkg/acquisition/modules/appsec/utils.go | 31 - pkg/appsec/appsec.go | 814 ++++- pkg/appsec/appsec_challenge_test.go | 440 +++ pkg/appsec/appsec_config_test.go | 158 +- pkg/appsec/appsec_rules_collection.go | 212 +- pkg/appsec/challenge/DESIGN.md | 888 +++++ pkg/appsec/challenge/challenge.go | 720 ++++ pkg/appsec/challenge/challenge.html.tmpl | 483 +++ pkg/appsec/challenge/challenge.js | 346 ++ pkg/appsec/challenge/challenge_test.go | 704 ++++ pkg/appsec/challenge/config.go | 151 + pkg/appsec/challenge/config_test.go | 169 + pkg/appsec/challenge/crypto.go | 277 ++ pkg/appsec/challenge/dynamic_module.go | 231 ++ pkg/appsec/challenge/dynamic_module.js.tmpl | 23 + pkg/appsec/challenge/fingerprint.go | 721 ++++ pkg/appsec/challenge/fingerprint_helpers.go | 417 +++ .../challenge/fingerprint_helpers_test.go | 158 + pkg/appsec/challenge/fingerprint_mismatch.go | 258 ++ .../challenge/fingerprint_mismatch_data.go | 170 + .../challenge/fingerprint_mismatch_test.go | 334 ++ pkg/appsec/challenge/fingerprint_proto.go | 568 ++++ pkg/appsec/challenge/grant_redirect.html.tmpl | 11 + pkg/appsec/challenge/initial_bundle.js.gz | Bin 0 -> 68663 bytes pkg/appsec/challenge/initial_bundle_test.go | 84 + pkg/appsec/challenge/js/README.md | 120 + pkg/appsec/challenge/js/challenge_code.js | 1 + pkg/appsec/challenge/js/cmd/bundle/main.go | 68 + .../challenge/js/cmd/initialbundle/main.go | 174 + pkg/appsec/challenge/js/cmd/obfuscate/main.go | 157 + pkg/appsec/challenge/js/fpscanner/fpscanner | 1 + .../challenge/js/fpscanner/fpscanner.js | 48 + pkg/appsec/challenge/js/fpscanner/global.js | 5 + pkg/appsec/challenge/js/generate.go | 5 + pkg/appsec/challenge/js/js.go | 27 + .../challenge/js/obfuscate/index.wasm.gz | Bin 0 -> 4268331 bytes .../js/obfuscate/javascript-obfuscator.js | 54 + .../challenge/js/obfuscate/obfuscate.js | 56 + pkg/appsec/challenge/keyring.go | 259 ++ .../challenge/keyring_integration_test.go | 474 +++ pkg/appsec/challenge/keyring_test.go | 172 + .../challenge/obfuscation_pools_test.go | 92 + pkg/appsec/challenge/obfuscator.go | 56 + pkg/appsec/challenge/pb/fingerprint.pb.go | 2969 +++++++++++++++++ pkg/appsec/challenge/pb/fingerprint.proto | 289 ++ pkg/appsec/challenge/pb/generate.go | 14 + pkg/appsec/challenge/pow-worker.js | 77 + pkg/appsec/challenge/secret.go | 75 + pkg/appsec/challenge/secret_test.go | 63 + pkg/appsec/challenge/spent_set.go | 48 + pkg/appsec/challenge/spent_set_test.go | 63 + pkg/appsec/challenge/split_bundle_test.go | 231 ++ pkg/appsec/challenge/static_bundle.go | 119 + pkg/appsec/challenge/ticket.go | 192 ++ pkg/appsec/challenge/tzcountry_data.go | 354 ++ pkg/appsec/cookie/cookie.go | 156 + pkg/appsec/event.go | 112 + pkg/appsec/event_test.go | 122 + pkg/appsec/fingerprint_dump.go | 120 + pkg/appsec/fingerprint_dump_test.go | 272 ++ .../fingerprint_mismatch_runtime_test.go | 259 ++ pkg/appsec/patcher.go | 26 + pkg/appsec/patcher_test.go | 48 + pkg/appsec/response_clone_test.go | 34 + pkg/appsec/waf_helpers.go | 258 +- pkg/appsec/waf_helpers_test.go | 86 + pkg/csconfig/crowdsec_service.go | 9 + pkg/csconfig/crowdsec_service_test.go | 29 + pkg/cwhub/hub.go | 5 + pkg/dnscache/dnscache.go | 172 + pkg/dnscache/dnscache_test.go | 213 ++ pkg/exprhelpers/botfile.go | 215 ++ pkg/exprhelpers/botfile_test.go | 336 ++ pkg/exprhelpers/expr_lib.go | 7 + pkg/exprhelpers/geoip.go | 30 + pkg/exprhelpers/helpers.go | 10 + pkg/exprhelpers/testdata/test_data_bots.json | 6 + .../testdata/test_data_bots_bad_cidr.json | 1 + .../testdata/test_data_bots_bad_ip.json | 1 + .../testdata/test_data_bots_bad_rdns.json | 1 + .../testdata/test_data_bots_bad_regex.json | 1 + .../testdata/test_data_bots_invalid.json | 1 + .../testdata/test_data_bots_no_identity.json | 1 + .../testdata/test_data_bots_no_name.json | 1 + .../test_data_bots_unknown_field.json | 1 + pkg/metrics/acquisition_appsec.go | 105 + pkg/parser/enrich_dns.go | 22 +- pkg/parser/enrich_dns_test.go | 68 + pkg/protobufs/notifier.pb.go | 130 +- pkg/protobufs/notifier_grpc.pb.go | 10 +- 114 files changed, 18566 insertions(+), 492 deletions(-) create mode 100644 cmd/crowdsec-cli/climetrics/statappsecchallenge.go create mode 100644 cmd/crowdsec-cli/climetrics/statappsecchallengeinfra.go create mode 100644 pkg/acquisition/modules/appsec/config_test.go create mode 100644 pkg/acquisition/modules/appsec/testhelpers_test.go create mode 100644 pkg/appsec/appsec_challenge_test.go create mode 100644 pkg/appsec/challenge/DESIGN.md create mode 100644 pkg/appsec/challenge/challenge.go create mode 100644 pkg/appsec/challenge/challenge.html.tmpl create mode 100644 pkg/appsec/challenge/challenge.js create mode 100644 pkg/appsec/challenge/challenge_test.go create mode 100644 pkg/appsec/challenge/config.go create mode 100644 pkg/appsec/challenge/config_test.go create mode 100644 pkg/appsec/challenge/crypto.go create mode 100644 pkg/appsec/challenge/dynamic_module.go create mode 100644 pkg/appsec/challenge/dynamic_module.js.tmpl create mode 100644 pkg/appsec/challenge/fingerprint.go create mode 100644 pkg/appsec/challenge/fingerprint_helpers.go create mode 100644 pkg/appsec/challenge/fingerprint_helpers_test.go create mode 100644 pkg/appsec/challenge/fingerprint_mismatch.go create mode 100644 pkg/appsec/challenge/fingerprint_mismatch_data.go create mode 100644 pkg/appsec/challenge/fingerprint_mismatch_test.go create mode 100644 pkg/appsec/challenge/fingerprint_proto.go create mode 100644 pkg/appsec/challenge/grant_redirect.html.tmpl create mode 100644 pkg/appsec/challenge/initial_bundle.js.gz create mode 100644 pkg/appsec/challenge/initial_bundle_test.go create mode 100644 pkg/appsec/challenge/js/README.md create mode 100644 pkg/appsec/challenge/js/challenge_code.js create mode 100644 pkg/appsec/challenge/js/cmd/bundle/main.go create mode 100644 pkg/appsec/challenge/js/cmd/initialbundle/main.go create mode 100644 pkg/appsec/challenge/js/cmd/obfuscate/main.go create mode 160000 pkg/appsec/challenge/js/fpscanner/fpscanner create mode 100644 pkg/appsec/challenge/js/fpscanner/fpscanner.js create mode 100644 pkg/appsec/challenge/js/fpscanner/global.js create mode 100644 pkg/appsec/challenge/js/generate.go create mode 100644 pkg/appsec/challenge/js/js.go create mode 100644 pkg/appsec/challenge/js/obfuscate/index.wasm.gz create mode 100644 pkg/appsec/challenge/js/obfuscate/javascript-obfuscator.js create mode 100644 pkg/appsec/challenge/js/obfuscate/obfuscate.js create mode 100644 pkg/appsec/challenge/keyring.go create mode 100644 pkg/appsec/challenge/keyring_integration_test.go create mode 100644 pkg/appsec/challenge/keyring_test.go create mode 100644 pkg/appsec/challenge/obfuscation_pools_test.go create mode 100644 pkg/appsec/challenge/obfuscator.go create mode 100644 pkg/appsec/challenge/pb/fingerprint.pb.go create mode 100644 pkg/appsec/challenge/pb/fingerprint.proto create mode 100644 pkg/appsec/challenge/pb/generate.go create mode 100644 pkg/appsec/challenge/pow-worker.js create mode 100644 pkg/appsec/challenge/secret.go create mode 100644 pkg/appsec/challenge/secret_test.go create mode 100644 pkg/appsec/challenge/spent_set.go create mode 100644 pkg/appsec/challenge/spent_set_test.go create mode 100644 pkg/appsec/challenge/split_bundle_test.go create mode 100644 pkg/appsec/challenge/static_bundle.go create mode 100644 pkg/appsec/challenge/ticket.go create mode 100644 pkg/appsec/challenge/tzcountry_data.go create mode 100644 pkg/appsec/cookie/cookie.go create mode 100644 pkg/appsec/event.go create mode 100644 pkg/appsec/event_test.go create mode 100644 pkg/appsec/fingerprint_dump.go create mode 100644 pkg/appsec/fingerprint_dump_test.go create mode 100644 pkg/appsec/fingerprint_mismatch_runtime_test.go create mode 100644 pkg/appsec/patcher.go create mode 100644 pkg/appsec/patcher_test.go create mode 100644 pkg/appsec/response_clone_test.go create mode 100644 pkg/appsec/waf_helpers_test.go create mode 100644 pkg/dnscache/dnscache.go create mode 100644 pkg/dnscache/dnscache_test.go create mode 100644 pkg/exprhelpers/botfile.go create mode 100644 pkg/exprhelpers/botfile_test.go create mode 100644 pkg/exprhelpers/testdata/test_data_bots.json create mode 100644 pkg/exprhelpers/testdata/test_data_bots_bad_cidr.json create mode 100644 pkg/exprhelpers/testdata/test_data_bots_bad_ip.json create mode 100644 pkg/exprhelpers/testdata/test_data_bots_bad_rdns.json create mode 100644 pkg/exprhelpers/testdata/test_data_bots_bad_regex.json create mode 100644 pkg/exprhelpers/testdata/test_data_bots_invalid.json create mode 100644 pkg/exprhelpers/testdata/test_data_bots_no_identity.json create mode 100644 pkg/exprhelpers/testdata/test_data_bots_no_name.json create mode 100644 pkg/exprhelpers/testdata/test_data_bots_unknown_field.json create mode 100644 pkg/parser/enrich_dns_test.go diff --git a/.github/workflows/bats.yml b/.github/workflows/bats.yml index 59976bad8..ed63133d0 100644 --- a/.github/workflows/bats.yml +++ b/.github/workflows/bats.yml @@ -19,9 +19,6 @@ on: paths-ignore: - "README.md" pull_request: - branches: - - master - - releases/** paths-ignore: - "README.md" diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 3549942e9..47e5a098e 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -8,9 +8,6 @@ on: paths-ignore: - "README.md" pull_request: - branches: - - master - - releases/** paths-ignore: - "README.md" diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index f67c15438..dfb6a1962 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -11,13 +11,10 @@ on: - master - releases/** paths-ignore: - - 'README.md' + - "README.md" pull_request: - branches: - - master - - releases/** paths-ignore: - - 'README.md' + - "README.md" # these env variables are for localstack, so we can emulate aws services env: @@ -36,13 +33,13 @@ jobs: localstack: image: localstack/localstack:3.0 ports: - - 4566:4566 # Localstack exposes all services on the same port + - 4566:4566 # Localstack exposes all services on the same port env: DEBUG: "" LAMBDA_EXECUTOR: "" KINESIS_ERROR_PROBABILITY: "" KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }} - LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly + LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly # e.g sqs url will get localhost if we don't set this env to map our service options: >- --name=localstack @@ -125,95 +122,100 @@ jobs: --health-retries 5 --health-start-period 30s steps: + - name: Check out CrowdSec repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + submodules: true - - name: Check out CrowdSec repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - submodules: false + - name: "Set up Go" + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: go.mod - - name: "Set up Go" - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 - with: - go-version-file: go.mod + - name: Run "go generate" and check for changes + run: | + set -e + # ensure the version of 'protoc' matches the one that generated the files + PROTOBUF_VERSION="35.0" + JAVY_VERSION="8.1.1" + # don't pollute the repo + pushd $HOME + curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip + unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d $HOME/.protoc + curl -OL https://github.com/bytecodealliance/javy/releases/download/v${JAVY_VERSION}/javy-x86_64-linux-v${JAVY_VERSION}.gz + mkdir -p $HOME/.javy + gzip -d javy-x86_64-linux-v${JAVY_VERSION}.gz + mv javy-x86_64-linux-v${JAVY_VERSION} $HOME/.javy/javy + chmod +x $HOME/.javy/javy + popd + export PATH="$HOME/.protoc/bin:$HOME/.javy/:$PATH" + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.2 + go generate ./... + protoc --version + if [[ $(git status --porcelain) ]]; then + echo "Error: Uncommitted changes found after running 'go generate'. Please commit all generated code." + git diff + exit 1 + else + echo "No changes detected after running 'go generate'." + fi - - name: Run "go generate" and check for changes - run: | - set -e - # ensure the version of 'protoc' matches the one that generated the files - PROTOBUF_VERSION="21.12" - # don't pollute the repo - pushd $HOME - curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip - unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d $HOME/.protoc - popd - export PATH="$HOME/.protoc/bin:$PATH" - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1 - go generate ./... - protoc --version - if [[ $(git status --porcelain) ]]; then - echo "Error: Uncommitted changes found after running 'go generate'. Please commit all generated code." - git diff - exit 1 - else - echo "No changes detected after running 'go generate'." - fi - - - name: Create localstack streams - run: | + - name: Create localstack streams + run: | aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1 aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2 - - name: Generate codecov configuration - run: | + - name: Generate codecov configuration + run: | .github/generate-codecov-yml.sh > .github/codecov.yml - - name: Ensure we can do a dynamic build - run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev - make build + - name: Ensure we can do a dynamic build + run: | + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev + make build - - name: Ensure we can do a static build - run: | - make clean build BUILD_STATIC=1 + - name: Ensure we can do a static build + run: | + make clean build BUILD_STATIC=1 - - name: Unit tests - run: | - go install gotest.tools/gotestsum@v1.12.3 - # make the repo read-only, with the exception of coverage output - touch coverage.out - chmod -R a-w . - chmod u+w coverage.out - make testcover - # ignore changes to codecov.yml - if [[ $(git status --porcelain -- . ":(exclude).github/codecov.yml") ]]; then - echo "Error: Unit tests should not create or alter files inside the repository. Please use the appropriate testing helpers or otherwise temporary locations." - git diff --name-only - exit 1 - fi - chmod -R u+w . + - name: Unit tests + run: | + go install gotest.tools/gotestsum@v1.12.3 + # make the repo read-only, with the exception of coverage output + touch coverage.out + chmod -R a-w . + chmod u+w coverage.out + make testcover + # ignore changes to codecov.yml + if [[ $(git status --porcelain -- . ":(exclude).github/codecov.yml") ]]; then + echo "Error: Unit tests should not create or alter files inside the repository. Please use the appropriate testing helpers or otherwise temporary locations." + git diff --name-only + exit 1 + fi + chmod -R u+w . - # check if some component stubs are missing - - name: "Build profile: minimal" - run: | - make build BUILD_PROFILE=minimal + # check if some component stubs are missing + - name: "Build profile: minimal" + run: | + make build BUILD_PROFILE=minimal - # check that the freebsd build is not broken - - name: "Build target: freebsd" - run: | - make build BUILD_STATIC=1 BUILD_RE2_WASM=1 BUILD_SQLITE=modernc GOOS=freebsd + # check that the freebsd build is not broken + - name: "Build target: freebsd" + run: | + make build BUILD_STATIC=1 BUILD_RE2_WASM=1 BUILD_SQLITE=modernc GOOS=freebsd - - name: Upload unit coverage to Codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 - with: - files: coverage.out - flags: unit-linux - token: ${{ secrets.CODECOV_TOKEN }} + - name: Upload unit coverage to Codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + with: + files: coverage.out + flags: unit-linux + token: ${{ secrets.CODECOV_TOKEN }} - - name: golangci-lint - uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1 - with: - version: v2.10 - args: --issues-exit-code=1 --timeout 10m - only-new-issues: false + - name: golangci-lint + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1 + with: + version: v2.10 + args: --issues-exit-code=1 --timeout 10m + only-new-issues: false diff --git a/.gitignore b/.gitignore index 2045d2b96..dfc750fc4 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,7 @@ msi __pycache__ *.py[cod] *.egg-info + +# local agent/tooling dirs (not part of the repo) +.agents/ +.claude/ diff --git a/.gitmodules b/.gitmodules index eb05693ce..e266612ae 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "tests/lib/bats-support"] path = test/lib/bats-support url = https://github.com/bats-core/bats-support.git +[submodule "pkg/appsec/challenge/js/fpscanner/fpscanner"] + path = pkg/appsec/challenge/js/fpscanner/fpscanner + url = https://github.com/antoinevastel/fpscanner diff --git a/.golangci.yml b/.golangci.yml index 74cb4d714..cc4f8bbf2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -337,6 +337,10 @@ linters: # A wildcard `*` can use as a struct name (ex: `*.MethodName`). exclusions: - "HCLogAdapter.SetLevel" + # Unmarshal methods must have a pointer receiver by nature; getters on + # the same type legitimately use value receivers (same rationale as the + # built-in Marshal* exclusions, which don't cover Unmarshal*). + - "*.UnmarshalJSON" revive: severity: error @@ -548,6 +552,14 @@ linters: path: pkg/acquisition/modules/victorialogs/internal/vlclient/vl_client.go text: 'confusing-naming: Method ''QueryRange'' differs only by capitalization to method ''queryRange'' in the same source file' + # pkg/dnscache lookups are deliberately detached from the request + # context: their results are shared across requests (singleflight + + # cache), so one caller's cancellation must not abort the lookup for + # the others; a fixed timeout bounds them instead. + - linters: + - contextcheck + text: '(lookupPTR|lookupForward)` should pass the context parameter' + # tolerate complex functions in tests for now - linters: - maintidx diff --git a/Makefile b/Makefile index eb8a27571..bd94d4759 100644 --- a/Makefile +++ b/Makefile @@ -320,6 +320,23 @@ cscli: ## Build cscli crowdsec: ## Build crowdsec @$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS) +# Regenerate the WAF challenge JS artifacts: the bundled fpscanner source +# (fpscanner/bundle.js), the obfuscator runtime (obfuscate/index.wasm.gz), +# and the build-time pre-obfuscated initial bundle (initial_bundle.js.gz). +# These are committed to the tree so a normal `make build` does NOT rebuild +# them — only run this target after intentionally changing challenge.js, +# fpscanner sources, the obfuscator wrapper, or the dynamic-module template. +# +# Requires `javy` (https://github.com/bytecodealliance/javy/releases) on +# PATH. The full obfuscation pass is slow (~1 minute) because it runs +# javascript-obfuscator's "high-obfuscation" preset over the full bundle; +# this is the cost we are paying once at generate time so the runtime +# doesn't pay it on every startup. +.PHONY: generate-challenge-js +generate-challenge-js: ## Regenerate WAF challenge JS bundles (requires javy on PATH) + @command -v javy >/dev/null 2>&1 || (echo "Error: javy is not installed. Download it from https://github.com/bytecodealliance/javy/releases and put it on PATH." && exit 1) + $(GO) generate ./pkg/appsec/challenge/js/... + testenv: ifeq ($(TEST_LOCAL_ONLY),) @echo 'NOTE: You need to run "make localstack" in a separate shell, "make localstack-stop" to terminate it; or define the envvar TEST_LOCAL_ONLY to some value.' diff --git a/cmd/crowdsec-cli/climetrics/show.go b/cmd/crowdsec-cli/climetrics/show.go index 6c8026dca..a6f6a0000 100644 --- a/cmd/crowdsec-cli/climetrics/show.go +++ b/cmd/crowdsec-cli/climetrics/show.go @@ -75,7 +75,9 @@ func expandAlias(args []string) []string { case "lapi": ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine") case "appsec": - ret = append(ret, "appsec-engine", "appsec-rule") + ret = append(ret, "appsec-engine", "appsec-rule", "appsec-challenge") + case "bot-detection": + ret = append(ret, "appsec-challenge", "appsec-challenge-infra") default: ret = append(ret, section) } diff --git a/cmd/crowdsec-cli/climetrics/statappsecchallenge.go b/cmd/crowdsec-cli/climetrics/statappsecchallenge.go new file mode 100644 index 000000000..4a3d5c77e --- /dev/null +++ b/cmd/crowdsec-cli/climetrics/statappsecchallenge.go @@ -0,0 +1,189 @@ +package climetrics + +import ( + "fmt" + "io" + "sort" + + "github.com/jedib0t/go-pretty/v6/table" + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/core/cstable" +) + +// statAppsecChallenge holds two views of the bot-detection counters: +// +// - Funnel: per-engine aggregates rendered as one row per appsec_engine, +// with columns for each lifecycle/outcome bucket. Mirrors the shape +// statAppsecEngine uses. +// - Reasons: per-engine per-kind per-reason breakdown so operators can see +// which RejectSubmission / GrantChallengeCookie strings (and which +// bounded protocol/cookie reasons) are firing without dropping into +// Prometheus. Rendered as a second table below the funnel. +type statAppsecChallenge struct { + Funnel map[string]map[string]int `json:"funnel"` + Reasons map[string]map[string]map[string]int `json:"reasons"` // engine -> kind -> reason -> count +} + +func newStatAppsecChallenge() *statAppsecChallenge { + return &statAppsecChallenge{ + Funnel: map[string]map[string]int{}, + Reasons: map[string]map[string]map[string]int{}, + } +} + +func (*statAppsecChallenge) Description() (string, string) { + return "Bot Detection Metrics", + `Measures the challenge lifecycle of the AppSec component.` +} + +func (s *statAppsecChallenge) Process(appsecEngine, metric string, val int) { + if _, ok := s.Funnel[appsecEngine]; !ok { + s.Funnel[appsecEngine] = make(map[string]int) + } + + s.Funnel[appsecEngine][metric] += val +} + +// ProcessReason records a per-kind/reason datapoint. Empty reasons are +// dropped. +func (s *statAppsecChallenge) ProcessReason(appsecEngine, kind, reason string, val int) { + if reason == "" { + return + } + + if _, ok := s.Reasons[appsecEngine]; !ok { + s.Reasons[appsecEngine] = map[string]map[string]int{} + } + + if _, ok := s.Reasons[appsecEngine][kind]; !ok { + s.Reasons[appsecEngine][kind] = map[string]int{} + } + + s.Reasons[appsecEngine][kind][reason] += val +} + +func (s *statAppsecChallenge) Table(out io.Writer, wantColor string, noUnit bool, showEmpty bool) { + t := cstable.New(out, wantColor).Writer + t.AppendHeader(table.Row{"Bot Detection", "Requested", "Submitted", "Solved", "Granted", "Protocol Failures", "Submissions Rejected", "Cookies Invalid"}) + + keys := []string{"requested", "submitted", "solved", "granted", "rejected_protocol", "rejected_submission", "rejected_cookie"} + + numRows, err := metricsToTable(t, s.Funnel, keys, noUnit) + if err != nil { + log.Warningf("while collecting appsec challenge stats: %s", err) + return + } + + if numRows == 0 && !showEmpty { + return + } + + title, _ := s.Description() + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) + + // Detail tables: per-engine breakdown of the user-supplied + // (GrantChallengeCookie / RejectSubmission) and bounded-vocabulary + // (protocol / cookie) reasons. One table per outcome (accepted / + // rejected) so the kind dimension only needs to show up where it + // actually varies. + s.renderAccepted(out, wantColor, noUnit) + s.renderRejected(out, wantColor, noUnit) +} + +// rejectedKindDisplay maps a rejected sub-kind to the label shown in the +// rejected breakdown table. +var rejectedKindDisplay = map[string]string{ + "protocol": "protocol", + "submission": "submission", + "cookie": "cookie", +} + +// rejectedKindOrder fixes the row order so the rendered table is stable +// across runs and groups related kinds together. +var rejectedKindOrder = []string{"protocol", "submission", "cookie"} + +// renderAccepted renders the per-engine breakdown of GrantChallengeCookie +// reasons. Only kind="granted" carries a reason (kind="solved" has none by +// design), so the Kind column is dropped here and the table footer carries +// a grand total across all engines and reasons. +func (s *statAppsecChallenge) renderAccepted(out io.Writer, wantColor string, noUnit bool) { + engines := make([]string, 0, len(s.Reasons)) + for e, byKind := range s.Reasons { + if _, ok := byKind["granted"]; ok { + engines = append(engines, e) + } + } + if len(engines) == 0 { + return + } + sort.Strings(engines) + + t := cstable.New(out, wantColor).Writer + t.AppendHeader(table.Row{"Appsec Engine", "Reason", "Count"}) + + total := int64(0) + for _, engine := range engines { + counts := s.Reasons[engine]["granted"] + reasons := make([]string, 0, len(counts)) + for r := range counts { + reasons = append(reasons, r) + } + sort.Strings(reasons) + for _, r := range reasons { + t.AppendRow(table.Row{engine, r, formatNumber(int64(counts[r]), !noUnit)}) + total += int64(counts[r]) + } + } + + t.AppendFooter(table.Row{"Total", "", formatNumber(total, !noUnit)}) + t.SetTitle("Bot Detection — Accepted") + fmt.Fprintln(out, t.Render()) +} + +// renderRejected renders the per-engine per-kind per-reason breakdown of +// rejected outcomes. Kind matters here (protocol vs submission vs cookie), +// and the table footer carries a grand total across all rows. +func (s *statAppsecChallenge) renderRejected(out io.Writer, wantColor string, noUnit bool) { + engines := make([]string, 0, len(s.Reasons)) + for e, byKind := range s.Reasons { + for _, kind := range rejectedKindOrder { + if _, ok := byKind[kind]; ok { + engines = append(engines, e) + break + } + } + } + if len(engines) == 0 { + return + } + sort.Strings(engines) + + t := cstable.New(out, wantColor).Writer + t.AppendHeader(table.Row{"Appsec Engine", "Kind", "Reason", "Count"}) + + total := int64(0) + for _, engine := range engines { + byKind := s.Reasons[engine] + for _, kind := range rejectedKindOrder { + counts, ok := byKind[kind] + if !ok { + continue + } + reasons := make([]string, 0, len(counts)) + for r := range counts { + reasons = append(reasons, r) + } + sort.Strings(reasons) + for _, r := range reasons { + t.AppendRow(table.Row{engine, rejectedKindDisplay[kind], r, formatNumber(int64(counts[r]), !noUnit)}) + total += int64(counts[r]) + } + } + } + + t.AppendFooter(table.Row{"Total", "", "", formatNumber(total, !noUnit)}) + t.SetTitle("Bot Detection — Rejected") + fmt.Fprintln(out, t.Render()) +} diff --git a/cmd/crowdsec-cli/climetrics/statappsecchallengeinfra.go b/cmd/crowdsec-cli/climetrics/statappsecchallengeinfra.go new file mode 100644 index 000000000..61bc63d63 --- /dev/null +++ b/cmd/crowdsec-cli/climetrics/statappsecchallengeinfra.go @@ -0,0 +1,62 @@ +package climetrics + +import ( + "fmt" + "io" + + "github.com/jedib0t/go-pretty/v6/table" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/core/cstable" +) + +// statAppsecChallengeInfra holds the process-global housekeeping counters of +// the bot-detection challenge runtime (key rotation, obfuscation, cache +// eviction). Unlike the lifecycle funnel these carry no appsec_engine label +// — the challenge runtime is a single per-process instance shared by every +// engine — so this is a flat metric -> count map rather than per-engine. +type statAppsecChallengeInfra map[string]int + +func (statAppsecChallengeInfra) Description() (string, string) { + return "Bot Detection Infrastructure Metrics", + `Tracks the internal upkeep of the AppSec challenge runtime: signing-key rotation, JS re-obfuscation and cache eviction.` +} + +func (s statAppsecChallengeInfra) Process(metric string, val int) { + s[metric] += val +} + +// infraRow pairs a metric key with the label shown in the table, in a fixed +// order so the rendered output is stable across runs. +var infraRows = []struct { + key string + label string +}{ + {"kepoch_generated", "Signing key regenerated"}, + {"kepoch_evicted", "Signing key evicted"}, + {"reobfuscation_dynamic", "Re-obfuscation (dynamic module)"}, + {"reobfuscation_library", "Re-obfuscation (library bundle)"}, + {"dynamic_module_evicted", "Dynamic module evicted"}, +} + +func (s statAppsecChallengeInfra) Table(out io.Writer, wantColor string, noUnit bool, showEmpty bool) { + t := cstable.New(out, wantColor).Writer + t.AppendHeader(table.Row{"Metric", "Count"}) + + numRows := 0 + for _, r := range infraRows { + val, ok := s[r.key] + if !ok { + continue + } + t.AppendRow(table.Row{r.label, formatNumber(int64(val), !noUnit)}) + numRows++ + } + + if numRows == 0 && !showEmpty { + return + } + + title, _ := s.Description() + t.SetTitle(title) + fmt.Fprintln(out, t.Render()) +} diff --git a/cmd/crowdsec-cli/climetrics/statappsecengine.go b/cmd/crowdsec-cli/climetrics/statappsecengine.go index a6f524e33..f6df683d5 100644 --- a/cmd/crowdsec-cli/climetrics/statappsecengine.go +++ b/cmd/crowdsec-cli/climetrics/statappsecengine.go @@ -27,9 +27,9 @@ func (s statAppsecEngine) Process(appsecEngine, metric string, val int) { func (s statAppsecEngine) Table(out io.Writer, wantColor string, noUnit bool, showEmpty bool) { t := cstable.New(out, wantColor).Writer - t.AppendHeader(table.Row{"Appsec Engine", "Processed", "Blocked"}) + t.AppendHeader(table.Row{"Appsec Engine", "Processed", "Blocked", "Ch. Requested", "Ch. Accepted", "Ch. Rejected"}) - keys := []string{"processed", "blocked"} + keys := []string{"processed", "blocked", "challenge_requested", "challenge_accepted", "challenge_rejected"} if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil { log.Warningf("while collecting appsec stats: %s", err) diff --git a/cmd/crowdsec-cli/climetrics/store.go b/cmd/crowdsec-cli/climetrics/store.go index e025d7d70..a517ea027 100644 --- a/cmd/crowdsec-cli/climetrics/store.go +++ b/cmd/crowdsec-cli/climetrics/store.go @@ -24,20 +24,22 @@ type metricStore map[string]metricSection func NewMetricStore() metricStore { return metricStore{ - "acquisition": statAcquis{}, - "alerts": statAlert{}, - "bouncers": &statBouncer{}, - "appsec-engine": statAppsecEngine{}, - "appsec-rule": statAppsecRule{}, - "decisions": statDecision{}, - "lapi": statLapi{}, - "lapi-bouncer": statLapiBouncer{}, - "lapi-decisions": statLapiDecision{}, - "lapi-machine": statLapiMachine{}, - "parsers": statParser{}, - "scenarios": statBucket{}, - "stash": statStash{}, - "whitelists": statWhitelist{}, + "acquisition": statAcquis{}, + "alerts": statAlert{}, + "bouncers": &statBouncer{}, + "appsec-engine": statAppsecEngine{}, + "appsec-rule": statAppsecRule{}, + "appsec-challenge": newStatAppsecChallenge(), + "appsec-challenge-infra": statAppsecChallengeInfra{}, + "decisions": statDecision{}, + "lapi": statLapi{}, + "lapi-bouncer": statLapiBouncer{}, + "lapi-decisions": statLapiDecision{}, + "lapi-machine": statLapiMachine{}, + "parsers": statParser{}, + "scenarios": statBucket{}, + "stash": statStash{}, + "whitelists": statWhitelist{}, } } @@ -57,6 +59,59 @@ func (ms metricStore) Fetch(ctx context.Context, url string, db *database.Client return nil } +// metricLabels holds the label values extracted from a single Prometheus point. +type metricLabels struct { + name string + source string + machine string + bouncer string + route string + method string + reason string + origin string + action string + appsecEngine string + appsecRule string + // kind is carried by the appsec-challenge accepted/rejected counters + // to distinguish sub-outcomes; empty for every other metric. + kind string + // bundle is carried by the challenge re-obfuscation counter + // (dynamic vs library); empty for every other metric. + bundle string + mtype string +} + +func extractLabels(p MetricPoint) metricLabels { + name, ok := p.Labels["name"] + if !ok { + log.Debugf("no name in Metric %v", p.Labels) + } + + source, ok := p.Labels["source"] + if !ok { + log.Debugf("no source in Metric %v for %s", p.Labels, p.Name) + } else if srctype, ok := p.Labels["type"]; ok { + source = srctype + ":" + source + } + + return metricLabels{ + name: name, + source: source, + machine: p.Labels["machine"], + bouncer: p.Labels["bouncer"], + route: p.Labels["route"], + method: p.Labels["method"], + reason: p.Labels["reason"], + origin: p.Labels["origin"], + action: p.Labels["action"], + appsecEngine: p.Labels["appsec_engine"], + appsecRule: p.Labels["rule_name"], + kind: p.Labels["kind"], + bundle: p.Labels["bundle"], + mtype: p.Labels["type"], + } +} + func (ms metricStore) processPrometheusMetrics(result []MetricPoint) { mAcquis := ms["acquisition"].(statAcquis) mAlert := ms["alerts"].(statAlert) @@ -77,115 +132,157 @@ func (ms metricStore) processPrometheusMetrics(result []MetricPoint) { continue } - name, ok := p.Labels["name"] - if !ok { - log.Debugf("no name in Metric %v", p.Labels) - } + l := extractLabels(p) - source, ok := p.Labels["source"] - if !ok { - log.Debugf("no source in Metric %v for %s", p.Labels, p.Name) - } else { - if srctype, ok := p.Labels["type"]; ok { - source = srctype + ":" + source - } - } + ival := int(p.Value) - machine := p.Labels["machine"] - bouncer := p.Labels["bouncer"] - - route := p.Labels["route"] - method := p.Labels["method"] - - reason := p.Labels["reason"] - origin := p.Labels["origin"] - action := p.Labels["action"] - - appsecEngine := p.Labels["appsec_engine"] - appsecRule := p.Labels["rule_name"] - - mtype := p.Labels["type"] - - ival := int(p.Value) - - switch p.Name { + switch p.Name { // // buckets // case metrics.BucketsInstantiationMetricName: - mBucket.Process(name, "instantiation", ival) + mBucket.Process(l.name, "instantiation", ival) case metrics.BucketsCurrentCountMetricName: - mBucket.Process(name, "curr_count", ival) + mBucket.Process(l.name, "curr_count", ival) case metrics.BucketsOverflowMetricName: - mBucket.Process(name, "overflow", ival) + mBucket.Process(l.name, "overflow", ival) case metrics.BucketPouredMetricName: - mBucket.Process(name, "pour", ival) - mAcquis.Process(source, "pour", ival) + mBucket.Process(l.name, "pour", ival) + mAcquis.Process(l.source, "pour", ival) case metrics.BucketsUnderflowMetricName: - mBucket.Process(name, "underflow", ival) + mBucket.Process(l.name, "underflow", ival) // // parsers // case metrics.GlobalParserHitsMetricName: - mAcquis.Process(source, "reads", ival) + mAcquis.Process(l.source, "reads", ival) case metrics.GlobalParserHitsOkMetricName: - mAcquis.Process(source, "parsed", ival) + mAcquis.Process(l.source, "parsed", ival) case metrics.GlobalParserHitsKoMetricName: - mAcquis.Process(source, "unparsed", ival) + mAcquis.Process(l.source, "unparsed", ival) case metrics.NodesHitsMetricName: - mParser.Process(name, "hits", ival) + mParser.Process(l.name, "hits", ival) case metrics.NodesHitsOkMetricName: - mParser.Process(name, "parsed", ival) + mParser.Process(l.name, "parsed", ival) case metrics.NodesHitsKoMetricName: - mParser.Process(name, "unparsed", ival) + mParser.Process(l.name, "unparsed", ival) // // whitelists // case metrics.NodesWlHitsMetricName: - mWhitelist.Process(name, reason, "hits", ival) + mWhitelist.Process(l.name, l.reason, "hits", ival) case metrics.NodesWlHitsOkMetricName: - mWhitelist.Process(name, reason, "whitelisted", ival) + mWhitelist.Process(l.name, l.reason, "whitelisted", ival) // track as well whitelisted lines at acquis level - mAcquis.Process(source, "whitelisted", ival) + mAcquis.Process(l.source, "whitelisted", ival) // // lapi // case metrics.LapiRouteHitsMetricName: - mLapi.Process(route, method, ival) + mLapi.Process(l.route, l.method, ival) case metrics.LapiMachineHitsMetricName: - mLapiMachine.Process(machine, route, method, ival) + mLapiMachine.Process(l.machine, l.route, l.method, ival) case metrics.LapiBouncerHitsMetricName: - mLapiBouncer.Process(bouncer, route, method, ival) + mLapiBouncer.Process(l.bouncer, l.route, l.method, ival) case metrics.LapiNilDecisionsMetricName, metrics.LapiNonNilDecisionsMetricName: - mLapiDecision.Process(bouncer, p.Name, ival) + mLapiDecision.Process(l.bouncer, p.Name, ival) // // decisions // case metrics.GlobalActiveDecisionsMetricName: - mDecision.Process(reason, origin, action, ival) + mDecision.Process(l.reason, l.origin, l.action, ival) case metrics.GlobalAlertsMetricName: - mAlert.Process(reason, ival) + mAlert.Process(l.reason, ival) // // stash // case metrics.CacheMetricName: - mStash.Process(name, mtype, ival) + mStash.Process(l.name, l.mtype, ival) // // appsec // case "cs_appsec_reqs_total": - mAppsecEngine.Process(appsecEngine, "processed", ival) + mAppsecEngine.Process(l.appsecEngine, "processed", ival) case "cs_appsec_block_total": - mAppsecEngine.Process(appsecEngine, "blocked", ival) + mAppsecEngine.Process(l.appsecEngine, "blocked", ival) case "cs_appsec_rule_hits": - mAppsecRule.Process(appsecEngine, appsecRule, "triggered", ival) + mAppsecRule.Process(l.appsecEngine, l.appsecRule, "triggered", ival) default: + // bot detection / challenge metrics live in their own handler to + // keep this switch's complexity in check. + if ms.processAppsecChallengeMetric(p.Name, l, ival) { + continue + } + log.Debugf("unknown: %+v", p.Name) continue } } } +// processAppsecChallengeMetric handles the bot-detection / challenge lifecycle +// and infrastructure metrics. It lives apart from processPrometheusMetrics +// because its nested kind/bundle switches would otherwise push that function +// over the cyclomatic-complexity limit. Returns false for unrecognized metrics. +func (ms metricStore) processAppsecChallengeMetric(name string, l metricLabels, ival int) bool { + mAppsecEngine := ms["appsec-engine"].(statAppsecEngine) + mAppsecChallenge := ms["appsec-challenge"].(*statAppsecChallenge) + mAppsecChallengeInfra := ms["appsec-challenge-infra"].(statAppsecChallengeInfra) + + switch name { + // + // bot detection / challenge lifecycle + // + case metrics.AppsecChallengeRequestedMetricName: + mAppsecEngine.Process(l.appsecEngine, "challenge_requested", ival) + mAppsecChallenge.Process(l.appsecEngine, "requested", ival) + case metrics.AppsecChallengeSubmittedMetricName: + mAppsecChallenge.Process(l.appsecEngine, "submitted", ival) + case metrics.AppsecChallengeAcceptedMetricName: + mAppsecEngine.Process(l.appsecEngine, "challenge_accepted", ival) + switch l.kind { + case "solved": + mAppsecChallenge.Process(l.appsecEngine, "solved", ival) + case "granted": + mAppsecChallenge.Process(l.appsecEngine, "granted", ival) + mAppsecChallenge.ProcessReason(l.appsecEngine, "granted", l.reason, ival) + } + case metrics.AppsecChallengeRejectedMetricName: + mAppsecEngine.Process(l.appsecEngine, "challenge_rejected", ival) + switch l.kind { + case "protocol": + mAppsecChallenge.Process(l.appsecEngine, "rejected_protocol", ival) + mAppsecChallenge.ProcessReason(l.appsecEngine, "protocol", l.reason, ival) + case "submission": + mAppsecChallenge.Process(l.appsecEngine, "rejected_submission", ival) + mAppsecChallenge.ProcessReason(l.appsecEngine, "submission", l.reason, ival) + case "cookie": + mAppsecChallenge.Process(l.appsecEngine, "rejected_cookie", ival) + mAppsecChallenge.ProcessReason(l.appsecEngine, "cookie", l.reason, ival) + } + // + // bot detection / challenge infrastructure (process-global, no engine label) + // + case metrics.AppsecChallengeKepochGeneratedMetricName: + mAppsecChallengeInfra.Process("kepoch_generated", ival) + case metrics.AppsecChallengeKepochEvictedMetricName: + mAppsecChallengeInfra.Process("kepoch_evicted", ival) + case metrics.AppsecChallengeReobfuscationMetricName: + switch l.bundle { + case "dynamic": + mAppsecChallengeInfra.Process("reobfuscation_dynamic", ival) + case "library": + mAppsecChallengeInfra.Process("reobfuscation_library", ival) + } + case metrics.AppsecChallengeDynamicModuleEvictedMetricName: + mAppsecChallengeInfra.Process("dynamic_module_evicted", ival) + default: + return false + } + + return true +} + func (ms metricStore) Format(out io.Writer, wantColor string, sections []string, outputFormat string, noUnit bool) error { // copy only the sections we want want := map[string]metricSection{} diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 968d8dd55..2d86ba134 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -17,6 +17,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/dnscache" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/metrics" @@ -31,6 +32,8 @@ func initCrowdsec(ctx context.Context, cConfig *csconfig.Config, hub *cwhub.Hub, return nil, nil, fmt.Errorf("while loading context: %w", err) } + configureDNSCache(cConfig.Crowdsec.DNSCache) + err = exprhelpers.GeoIPInit(hub.GetDataDir()) if err != nil { // GeoIP databases are not mandatory, do not make crowdsec fail if they are not present @@ -70,6 +73,31 @@ func initCrowdsec(ctx context.Context, cConfig *csconfig.Config, hub *cwhub.Hub, return csParsers, datasources, nil } +func configureDNSCache(cfg *csconfig.DNSCacheCfg) { + if cfg == nil { + return + } + + var ( + ttl, negTTL time.Duration + size int + ) + + if cfg.TTL != nil { + ttl = *cfg.TTL + } + + if cfg.NegativeTTL != nil { + negTTL = *cfg.NegativeTTL + } + + if cfg.Size != nil { + size = *cfg.Size + } + + dnscache.Configure(ttl, negTTL, size) +} + func startParserRoutines(ctx context.Context, g *errgroup.Group, cConfig *csconfig.Config, parsers *parser.Parsers, stageCollector *parser.StageParseCollector) { for idx := range cConfig.Crowdsec.ParserRoutinesCount { log.WithField("idx", idx).Info("Starting parser routine") diff --git a/go.mod b/go.mod index 5b0e61afe..04fa5197e 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/crowdsecurity/grokky v0.2.2 github.com/crowdsecurity/machineid v1.0.3 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc + github.com/evanw/esbuild v0.27.2 github.com/expr-lang/expr v1.17.8 github.com/fatih/color v1.19.0 github.com/fsnotify/fsnotify v1.10.1 @@ -80,6 +81,7 @@ require ( github.com/slack-go/slack v0.23.0 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 + github.com/tetratelabs/wazero v1.11.0 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v1.11.0 github.com/xhit/go-simple-mail/v2 v2.16.0 @@ -191,6 +193,7 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.18.5 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect github.com/magefile/mage v1.17.1 // indirect @@ -226,7 +229,6 @@ require ( github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/tetratelabs/wazero v1.11.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.1 // indirect diff --git a/go.sum b/go.sum index b364cafa9..04763d5a5 100644 --- a/go.sum +++ b/go.sum @@ -157,6 +157,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/evanw/esbuild v0.27.2 h1:3xBEws9y/JosfewXMM2qIyHAi+xRo8hVx475hVkJfNg= +github.com/evanw/esbuild v0.27.2/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48= github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM= github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -734,6 +736,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/acquisition/modules/appsec/appsec_hooks_test.go b/pkg/acquisition/modules/appsec/appsec_hooks_test.go index 38334f4f1..42af73b35 100644 --- a/pkg/acquisition/modules/appsec/appsec_hooks_test.go +++ b/pkg/acquisition/modules/appsec/appsec_hooks_test.go @@ -4,6 +4,8 @@ import ( "io" "net/http" "net/url" + "os" + "path/filepath" "strings" "testing" @@ -13,6 +15,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/appsec/challenge" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/pipeline" ) @@ -1348,6 +1352,275 @@ func TestAppsecPhaseScopedHooks(t *testing.T) { runTests(t, tests) } +func TestAppsecOnChallengeHooks(t *testing.T) { + powWorkerURL, err := url.Parse(challenge.ChallengePowWorkerPath) + require.NoError(t, err) + + tests := []appsecRuleTest{ + { + name: "post_eval issues a challenge when no cookie is present", + expected_load_ok: true, + post_eval: []appsec.Hook{ + {Apply: []string{"SendChallenge()"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/protected", + HTTPRequest: &http.Request{Host: "example.com"}, + }, + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.Equal(t, appsec.ChallengeRemediation, responses[0].Action) + require.NotEmpty(t, responses[0].UserHTTPBodyContent) + require.Contains(t, responses[0].UserHeaders["Content-Type"], "text/html") + // Issuing the challenge page must emit a single "requested" event + // on a source distinct from the WAF one. + require.Len(t, events, 1) + require.Equal(t, pipeline.LOG, events[0].Type) + require.Equal(t, appsec.SourceChallenge, events[0].Parsed["source"]) + require.Equal(t, string(appsec.ChallengeReasonRequested), events[0].Parsed["challenge_event"]) + }, + }, + { + name: "on_challenge: no cookie → user hooks skipped (filter would nil-deref)", + expected_load_ok: true, + // This filter would nil-deref if it ran without a fingerprint. The + // dispatcher must skip it when no cookie is present. + on_challenge: []appsec.Hook{ + {Filter: "fingerprint.Bot.MismatchWebGLInWorker", Apply: []string{"DropRequest('bot')"}}, + }, + // Must reference SendChallenge() somewhere to force ChallengeRuntime init. + post_eval: []appsec.Hook{ + {Filter: "false", Apply: []string{"SendChallenge()"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/protected", + HTTPRequest: &http.Request{Host: "example.com"}, + }, + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.False(t, responses[0].InBandInterrupt, "on_challenge must not fire without a fingerprint") + }, + }, + { + name: "on_challenge: PoW worker path served, WAF evaluation skipped", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + // Force the challenge runtime to be initialized by referencing SendChallenge() + // in a hook that won't match this request. + on_challenge: []appsec.Hook{ + {Filter: "false", Apply: []string{"SendChallenge()"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: challenge.ChallengePowWorkerPath, + Args: url.Values{"foo": []string{"toto"}}, // would normally trigger rule1 + HTTPRequest: &http.Request{Host: "example.com", URL: powWorkerURL}, + }, + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Empty(t, events, "WAF should not have evaluated the infrastructure path") + require.Len(t, responses, 1) + require.Equal(t, appsec.ChallengeRemediation, responses[0].Action) + require.Equal(t, challenge.PowWorkerJS, responses[0].UserHTTPBodyContent) + require.Contains(t, responses[0].UserHeaders["Content-Type"], "application/javascript") + }, + }, + { + name: "on_challenge: invalid submission returns failed body, no hooks run", + expected_load_ok: true, + // Unconditional DropRequest in on_challenge must NOT fire on an + // invalid submission — the dispatcher returns the failed JSON + // body and skips user hooks. + on_challenge: []appsec.Hook{ + {Apply: []string{"DropRequest('should not fire')"}}, + }, + // Force ChallengeRuntime init. + post_eval: []appsec.Hook{ + {Filter: "false", Apply: []string{"SendChallenge()"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "POST", + URI: challenge.ChallengeSubmitPath, + HTTPRequest: func() *http.Request { + u, _ := url.Parse(challenge.ChallengeSubmitPath) + return &http.Request{Host: "example.com", URL: u, Method: http.MethodPost} + }(), + }, + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.Equal(t, appsec.ChallengeRemediation, responses[0].Action) + require.JSONEq(t, `{"status":"failed"}`, responses[0].UserHTTPBodyContent) + require.False(t, responses[0].InBandInterrupt, "on_challenge hooks must not run on invalid submission") + // A submission attempt emits "submitted", then "failed" (with a reason). + require.Len(t, events, 2) + require.Equal(t, appsec.SourceChallenge, events[0].Parsed["source"]) + require.Equal(t, string(appsec.ChallengeReasonSubmitted), events[0].Parsed["challenge_event"]) + require.Equal(t, string(appsec.ChallengeReasonFailed), events[1].Parsed["challenge_event"]) + require.NotEmpty(t, events[1].Parsed["challenge_fail_reason"]) + }, + }, + { + // Mock LAPI (see testAppSecEngine in appsec_test.go) exposes 5.4.3.2 + // as an allowlisted IP. SendChallenge() must be a no-op for it: no + // challenge HTML, no Set-Cookie, response stays at the default pass + // action so the request goes through cleanly. + name: "allowlisted IP: post_eval SendChallenge() is suppressed", + expected_load_ok: true, + post_eval: []appsec.Hook{ + {Apply: []string{"SendChallenge()"}}, + }, + input_request: appsec.ParsedRequest{ + ClientIP: "5.4.3.2", + RemoteAddr: "5.4.3.2", + Method: "GET", + URI: "/protected", + HTTPRequest: &http.Request{Host: "example.com"}, + }, + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action, + "allowlisted IP must not be challenged") + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + require.Empty(t, responses[0].UserHTTPBodyContent, "no challenge HTML must be served") + require.Empty(t, responses[0].UserHTTPCookies, "no challenge cookie must be issued") + require.False(t, responses[0].InBandInterrupt) + }, + }, + { + // Same allowlisted IP, but inside a /24 CIDR entry (5.4.4.0/24) — + // confirms range matches are honored the same way as exact IPs. + name: "allowlisted CIDR: post_eval SendChallenge() is suppressed", + expected_load_ok: true, + post_eval: []appsec.Hook{ + {Apply: []string{"SendChallenge()"}}, + }, + input_request: appsec.ParsedRequest{ + ClientIP: "5.4.4.42", + RemoteAddr: "5.4.4.42", + Method: "GET", + URI: "/protected", + HTTPRequest: &http.Request{Host: "example.com"}, + }, + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + require.Empty(t, responses[0].UserHTTPBodyContent) + require.Empty(t, responses[0].UserHTTPCookies) + }, + }, + { + // Allowlisted IPs hitting infrastructure paths must not get the PoW + // worker JS served either — ProcessOnChallengeRules short-circuits + // before the path-based branch, so the request flows through normal + // WAF processing. + name: "allowlisted IP: PoW worker path is not served", + expected_load_ok: true, + // Reference SendChallenge() so ChallengeRuntime initializes. + on_challenge: []appsec.Hook{ + {Filter: "false", Apply: []string{"SendChallenge()"}}, + }, + input_request: appsec.ParsedRequest{ + ClientIP: "5.4.3.2", + RemoteAddr: "5.4.3.2", + Method: "GET", + URI: challenge.ChallengePowWorkerPath, + HTTPRequest: &http.Request{Host: "example.com", URL: powWorkerURL}, + }, + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action, + "allowlisted IP must not receive the PoW worker JS") + require.NotEqual(t, challenge.PowWorkerJS, responses[0].UserHTTPBodyContent) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + }, + }, + { + // SendChallenge is restricted to on_challenge + in-band post_eval + name: "pre_eval SendChallenge() fails to load", + expected_load_ok: false, + pre_eval: []appsec.Hook{ + {Apply: []string{"SendChallenge()"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/protected", + HTTPRequest: &http.Request{Host: "example.com"}, + }, + }, + { + // SendChallenge in an out-of-band post_eval hook is rejected at runtime + name: "outofband post_eval SendChallenge() is rejected", + expected_load_ok: true, + outofband_post_eval: []appsec.Hook{ + {Apply: []string{"SendChallenge()"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/protected", + HTTPRequest: &http.Request{Host: "example.com"}, + }, + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action, + "out-of-band SendChallenge must not issue a challenge") + require.Empty(t, responses[0].UserHTTPBodyContent, "no challenge HTML must be served") + }, + }, + { + // A challenged request is served the challenge and never reaches the + // backend; + name: "challenge skips out-of-band evaluation", + expected_load_ok: true, + post_eval: []appsec.Hook{ + {Apply: []string{"SendChallenge()"}}, + }, + outofband_rules: []appsec_rule.CustomRule{ + { + Name: "oob-rule", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/protected", + Args: url.Values{"foo": []string{"toto"}}, // would match oob-rule + HTTPRequest: &http.Request{Host: "example.com"}, + }, + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.Equal(t, appsec.ChallengeRemediation, responses[0].Action) + // Only the challenge "requested" event must be present; the OOB rule + // was skipped, so no appsec match event is emitted. + require.Len(t, events, 1) + require.Equal(t, appsec.SourceChallenge, events[0].Parsed["source"]) + require.Equal(t, string(appsec.ChallengeReasonRequested), events[0].Parsed["challenge_event"]) + }, + }, + } + + runTests(t, tests) +} + // minimal schema used by the hook_vars-in-event tests: /users only accepts // a POST whose body has a required username field. const testUsersSchema = `openapi: 3.0.0 @@ -1602,3 +1875,91 @@ func mustParseURL(raw string) *url.URL { } return u } + +// TestAppsecLegitimateBotHooks exercises the IsLegitimateBot/ExemptFromChallenge +// helpers through the full runner: a bot definition is loaded from a +// legit_bots directory (as AppsecConfig.Build does with the hub data dir), +// an in-band rule matches the request, and a pre_eval hook downgrades the +// remediation for verified bots only. +func TestAppsecLegitimateBotHooks(t *testing.T) { + require.NoError(t, exprhelpers.Init(nil)) + + datadir := t.TempDir() + botsDir := filepath.Join(datadir, "legit_bots") + require.NoError(t, os.Mkdir(botsDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(botsDir, "bots.json"), + []byte(`{"name":"testbot","user_agent":"testbot","ranges":["192.0.2.0/24"]}`), 0o644)) + require.NoError(t, exprhelpers.FileInit(botsDir, "bots.json", "bots")) + + banRule := appsec_rule.CustomRule{ + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + } + bypassHook := appsec.Hook{ + Filter: `IsLegitimateBot(req.RemoteAddr, req.UserAgent(), req.URL.Path)`, + Apply: []string{`SetRemediation("allow")`}, + } + botRequest := func(remoteAddr string, ua string) appsec.ParsedRequest { + return appsec.ParsedRequest{ + RemoteAddr: remoteAddr, + Method: "GET", + URI: "/crawl", + Args: url.Values{"foo": []string{"toto"}}, + HTTPRequest: &http.Request{ + Host: "example.com", + RemoteAddr: remoteAddr, + URL: mustParseURL("http://example.com/crawl"), + Header: http.Header{"User-Agent": []string{ua}}, + }, + } + } + + tests := []appsecRuleTest{ + { + name: "verified bot bypasses the ban", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{banRule}, + pre_eval: []appsec.Hook{bypassHook}, + // the IP is in the declared range; ip:port proves address normalization end to end + input_request: botRequest("192.0.2.10:34567", "Mozilla/5.0 (compatible; TestBot/1.0)"), + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.True(t, responses[0].InBandInterrupt, "the rule still matches, only the remediation changes") + require.Equal(t, appsec.AllowRemediation, responses[0].Action) + }, + }, + { + name: "spoofed UA from a foreign IP is still banned", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{banRule}, + pre_eval: []appsec.Hook{bypassHook}, + input_request: botRequest("203.0.113.9:34567", "Mozilla/5.0 (compatible; TestBot/1.0)"), + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.True(t, responses[0].InBandInterrupt) + require.Equal(t, appsec.BanRemediation, responses[0].Action) + }, + }, + { + name: "ExemptFromChallenge escape hatch short-circuits the checks", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{banRule}, + pre_eval: []appsec.Hook{ + {Apply: []string{"ExemptFromChallenge()"}}, + bypassHook, + }, + // neither the UA nor the IP matches any definition + input_request: botRequest("203.0.113.9:34567", "curl/8.0"), + output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + require.True(t, responses[0].InBandInterrupt) + require.Equal(t, appsec.AllowRemediation, responses[0].Action) + }, + }, + } + + runTests(t, tests) +} diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index e410b353a..746104c85 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -5,6 +5,7 @@ import ( "fmt" "maps" "os" + "path/filepath" "slices" "strings" "time" @@ -129,6 +130,15 @@ func (r *AppsecRunner) Init(datadir string) error { r.logger.Tracef("Loaded inband rules: %+v", r.AppsecInbandEngine.GetRuleGroup().GetRules()) r.logger.Tracef("Loaded outband rules: %+v", r.AppsecOutbandEngine.GetRuleGroup().GetRules()) + // Materialize the fingerprint-dump dir under data_dir. Failure doesn't prevent startup. + r.AppsecRuntime.FingerprintDumpDir = filepath.Join(datadir, "fingerprint_dumps") + if err := os.MkdirAll(r.AppsecRuntime.FingerprintDumpDir, 0o700); err != nil { + r.logger.Warnf("could not create fingerprint dump dir %q: %s; DumpFingerprint() will be a no-op", r.AppsecRuntime.FingerprintDumpDir, err) + r.AppsecRuntime.FingerprintDumpDir = "" + } else if err := os.Chmod(r.AppsecRuntime.FingerprintDumpDir, 0o700); err != nil { + r.logger.Warnf("could not chmod fingerprint dump dir %q to 0700: %s", r.AppsecRuntime.FingerprintDumpDir, err) + } + return nil } @@ -142,10 +152,8 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR } defer func() { - state.Tx.ProcessLogging() - //We don't close the transaction here, as it will reset coraza internal state and break variable tracking - - err := r.AppsecRuntime.ProcessPostEvalRules(state, request) + // We don't close the transaction here, as it would reset coraza internal state and break variable tracking. + err := r.AppsecRuntime.ProcessPostEvalRules(ctx, state, request) if err != nil { r.logger.Errorf("unable to process PostEval rules: %s", err) } @@ -158,6 +166,13 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR //FIXME: should we abort here ? } + // User has requested valid challenge, but we did not find a valid cookie + // Immediately return, everything has been set already + if state.RequireChallenge { + r.logger.Debug("serving challenge") + return nil + } + if state.DropInfo(request) != nil { r.logger.Debug("drop helper triggered during pre_eval, skipping WAF evaluation") return nil @@ -177,6 +192,11 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR r.logger.Debugf("request body exceeded maximum allowed size but body inspection is disabled, allowing request") } + defer func() { + state.Tx.ProcessLogging() + // We don't close the transaction here, as it would reset coraza internal state and break variable tracking. + }() + state.Tx.ProcessConnection(request.ClientIP, 0, "", 0) for k, v := range request.Args { @@ -241,14 +261,35 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR func (r *AppsecRunner) ProcessInBandRules(ctx context.Context, state *appsec.AppsecRequestState, request *appsec.ParsedRequest) error { tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID) state.Tx = tx - // Even if we have no inband rules, we might have pre-eval or post-eval rules to process + // Even if we have no inband rules, we might have pre-eval, post-eval or on_challenge hooks to process if len(r.AppsecRuntime.InBandRules) == 0 && len(r.AppsecRuntime.CommonHooks.PreEval) == 0 && len(r.AppsecRuntime.InBandHooks.PreEval) == 0 && len(r.AppsecRuntime.CommonHooks.PostEval) == 0 && - len(r.AppsecRuntime.InBandHooks.PostEval) == 0 { + len(r.AppsecRuntime.InBandHooks.PostEval) == 0 && + len(r.AppsecRuntime.CompiledOnChallenge) == 0 && + r.AppsecRuntime.ChallengeRuntime == nil { return nil } + + // on_challenge runs before any WAF work: it serves PoW infrastructure paths, + // validates submissions, and populates state.Fingerprint from the cookie. + if err := r.AppsecRuntime.ProcessOnChallengeRules(ctx, state, request); err != nil { + r.logger.Errorf("unable to process OnChallenge rules: %s", err) + } + + // Infrastructure paths (PoW worker, challenge submit) already set up the + // full response — skip pre_eval, WAF evaluation and post_eval entirely. + if state.RequireChallenge { + r.logger.Debugf("challenge response set by on_challenge, skipping WAF evaluation") + return nil + } + + if state.DropInfo(request) != nil { + r.logger.Debug("drop helper triggered during on_challenge, skipping WAF evaluation") + return nil + } + err := r.processRequest(ctx, state, request) return err } @@ -268,13 +309,8 @@ func (r *AppsecRunner) ProcessOutOfBandRules(ctx context.Context, state *appsec. } func (r *AppsecRunner) handleInBandInterrupt(ctx context.Context, state *appsec.AppsecRequestState, request *appsec.ParsedRequest) { - if allowed, reason := r.appsecAllowlistsClient.IsAllowlisted(request.ClientIP); allowed { - r.logger.Infof("%s is allowlisted by %s, skipping", request.ClientIP, reason) - return - } - //create the associated event for crowdsec itself - evt, err := EventFromRequest(request, r.Labels, state.Tx.ID()) + evt, err := appsec.EventFromRequest(request, r.Labels, state.Tx.ID()) if err != nil { //let's not interrupt the pipeline for this r.logger.Errorf("unable to create event from request : %s", err) @@ -361,13 +397,7 @@ func copyHookVars(evt *pipeline.Event, state *appsec.AppsecRequestState) { } func (r *AppsecRunner) handleOutBandInterrupt(ctx context.Context, state *appsec.AppsecRequestState, request *appsec.ParsedRequest) { - - if allowed, reason := r.appsecAllowlistsClient.IsAllowlisted(request.ClientIP); allowed { - r.logger.Infof("%s is allowlisted by %s, skipping", request.ClientIP, reason) - return - } - - evt, err := EventFromRequest(request, r.Labels, state.Tx.ID()) + evt, err := appsec.EventFromRequest(request, r.Labels, state.Tx.ID()) if err != nil { //let's not interrupt the pipeline for this r.logger.Errorf("unable to create event from request : %s", err) @@ -435,6 +465,19 @@ func (r *AppsecRunner) handleRequest(ctx context.Context, request *appsec.Parsed logger.Debug("Request received in runner") r.AppsecRuntime.ClearResponse(&state) + // Allowlisted IPs bypass every form of appsec processing (in-band rules, + // hooks, challenge issuance, on_challenge cookie validation, out-of-band + // rules). We send the default pass response straight back to the bouncer. + if r.appsecAllowlistsClient != nil { + if allowed, reason := r.appsecAllowlistsClient.IsAllowlisted(request.ClientIP); allowed { + logger.Infof("%s is allowlisted by %s, skipping WAF processing", request.ClientIP, reason) + // No Clone() needed (unlike the in-band send below): we return + // immediately, so no out-of-band phase races this response. + request.ResponseChannel <- state.Response + return + } + } + request.IsInBand = true request.IsOutBand = false @@ -468,8 +511,20 @@ func (r *AppsecRunner) handleRequest(ctx context.Context, request *appsec.Parsed r.logger.Errorf("unable to close inband transaction: %s", err) } - // send back the result to the HTTP handler for the InBand part - request.ResponseChannel <- state.Response + // Clone as the out-of-band phase might mutate the response + request.ResponseChannel <- state.Response.Clone() + + // A challenge was served, so the request never reaches the backend; + if state.RequireChallenge { + globalParsingElapsed := time.Since(startGlobalParsing) + metrics.AppsecGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(globalParsingElapsed.Seconds()) + return + } + + // Challenge remediation is intentionally a no-op for OOB matches: the inband + // response has already been sent to the visitor at this point, so there is + // nothing left to challenge. OOB matches still feed alerts/events. + // (captcha gets the same treatment for the same reason.) //Now let's process the out of band rules diff --git a/pkg/acquisition/modules/appsec/appsec_test.go b/pkg/acquisition/modules/appsec/appsec_test.go index ed1780d01..fd27c244b 100644 --- a/pkg/acquisition/modules/appsec/appsec_test.go +++ b/pkg/acquisition/modules/appsec/appsec_test.go @@ -13,11 +13,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/appsec/allowlists" "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/pipeline" ) @@ -31,15 +31,17 @@ type appsecRuleTest struct { // schemas registers OpenAPI schemas (ref → YAML content) on the // RequestValidator after Build, so hooks can refer to them via // ValidateRequestWithSchema(ref). - schemas map[string]string - on_load []appsec.Hook - pre_eval []appsec.Hook - post_eval []appsec.Hook - on_match []appsec.Hook + schemas map[string]string + on_load []appsec.Hook + pre_eval []appsec.Hook + post_eval []appsec.Hook + on_match []appsec.Hook + on_challenge []appsec.Hook // Phase-scoped hooks (dispatched only during the matching phase) inband_on_match []appsec.Hook inband_pre_eval []appsec.Hook inband_post_eval []appsec.Hook + inband_on_challenge []appsec.Hook outofband_on_match []appsec.Hook outofband_pre_eval []appsec.Hook outofband_post_eval []appsec.Hook @@ -119,6 +121,7 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) { PreEval: test.pre_eval, PostEval: test.post_eval, OnMatch: test.on_match, + OnChallenge: test.on_challenge, BouncerBlockedHTTPCode: test.BouncerBlockedHTTPCode, UserBlockedHTTPCode: test.UserBlockedHTTPCode, UserPassedHTTPCode: test.UserPassedHTTPCode, @@ -127,11 +130,12 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) { } // Set phase-scoped hooks if any are provided - if len(test.inband_on_match) > 0 || len(test.inband_pre_eval) > 0 || len(test.inband_post_eval) > 0 { + if len(test.inband_on_match) > 0 || len(test.inband_pre_eval) > 0 || len(test.inband_post_eval) > 0 || len(test.inband_on_challenge) > 0 { appsecCfg.InBand = &appsec.AppsecPhaseConfig{ - OnMatch: test.inband_on_match, - PreEval: test.inband_pre_eval, - PostEval: test.inband_post_eval, + OnMatch: test.inband_on_match, + PreEval: test.inband_pre_eval, + PostEval: test.inband_post_eval, + OnChallenge: test.inband_on_challenge, } } @@ -146,6 +150,9 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) { hub := cwhub.Hub{} AppsecRuntime, err := appsecCfg.Build(t.Context(), &hub) if err != nil { + if !test.expected_load_ok { + return + } t.Fatalf("unable to build appsec runtime : %s", err) } for ref, schemaYAML := range test.schemas { @@ -155,10 +162,23 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) { } AppsecRuntime.InBandRules = []appsec.AppsecCollection{{Rules: inbandRules, NativeRules: nativeInbandRules}} AppsecRuntime.OutOfBandRules = []appsec.AppsecCollection{{Rules: outofbandRules, NativeRules: nativeOutofbandRules}} + + // Hooks using SendChallenge() or on_challenge hooks require the WASM + // challenge runtime; mirror what pkg/acquisition/modules/appsec/config.go + // does in production. We share a single runtime across tests — building + // one is expensive (~15-20s for WASM obfuscator warm-up), and none of the + // tests mutate runtime-level state like default difficulty. + if AppsecRuntime.NeedWASMVM { + AppsecRuntime.ChallengeRuntime = getSharedChallengeRuntime(t) + } appsecRunnerUUID := uuid.New().String() //we copy AppsecRutime for each runner wrt := *AppsecRuntime wrt.Logger = logger + // mirror production wiring (run.go / config.go) so challenge lifecycle + // events are emitted on the same channel the harness collects from. + wrt.OutChan = OutChan + wrt.Labels = map[string]string{"foo": "bar"} mux, urlx, teardown := setupLapi() defer teardown() @@ -242,7 +262,7 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) { idle.Reset(idleDuration) } - responses :=[]appsec.AppsecTempResponse{} + responses := []appsec.AppsecTempResponse{} events := []pipeline.Event{} done := make(chan struct{}) diff --git a/pkg/acquisition/modules/appsec/config.go b/pkg/acquisition/modules/appsec/config.go index 9bea48906..9f85eb52d 100644 --- a/pkg/acquisition/modules/appsec/config.go +++ b/pkg/acquisition/modules/appsec/config.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http" "os" + "strings" "time" yaml "github.com/goccy/go-yaml" @@ -20,6 +21,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/apiclient/useragent" "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/appsec/allowlists" + "github.com/crowdsecurity/crowdsec/pkg/appsec/challenge" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/metrics" ) @@ -121,6 +125,61 @@ func loadCertPool(caCertPath string, logger log.FieldLogger) (*x509.CertPool, er return caCertPool, nil } +// expandAppsecConfigEntry resolves a single appsec_config(s) entry into the list +// of appsec-config item names to load. A literal entry is returned untouched. An entry containing a glob meta-character +// is matched against the installed appsec-configs with the same matcher used +// to expand appsec-rule patterns; it errors when no installed config matches. +func expandAppsecConfigEntry(entry string, hub *cwhub.Hub) ([]string, error) { + if !strings.ContainsAny(entry, "*?") { + return []string{entry}, nil + } + + var matches []string + + for _, item := range hub.GetInstalledByType(cwhub.APPSEC_CONFIGS, true) { + matched, err := exprhelpers.Match(entry, item.Name) + if err != nil { + return nil, fmt.Errorf("while matching %q against %q: %w", entry, item.Name, err) + } + + if matched.(bool) { + matches = append(matches, item.Name) + } + } + + if len(matches) == 0 { + return nil, fmt.Errorf("no installed appsec-config matches pattern %q", entry) + } + + return matches, nil +} + +// resolveAppsecConfigEntries expands every appsec_config(s) entry and returns +// the de-duplicated list of config names to load, in first-seen order. +func resolveAppsecConfigEntries(entries []string, hub *cwhub.Hub) ([]string, error) { + seen := make(map[string]struct{}) + + var toLoad []string + + for _, entry := range entries { + names, err := expandAppsecConfigEntry(entry, hub) + if err != nil { + return nil, fmt.Errorf("unable to resolve appsec_config %q: %w", entry, err) + } + + for _, name := range names { + if _, ok := seen[name]; ok { + continue + } + + seen[name] = struct{}{} + toLoad = append(toLoad, name) + } + } + + return toLoad, nil +} + func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.Entry, _ metrics.AcquisitionMetricsLevel) error { if w.hub == nil { return errors.New("appsec datasource requires a hub. this is a bug, please report") @@ -177,13 +236,19 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E if err := appsecCfg.LoadByPath(w.config.AppsecConfigPath); err != nil { return fmt.Errorf("unable to load appsec_config: %w", err) } - } else if w.config.AppsecConfig != "" { - if err := appsecCfg.Load(w.config.AppsecConfig, w.hub); err != nil { - return fmt.Errorf("unable to load appsec_config: %w", err) + } else if w.config.AppsecConfig != "" || len(w.config.AppsecConfigs) > 0 { + entries := w.config.AppsecConfigs + if w.config.AppsecConfig != "" { + entries = []string{w.config.AppsecConfig} } - } else if len(w.config.AppsecConfigs) > 0 { - for _, appsecConfig := range w.config.AppsecConfigs { - if err := appsecCfg.Load(appsecConfig, w.hub); err != nil { + + names, err := resolveAppsecConfigEntries(entries, w.hub) + if err != nil { + return err + } + + for _, name := range names { + if err := appsecCfg.Load(name, w.hub); err != nil { return fmt.Errorf("unable to load appsec_config: %w", err) } } @@ -199,7 +264,23 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E return fmt.Errorf("unable to build appsec_config: %w", err) } + if appsecRuntime.NeedWASMVM { + logger.Info("Initializing WASM runtime for challenge obfuscation") + + challengeOpts, err := challenge.BuildOptions(appsecCfg.Challenge, appsecCfg.Logger) + if err != nil { + return fmt.Errorf("unable to build challenge options: %w", err) + } + + challengeRuntime, err := challenge.NewChallengeRuntime(ctx, challengeOpts...) + if err != nil { + return fmt.Errorf("unable to create challenge runtime: %w", err) + } + appsecRuntime.ChallengeRuntime = challengeRuntime + } + w.AppsecRuntime = appsecRuntime + w.AppsecRuntime.Labels = w.config.Labels w.AppsecRuntime.DataDir = w.hub.GetDataDir() err = w.AppsecRuntime.ProcessOnLoadRules() @@ -213,14 +294,11 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E for nbRoutine := range w.config.Routines { appsecRunnerUUID := uuid.New().String() - // we copy AppsecRuntime for each runner - wrt := *w.AppsecRuntime - wrt.Logger = w.logger.Dup().WithField("runner_uuid", appsecRunnerUUID) runner := AppsecRunner{ inChan: w.InChan, UUID: appsecRunnerUUID, logger: w.logger.WithField("runner_uuid", appsecRunnerUUID), - AppsecRuntime: &wrt, + AppsecRuntime: w.AppsecRuntime, Labels: w.config.Labels, appsecAllowlistsClient: w.appsecAllowlistClient, } diff --git a/pkg/acquisition/modules/appsec/config_test.go b/pkg/acquisition/modules/appsec/config_test.go new file mode 100644 index 000000000..3f06db844 --- /dev/null +++ b/pkg/acquisition/modules/appsec/config_test.go @@ -0,0 +1,157 @@ +package appsecacquisition + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// hubWithConfigs builds an in-memory hub holding the given appsec-configs and +// marks the listed ones as installed (an installed item is one with a +// LocalPath, see cwhub.ItemState.IsInstalled). +func hubWithConfigs(t *testing.T, all, installed []string) *cwhub.Hub { + t.Helper() + + tempDir := t.TempDir() + local := &csconfig.LocalHubCfg{ + HubDir: filepath.Join(tempDir, "hub"), + HubIndexFile: filepath.Join(tempDir, "hub", ".index.json"), + InstallDir: filepath.Join(tempDir, "install"), + InstallDataDir: filepath.Join(tempDir, "data"), + } + + require.NoError(t, os.MkdirAll(local.HubDir, 0o755)) + require.NoError(t, os.MkdirAll(local.InstallDir, 0o755)) + require.NoError(t, os.MkdirAll(local.InstallDataDir, 0o755)) + + index := `{"appsec-configs": {` + for i, name := range all { + if i > 0 { + index += "," + } + index += `"` + name + `": {"path": "appsec-configs/` + name + `.yaml", "version": "1.0", "versions": {"1.0": {"digest": "aa"}}}` + } + index += "}}" + + require.NoError(t, os.WriteFile(local.HubIndexFile, []byte(index), 0o644)) + + hub, err := cwhub.NewHub(local, nil) + require.NoError(t, err) + require.NoError(t, hub.Load()) + + for _, name := range installed { + item := hub.GetItem(cwhub.APPSEC_CONFIGS, name) + require.NotNilf(t, item, "appsec-config %q missing from test hub", name) + item.State.LocalPath = filepath.Join(local.InstallDir, name+".yaml") + } + + return hub +} + +func TestExpandAppsecConfigEntry(t *testing.T) { + all := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "crowdsecurity/uninstalled", "custom/mine"} + installed := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "custom/mine"} + + tests := []struct { + name string + entry string + expected []string + expectedErr string + }{ + { + // Literals pass through untouched, even unknown ones, so the + // per-name error still surfaces later from AppsecConfig.Load. + name: "literal passthrough", + entry: "crowdsecurity/vpatch", + expected: []string{"crowdsecurity/vpatch"}, + }, + { + name: "unknown literal passthrough", + entry: "does/not-exist", + expected: []string{"does/not-exist"}, + }, + { + // Sorted (case-insensitive) order from the hub; uninstalled is skipped. + name: "wildcard expands installed only", + entry: "crowdsecurity/*", + expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch"}, + }, + { + name: "wildcard matches all", + entry: "*", + expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch", "custom/mine"}, + }, + { + name: "wildcard with no match errors", + entry: "nope/*", + expectedErr: `no installed appsec-config matches pattern "nope/*"`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + hub := hubWithConfigs(t, all, installed) + + got, err := expandAppsecConfigEntry(tc.entry, hub) + if tc.expectedErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.expectedErr) + return + } + + require.NoError(t, err) + assert.Equal(t, tc.expected, got) + }) + } +} + +func TestResolveAppsecConfigEntries(t *testing.T) { + all := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "crowdsecurity/uninstalled", "custom/mine"} + installed := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "custom/mine"} + + tests := []struct { + name string + entries []string + expected []string + }{ + { + // "*" already pulls in vpatch; the explicit literal must not load it + // a second time. First-seen order from the wildcard expansion wins. + name: "wildcard plus overlapping literal dedups", + entries: []string{"*", "crowdsecurity/vpatch"}, + expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch", "custom/mine"}, + }, + { + name: "same literal twice collapses to one", + entries: []string{"crowdsecurity/vpatch", "crowdsecurity/vpatch"}, + expected: []string{"crowdsecurity/vpatch"}, + }, + { + name: "unknown literal passes through once", + entries: []string{"does/not-exist", "does/not-exist"}, + expected: []string{"does/not-exist"}, + }, + { + // Overlapping patterns: generic matches both, must appear once. + name: "overlapping patterns dedup", + entries: []string{"crowdsecurity/*", "*/generic"}, + expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + hub := hubWithConfigs(t, all, installed) + + got, err := resolveAppsecConfigEntries(tc.entries, hub) + require.NoError(t, err) + assert.Equal(t, tc.expected, got) + }) + } +} diff --git a/pkg/acquisition/modules/appsec/init.go b/pkg/acquisition/modules/appsec/init.go index c7190e155..477a4200a 100644 --- a/pkg/acquisition/modules/appsec/init.go +++ b/pkg/acquisition/modules/appsec/init.go @@ -3,6 +3,7 @@ package appsecacquisition import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition/registry" "github.com/crowdsecurity/crowdsec/pkg/acquisition/types" + "github.com/crowdsecurity/crowdsec/pkg/appsec" ) var ( @@ -14,7 +15,7 @@ var ( _ types.LAPIClientAware = (*Source)(nil) ) -const ModuleName = "appsec" +const ModuleName = appsec.ModuleName //nolint:gochecknoinits func init() { diff --git a/pkg/acquisition/modules/appsec/metrics.go b/pkg/acquisition/modules/appsec/metrics.go index b798c66b5..d51ba7434 100644 --- a/pkg/acquisition/modules/appsec/metrics.go +++ b/pkg/acquisition/modules/appsec/metrics.go @@ -11,6 +11,15 @@ func (*Source) GetMetrics() []prometheus.Collector { metrics.AppsecReqCounter, metrics.AppsecBlockCounter, metrics.AppsecRuleHits, + metrics.AppsecFingerprintMismatch, + metrics.AppsecChallengeRequested, + metrics.AppsecChallengeSubmitted, + metrics.AppsecChallengeAccepted, + metrics.AppsecChallengeRejected, + metrics.AppsecChallengeKepochGenerated, + metrics.AppsecChallengeKepochEvicted, + metrics.AppsecChallengeReobfuscation, + metrics.AppsecChallengeDynamicModuleEvicted, metrics.AppsecValidationOKCounter, metrics.AppsecValidationFailedCounter, metrics.AppsecOutbandParsingHistogram, @@ -24,9 +33,18 @@ func (*Source) GetAggregMetrics() []prometheus.Collector { metrics.AppsecReqCounter, metrics.AppsecBlockCounter, metrics.AppsecRuleHits, + metrics.AppsecFingerprintMismatch, + metrics.AppsecChallengeRequested, + metrics.AppsecChallengeSubmitted, + metrics.AppsecChallengeAccepted, + metrics.AppsecChallengeRejected, + metrics.AppsecChallengeKepochGenerated, + metrics.AppsecChallengeKepochEvicted, + metrics.AppsecChallengeReobfuscation, + metrics.AppsecChallengeDynamicModuleEvicted, + metrics.AppsecOutbandParsingHistogram, metrics.AppsecValidationOKCounter, metrics.AppsecValidationFailedCounter, - metrics.AppsecOutbandParsingHistogram, metrics.AppsecInbandParsingHistogram, metrics.AppsecGlobalParsingHistogram, } diff --git a/pkg/acquisition/modules/appsec/run.go b/pkg/acquisition/modules/appsec/run.go index 6e68c55f7..df8c62191 100644 --- a/pkg/acquisition/modules/appsec/run.go +++ b/pkg/acquisition/modules/appsec/run.go @@ -105,6 +105,12 @@ func (w *Source) listenAndServe(ctx context.Context, t *tomb.Tomb) error { w.logger.Errorf("Error shutting down Appsec server: %s", err.Error()) } + if w.AppsecRuntime != nil && w.AppsecRuntime.ChallengeRuntime != nil { + if err := w.AppsecRuntime.ChallengeRuntime.Close(ctx); err != nil { + w.logger.Errorf("Error closing challenge runtime: %s", err) + } + } + if w.config.ListenSocket != "" { if err := os.Remove(w.config.ListenSocket); err != nil { if !errors.Is(err, fs.ErrNotExist) { @@ -133,6 +139,8 @@ func (w *Source) StreamingAcquisition(ctx context.Context, out chan pipeline.Eve t.Go(func() error { defer trace.ReportPanic() + w.AppsecRuntime.OutChan = out + for _, runner := range w.AppsecRunners { runner.outChan = out diff --git a/pkg/acquisition/modules/appsec/testhelpers_test.go b/pkg/acquisition/modules/appsec/testhelpers_test.go new file mode 100644 index 000000000..690b3e2d2 --- /dev/null +++ b/pkg/acquisition/modules/appsec/testhelpers_test.go @@ -0,0 +1,30 @@ +package appsecacquisition + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/appsec/challenge" +) + +// sharedChallengeRuntime lazily creates a single ChallengeRuntime for the +// whole package test run. NewChallengeRuntime runs the obfuscator WASM to +// generate a challenge JS bundle (~15-20s), so spinning one up per test is +// prohibitively slow and unnecessary for integration tests that don't mutate +// runtime-level state. +var ( + sharedChallengeRuntimeOnce sync.Once + sharedChallengeRuntimeInst *challenge.ChallengeRuntime + sharedChallengeRuntimeErr error +) + +func getSharedChallengeRuntime(t *testing.T) *challenge.ChallengeRuntime { + t.Helper() + sharedChallengeRuntimeOnce.Do(func() { + sharedChallengeRuntimeInst, sharedChallengeRuntimeErr = challenge.NewChallengeRuntime(t.Context()) + }) + require.NoError(t, sharedChallengeRuntimeErr) + return sharedChallengeRuntimeInst +} diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 225c105c2..7fb067427 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -288,37 +288,6 @@ func containsAll(excludedZones []string, matchedZones []string) bool { return true } -func EventFromRequest(r *appsec.ParsedRequest, labels map[string]string, txUuid string) (pipeline.Event, error) { - evt := pipeline.MakeEvent(false, pipeline.LOG, true) - // def needs fixing - evt.Stage = "s00-raw" - evt.Parsed = map[string]string{ - "source_ip": r.ClientIP, - "target_host": r.Host, - "target_uri": r.URI, - "method": r.Method, - "req_uuid": txUuid, - "source": "crowdsec-appsec", - "remediation_cmpt_ip": r.RemoteAddrNormalized, - // TBD: - // http_status - // user_agent - - } - evt.Line = pipeline.Line{ - Time: time.Now(), - // should we add some info like listen addr/port/path ? - Labels: labels, - Process: true, - Module: ModuleName, - Src: ModuleName, - Raw: "dummy-appsec-data", // we discard empty Line.Raw items :) - } - evt.Appsec = pipeline.AppsecEvent{} - - return evt, nil -} - type ruleData struct { ID int Name string diff --git a/pkg/appsec/appsec.go b/pkg/appsec/appsec.go index fd31106d5..98a66a631 100644 --- a/pkg/appsec/appsec.go +++ b/pkg/appsec/appsec.go @@ -11,6 +11,7 @@ import ( "path/filepath" "regexp" "strings" + "time" corazatypes "github.com/corazawaf/coraza/v3/types" apivalidation "github.com/crowdsecurity/crowdsec/pkg/appsec/api_validation" @@ -20,6 +21,8 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/crowdsec/pkg/appsec/challenge" + "github.com/crowdsecurity/crowdsec/pkg/appsec/cookie" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/metrics" @@ -42,6 +45,8 @@ const ( hookPreEval hookPostEval hookOnMatch + hookOnChallenge + hookOnChallengeSubmit ) func (s hookStage) String() string { @@ -54,6 +59,10 @@ func (s hookStage) String() string { return "post_eval" case hookOnMatch: return "on_match" + case hookOnChallenge: + return "on_challenge" + case hookOnChallengeSubmit: + return "on_challenge_submit" default: return "unknown" } @@ -84,11 +93,22 @@ func (p *PhaseHooks) get(stage hookStage) []Hook { } const ( - BanRemediation = "ban" - CaptchaRemediation = "captcha" - AllowRemediation = "allow" + BanRemediation = "ban" + CaptchaRemediation = "captcha" + AllowRemediation = "allow" + ChallengeRemediation = "challenge" ) +const bodyChallengeOK = `{"status":"ok"}` +const bodyChallengeFailed = `{"status":"failed"}` + +// bodyChallengeRejected is returned when an on_challenge_submit hook calls +// RejectSubmission to refuse cookie issuance after a cryptographically +// valid submission. The client-side JS handler renders a terminal "your +// browser was rejected" UI with no retry CTA (vs. "failed" which prompts +// a retry). The server-side reason string is logged but NOT echoed to +// the client to avoid leaking detection logic. +const bodyChallengeRejected = `{"status":"rejected"}` const ( // BodySizeActionDrop drops the request when the body exceeds the maximum size. BodySizeActionDrop = "drop" @@ -108,7 +128,7 @@ const ( PhaseOutOfBand ) -func (h *Hook) Build(ctx context.Context, stage hookStage) error { +func (h *Hook) Build(ctx context.Context, stage hookStage, patcher *appsecExprPatcher) error { env := map[string]any{} // Env builders for phase hooks dereference state (e.g. state.HookVars), so @@ -122,12 +142,19 @@ func (h *Hook) Build(ctx context.Context, stage hookStage) error { case hookPreEval: env = GetPreEvalEnv(ctx, &AppsecRuntimeConfig{}, placeholderState, &ParsedRequest{}) case hookPostEval: - env = GetPostEvalEnv(&AppsecRuntimeConfig{}, placeholderState, &ParsedRequest{}) + env = GetPostEvalEnv(ctx, &AppsecRuntimeConfig{}, placeholderState, &ParsedRequest{}) case hookOnMatch: env = GetOnMatchEnv(&AppsecRuntimeConfig{}, placeholderState, &ParsedRequest{}, pipeline.Event{}) + case hookOnChallenge: + env = GetOnChallengeEnv(ctx, &AppsecRuntimeConfig{}, placeholderState, &ParsedRequest{}) + case hookOnChallengeSubmit: + env = GetOnChallengeSubmitEnv(&AppsecRuntimeConfig{}, placeholderState, &ParsedRequest{}) } opts := exprhelpers.GetExprOptions(env) + if patcher != nil { + opts = append(opts, expr.Patch(patcher)) + } if h.Filter != "" { program, err := expr.Compile(h.Filter, opts...) // FIXME: opts if err != nil { @@ -152,11 +179,28 @@ func (h *Hook) Build(ctx context.Context, stage hookStage) error { type AppsecTempResponse struct { InBandInterrupt bool OutOfBandInterrupt bool - Action string // allow, deny, captcha, log - UserHTTPResponseCode int // The response code to send to the user - BouncerHTTPResponseCode int // The response code to send to the remediation component - SendEvent bool // do we send an internal event on rule match - SendAlert bool // do we send an alert on rule match + Action string // allow, deny, captcha, challenge, log + UserHTTPResponseCode int // The response code to send to the user + UserHTTPBodyContent string // The body content to send to the user, only for challenge response + UserHTTPCookies []cookie.AppsecCookie // Raw Set-Cookie headers to send to the user. + UserHeaders map[string][]string // Headers to send to the user + BouncerHTTPResponseCode int // The response code to send to the remediation component + SendEvent bool // do we send an internal event on rule match + SendAlert bool // do we send an alert on rule match +} + +func (r AppsecTempResponse) Clone() AppsecTempResponse { + clone := r + if r.UserHeaders != nil { + clone.UserHeaders = make(map[string][]string, len(r.UserHeaders)) + for k, v := range r.UserHeaders { + clone.UserHeaders[k] = append([]string(nil), v...) + } + } + if r.UserHTTPCookies != nil { + clone.UserHTTPCookies = append([]cookie.AppsecCookie(nil), r.UserHTTPCookies...) + } + return clone } type AppsecDropInfo struct { @@ -164,6 +208,14 @@ type AppsecDropInfo struct { Interruption *corazatypes.Interruption } +// SubmissionRejectInfo signals that an on_challenge_submit hook called +// RejectSubmission to refuse cookie issuance for a cryptographically valid +// submission. ProcessOnChallengeRules inspects this on the submit-path +// branch and, if set, serves bodyChallengeRejected with no Set-Cookie. +type SubmissionRejectInfo struct { + Reason string +} + type AppsecRequestState struct { Tx ExtendedTransaction CurrentPhase phase @@ -175,6 +227,43 @@ type AppsecRequestState struct { PendingAction *string PendingHTTPCode *int + RequireChallenge bool + Fingerprint *challenge.FingerprintData + CookiePowDifficulty int // PoW difficulty proven by the client for the current cookie (0 if no/invalid cookie) + ChallengeDifficulty *int // per-request PoW difficulty override (nil = use runtime default) + + // SubmissionRejection is set by RejectSubmission inside an + // on_challenge_submit hook to refuse cookie issuance for the current + // challenge submission. nil for any other phase / outcome. + SubmissionRejection *SubmissionRejectInfo + + // ChallengeBypassed is set by GrantChallengeCookie to suppress later + // SendChallenge calls in the same request. Per-request only; cleared + // on ResetResponse. The bypass for subsequent requests is carried by + // the allowlist cookie itself, not by this flag. + ChallengeBypassed bool + + // ChallengeExempt is set by the ExemptFromChallenge expr helper to exempt + // the current request from the bot challenge. It works by short-circuiting + // every later IsLegitimateBot call to true (without datafile or DNS checks); + // It only affects thhe current request and doesn't mint a cookie (unlike GrantChallengeCookie) + ChallengeExempt bool + + // HooksHalted is flipped by terminal hook actions (currently + // RejectSubmission and the inline GrantChallengeCookie variant + // exposed in on_challenge_submit) to short-circuit later rules in + // the same phase. Without this, a `LogAccepted` rule following a + // `RejectSubmission` rule with `filter: "true"` would emit a + // contradictory accept-log line for an already-rejected submission. + // Per-request only; cleared by ResetResponse. + HooksHalted bool + + // LastMismatchReport caches the result of the EvaluateMismatches expr + // closure for the current request, so repeated calls from a single + // rule expression don't redo the work (or re-emit observability). + // nil until the first call. + LastMismatchReport *challenge.MismatchReport + // HookVars is a per-request scratch space exposed to expr hooks as // `hook_vars`. Helpers (e.g. ValidateRequestWithSchema) publish string // values here so that later hook expressions — including the `apply` @@ -197,8 +286,14 @@ func (s *AppsecRequestState) ResetResponse(cfg *AppsecConfig) { s.Response.UserHTTPResponseCode = cfg.UserPassedHTTPCode s.Response.SendEvent = true s.Response.SendAlert = true + s.Response.UserHTTPBodyContent = "" + s.Response.UserHTTPCookies = nil s.PendingAction = nil s.PendingHTTPCode = nil + s.RequireChallenge = false + s.SubmissionRejection = nil + s.ChallengeBypassed = false + s.HooksHalted = false } func (s *AppsecRequestState) DropInfo(request *ParsedRequest) *AppsecDropInfo { @@ -240,11 +335,15 @@ type BodySettings struct { // AppsecPhaseConfig holds configuration scoped to a specific phase (inband or outofband). // Hooks defined here are automatically dispatched only during the corresponding phase. +// on_challenge and on_challenge_submit are in-band only; setting them under +// `outofband:` is rejected at Build() time. type AppsecPhaseConfig struct { Rules []string `yaml:"rules"` OnMatch []Hook `yaml:"on_match"` PreEval []Hook `yaml:"pre_eval"` PostEval []Hook `yaml:"post_eval"` + OnChallenge []Hook `yaml:"on_challenge"` + OnChallengeSubmit []Hook `yaml:"on_challenge_submit"` Options AppsecSubEngineOpts `yaml:"options"` VariablesTracking []string `yaml:"variables_tracking"` } @@ -260,10 +359,12 @@ type AppsecRuntimeConfig struct { RemediationByTag map[string]string // Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME RemediationById map[int]string - CompiledOnLoad []Hook // runs once at startup, not phase-scoped - CommonHooks PhaseHooks // apply to both phases - InBandHooks PhaseHooks // only run during in-band - OutOfBandHooks PhaseHooks // only run during out-of-band + CompiledOnLoad []Hook // runs once at startup, not phase-scoped + CompiledOnChallenge []Hook // in-band only; runs before pre_eval + CompiledOnChallengeSubmit []Hook // in-band only; runs at /submit POST after validation + CommonHooks PhaseHooks // apply to both phases + InBandHooks PhaseHooks // only run during in-band + OutOfBandHooks PhaseHooks // only run during out-of-band CompiledVariablesTracking []*regexp.Regexp Config *AppsecConfig @@ -278,12 +379,113 @@ type AppsecRuntimeConfig struct { DisabledOutOfBandRuleIds []int DisabledOutOfBandRulesTags []string // Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME + // True if at least one of the hooks use `RequireValidChallenge` + NeedWASMVM bool + ChallengeRuntime *challenge.ChallengeRuntime + + // OutChan is the pipeline output channel challenge lifecycle events are sent + // on, and Labels are the datasource labels stamped on those events. Both are + // wired by the appsec datasource at startup; nil OutChan disables emission + // (e.g. in unit tests with no datasource). + OutChan chan pipeline.Event + Labels map[string]string + + // FingerprintDumpDir is the on-disk directory the DumpFingerprint + FingerprintDumpDir string + RequestValidator *apivalidation.RequestValidator DataDir string // BodySettings controls how oversized request bodies are handled. Settable via on_load hooks. BodySettings BodySettings } +// emitChallengeEvent builds and sends a challenge lifecycle event to the +// pipeline. It is a no-op when no output channel is wired. Challenge handling is +// an in-band concern, so we never emit during the out-of-band phase — a common +// pre_eval/post_eval hook calling SendChallenge() runs in both phases, and the +// out-of-band invocation is already a no-op for the client. +func (w *AppsecRuntimeConfig) emitChallengeEvent(request *ParsedRequest, info ChallengeEventInfo) { + if w.OutChan == nil || !request.IsInBand { + return + } + + labels := prometheus.Labels{ + "source": request.RemoteAddrNormalized, + "appsec_engine": request.AppsecEngine, + } + + switch info.Reason { + case ChallengeReasonRequested: + metrics.AppsecChallengeRequested.With(labels).Inc() + case ChallengeReasonSubmitted: + metrics.AppsecChallengeSubmitted.With(labels).Inc() + case ChallengeReasonSolved: + acceptedLabels := prometheus.Labels{ + "source": labels["source"], + "appsec_engine": labels["appsec_engine"], + "kind": "solved", + "reason": "", // regular submissions have no per-issue reason + } + metrics.AppsecChallengeAccepted.With(acceptedLabels).Inc() + case ChallengeReasonFailed: + rejectedLabels := prometheus.Labels{ + "source": labels["source"], + "appsec_engine": labels["appsec_engine"], + "kind": "protocol", + "reason": classifyProtocolErr(info.FailErr), + } + metrics.AppsecChallengeRejected.With(rejectedLabels).Inc() + case ChallengeReasonRejected: + rejectedLabels := prometheus.Labels{ + "source": labels["source"], + "appsec_engine": labels["appsec_engine"], + "kind": "submission", + "reason": info.FailReason, + } + metrics.AppsecChallengeRejected.With(rejectedLabels).Inc() + } + + w.OutChan <- ChallengeEventFromRequest(request, w.Labels, request.UUID, info) +} + +// classifyProtocolErr maps a challenge-validation error to one of a small +// closed vocabulary. +func classifyProtocolErr(err error) string { + switch { + case err == nil: + return "other" + case errors.Is(err, challenge.ErrChallengeHMAC): + return "hmac" + case errors.Is(err, challenge.ErrChallengeTicket): + return "ticket" + case errors.Is(err, challenge.ErrChallengePoW): + return "pow" + case errors.Is(err, challenge.ErrChallengeFields), errors.Is(err, challenge.ErrChallengePayload): + return "payload" + case errors.Is(err, challenge.ErrChallengeDifficulty): + return "difficulty" + default: + return "other" + } +} + +// classifyCookieErr maps a ValidCookie error to one of a small closed +// vocabulary +func classifyCookieErr(err error) string { + switch { + case err == nil: + return "other" + case errors.Is(err, challenge.ErrCookieSignature): + return "signature" + case errors.Is(err, challenge.ErrCookiePayload), errors.Is(err, challenge.ErrCookieMalformed): + return "payload" + case errors.Is(err, challenge.ErrCookieExpired), errors.Is(err, challenge.ErrCookieVersion): + return "epoch" + default: + return "other" + } +} + type AppsecConfig struct { Name string `yaml:"name"` OutOfBandRules []string `yaml:"outofband_rules"` @@ -299,6 +501,8 @@ type AppsecConfig struct { PreEval []Hook `yaml:"pre_eval"` PostEval []Hook `yaml:"post_eval"` OnMatch []Hook `yaml:"on_match"` + OnChallenge []Hook `yaml:"on_challenge"` + OnChallengeSubmit []Hook `yaml:"on_challenge_submit"` VariablesTracking []string `yaml:"variables_tracking"` InbandOptions AppsecSubEngineOpts `yaml:"inband_options"` OutOfBandOptions AppsecSubEngineOpts `yaml:"outofband_options"` @@ -306,6 +510,13 @@ type AppsecConfig struct { InBand *AppsecPhaseConfig `yaml:"inband"` OutOfBand *AppsecPhaseConfig `yaml:"outofband"` + // Challenge carries the WAF challenge / bot-detection runtime tuning. + // All fields are optional; unset fields fall back to the runtime + // defaults at NewChallengeRuntime time. When multiple appsec-configs + // are loaded, each later config's non-nil fields override the earlier + // values (see LoadByPath). + Challenge *challenge.Config `yaml:"challenge"` + LogLevel *log.Level `yaml:"log_level"` Logger *log.Entry `yaml:"-"` } @@ -425,6 +636,14 @@ func (wc *AppsecConfig) LoadByPath(file string) error { wc.OnMatch = append(wc.OnMatch, tmp.OnMatch...) } + if tmp.OnChallenge != nil { + wc.OnChallenge = append(wc.OnChallenge, tmp.OnChallenge...) + } + + if tmp.OnChallengeSubmit != nil { + wc.OnChallengeSubmit = append(wc.OnChallengeSubmit, tmp.OnChallengeSubmit...) + } + if tmp.VariablesTracking != nil { wc.VariablesTracking = append(wc.VariablesTracking, tmp.VariablesTracking...) } @@ -438,6 +657,8 @@ func (wc *AppsecConfig) LoadByPath(file string) error { wc.InBand.OnMatch = append(wc.InBand.OnMatch, tmp.InBand.OnMatch...) wc.InBand.PreEval = append(wc.InBand.PreEval, tmp.InBand.PreEval...) wc.InBand.PostEval = append(wc.InBand.PostEval, tmp.InBand.PostEval...) + wc.InBand.OnChallenge = append(wc.InBand.OnChallenge, tmp.InBand.OnChallenge...) + wc.InBand.OnChallengeSubmit = append(wc.InBand.OnChallengeSubmit, tmp.InBand.OnChallengeSubmit...) } if tmp.OutOfBand != nil { @@ -448,6 +669,8 @@ func (wc *AppsecConfig) LoadByPath(file string) error { wc.OutOfBand.OnMatch = append(wc.OutOfBand.OnMatch, tmp.OutOfBand.OnMatch...) wc.OutOfBand.PreEval = append(wc.OutOfBand.PreEval, tmp.OutOfBand.PreEval...) wc.OutOfBand.PostEval = append(wc.OutOfBand.PostEval, tmp.OutOfBand.PostEval...) + wc.OutOfBand.OnChallenge = append(wc.OutOfBand.OnChallenge, tmp.OutOfBand.OnChallenge...) + wc.OutOfBand.OnChallengeSubmit = append(wc.OutOfBand.OnChallengeSubmit, tmp.OutOfBand.OnChallengeSubmit...) } // override other options @@ -476,6 +699,16 @@ func (wc *AppsecConfig) LoadByPath(file string) error { wc.OutOfBandOptions.RequestBodyInMemoryLimit = tmp.OutOfBandOptions.RequestBodyInMemoryLimit } + // Merge challenge tuning field by field so multiple appsec-configs can + // each contribute a disjoint subset without one wiping out the others. + // Each non-nil field in tmp overrides the corresponding field in wc. + if tmp.Challenge != nil { + if wc.Challenge == nil { + wc.Challenge = &challenge.Config{} + } + wc.Challenge.MergeFrom(tmp.Challenge) + } + return nil } @@ -517,7 +750,7 @@ func (wc *AppsecConfig) normalizePhaseScoped() { } // buildHookList validates and compiles a list of hooks of the given stage. -func buildHookList(ctx context.Context, hooks []Hook, stage hookStage) ([]Hook, error) { +func buildHookList(ctx context.Context, hooks []Hook, stage hookStage, patcher *appsecExprPatcher) ([]Hook, error) { var compiled []Hook for _, hook := range hooks { @@ -525,7 +758,7 @@ func buildHookList(ctx context.Context, hooks []Hook, stage hookStage) ([]Hook, return nil, fmt.Errorf("invalid 'on_success' for %s hook : %s", stage, hook.OnSuccess) } - if err := hook.Build(ctx, stage); err != nil { + if err := hook.Build(ctx, stage, patcher); err != nil { return nil, fmt.Errorf("unable to build %s hook : %w", stage, err) } @@ -537,7 +770,7 @@ func buildHookList(ctx context.Context, hooks []Hook, stage hookStage) ([]Hook, // buildPhaseHooks compiles pre_eval / post_eval / on_match hook lists into a // PhaseHooks. phaseName is only used to wrap errors ("" for the shared section). -func buildPhaseHooks(ctx context.Context, phaseName string, pre, post, onMatch []Hook) (PhaseHooks, error) { +func buildPhaseHooks(ctx context.Context, phaseName string, pre, post, onMatch []Hook, patcher *appsecExprPatcher) (PhaseHooks, error) { var ( out PhaseHooks err error @@ -550,15 +783,15 @@ func buildPhaseHooks(ctx context.Context, phaseName string, pre, post, onMatch [ return fmt.Errorf("%s: %w", phaseName, e) } - if out.PreEval, err = buildHookList(ctx, pre, hookPreEval); err != nil { + if out.PreEval, err = buildHookList(ctx, pre, hookPreEval, patcher); err != nil { return PhaseHooks{}, wrap(err) } - if out.PostEval, err = buildHookList(ctx, post, hookPostEval); err != nil { + if out.PostEval, err = buildHookList(ctx, post, hookPostEval, patcher); err != nil { return PhaseHooks{}, wrap(err) } - if out.OnMatch, err = buildHookList(ctx, onMatch, hookOnMatch); err != nil { + if out.OnMatch, err = buildHookList(ctx, onMatch, hookOnMatch, patcher); err != nil { return PhaseHooks{}, wrap(err) } @@ -613,10 +846,11 @@ func (wc *AppsecConfig) Build(ctx context.Context, hub *cwhub.Hub) (*AppsecRunti // set the defaults switch wc.DefaultRemediation { - case BanRemediation, CaptchaRemediation, AllowRemediation: + case BanRemediation, CaptchaRemediation, AllowRemediation, ChallengeRemediation: // those are the officially supported remediation(s) default: - wc.Logger.Warningf("default '%s' remediation of %s is none of [%s,%s,%s] ensure bouncer compatbility!", wc.DefaultRemediation, wc.Name, BanRemediation, CaptchaRemediation, AllowRemediation) + wc.Logger.Warningf("default '%s' remediation of %s is none of [%s,%s,%s,%s] ensure bouncer compatibility!", + wc.DefaultRemediation, wc.Name, BanRemediation, CaptchaRemediation, AllowRemediation, ChallengeRemediation) } ret.Name = wc.Name @@ -655,29 +889,70 @@ func (wc *AppsecConfig) Build(ctx context.Context, hub *cwhub.Hub) (*AppsecRunti wc.Logger.Infof("Loaded %d inband rules", len(ret.InBandRules)) + patcher := &appsecExprPatcher{} + // load hooks var err error - if ret.CompiledOnLoad, err = buildHookList(ctx, wc.OnLoad, hookOnLoad); err != nil { + if ret.CompiledOnLoad, err = buildHookList(ctx, wc.OnLoad, hookOnLoad, nil); err != nil { return nil, err } - if ret.CommonHooks, err = buildPhaseHooks(ctx, "", wc.PreEval, wc.PostEval, wc.OnMatch); err != nil { + if ret.CommonHooks, err = buildPhaseHooks(ctx, "", wc.PreEval, wc.PostEval, wc.OnMatch, patcher); err != nil { return nil, err } if wc.InBand != nil { if ret.InBandHooks, err = buildPhaseHooks(ctx, "inband", - wc.InBand.PreEval, wc.InBand.PostEval, wc.InBand.OnMatch); err != nil { + wc.InBand.PreEval, wc.InBand.PostEval, wc.InBand.OnMatch, patcher); err != nil { return nil, err } } if wc.OutOfBand != nil { if ret.OutOfBandHooks, err = buildPhaseHooks(ctx, "outofband", - wc.OutOfBand.PreEval, wc.OutOfBand.PostEval, wc.OutOfBand.OnMatch); err != nil { + wc.OutOfBand.PreEval, wc.OutOfBand.PostEval, wc.OutOfBand.OnMatch, patcher); err != nil { return nil, err } + + if len(wc.OutOfBand.OnChallenge) > 0 { + return nil, errors.New("on_challenge hooks are only valid in-band, not under outofband") + } + + if len(wc.OutOfBand.OnChallengeSubmit) > 0 { + return nil, errors.New("on_challenge_submit hooks are only valid in-band, not under outofband") + } + } + + // on_challenge hooks: merge top-level and inband-scoped (both are in-band only). + onChallengeHooks := wc.OnChallenge + if wc.InBand != nil { + onChallengeHooks = append(onChallengeHooks, wc.InBand.OnChallenge...) + } + + if ret.CompiledOnChallenge, err = buildHookList(ctx, onChallengeHooks, hookOnChallenge, patcher); err != nil { + return nil, err + } + + // Defining any on_challenge hook implies we need the challenge runtime to + // validate cookies and submissions, even if the hook bodies never call + // SendChallenge() themselves. + if len(ret.CompiledOnChallenge) > 0 { + patcher.NeedWASMVM = true + } + + // on_challenge_submit hooks: same merge pattern; in-band only. + onChallengeSubmitHooks := wc.OnChallengeSubmit + if wc.InBand != nil { + onChallengeSubmitHooks = append(onChallengeSubmitHooks, wc.InBand.OnChallengeSubmit...) + } + + if ret.CompiledOnChallengeSubmit, err = buildHookList(ctx, onChallengeSubmitHooks, hookOnChallengeSubmit, patcher); err != nil { + return nil, err + } + + if len(ret.CompiledOnChallengeSubmit) > 0 { + patcher.NeedWASMVM = true } // variable tracking @@ -690,14 +965,28 @@ func (wc *AppsecConfig) Build(ctx context.Context, hub *cwhub.Hub) (*AppsecRunti ret.CompiledVariablesTracking = append(ret.CompiledVariablesTracking, compiledVariableRule) } + ret.NeedWASMVM = patcher.NeedWASMVM + return ret, nil } // processHooks runs a list of compiled hooks with the given environment. -func (w *AppsecRuntimeConfig) processHooks(hooks []Hook, env map[string]interface{}, hookType string) error { +// +// state, when non-nil, is consulted between rule iterations: if +// state.HooksHalted is true (set by a terminal expr helper such as +// RejectSubmission or the on_challenge_submit GrantChallengeCookie), +// remaining rules in this phase are skipped. ProcessOnLoadRules and +// the non-submit phases pass nil — they have no terminal actions +// today. +func (w *AppsecRuntimeConfig) processHooks(hooks []Hook, env map[string]interface{}, hookType string, state *AppsecRequestState) error { has_match := false for _, rule := range hooks { + if state != nil && state.HooksHalted { + w.Logger.Debugf("hooks halted by a terminal action; skipping remaining %s rules", hookType) + break + } + if rule.FilterExpr != nil { output, err := exprhelpers.Run(rule.FilterExpr, env, w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { @@ -742,7 +1031,7 @@ func (w *AppsecRuntimeConfig) processHooks(hooks []Hook, env map[string]interfac } func (w *AppsecRuntimeConfig) ProcessOnLoadRules() error { - return w.processHooks(w.CompiledOnLoad, GetOnLoadEnv(w), "on_load") + return w.processHooks(w.CompiledOnLoad, GetOnLoadEnv(w), "on_load", nil) } // runPhaseHooks runs the common hooks for the given stage, then dispatches to @@ -750,15 +1039,15 @@ func (w *AppsecRuntimeConfig) ProcessOnLoadRules() error { func (w *AppsecRuntimeConfig) runPhaseHooks(stage hookStage, env map[string]interface{}, request *ParsedRequest) error { label := stage.String() - if err := w.processHooks(w.CommonHooks.get(stage), env, label); err != nil { + if err := w.processHooks(w.CommonHooks.get(stage), env, label, nil); err != nil { return err } switch { case request.IsInBand: - return w.processHooks(w.InBandHooks.get(stage), env, label+"[inband]") + return w.processHooks(w.InBandHooks.get(stage), env, label+"[inband]", nil) case request.IsOutBand: - return w.processHooks(w.OutOfBandHooks.get(stage), env, label+"[outofband]") + return w.processHooks(w.OutOfBandHooks.get(stage), env, label+"[outofband]", nil) } return nil @@ -768,12 +1057,138 @@ func (w *AppsecRuntimeConfig) ProcessOnMatchRules(state *AppsecRequestState, req return w.runPhaseHooks(hookOnMatch, GetOnMatchEnv(w, state, request, evt), request) } +// ProcessOnChallengeRules is the in-band-only challenge entry point. It +// handles the PoW worker JS path and the challenge submission path internally, +// validates any existing challenge cookie to populate state.Fingerprint, and +// runs the user-defined on_challenge hook expressions ONLY when there is a +// fingerprint to inspect — i.e. on a valid submission or when a valid cookie +// was presented. Requests with no cookie / invalid cookie / invalid submission +// skip user hooks entirely (there's nothing to evaluate). +func (w *AppsecRuntimeConfig) ProcessOnChallengeRules(ctx context.Context, state *AppsecRequestState, request *ParsedRequest) error { + if w.ChallengeRuntime == nil { + return nil + } + + var path string + if request.HTTPRequest.URL != nil { + path = request.HTTPRequest.URL.Path + } + + // Serve the PoW worker JS (static asset). Skip user expressions. + if path == challenge.ChallengePowWorkerPath { + return w.setChallengeResponse(state, http.StatusOK, challenge.PowWorkerJS, + map[string]string{"Content-Type": "application/javascript", "Cache-Control": "public, max-age=3600"}, nil) + } + + // Serve the fpscanner bundle (static, public, unobfuscated asset). Skip user expressions. + if path == challenge.ChallengeFPScannerPath { + return w.setChallengeResponse(state, http.StatusOK, challenge.FPScannerJS, + map[string]string{"Content-Type": "application/javascript", "Cache-Control": "public, max-age=3600"}, nil) + } + + // Challenge submission: validate, give on_challenge_submit hooks a chance + // to reject the submission, then issue (or deny) the cookie. Per-route + // on_challenge inspection happens on subsequent cookie-bearing requests. + if path == challenge.ChallengeSubmitPath && request.HTTPRequest.Method == http.MethodPost { + w.Logger.Debugf("validating challenge response") + w.emitChallengeEvent(request, ChallengeEventInfo{Reason: ChallengeReasonSubmitted}) + + ck, fpData, provenDifficulty, err := w.ChallengeRuntime.ValidateChallengeResponse(request.HTTPRequest, request.Body) + if err != nil { + w.Logger.Errorf("challenge validation failed: %s", err) + w.emitChallengeEvent(request, ChallengeEventInfo{ + Reason: ChallengeReasonFailed, + FailReason: err.Error(), + FailErr: err, + }) + return w.setChallengeResponse(state, http.StatusOK, bodyChallengeFailed, + map[string]string{"Content-Type": "application/json", "Cache-Control": "no-cache, no-store"}, nil) + } + + // Populate state.Fingerprint so on_challenge_submit expressions see + // the freshly-decrypted fingerprint via the env. CookiePowDifficulty + // records the difficulty just proven so the Solved event and + // on_challenge_submit expressions see the real value, not 0. + state.Fingerprint = &fpData + state.CookiePowDifficulty = provenDifficulty + + if err := w.processHooks(w.CompiledOnChallengeSubmit, GetOnChallengeSubmitEnv(w, state, request), "on_challenge_submit", state); err != nil { + w.Logger.Errorf("unable to process on_challenge_submit rules: %s", err) + } + + if state.SubmissionRejection != nil { + // The expr-side RejectSubmission helper emits the reject log + // itself (with the operator-chosen verbosity), so we don't + // re-log here — just serve the rejection envelope. + w.emitChallengeEvent(request, ChallengeEventInfo{ + Reason: ChallengeReasonRejected, + FailReason: state.SubmissionRejection.Reason, + Fingerprint: &fpData, + }) + return w.setChallengeResponse(state, http.StatusOK, bodyChallengeRejected, + map[string]string{"Content-Type": "application/json", "Cache-Control": "no-cache, no-store"}, nil) + } + + w.emitChallengeEvent(request, ChallengeEventInfo{ + Reason: ChallengeReasonSolved, + Difficulty: state.CookiePowDifficulty, + Fingerprint: &fpData, + }) + + return w.setChallengeResponse(state, http.StatusOK, bodyChallengeOK, + map[string]string{"Content-Type": "application/json", "Cache-Control": "no-cache, no-store"}, ck) + } + + // Regular request: validate the existing cookie (if any) to populate + // fingerprint and remember the difficulty the client proved. + if httpCookie, err := request.HTTPRequest.Cookie(challenge.ChallengeCookieName); err == nil { + cookieData, validErr := w.ChallengeRuntime.ValidCookie(httpCookie, request.HTTPRequest.UserAgent()) + switch { + case validErr != nil: + // Tampered / replayed / past-window cookies show up here. Count + // them on the rejected counter with kind=cookie so dashboards + // can separate "active submission was rejected" from "a stale + // or forged cookie was presented on a regular request". + metrics.AppsecChallengeRejected.With(prometheus.Labels{ + "source": request.RemoteAddrNormalized, + "appsec_engine": request.AppsecEngine, + "kind": "cookie", + "reason": classifyCookieErr(validErr), + }).Inc() + default: + fp := cookieData.Fingerprint + fp.Allowlisted = cookieData.Allowlisted + fp.AllowlistReason = cookieData.AllowlistReason + state.Fingerprint = &fp + state.CookiePowDifficulty = cookieData.PowDifficulty + // An allowlist cookie minted on a prior request must short-circuit + // SendChallenge on every replay, exactly like a GrantChallengeCookie + // call within the current request would. Without this, the visitor + // gets re-challenged on each hop and the cookie achieves nothing. + msg := "valid challenge cookie" + if cookieData.Allowlisted { + state.ChallengeBypassed = true + msg = "valid allowlist challenge cookie — bypassing challenge" + } + fp.LogAccepted(w.Logger, log.DebugLevel, request.ClientIP, request.RemoteAddrNormalized, msg) + } + } + + // No fingerprint to inspect — skip user hooks. This avoids nil-deref inside + // expr filters like `fingerprint.Bot.X` when no cookie was presented. + if state.Fingerprint == nil { + return nil + } + + return w.processHooks(w.CompiledOnChallenge, GetOnChallengeEnv(ctx, w, state, request), "on_challenge", nil) +} + func (w *AppsecRuntimeConfig) ProcessPreEvalRules(ctx context.Context, state *AppsecRequestState, request *ParsedRequest) error { return w.runPhaseHooks(hookPreEval, GetPreEvalEnv(ctx, w, state, request), request) } -func (w *AppsecRuntimeConfig) ProcessPostEvalRules(state *AppsecRequestState, request *ParsedRequest) error { - return w.runPhaseHooks(hookPostEval, GetPostEvalEnv(w, state, request), request) +func (w *AppsecRuntimeConfig) ProcessPostEvalRules(ctx context.Context, state *AppsecRequestState, request *ParsedRequest) error { + return w.runPhaseHooks(hookPostEval, GetPostEvalEnv(ctx, w, state, request), request) } func (w *AppsecRuntimeConfig) RemoveInbandRuleByID(state *AppsecRequestState, id int) error { @@ -939,6 +1354,75 @@ func (w *AppsecRuntimeConfig) SetHTTPCode(state *AppsecRequestState, code int) e return nil } +func (w *AppsecRuntimeConfig) SetChallengeBody(state *AppsecRequestState, content string) error { + w.Logger.Debugf("setting challenge body content") + state.Response.UserHTTPBodyContent = content + return nil +} + +func (w *AppsecRuntimeConfig) SetChallengeCookie(state *AppsecRequestState, cookie cookie.AppsecCookie) error { + w.Logger.Debugf("adding challenge cookie") + state.Response.UserHTTPCookies = append(state.Response.UserHTTPCookies, cookie) + return nil +} + +func (w *AppsecRuntimeConfig) SetChallengeHeader(state *AppsecRequestState, name string, value string) error { + w.Logger.Debugf("adding challenge headers") + if state.Response.UserHeaders == nil { + state.Response.UserHeaders = make(map[string][]string) + } + state.Response.UserHeaders[name] = append(state.Response.UserHeaders[name], value) + return nil +} + +func (w *AppsecRuntimeConfig) setChallengeResponse(state *AppsecRequestState, code int, body string, headers map[string]string, cookie *cookie.AppsecCookie) error { + if err := w.SetAction(state, ChallengeRemediation); err != nil { + return err + } + if err := w.SetHTTPCode(state, code); err != nil { + return err + } + // Initial response state defaults BouncerHTTPResponseCode to the "passed" code (see InitRequestState); + // override it here so the bouncer gets the blocked code while the visitor still receives the challenge page. + state.Response.BouncerHTTPResponseCode = w.Config.BouncerBlockedHTTPCode + if err := w.SetChallengeBody(state, body); err != nil { + return err + } + for name, value := range headers { + if err := w.SetChallengeHeader(state, name, value); err != nil { + return err + } + } + if cookie != nil { + if err := w.SetChallengeCookie(state, *cookie); err != nil { + return err + } + } + state.RequireChallenge = true + return nil +} + +// SetChallengeDifficulty sets the default PoW difficulty on the runtime (used from on_load). +func (w *AppsecRuntimeConfig) SetChallengeDifficulty(level string) error { + if w.ChallengeRuntime == nil { + return errors.New("challenge runtime not initialized") + } + + return w.ChallengeRuntime.SetDifficulty(level) +} + +// SetChallengeDifficultyPerRequest sets a per-request PoW difficulty override (used from pre_eval/post_eval). +func (*AppsecRuntimeConfig) SetChallengeDifficultyPerRequest(state *AppsecRequestState, level string) error { + bits, err := challenge.DifficultyFromLevel(level) + if err != nil { + return err + } + + state.ChallengeDifficulty = &bits + + return nil +} + // SetMaxBodySize sets the maximum allowed body size in bytes. Intended for use in on_load hooks. func (w *AppsecRuntimeConfig) SetMaxBodySize(size int64) error { if size <= 0 { @@ -947,6 +1431,229 @@ func (w *AppsecRuntimeConfig) SetMaxBodySize(size int64) error { w.Logger.Debugf("setting max body size to %d bytes", size) w.BodySettings.MaxSize = size + return nil +} + +// SendChallenge issues a challenge HTML page for the current request. Cookie +// and submission handling live in ProcessOnChallengeRules; by the time this +// runs, state.Fingerprint has already been populated if a valid cookie was +// presented. If the client already proved a PoW at least as hard as the +// target difficulty for this request, SendChallenge is a no-op. When the +// target difficulty is raised (e.g. on_challenge calls SetChallengeDifficulty +// to punish a suspect fingerprint), the stored difficulty is lower than the +// target and a fresh challenge is issued. +// EvaluateMismatches runs all library-native + custom fingerprint mismatch +// checks, caches the result on state, and emits one structured Debug log +// line + one metric bump per fired signal on the first call of a given +// request. Subsequent calls return the cached pointer so rules can reference +// the report multiple times without redoing the work. +func (w *AppsecRuntimeConfig) EvaluateMismatches(state *AppsecRequestState, request *ParsedRequest) *challenge.MismatchReport { + if state.LastMismatchReport != nil { + return state.LastMismatchReport + } + + country := exprhelpers.IPToCountryString(request.ClientIP) + report := state.Fingerprint.ComputeMismatchReport(request.HTTPRequest, country) + + state.LastMismatchReport = report + + if !report.Empty() { + w.emitMismatchObservability(state, request, report) + } + + return report +} + +// emitMismatchObservability logs the report at Debug level and bumps the +// per-reason/severity Prometheus counter. Called exactly once per request +// from EvaluateMismatches (guarded by state.LastMismatchReport being nil +// on entry). +func (w *AppsecRuntimeConfig) emitMismatchObservability( + state *AppsecRequestState, + request *ParsedRequest, + report *challenge.MismatchReport, +) { + fsid := "" + if state.Fingerprint != nil { + fsid = state.Fingerprint.FSID + } + + if w.Logger != nil { + w.Logger.WithFields(log.Fields{ + "fsid": fsid, + "source": request.ClientIP, + "bouncer": request.RemoteAddrNormalized, + "reasons": report.Reasons(), + "high": report.High(), + "medium": report.Medium(), + "low": report.Low(), + "count": report.Count(), + }).Debug("fingerprint mismatch") + } + + for _, sig := range report.Signals { + metrics.AppsecFingerprintMismatch.With(prometheus.Labels{ + "reason": sig.Reason, + "severity": sig.Severity, + "appsec_engine": request.AppsecEngine, + }).Inc() + } +} + +func (w *AppsecRuntimeConfig) SendChallenge(ctx context.Context, state *AppsecRequestState, request *ParsedRequest) error { + if w.ChallengeRuntime == nil { + return errors.New("challenge runtime not initialized") + } + + // SendChallenge can only be called from inband.post_eval and inband.on_challenge. + // as it's the same expr-env, we need to detect here. + if state.CurrentPhase != PhaseInBand { + return errors.New("SendChallenge can only be called from an in-band hook (on_challenge or post_eval)") + } + + // GrantChallengeCookie earlier in the same request already minted an + // allowlist cookie; refuse to overwrite it with a challenge page. + if state.ChallengeBypassed { + w.Logger.Debugf("SendChallenge no-op: allowlist cookie already granted this request") + return nil + } + + target := w.ChallengeRuntime.Difficulty() + if state.ChallengeDifficulty != nil { + target = *state.ChallengeDifficulty + } + + if state.Fingerprint != nil && state.CookiePowDifficulty >= target { + w.Logger.Debugf("client already proved difficulty %d >= target %d, skipping challenge issue", + state.CookiePowDifficulty, target) + return nil + } + + w.Logger.Debugf("sending challenge at difficulty %d (client proved %d)", target, state.CookiePowDifficulty) + + challengePage, err := w.ChallengeRuntime.GetChallengePage(ctx, request.HTTPRequest.UserAgent(), target) + if err != nil { + return fmt.Errorf("unable to get challenge page: %w", err) + } + + if err := w.setChallengeResponse(state, http.StatusOK, challengePage, map[string]string{"Content-Type": "text/html", "Cache-Control": "no-cache, no-store"}, nil); err != nil { + return err + } + + w.emitChallengeEvent(request, ChallengeEventInfo{ + Reason: ChallengeReasonRequested, + Difficulty: target, + Fingerprint: state.Fingerprint, + }) + + return nil +} + +// RejectSubmission flags the in-flight challenge submission so the +// ProcessOnChallengeRules submit-path branch refuses to issue a cookie and +// returns bodyChallengeRejected. Exposed ONLY in the on_challenge_submit +// hook env — calling it from any other phase is a no-op (the field is +// inspected only at submit time). +// +// The reason string is logged server-side and NOT echoed to the client. +func (*AppsecRuntimeConfig) RejectSubmission(state *AppsecRequestState, reason string) error { + reason = strings.TrimSpace(reason) + if reason == "" { + reason = "submission rejected by on_challenge_submit" + } + state.SubmissionRejection = &SubmissionRejectInfo{Reason: reason} + return nil +} + +// mintAllowlistCookie seals a v0 allowlist cookie, stamps a synthetic +// Allowlisted fingerprint, and flips state.ChallengeBypassed so any later +// SendChallenge in the same request is a no-op. The caller decides how the +// cookie reaches the visitor (redirect vs. inline envelope). +// +// It DELIBERATELY overwrites any prior state.Fingerprint from a real +// submission: an operator allowlist wins. ttlOverride (non-nil) overrides the +// runtime cookie_ttl. Returns ErrAllowlistReasonSize if reason is too long. +func (w *AppsecRuntimeConfig) mintAllowlistCookie(state *AppsecRequestState, request *ParsedRequest, reason string, ttlOverride *time.Duration) (*cookie.AppsecCookie, error) { + if w.ChallengeRuntime == nil { + return nil, errors.New("challenge runtime not initialized") + } + + ck, err := w.ChallengeRuntime.SealAllowlistCookie(request.HTTPRequest, reason, ttlOverride) + if err != nil { + return nil, fmt.Errorf("unable to seal allowlist cookie: %w", err) + } + + state.Fingerprint = &challenge.FingerprintData{ + Allowlisted: true, + AllowlistReason: reason, + } + state.ChallengeBypassed = true + + // One increment here covers both GrantChallengeCookie (307 redirect from + // pre_eval/post_eval) and GrantAllowlistCookieInline (inline on the + // challenge-submit response). Both delegate to this function. + metrics.AppsecChallengeAccepted.With(prometheus.Labels{ + "source": request.RemoteAddrNormalized, + "appsec_engine": request.AppsecEngine, + "kind": "granted", + "reason": reason, // operator-supplied string passed to GrantChallengeCookie + }).Inc() + + return ck, nil +} + +// GrantChallengeCookie mints an allowlist-bypass cookie and issues an HTTP 307 +// redirect carrying it back to the visitor. Used from pre_eval/post_eval. +// +// Why a 307 and not a silent allow: the bouncer only serializes cookies on a +// ChallengeRemediation response (see GenerateResponse), so a plain allow drops +// the Set-Cookie. The redirect to the same URL preserves method+body and +// bounces the visitor back through the WAF with the cookie present, so +// ProcessOnChallengeRules' allowlist branch lets them through on the next hop. +// +// Precedence: see mintAllowlistCookie (operator allowlist overwrites a real +// fingerprint). ttlOverride overrides cookie_ttl. Returns ErrAllowlistReasonSize +// if reason is too long. +func (w *AppsecRuntimeConfig) GrantChallengeCookie(state *AppsecRequestState, request *ParsedRequest, reason string, ttlOverride *time.Duration) error { + ck, err := w.mintAllowlistCookie(state, request, reason, ttlOverride) + if err != nil { + return err + } + + headers := map[string]string{ + "Location": request.HTTPRequest.URL.RequestURI(), + "Content-Type": "text/html; charset=utf-8", + "Cache-Control": "no-store", + } + + state.Fingerprint.LogAccepted( + w.Logger.WithField("location", headers["Location"]), + log.InfoLevel, + request.ClientIP, + request.RemoteAddrNormalized, + "granted allowlist challenge cookie via 307 redirect", + ) + + return w.setChallengeResponse(state, http.StatusTemporaryRedirect, challenge.GrantRedirectBody, headers, ck) +} + +// GrantAllowlistCookieInline mints an allowlist-bypass cookie and attaches it +// to the in-flight challenge-submit envelope. Used from on_challenge_submit, +// where the client awaits the submit JSON envelope and a redirect would break +// its state machine (the envelope is already a ChallengeRemediation, so +// UserCookies is serialized). Same precedence/ttl/error semantics as +// mintAllowlistCookie. +func (w *AppsecRuntimeConfig) GrantAllowlistCookieInline(state *AppsecRequestState, request *ParsedRequest, reason string, ttlOverride *time.Duration) error { + ck, err := w.mintAllowlistCookie(state, request, reason, ttlOverride) + if err != nil { + return err + } + + if err := w.SetChallengeCookie(state, *ck); err != nil { + return err + } + + state.Fingerprint.LogAccepted(w.Logger, log.InfoLevel, request.ClientIP, request.RemoteAddrNormalized, "granted allowlist challenge cookie inline (submit phase)") return nil } @@ -976,18 +1683,51 @@ func (w *AppsecRuntimeConfig) DisableBodyInspection(state *AppsecRequestState) e } type BodyResponse struct { - Action string `json:"action"` - HTTPStatus int `json:"http_status"` + Action string `json:"action"` + HTTPStatus int `json:"http_status"` + UserBodyContent string `json:"user_body_content,omitempty"` + UserCookies []string `json:"user_cookies,omitempty"` + UserHeaders map[string][]string `json:"user_headers,omitempty"` } func (w *AppsecRuntimeConfig) GenerateResponse(response AppsecTempResponse, logger *log.Entry) (int, BodyResponse) { var bouncerStatusCode int resp := BodyResponse{Action: response.Action} - if response.Action == AllowRemediation { + + switch response.Action { + case AllowRemediation: resp.HTTPStatus = w.Config.UserPassedHTTPCode bouncerStatusCode = w.Config.BouncerPassedHTTPCode - } else { // ban, captcha and anything else + case ChallengeRemediation: + resp.UserBodyContent = response.UserHTTPBodyContent + resp.UserCookies = make([]string, 0, len(response.UserHTTPCookies)) + for _, cookie := range response.UserHTTPCookies { + resp.UserCookies = append(resp.UserCookies, cookie.String()) + } + resp.UserHeaders = response.UserHeaders + if resp.UserHeaders == nil { + // default_remediation: challenge reaches here without any headers set + // (the internal challenge-serving paths always set Content-Type). + resp.UserHeaders = make(map[string][]string) + } + // If there's no Content-Security-Policy header, add a default one to make sure that the JS code can be evaluated + if _, ok := resp.UserHeaders["Content-Security-Policy"]; !ok { + resp.UserHeaders["Content-Security-Policy"] = []string{challenge.DefaultChallengeCSP} + } + // Challenge shares the ban/captcha status-code logic below. + fallthrough + case BanRemediation, CaptchaRemediation: + resp.HTTPStatus = response.UserHTTPResponseCode + if resp.HTTPStatus == 0 { + resp.HTTPStatus = w.Config.UserBlockedHTTPCode + } + bouncerStatusCode = response.BouncerHTTPResponseCode + if bouncerStatusCode == 0 { + bouncerStatusCode = w.Config.BouncerBlockedHTTPCode + } + default: + // Custom remediations use the same status code logic as ban/captcha resp.HTTPStatus = response.UserHTTPResponseCode if resp.HTTPStatus == 0 { resp.HTTPStatus = w.Config.UserBlockedHTTPCode diff --git a/pkg/appsec/appsec_challenge_test.go b/pkg/appsec/appsec_challenge_test.go new file mode 100644 index 000000000..b8c8a9aac --- /dev/null +++ b/pkg/appsec/appsec_challenge_test.go @@ -0,0 +1,440 @@ +package appsec + +import ( + "net/http" + "net/url" + "sync" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/appsec/challenge" +) + +// sharedChallengeRuntime lazily creates a single ChallengeRuntime for the +// whole package test run. NewChallengeRuntime runs the obfuscator WASM to +// generate a challenge JS bundle (~15-20s), so spinning one up per test is +// prohibitively slow and unnecessary for logic-only dispatcher tests. +var ( + sharedChallengeRuntimeOnce sync.Once + sharedChallengeRuntimeInst *challenge.ChallengeRuntime + sharedChallengeRuntimeErr error +) + +func getSharedChallengeRuntime(t *testing.T) *challenge.ChallengeRuntime { + t.Helper() + sharedChallengeRuntimeOnce.Do(func() { + // Use context.Background so the runtime survives across tests. + // Goroutines it spawns leak for the duration of the test binary; that's + // acceptable here since the binary exits after tests complete. + sharedChallengeRuntimeInst, sharedChallengeRuntimeErr = challenge.NewChallengeRuntime(t.Context()) + }) + require.NoError(t, sharedChallengeRuntimeErr) + return sharedChallengeRuntimeInst +} + +// newChallengeTestRuntime builds an AppsecRuntimeConfig wired with a real +// ChallengeRuntime for dispatcher tests. Provided on_challenge hooks are +// compiled via buildHookList, so tests can assert that user expressions run. +func newChallengeTestRuntime(t *testing.T, hooks []Hook) *AppsecRuntimeConfig { + t.Helper() + + cr := getSharedChallengeRuntime(t) + + compiled, err := buildHookList(t.Context(), hooks, hookOnChallenge, &appsecExprPatcher{}) + require.NoError(t, err) + + logger := log.NewEntry(log.StandardLogger()) + + cfg := &AppsecConfig{ + Name: "test-challenge", + BouncerBlockedHTTPCode: http.StatusForbidden, + BouncerPassedHTTPCode: http.StatusOK, + UserBlockedHTTPCode: http.StatusForbidden, + UserPassedHTTPCode: http.StatusOK, + DefaultPassAction: AllowRemediation, + DefaultRemediation: BanRemediation, + } + + return &AppsecRuntimeConfig{ + Name: "test-challenge", + Config: cfg, + Logger: logger, + ChallengeRuntime: cr, + CompiledOnChallenge: compiled, + } +} + +func newInBandRequest(method, path string, body []byte) *ParsedRequest { + u, _ := url.Parse(path) + return &ParsedRequest{ + HTTPRequest: &http.Request{ + Method: method, + URL: u, + Header: http.Header{"User-Agent": []string{"go-test"}}, + }, + Body: body, + IsInBand: true, + } +} + +func TestProcessOnChallengeRulesNilRuntime(t *testing.T) { + rt := &AppsecRuntimeConfig{ + Logger: log.NewEntry(log.StandardLogger()), + Config: &AppsecConfig{}, + } + state := &AppsecRequestState{} + req := newInBandRequest(http.MethodGet, "/", nil) + + require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req)) + assert.False(t, state.RequireChallenge) + assert.Nil(t, state.Fingerprint) +} + +func TestProcessOnChallengeRulesServesPowWorker(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + req := newInBandRequest(http.MethodGet, challenge.ChallengePowWorkerPath, nil) + + require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req)) + assert.True(t, state.RequireChallenge) + assert.Equal(t, ChallengeRemediation, state.Response.Action) + assert.Equal(t, http.StatusOK, state.Response.UserHTTPResponseCode) + assert.Equal(t, challenge.PowWorkerJS, state.Response.UserHTTPBodyContent) + assert.Contains(t, state.Response.UserHeaders["Content-Type"], "application/javascript") +} + +func TestProcessOnChallengeRulesSubmitInvalidBody(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + // Empty body → ValidateChallengeResponse fails; dispatcher returns failed JSON body. + req := newInBandRequest(http.MethodPost, challenge.ChallengeSubmitPath, nil) + + require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req)) + assert.True(t, state.RequireChallenge) + assert.JSONEq(t, bodyChallengeFailed, state.Response.UserHTTPBodyContent) + assert.Contains(t, state.Response.UserHeaders["Content-Type"], "application/json") + // No cookie is issued on failure. + assert.Empty(t, state.Response.UserHTTPCookies) +} + +func TestProcessOnChallengeRulesNoCookieSkipsUserHooks(t *testing.T) { + // A hook that would unconditionally flip PendingAction must NOT run when + // there is no fingerprint to inspect — this prevents nil-deref panics on + // filters like `fingerprint.Bot.X`. + rt := newChallengeTestRuntime(t, []Hook{ + {Apply: []string{"SetRemediation('allow')"}}, + }) + + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + req := newInBandRequest(http.MethodGet, "/", nil) + + require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req)) + assert.Nil(t, state.Fingerprint) + assert.Nil(t, state.PendingAction, "user hooks must not run when fingerprint is nil") +} + +func TestProcessOnChallengeRulesInvalidCookieSkipsUserHooks(t *testing.T) { + rt := newChallengeTestRuntime(t, []Hook{ + {Apply: []string{"SetReturnCode(418)"}}, + }) + + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + // Attach an invalid cookie; ValidCookie fails silently and fingerprint stays nil. + u, _ := url.Parse("/") + httpReq := &http.Request{ + Method: http.MethodGet, + URL: u, + Header: http.Header{ + "User-Agent": []string{"go-test"}, + "Cookie": []string{challenge.ChallengeCookieName + "=garbage"}, + }, + } + req := &ParsedRequest{HTTPRequest: httpReq, IsInBand: true} + + require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req)) + assert.Nil(t, state.Fingerprint) + assert.Nil(t, state.PendingHTTPCode, "user hooks must not run when fingerprint is nil") +} + +func TestSendChallengeNoOpWhenStoredDifficultyMeetsTarget(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + // Simulate a client that already proved the runtime-default difficulty. + state.Fingerprint = &challenge.FingerprintData{} + state.CookiePowDifficulty = rt.ChallengeRuntime.Difficulty() + + req := newInBandRequest(http.MethodGet, "/protected", nil) + + require.NoError(t, rt.SendChallenge(t.Context(), state, req)) + assert.False(t, state.RequireChallenge) + assert.Empty(t, state.Response.UserHTTPBodyContent) + assert.NotEqual(t, ChallengeRemediation, state.Response.Action) +} + +func TestSendChallengeReIssuesWhenTargetExceedsStored(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + // Client proved medium; this request demands impossible. + state.Fingerprint = &challenge.FingerprintData{} + state.CookiePowDifficulty = challenge.PowDifficultyMedium + impossible := challenge.PowDifficultyImpossible + state.ChallengeDifficulty = &impossible + + req := newInBandRequest(http.MethodGet, "/protected", nil) + + require.NoError(t, rt.SendChallenge(t.Context(), state, req)) + assert.True(t, state.RequireChallenge) + assert.Equal(t, ChallengeRemediation, state.Response.Action) + assert.Contains(t, state.Response.UserHeaders["Content-Type"], "text/html") + assert.NotEmpty(t, state.Response.UserHTTPBodyContent) +} + +func TestSendChallengeNoOpWhenTargetLoweredBelowStored(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + // Client proved high; admin lowered the global target. + state.Fingerprint = &challenge.FingerprintData{} + state.CookiePowDifficulty = challenge.PowDifficultyHigh + low := challenge.PowDifficultyLow + state.ChallengeDifficulty = &low + + req := newInBandRequest(http.MethodGet, "/protected", nil) + + require.NoError(t, rt.SendChallenge(t.Context(), state, req)) + assert.False(t, state.RequireChallenge) +} + +func TestProcessOnChallengeRulesInvalidSubmissionSkipsUserHooks(t *testing.T) { + // A hook that would unconditionally flip PendingAction must NOT run when + // the submission is invalid — the failed JSON body is the only response. + rt := newChallengeTestRuntime(t, []Hook{ + {Apply: []string{"SetRemediation('allow')"}}, + }) + + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + // Empty body → ValidateChallengeResponse fails. + req := newInBandRequest(http.MethodPost, challenge.ChallengeSubmitPath, nil) + + require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req)) + assert.True(t, state.RequireChallenge) + assert.JSONEq(t, bodyChallengeFailed, state.Response.UserHTTPBodyContent) + assert.Nil(t, state.PendingAction, "user hooks must not run on invalid submission") +} + +// TestRejectSubmissionSetsState confirms the helper sets the state field; +// downstream ProcessOnChallengeRules submit branch keys off this to refuse +// cookie issuance. +func TestRejectSubmissionSetsState(t *testing.T) { + rt := &AppsecRuntimeConfig{ + Logger: log.NewEntry(log.StandardLogger()), + Config: &AppsecConfig{}, + } + state := &AppsecRequestState{} + + require.NoError(t, rt.RejectSubmission(state, "bot detected")) + require.NotNil(t, state.SubmissionRejection) + assert.Equal(t, "bot detected", state.SubmissionRejection.Reason) + + // Empty reason gets a default placeholder. + state2 := &AppsecRequestState{} + require.NoError(t, rt.RejectSubmission(state2, "")) + assert.Equal(t, "submission rejected by on_challenge_submit", state2.SubmissionRejection.Reason) +} + +// TestGrantChallengeCookieIssuesRedirect confirms the helper mints a +// cookie, stamps allowlist fingerprint + bypass state, and produces a +// 307 challenge response carrying the Location header and Set-Cookie +// back to the visitor. The 307 is necessary because GenerateResponse +// only serializes UserCookies on ChallengeRemediation responses. +func TestGrantChallengeCookieIssuesRedirect(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + req := newInBandRequest(http.MethodGet, "/protected?a=1", nil) + + require.NoError(t, rt.GrantChallengeCookie(state, req, "Googlebot/2.1", nil)) + + require.NotNil(t, state.Fingerprint) + assert.True(t, state.Fingerprint.Allowlisted) + assert.Equal(t, "Googlebot/2.1", state.Fingerprint.AllowlistReason) + assert.True(t, state.ChallengeBypassed, "ChallengeBypassed must be set to suppress later SendChallenge") + + assert.Equal(t, ChallengeRemediation, state.Response.Action, "must produce a challenge-action response so the cookie is serialized") + assert.Equal(t, http.StatusTemporaryRedirect, state.Response.UserHTTPResponseCode, "must be a 307 redirect") + assert.True(t, state.RequireChallenge, "RequireChallenge must be flagged so the wire response carries the cookie") + + require.Len(t, state.Response.UserHTTPCookies, 1, "allowlist cookie must be appended") + require.NotNil(t, state.Response.UserHeaders) + require.Contains(t, state.Response.UserHeaders, "Location") + assert.Equal(t, []string{"/protected?a=1"}, state.Response.UserHeaders["Location"]) + assert.Equal(t, challenge.GrantRedirectBody, state.Response.UserHTTPBodyContent, "body must be the redirect fallback page") +} + +// TestGrantAllowlistCookieInlineNoRedirect confirms the submit-phase +// variant attaches the cookie without producing a redirect response. +// on_challenge_submit returns the challenge-submit JSON envelope the +// client is already awaiting; a 307 would break the client JS state +// machine. +func TestGrantAllowlistCookieInlineNoRedirect(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + req := newInBandRequest(http.MethodPost, challenge.ChallengeSubmitPath, nil) + + require.NoError(t, rt.GrantAllowlistCookieInline(state, req, "submit-allowlist", nil)) + + require.NotNil(t, state.Fingerprint) + assert.True(t, state.Fingerprint.Allowlisted) + assert.Equal(t, "submit-allowlist", state.Fingerprint.AllowlistReason) + assert.True(t, state.ChallengeBypassed) + require.Len(t, state.Response.UserHTTPCookies, 1, "cookie must ride on the existing envelope") + assert.NotEqual(t, http.StatusTemporaryRedirect, state.Response.UserHTTPResponseCode, "inline variant must NOT emit a redirect status") + assert.NotContains(t, state.Response.UserHeaders, "Location", "inline variant must not set Location") +} + +// TestSendChallengeNoOpAfterGrantChallengeCookie covers the bypass guard: +// after GrantChallengeCookie sets ChallengeBypassed, a subsequent +// SendChallenge call in the same request is a no-op (must not overwrite +// the 307 redirect response body with a challenge page). +func TestSendChallengeNoOpAfterGrantChallengeCookie(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + req := newInBandRequest(http.MethodGet, "/", nil) + require.NoError(t, rt.GrantChallengeCookie(state, req, "ua-pass", nil)) + require.True(t, state.ChallengeBypassed) + + cookiesBefore := len(state.Response.UserHTTPCookies) + bodyBefore := state.Response.UserHTTPBodyContent + statusBefore := state.Response.UserHTTPResponseCode + + require.NoError(t, rt.SendChallenge(t.Context(), state, req)) + + assert.Len(t, state.Response.UserHTTPCookies, cookiesBefore, "cookie set must be unchanged") + assert.Equal(t, bodyBefore, state.Response.UserHTTPBodyContent, "redirect body must not be overwritten") + assert.Equal(t, statusBefore, state.Response.UserHTTPResponseCode, "redirect status must not be overwritten") +} + +// TestProcessOnChallengeRulesAllowlistCookiePropagatesFlag asserts the +// cookie-valid branch of ProcessOnChallengeRules copies the Allowlisted + +// AllowlistReason fields from the decoded cookie into state.Fingerprint. +func TestProcessOnChallengeRulesAllowlistCookiePropagatesFlag(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + + // Mint an allowlist cookie via the public helper, then re-attach it to a + // regular request and run the dispatcher. + mintReq := newInBandRequest(http.MethodGet, "/", nil) + ck, err := rt.ChallengeRuntime.SealAllowlistCookie(mintReq.HTTPRequest, "test-bypass", nil) + require.NoError(t, err) + + u, _ := url.Parse("/protected") + getReq := &ParsedRequest{ + HTTPRequest: &http.Request{ + Method: http.MethodGet, + URL: u, + Header: http.Header{ + "User-Agent": []string{"go-test"}, + "Cookie": []string{challenge.ChallengeCookieName + "=" + ck.Val}, + }, + }, + IsInBand: true, + } + + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, getReq)) + require.NotNil(t, state.Fingerprint, "valid allowlist cookie must populate fingerprint") + assert.True(t, state.Fingerprint.Allowlisted) + assert.Equal(t, "test-bypass", state.Fingerprint.AllowlistReason) + assert.True(t, state.ChallengeBypassed, + "allowlist cookie must set ChallengeBypassed so later SendChallenge is a no-op") +} + +// TestAllowlistCookieRoundtripBypassesSendChallenge covers the actual bug +// reported against GrantChallengeCookie: a follow-up request carrying the +// minted allowlist cookie must not be re-challenged. Without the +// ChallengeBypassed propagation in the cookie-valid branch, SendChallenge +// would happily render a challenge page on top of the cookie. +func TestAllowlistCookieRoundtripBypassesSendChallenge(t *testing.T) { + rt := newChallengeTestRuntime(t, nil) + + mintReq := newInBandRequest(http.MethodGet, "/", nil) + ck, err := rt.ChallengeRuntime.SealAllowlistCookie(mintReq.HTTPRequest, "roundtrip", nil) + require.NoError(t, err) + + u, _ := url.Parse("/protected") + replayReq := &ParsedRequest{ + HTTPRequest: &http.Request{ + Method: http.MethodGet, + URL: u, + Header: http.Header{ + "User-Agent": []string{"go-test"}, + "Cookie": []string{challenge.ChallengeCookieName + "=" + ck.Val}, + }, + }, + IsInBand: true, + } + + state := &AppsecRequestState{} + state.ResetResponse(rt.Config) + + require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, replayReq)) + require.NotNil(t, state.Fingerprint) + require.True(t, state.ChallengeBypassed, "replayed allowlist cookie must flip ChallengeBypassed") + + // Anything downstream calling SendChallenge must now be a no-op. + bodyBefore := state.Response.UserHTTPBodyContent + statusBefore := state.Response.UserHTTPResponseCode + actionBefore := state.Response.Action + + require.NoError(t, rt.SendChallenge(t.Context(), state, replayReq)) + + assert.Equal(t, actionBefore, state.Response.Action, "SendChallenge must not set ChallengeRemediation") + assert.NotEqual(t, ChallengeRemediation, state.Response.Action, + "a replayed allowlist cookie must not produce a challenge response") + assert.Equal(t, bodyBefore, state.Response.UserHTTPBodyContent, "challenge body must not be written") + assert.Equal(t, statusBefore, state.Response.UserHTTPResponseCode, "challenge status must not overwrite") + assert.False(t, state.RequireChallenge, "RequireChallenge must stay false on bypassed replay") +} + +// TestGenerateResponseChallengeNilHeaders guards against a regression where a +// `default_remediation: challenge` match reaches GenerateResponse with no +// UserHeaders map set (only the internal challenge-serving paths set headers). +// Writing the default CSP into that nil map used to panic. +func TestGenerateResponseChallengeNilHeaders(t *testing.T) { + rt := &AppsecRuntimeConfig{ + Logger: log.NewEntry(log.StandardLogger()), + Config: &AppsecConfig{ + UserBlockedHTTPCode: http.StatusForbidden, + BouncerBlockedHTTPCode: http.StatusForbidden, + }, + } + + resp := AppsecTempResponse{Action: ChallengeRemediation, UserHeaders: nil} + + require.NotPanics(t, func() { + _, body := rt.GenerateResponse(resp, rt.Logger) + require.NotNil(t, body.UserHeaders) + assert.Equal(t, []string{challenge.DefaultChallengeCSP}, body.UserHeaders["Content-Security-Policy"]) + }) +} diff --git a/pkg/appsec/appsec_config_test.go b/pkg/appsec/appsec_config_test.go index bbf113ae0..234e7fd4e 100644 --- a/pkg/appsec/appsec_config_test.go +++ b/pkg/appsec/appsec_config_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "testing" + "time" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -326,24 +327,130 @@ func TestBuildNilPhaseConfig(t *testing.T) { assert.Empty(t, rt.OutOfBandHooks.OnMatch) } -func TestLoadAPISchemaRejectsPathTraversal(t *testing.T) { - cases := []string{ - "../escape.yaml", - "../../etc/passwd", - "/etc/passwd", +func TestBuildOnChallengeTopLevelAndInband(t *testing.T) { + cfg := AppsecConfig{ + Logger: log.NewEntry(log.StandardLogger()), + DefaultRemediation: "ban", + OnChallenge: []Hook{ + {Apply: []string{"SetRemediation('allow')"}}, + }, + InBand: &AppsecPhaseConfig{ + OnChallenge: []Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetReturnCode(418)"}}, + }, + }, } - for _, filename := range cases { - t.Run(filename, func(t *testing.T) { - rt := &AppsecRuntimeConfig{ - Logger: log.NewEntry(log.StandardLogger()), - DataDir: t.TempDir(), - } + hub := &cwhub.Hub{} + rt, err := cfg.Build(t.Context(), hub) + require.NoError(t, err) - err := rt.LoadAPISchemaWithName("ref", filename) - require.Error(t, err) - require.Contains(t, err.Error(), "must be relative to schemas") - }) + // Both top-level and inband on_challenge hooks end up in CompiledOnChallenge. + require.Len(t, rt.CompiledOnChallenge, 2) + assert.NotNil(t, rt.CompiledOnChallenge[0].ApplyExpr) + assert.NotNil(t, rt.CompiledOnChallenge[1].FilterExpr) + + // Defining on_challenge must force the challenge runtime on. + assert.True(t, rt.NeedWASMVM) +} + +func TestBuildOnChallengeUnderOutofbandRejected(t *testing.T) { + cfg := AppsecConfig{ + Logger: log.NewEntry(log.StandardLogger()), + DefaultRemediation: "ban", + OutOfBand: &AppsecPhaseConfig{ + OnChallenge: []Hook{{Apply: []string{"SetRemediation('allow')"}}}, + }, + } + + hub := &cwhub.Hub{} + _, err := cfg.Build(t.Context(), hub) + require.Error(t, err) + assert.Contains(t, err.Error(), "on_challenge hooks are only valid in-band") +} + +func TestLoadByPathOnChallengeYAML(t *testing.T) { + cfg := newTestConfig() + f := writeTempYAML(t, ` +name: test-on-challenge +on_challenge: + - filter: "fingerprint != nil" + apply: + - SetRemediation('allow') +inband: + on_challenge: + - apply: + - SendChallenge() +`) + + require.NoError(t, cfg.LoadByPath(f)) + assert.Len(t, cfg.OnChallenge, 1) + require.NotNil(t, cfg.InBand) + assert.Len(t, cfg.InBand.OnChallenge, 1) +} + +// TestLoadByPathChallengeBlockMergesAcrossFiles confirms that a `challenge:` +// block in an appsec-config YAML is parsed onto AppsecConfig.Challenge, and +// that loading a second config with a disjoint subset of fields *merges* +// rather than replacing — fields set in earlier configs survive when not +// touched by later ones, and overlapping fields take the later value. +func TestLoadByPathChallengeBlockMergesAcrossFiles(t *testing.T) { + cfg := newTestConfig() + + first := writeTempYAML(t, ` +name: base +challenge: + master_secret: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + key_rotation_interval: 5m + max_live_epochs: 3 + cookie_ttl: 12h +`) + require.NoError(t, cfg.LoadByPath(first)) + require.NotNil(t, cfg.Challenge) + require.NotNil(t, cfg.Challenge.MasterSecret) + assert.Equal(t, 5*time.Minute, *cfg.Challenge.KeyRotationInterval) + assert.Equal(t, 3, *cfg.Challenge.MaxLiveEpochs) + assert.Equal(t, 12*time.Hour, *cfg.Challenge.CookieTTL) + + // Second config overrides CookieTTL and adds crypto/spent-set fields, + // leaving master_secret / rotation / max_live_epochs from the first + // config untouched. This is the multi-config last-wins-per-field + // behavior collections rely on. + second := writeTempYAML(t, ` +name: overlay +challenge: + cookie_ttl: 1h + crypto_obfuscation_pool_size: 2 + spent_set_max_entries: 250000 +`) + require.NoError(t, cfg.LoadByPath(second)) + require.NotNil(t, cfg.Challenge) + assert.Equal(t, 1*time.Hour, *cfg.Challenge.CookieTTL, "later config overrides cookie_ttl") + require.NotNil(t, cfg.Challenge.MasterSecret, "master_secret from first config must survive") + assert.Equal(t, 5*time.Minute, *cfg.Challenge.KeyRotationInterval) + require.NotNil(t, cfg.Challenge.CryptoObfuscationPoolSize) + assert.Equal(t, 2, *cfg.Challenge.CryptoObfuscationPoolSize, "new field from second config appears") + require.NotNil(t, cfg.Challenge.SpentSetMaxEntries) + assert.Equal(t, 250000, *cfg.Challenge.SpentSetMaxEntries) +} + +// TestLoadByPathRejectsRemovedLibraryObfuscationKeys confirms that an +// appsec-config carrying any of the removed `library_*` obfuscation keys +// fails to parse (strict YAML mode). fpscanner is now served unobfuscated, +// so these knobs are gone and stale configs must fail loudly rather than +// silently doing nothing. +func TestLoadByPathRejectsRemovedLibraryObfuscationKeys(t *testing.T) { + for _, key := range []string{ + "library_runtime_obfuscation_enabled: true", + "library_obfuscation_pool_size: 2", + "library_obfuscation_refresh_interval: 30m", + } { + cfg := newTestConfig() + f := writeTempYAML(t, "name: stale\nchallenge:\n "+key+"\n") + err := cfg.LoadByPath(f) + require.Error(t, err, "stale key %q must be rejected", key) + assert.Contains(t, err.Error(), "library_", + "strict YAML parse error must name the unknown field") } } @@ -366,3 +473,24 @@ func TestBuildOnLoadStaysOutOfPhaseHooks(t *testing.T) { assert.Empty(t, rt.CommonHooks.PostEval) assert.Empty(t, rt.CommonHooks.OnMatch) } + +func TestLoadAPISchemaRejectsPathTraversal(t *testing.T) { + cases := []string{ + "../escape.yaml", + "../../etc/passwd", + "/etc/passwd", + } + + for _, filename := range cases { + t.Run(filename, func(t *testing.T) { + rt := &AppsecRuntimeConfig{ + Logger: log.NewEntry(log.StandardLogger()), + DataDir: t.TempDir(), + } + + err := rt.LoadAPISchemaWithName("ref", filename) + require.Error(t, err) + require.Contains(t, err.Error(), "must be relative to schemas") + }) + } +} diff --git a/pkg/appsec/appsec_rules_collection.go b/pkg/appsec/appsec_rules_collection.go index 41c0b1229..d7ddfc950 100644 --- a/pkg/appsec/appsec_rules_collection.go +++ b/pkg/appsec/appsec_rules_collection.go @@ -10,12 +10,13 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/enrichment" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" ) type AppsecCollection struct { - Rules []string - NativeRules []string + Rules []string + NativeRules []string } const APPSEC_RULE = "appsec-rule" @@ -33,7 +34,7 @@ type AppsecCollectionConfig struct { Labels map[string]any `yaml:"labels"` // Labels is K:V list aiming at providing context the overflow - Data any `yaml:"data"` // Ignore it + Data []*enrichment.DataProvider `yaml:"data"` hash string version string } @@ -49,103 +50,148 @@ type RulesDetails struct { // Is using the id is a good idea ? might be too specific to coraza and not easily reusable var AppsecRulesDetails = make(map[int]RulesDetails) +// matchPattern reports whether appsecRule.Name matches pattern. +func matchPattern(pattern, name string, logger *log.Entry) bool { + tmpMatch, err := exprhelpers.Match(pattern, name) + if err != nil { + logger.Errorf("unable to match %s with %s : %s", name, pattern, err) + return false + } + + matched, ok := tmpMatch.(bool) + if !ok { + logger.Errorf("unable to match %s with %s : result is not a boolean", name, pattern) + return false + } + + return matched +} + +// loadNativeRulesFromFile reads a single seclang rules file and returns its non-comment, non-empty lines. +func loadNativeRulesFromFile(fullPath string, logger *log.Entry) []string { + c, err := os.ReadFile(fullPath) + if err != nil { + logger.Errorf("unable to read file %s : %s", fullPath, err) + return nil + } + + rules := make([]string, 0) + + for line := range strings.SplitSeq(string(c), "\n") { + if strings.HasPrefix(line, "#") { + continue + } + + if strings.TrimSpace(line) == "" { + continue + } + + rules = append(rules, line) + } + + return rules +} + +// loadSecLangFilesRules resolves the seclang file globs and appends their rules to appsecCol. +func loadSecLangFilesRules(appsecRule AppsecCollectionConfig, hub *cwhub.Hub, logger *log.Entry, appsecCol *AppsecCollection) { + for _, rulesFile := range appsecRule.SecLangFilesRules { + logger.Debugf("Adding rules from %s", rulesFile) + globPattern := filepath.Join(hub.GetDataDir(), rulesFile) + + matches, err := filepath.Glob(globPattern) + if err != nil { + logger.Errorf("unable to glob %s : %s", rulesFile, err) + continue + } + + if len(matches) == 0 { + logger.Warnf("no file matched pattern %s", globPattern) + continue + } + + for _, fullPath := range matches { + appsecCol.NativeRules = append(appsecCol.NativeRules, loadNativeRulesFromFile(fullPath, logger)...) + } + } +} + +// registerRuleDetails records the metadata and debug flags for a converted custom rule. +func registerRuleDetails(appsecRule AppsecCollectionConfig, ruleName string, rulesId []uint32, logger *log.Entry) { + // We only take the first id, as it's the one of the "main" rule + if _, ok := AppsecRulesDetails[int(rulesId[0])]; !ok { + AppsecRulesDetails[int(rulesId[0])] = RulesDetails{ + LogLevel: log.InfoLevel, + Hash: appsecRule.hash, + Version: appsecRule.version, + Name: appsecRule.Name, + } + } else { + logger.Warnf("conflicting id %d for rule %s !", rulesId[0], ruleName) + } + + for _, id := range rulesId { + SetRuleDebug(int(id), appsecRule.Debug) + } +} + +// loadCustomRules converts the YAML custom rules and appends the resulting seclang rules to appsecCol. +func loadCustomRules(appsecRule AppsecCollectionConfig, logger *log.Entry, appsecCol *AppsecCollection) error { + for _, rule := range appsecRule.Rules { + rule.Severity = appsecRule.Severity + + strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name, appsecRule.Description) + if err != nil { + logger.Errorf("unable to convert rule %s : %s", appsecRule.Name, err) + return err + } + + logger.Debugf("Adding rule %s", strRule) + appsecCol.Rules = append(appsecCol.Rules, strRule) + + registerRuleDetails(appsecRule, rule.Name, rulesId, logger) + } + + return nil +} + +// initRuleData initializes the data files referenced by an appsec rule. +func initRuleData(appsecRule AppsecCollectionConfig, hub *cwhub.Hub, logger *log.Entry) { + for _, appsecRuleData := range appsecRule.Data { + if appsecRuleData.DestPath == "" { + logger.Errorf("missing dest_path for rule %s : %+v", appsecRule.Name, appsecRuleData) + continue + } + + if err := exprhelpers.FileInit(hub.GetDataDir(), appsecRuleData.DestPath, appsecRuleData.Type); err != nil { + logger.Errorf("unable to initialize data file %s : %s", appsecRuleData.DestPath, err) + continue + } + } +} + func LoadCollection(pattern string, logger *log.Entry, hub *cwhub.Hub) ([]AppsecCollection, error) { ret := make([]AppsecCollection, 0) for _, appsecRule := range appsecRules { - tmpMatch, err := exprhelpers.Match(pattern, appsecRule.Name) - if err != nil { - logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) - continue - } - - matched, ok := tmpMatch.(bool) - - if !ok { - logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) - continue - } - - if !matched { + if !matchPattern(pattern, appsecRule.Name, logger) { continue } appsecCol := AppsecCollection{} - if appsecRule.SecLangFilesRules != nil { - for _, rulesFile := range appsecRule.SecLangFilesRules { - logger.Debugf("Adding rules from %s", rulesFile) - globPattern := filepath.Join(hub.GetDataDir(), rulesFile) - - matches, err := filepath.Glob(globPattern) - - if err != nil { - logger.Errorf("unable to glob %s : %s", rulesFile, err) - continue - } - - if len(matches) == 0 { - logger.Warnf("no file matched pattern %s", globPattern) - continue - } - - for _, fullPath := range matches { - c, err := os.ReadFile(fullPath) - if err != nil { - logger.Errorf("unable to read file %s : %s", rulesFile, err) - continue - } - - for line := range strings.SplitSeq(string(c), "\n") { - if strings.HasPrefix(line, "#") { - continue - } - - if strings.TrimSpace(line) == "" { - continue - } - - appsecCol.NativeRules = append(appsecCol.NativeRules, line) - } - } - } - } + loadSecLangFilesRules(appsecRule, hub, logger, &appsecCol) if appsecRule.SecLangRules != nil { logger.Tracef("Adding inline rules %+v", appsecRule.SecLangRules) appsecCol.NativeRules = append(appsecCol.NativeRules, appsecRule.SecLangRules...) } - if appsecRule.Rules != nil { - for _, rule := range appsecRule.Rules { - rule.Severity = appsecRule.Severity - strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name, appsecRule.Description) - if err != nil { - logger.Errorf("unable to convert rule %s : %s", appsecRule.Name, err) - return nil, err - } - - logger.Debugf("Adding rule %s", strRule) - appsecCol.Rules = append(appsecCol.Rules, strRule) - - // We only take the first id, as it's the one of the "main" rule - if _, ok := AppsecRulesDetails[int(rulesId[0])]; !ok { - AppsecRulesDetails[int(rulesId[0])] = RulesDetails{ - LogLevel: log.InfoLevel, - Hash: appsecRule.hash, - Version: appsecRule.version, - Name: appsecRule.Name, - } - } else { - logger.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name) - } - - for _, id := range rulesId { - SetRuleDebug(int(id), appsecRule.Debug) - } - } + if err := loadCustomRules(appsecRule, logger, &appsecCol); err != nil { + return nil, err } + initRuleData(appsecRule, hub, logger) + ret = append(ret, appsecCol) } diff --git a/pkg/appsec/challenge/DESIGN.md b/pkg/appsec/challenge/DESIGN.md new file mode 100644 index 000000000..3d3132d63 --- /dev/null +++ b/pkg/appsec/challenge/DESIGN.md @@ -0,0 +1,888 @@ +# WAF Bot Detection / Challenge — Design, Review, Usage + +This document covers the CrowdSec WAF challenge subsystem: how it is built, what +reviewers should look for when touching it, and how operators / collection +authors are expected to use it. + +The implementation lives under [pkg/appsec/challenge](.) and is wired into the +acquisition pipeline by [pkg/acquisition/modules/appsec](../../acquisition/modules/appsec). + +--- + +## 1. Design + +### 1.1 What the feature does + +The challenge subsystem lets the WAF interpose a browser-side proof-of-work + +device-fingerprint check between a visitor and the protected origin. A visitor +who solves the challenge receives a sealed cookie carrying their measured +fingerprint; subsequent requests from the same browser can be evaluated by +operator-defined hooks (`on_challenge`, `on_challenge_submit`) without +re-challenging. + +The challenge runtime is built **lazily**: it only spins up if at least one +loaded hook calls `SendChallenge()`, `GrantChallengeCookie()`, or +`RejectSubmission()`. This is detected by [`appsecExprPatcher`](../patcher.go) +walking the compiled expression AST and setting +`AppsecRuntimeConfig.NeedWASMVM`. + +### 1.2 Component map + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ Acquisition module (pkg/acquisition/modules/appsec/config.go) │ +│ - HTTP listener, ParsedRequest plumbing │ +│ - Loads appsec-config(s), builds AppsecRuntimeConfig │ +│ - If NeedWASMVM: challenge.BuildOptions → NewChallengeRuntime │ +└──────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────────┐ +│ pkg/appsec (AppsecConfig / AppsecRuntimeConfig) │ +│ - YAML schema, hook compilation, request-lifecycle dispatcher │ +│ - ProcessOnChallengeRules: cookie validation, /submit handling │ +│ - SendChallenge / GrantChallengeCookie / RejectSubmission helpers │ +└──────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────────┐ +│ pkg/appsec/challenge (ChallengeRuntime) │ +│ - KeyRing: HKDF-derived per-epoch keys + long-lived cookie key │ +│ - Crypto: AES-GCM cookie seal/open, per-challenge + PoW MAC HMACs │ +│ - Obfuscator: wazero-hosted WASM running javascript-obfuscator │ +│ - Challenge code: single static build-time obfuscated variant │ +│ - Dynamic module pool: per-epoch key script, obfuscated variants │ +│ - Fingerprint mismatch report aggregator │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +### 1.3 Configuration model + +Challenge tuning lives **inside the appsec-config**. The relevant fields are on [`challenge.Config`](config.go), referenced +from `AppsecConfig.Challenge`: + +```go +type Config struct { + MasterSecret *string + KeyRotationInterval *time.Duration + MaxLiveEpochs *int + CookieTTL *time.Duration + CryptoObfuscationPoolSize *int +} +``` + +`challenge.BuildOptions(c *Config) ([]Option, error)` translates the merged +config into the option list consumed by `NewChallengeRuntime`. + +### 1.4 Cryptographic core + +**Notation.** These names are used consistently throughout this doc, the wire +protocol, and the code. Note the two distinct "nonces": the **challenge nonce +`r`** and the **PoW nonce `n`** (the AES-GCM nonce inside the cookie envelope is +a third, unrelated value — written `gcm_nonce`). + +| Symbol | Name | On the wire? | Definition | +|---|---|---|---| +| `master_secret` | master secret | no (config) | long-lived root secret, ≥ 32 B | +| `K_epoch` | per-epoch key | no (derived) | `HKDF(master_secret, epoch)`; rotates each interval | +| `r` | challenge nonce | yes (`r`) | random 16 B per challenge; identifies it for single-use | +| `ts` | timestamp | yes (`ts`) | `time.Now().UnixNano()`; selects the epoch | +| `s` | per-challenge secret | **no** | `HMAC(K_epoch, r)`; signs the submission | +| `p` | PoW salt | yes (`p`) | random 16 B the client hashes for the PoW | +| `n` | PoW nonce | yes (`n`) | PoW solution — makes `SHA256(p‖n)` have ≥ N leading zero bits | +| `m` | PoW MAC | yes (`m`) | `HMAC(K_epoch, p‖r‖ts)`; proves `p` is server-issued | +| `f` | fingerprint payload | yes (`f`) | obfuscated + base64 device fingerprint | +| `encKey` | fingerprint obfuscation key | **no** | `HMAC(s, "fpenc"‖r)` | +| `sig` | submission signature | yes (`sig`) | `HMAC(s, r‖ts‖n‖f)` | + +The submit POST therefore carries exactly `f, r, ts, sig, n, p, m` (§1.6); `s` +and `encKey` are derived on both ends and never transmitted. + +**At a glance** — every cryptographic operation (all keys are 256-bit; every +MAC/signature comparison uses constant-time `hmac.Equal`): + +| Operation | Primitive | Key | Input → output | +|---|---|---|---| +| Derive operational keys | HKDF-SHA256 (salt `crowdsec-challenge-keyring-v1`) | `master_secret` | info `epoch-sign:` → `K_epoch`; info `cookie-master` → master cookie key | +| Per-challenge secret `s` | HMAC-SHA256 | `K_epoch` | `r` → `s` | +| PoW MAC `m` | HMAC-SHA256 | `K_epoch` | `p‖r‖ts` → `m` | +| Proof-of-work | SHA-256 | — | `p‖n` accepted iff ≥ N leading zero bits | +| Fingerprint obf. key `encKey` | HMAC-SHA256 | `s` | `"fpenc"‖r` → `encKey` | +| Fingerprint payload `f` | repeating-key XOR + base64 | `encKey` | fingerprint JSON → `f` (light obfuscation, **not** confidentiality — §2.6) | +| Submission signature `sig` | HMAC-SHA256 | `s` | `r‖ts‖n‖f` → `sig` | +| Cookie seal / open | AES-256-GCM | HKDF-derived from master cookie key | proto envelope, AAD = `User-Agent` → sealed cookie | + +The subsections below detail each; the validation order is in §1.6. + +#### Master secret + +- Configured as hex (preferred — even-length, hex digits only) or as a + passphrase. Both forms must be at least **32 bytes** decoded — the natural + minimum for HMAC-SHA256 with full security margin. +- If unconfigured, [`generateRandomSecret`](secret.go) produces a fresh 32-byte + secret and the runtime logs a warning. Suitable only for single-instance + deployments: a restart invalidates outstanding cookies, and distributed + WAFs can't agree on the same value. + +#### KeyRing — per-epoch derivation + +[`pkg/appsec/challenge/keyring.go`](keyring.go) derives two independent key +families from the master secret via HKDF-SHA256 (salt +`"crowdsec-challenge-keyring-v1"`): + +| Family | HKDF info | Lifetime | Use | +|---|---|---|---| +| Per-epoch key (`K_epoch`) | `"epoch-sign"` | One rotation interval | Per-challenge secret `s` + PoW MAC `m` | +| Master cookie key | `"cookie-master"` | Lifetime of master secret | AES-GCM seal/open of challenge cookies | + +An **epoch** is a time bucket of `rotationInterval` seconds, computed as +`now.Unix() / rotation_interval_seconds`. All instances sharing the same +master secret + rotation interval derive bit-identical epoch keys with no +coordination — the basis for distributed verification of the per-challenge +crypto (replay protection is separately stateful; see §2.1). + +The keyring keeps a sliding window of `MaxLiveEpochs` past epochs plus the +current epoch plus a one-step clock-skew slack. Submissions signed by any key +inside the live window verify; older ones are rejected. Cached entries +outside the window are evicted whenever a new epoch is derived. + +Defaults: `keyringDefaultRotation = 5 minutes`, `keyringDefaultMaxLive = 3`, +`keyringMinRotation = 30 seconds`. + +#### Cookie envelope (v0) + +[`pkg/appsec/challenge/crypto.go`](crypto.go) seals cookies under the +long-lived master cookie key with AES-256-GCM: + +``` +wire format: version_byte(1) || gcm_nonce(12) || ciphertext +plaintext : not_after_be8(8) || flags(1) || reason_len_be(2) + || reason_bytes(0..256) + || protobuf_envelope +AAD : user-agent bytes +``` + +Three things to note: + +1. **Cookie expiry is enforced by the embedded `not_after` timestamp**, not + by keyring eviction. This intentional decouples cookie TTL (operator + policy, e.g. 12h–24h) from the keyring live window (the window in which an + extracted per-epoch key stays usable, e.g. 15m). Long-lived cookies don't + widen that signing window. +2. **User-Agent is the GCM authentication tag.** A cookie cannot be replayed + across UAs — `openCookie` fails authentication on mismatch. +3. **Allowlist cookies** set `cookieFlagAllowlisted = 0x01` and carry an + operator reason (`MaxAllowlistReasonLen = 256` bytes — bounded to keep the + final cookie comfortably under the browser 4 KB limit). Real-submission + cookies leave the flag clear and the reason empty. + +#### Per-challenge secret, PoW MAC, and single-use + +Each challenge render carries a fresh 16-byte challenge nonce `r`. From it the +client derives the **per-challenge secret** `s`, which is *never transmitted*: + +- `s = HMAC(K_epoch, r)` — the per-challenge secret. Both client and server + derive it independently (the client from the `K_epoch` delivered in the + obfuscated dynamic module; the server from the keyring). It never appears on + the wire. +- `m = HMAC(K_epoch, p || r || ts)` — the PoW MAC; authenticates the PoW salt + `p` as server-issued and bound to this challenge, so a client can't pick a + favourable salt. + +The submission is signed `sig = HMAC(s, r || ts || n || f)`. A valid `sig` proves +the submitter knows `K_epoch` — normally by running our obfuscated JS to read it +from the dynamic module, though a determined attacker can extract it (§2.6). See +[ticket.go](ticket.go) (`deriveChallengeSecret`, `computePowMAC`, +`verifyChallenge`). + +`ts` is `time.Now().UnixNano()` formatted as a string; the verifier derives the +epoch from `ts` itself and looks up the same key. **Challenge *issuance* is +stateless** (any instance with the same master secret can render and verify the +crypto), but **submission is single-use**: the validated `r` is burned in an +in-memory spent-set ([spent_set.go](spent_set.go)), so a captured submission +cannot be replayed against that instance. This is the load-bearing replay +control; the per-epoch key obfuscation is only a non-load-bearing speed-bump +(§2.6). The spent-set is **per-instance**, so single-use assumes all reverse +proxies route to one appsec instance — see §2.1 / §3.4. + +Freshness is doubly gated: keyring liveness (epoch in live window) and a loose +backstop of `ticketAgeBackstop = 20 minutes` — which also bounds the spent-set +TTL. + +#### Single-use tickets + +[`spent_set.go`](spent_set.go) records consumed challenge nonces (`r`) to +enforce single-use. It wraps **gcache** (already used by `pkg/cache`): a bounded +LRU (`spent_set_max_entries`, default `spentSetDefaultMaxEntries`) with a per-entry TTL equal to the freshness window, +so there is no janitor goroutine or hand-rolled cap. A single mutex makes the +`Has`/`SetWithExpire` pair in `checkAndInsert` atomic, so two concurrent +identical submissions can't both win. The burn happens **last** in +`ValidateChallengeResponse` (after freshness, PoW-MAC, PoW, and `sig` all pass), +so the set only ever grows for crypto+PoW-valid submissions — its growth is +sig+PoW-gated and TTL-bounded. + +PoW levels (in leading SHA-256 zero bits) live as constants in +[`ticket.go`](ticket.go): + +| Level | Bits | Approx work | +|---|---|---| +| `Disabled` | 0 | nonce `"0"` always valid | +| `Low` | 10 | ~1k hashes, 0.2–2 s | +| `Medium` (default) | 12 | ~4k hashes, 1–8 s | +| `High` | 15 | ~32k hashes, 7–60 s | +| `Impossible` | 256 | unsolvable — hard block | + +### 1.5 JS obfuscation pipeline + +The browser receives four logical pieces of JS at every challenge: + +1. **Obfuscated challenge code** — the small crypto/glue bundle (SHA-256, HMAC, + PoW orchestration, fingerprint XOR, submission, hook registration). Injected + **inline** on the challenge page. A single static build-time obfuscated + variant (no runtime pool — see below). +2. **Dynamic key module** — a ~30-line script embedding the current epoch's key + `K_epoch` as a hex literal, so the client can derive the per-challenge secret + `s = HMAC(K_epoch, r)` and sign the submission. Injected inline. +3. **fpscanner bundle** — the public `fpscanner` device-fingerprint scanner, + served **unobfuscated** as its own file at + `/crowdsec-internal/challenge/fpscanner.js` and loaded via a plain + ` + + + + + + + + + diff --git a/pkg/appsec/challenge/challenge.js b/pkg/appsec/challenge/challenge.js new file mode 100644 index 000000000..c8919954f --- /dev/null +++ b/pkg/appsec/challenge/challenge.js @@ -0,0 +1,346 @@ +// --- Pure JS SHA-256 (RFC 6234) --- + +const K = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]; + +function sha256(data) { + let h0 = 0x6a09e667, + h1 = 0xbb67ae85, + h2 = 0x3c6ef372, + h3 = 0xa54ff53a, + h4 = 0x510e527f, + h5 = 0x9b05688c, + h6 = 0x1f83d9ab, + h7 = 0x5be0cd19; + + const len = data.length; + const bitLen = len * 8; + + const padLen = 64 - ((len + 9) % 64); + const totalLen = len + 1 + (padLen === 64 ? 0 : padLen) + 8; + const msg = new Uint8Array(totalLen); + msg.set(data); + msg[len] = 0x80; + const dv = new DataView(msg.buffer); + dv.setUint32(totalLen - 4, bitLen, false); + + const w = new Int32Array(64); + for (let offset = 0; offset < totalLen; offset += 64) { + for (let i = 0; i < 16; i++) { + w[i] = dv.getInt32(offset + i * 4, false); + } + for (let i = 16; i < 64; i++) { + const s0 = + (((w[i - 15] >>> 7) | (w[i - 15] << 25)) ^ + ((w[i - 15] >>> 18) | (w[i - 15] << 14)) ^ + (w[i - 15] >>> 3)) | + 0; + const s1 = + (((w[i - 2] >>> 17) | (w[i - 2] << 15)) ^ + ((w[i - 2] >>> 19) | (w[i - 2] << 13)) ^ + (w[i - 2] >>> 10)) | + 0; + w[i] = (w[i - 16] + s0 + w[i - 7] + s1) | 0; + } + + let a = h0, + b = h1, + c = h2, + d = h3, + e = h4, + f = h5, + g = h6, + h = h7; + + for (let i = 0; i < 64; i++) { + const S1 = + (((e >>> 6) | (e << 26)) ^ + ((e >>> 11) | (e << 21)) ^ + ((e >>> 25) | (e << 7))) | + 0; + const ch = ((e & f) ^ (~e & g)) | 0; + const temp1 = (h + S1 + ch + K[i] + w[i]) | 0; + const S0 = + (((a >>> 2) | (a << 30)) ^ + ((a >>> 13) | (a << 19)) ^ + ((a >>> 22) | (a << 10))) | + 0; + const maj = ((a & b) ^ (a & c) ^ (b & c)) | 0; + const temp2 = (S0 + maj) | 0; + + h = g; + g = f; + f = e; + e = (d + temp1) | 0; + d = c; + c = b; + b = a; + a = (temp1 + temp2) | 0; + } + + h0 = (h0 + a) | 0; + h1 = (h1 + b) | 0; + h2 = (h2 + c) | 0; + h3 = (h3 + d) | 0; + h4 = (h4 + e) | 0; + h5 = (h5 + f) | 0; + h6 = (h6 + g) | 0; + h7 = (h7 + h) | 0; + } + + const out = new Uint8Array(32); + const ov = new DataView(out.buffer); + ov.setUint32(0, h0, false); + ov.setUint32(4, h1, false); + ov.setUint32(8, h2, false); + ov.setUint32(12, h3, false); + ov.setUint32(16, h4, false); + ov.setUint32(20, h5, false); + ov.setUint32(24, h6, false); + ov.setUint32(28, h7, false); + return out; +} + +// --- Helpers --- + +function toHex(bytes) { + let hex = ""; + for (let i = 0; i < bytes.length; i++) { + hex += (bytes[i] >> 4).toString(16) + (bytes[i] & 0xf).toString(16); + } + return hex; +} + +function encode(str) { + return new TextEncoder().encode(str); +} + +function sha256Hex(str) { + return toHex(sha256(encode(str))); +} + +// --- HMAC-SHA256 --- + +function hmacSHA256(key, message) { + if (key.length > 64) { + key = sha256(key); + } + const paddedKey = new Uint8Array(64); + paddedKey.set(key); + + const ipad = new Uint8Array(64); + const opad = new Uint8Array(64); + for (let i = 0; i < 64; i++) { + ipad[i] = paddedKey[i] ^ 0x36; + opad[i] = paddedKey[i] ^ 0x5c; + } + + const inner = new Uint8Array(64 + message.length); + inner.set(ipad); + inner.set(message, 64); + const innerHash = sha256(inner); + + const outer = new Uint8Array(64 + 32); + outer.set(opad); + outer.set(innerHash, 64); + return sha256(outer); +} + +function hmacSHA256Hex(keyStr, msgStr) { + return toHex(hmacSHA256(encode(keyStr), encode(msgStr))); +} + +// hexToBytes decodes a hex string into a Uint8Array. Used to materialize the +// per-epoch HMAC key delivered as hex by the dynamic module. +function hexToBytes(hex) { + const out = new Uint8Array(hex.length / 2); + for (let i = 0; i < out.length; i++) { + out[i] = parseInt(hex.substr(i * 2, 2), 16); + } + return out; +} + +function hmacSHA256HexKey(keyHex, msgStr) { + return toHex(hmacSHA256(hexToBytes(keyHex), encode(msgStr))); +} + +// --- Proof-of-Work (offloaded to Web Worker) --- + +const powWorkerPath = "__CROWDSEC_POW_WORKER_PATH__"; + +function solvePoWAsync(prefix, difficulty) { + if (difficulty <= 0) { + return Promise.resolve("0"); + } + + try { + const worker = new Worker(powWorkerPath); + + return new Promise((resolve) => { + worker.onmessage = (e) => { + worker.terminate(); + resolve(e.data); + }; + worker.onerror = () => { + worker.terminate(); + resolve(solvePoWMainThread(prefix, difficulty)); + }; + worker.postMessage({ p: prefix, d: difficulty }); + }); + } catch (_) { + return Promise.resolve(solvePoWMainThread(prefix, difficulty)); + } +} + +function solvePoWMainThread(prefix, difficulty) { + let nonce = 0; + while (true) { + const candidate = prefix + nonce.toString(36); + const hash = sha256(encode(candidate)); + const fullBytes = difficulty >> 3; + const remainBits = difficulty & 7; + let ok = true; + for (let i = 0; i < fullBytes; i++) { + if (hash[i] !== 0) { + ok = false; + break; + } + } + if (ok && remainBits > 0) { + const mask = (0xff << (8 - remainBits)) & 0xff; + if ((hash[fullBytes] & mask) !== 0) ok = false; + } + if (ok) return nonce.toString(36); + nonce++; + } +} + +// --- Fingerprint encryption (XOR) --- + +function encryptFingerprint(key, fingerprint) { + const keyBytes = encode(key); + const textBytes = encode(fingerprint); + const encrypted = new Uint8Array(textBytes.length); + + for (let i = 0; i < textBytes.length; i++) { + encrypted[i] = textBytes[i] ^ keyBytes[i % keyBytes.length]; + } + + const binaryString = String.fromCharCode(...encrypted); + return btoa(binaryString); +} + +// --- Non-secret per-request values (plain template injection) --- +// +// Set on `globalThis` by a plain