mirror of
https://github.com/crowdsecurity/crowdsec.git
synced 2026-10-12 02:12:31 +00:00
Waf challenge mode (#4268)
This commit is contained in:
@@ -19,9 +19,6 @@ on:
|
||||
paths-ignore:
|
||||
- "README.md"
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- releases/**
|
||||
paths-ignore:
|
||||
- "README.md"
|
||||
|
||||
|
||||
@@ -8,9 +8,6 @@ on:
|
||||
paths-ignore:
|
||||
- "README.md"
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- releases/**
|
||||
paths-ignore:
|
||||
- "README.md"
|
||||
|
||||
|
||||
@@ -11,13 +11,10 @@ on:
|
||||
- master
|
||||
- releases/**
|
||||
paths-ignore:
|
||||
- 'README.md'
|
||||
- "README.md"
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- releases/**
|
||||
paths-ignore:
|
||||
- 'README.md'
|
||||
- "README.md"
|
||||
|
||||
# these env variables are for localstack, so we can emulate aws services
|
||||
env:
|
||||
@@ -36,13 +33,13 @@ jobs:
|
||||
localstack:
|
||||
image: localstack/localstack:3.0
|
||||
ports:
|
||||
- 4566:4566 # Localstack exposes all services on the same port
|
||||
- 4566:4566 # Localstack exposes all services on the same port
|
||||
env:
|
||||
DEBUG: ""
|
||||
LAMBDA_EXECUTOR: ""
|
||||
KINESIS_ERROR_PROBABILITY: ""
|
||||
KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
|
||||
LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
|
||||
LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
|
||||
# e.g sqs url will get localhost if we don't set this env to map our service
|
||||
options: >-
|
||||
--name=localstack
|
||||
@@ -125,95 +122,100 @@ jobs:
|
||||
--health-retries 5
|
||||
--health-start-period 30s
|
||||
steps:
|
||||
- name: Check out CrowdSec repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
|
||||
- name: Check out CrowdSec repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: false
|
||||
- name: "Set up Go"
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: "Set up Go"
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: Run "go generate" and check for changes
|
||||
run: |
|
||||
set -e
|
||||
# ensure the version of 'protoc' matches the one that generated the files
|
||||
PROTOBUF_VERSION="35.0"
|
||||
JAVY_VERSION="8.1.1"
|
||||
# don't pollute the repo
|
||||
pushd $HOME
|
||||
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
|
||||
unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d $HOME/.protoc
|
||||
curl -OL https://github.com/bytecodealliance/javy/releases/download/v${JAVY_VERSION}/javy-x86_64-linux-v${JAVY_VERSION}.gz
|
||||
mkdir -p $HOME/.javy
|
||||
gzip -d javy-x86_64-linux-v${JAVY_VERSION}.gz
|
||||
mv javy-x86_64-linux-v${JAVY_VERSION} $HOME/.javy/javy
|
||||
chmod +x $HOME/.javy/javy
|
||||
popd
|
||||
export PATH="$HOME/.protoc/bin:$HOME/.javy/:$PATH"
|
||||
go install google.golang.org/protobuf/cmd/[email protected]
|
||||
go install google.golang.org/grpc/cmd/[email protected]
|
||||
go generate ./...
|
||||
protoc --version
|
||||
if [[ $(git status --porcelain) ]]; then
|
||||
echo "Error: Uncommitted changes found after running 'go generate'. Please commit all generated code."
|
||||
git diff
|
||||
exit 1
|
||||
else
|
||||
echo "No changes detected after running 'go generate'."
|
||||
fi
|
||||
|
||||
- name: Run "go generate" and check for changes
|
||||
run: |
|
||||
set -e
|
||||
# ensure the version of 'protoc' matches the one that generated the files
|
||||
PROTOBUF_VERSION="21.12"
|
||||
# don't pollute the repo
|
||||
pushd $HOME
|
||||
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
|
||||
unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d $HOME/.protoc
|
||||
popd
|
||||
export PATH="$HOME/.protoc/bin:$PATH"
|
||||
go install google.golang.org/protobuf/cmd/[email protected]
|
||||
go install google.golang.org/grpc/cmd/[email protected]
|
||||
go generate ./...
|
||||
protoc --version
|
||||
if [[ $(git status --porcelain) ]]; then
|
||||
echo "Error: Uncommitted changes found after running 'go generate'. Please commit all generated code."
|
||||
git diff
|
||||
exit 1
|
||||
else
|
||||
echo "No changes detected after running 'go generate'."
|
||||
fi
|
||||
|
||||
- name: Create localstack streams
|
||||
run: |
|
||||
- name: Create localstack streams
|
||||
run: |
|
||||
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
|
||||
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
|
||||
|
||||
- name: Generate codecov configuration
|
||||
run: |
|
||||
- name: Generate codecov configuration
|
||||
run: |
|
||||
.github/generate-codecov-yml.sh > .github/codecov.yml
|
||||
|
||||
- name: Ensure we can do a dynamic build
|
||||
run: |
|
||||
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
|
||||
make build
|
||||
- name: Ensure we can do a dynamic build
|
||||
run: |
|
||||
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
|
||||
make build
|
||||
|
||||
- name: Ensure we can do a static build
|
||||
run: |
|
||||
make clean build BUILD_STATIC=1
|
||||
- name: Ensure we can do a static build
|
||||
run: |
|
||||
make clean build BUILD_STATIC=1
|
||||
|
||||
- name: Unit tests
|
||||
run: |
|
||||
go install gotest.tools/[email protected]
|
||||
# make the repo read-only, with the exception of coverage output
|
||||
touch coverage.out
|
||||
chmod -R a-w .
|
||||
chmod u+w coverage.out
|
||||
make testcover
|
||||
# ignore changes to codecov.yml
|
||||
if [[ $(git status --porcelain -- . ":(exclude).github/codecov.yml") ]]; then
|
||||
echo "Error: Unit tests should not create or alter files inside the repository. Please use the appropriate testing helpers or otherwise temporary locations."
|
||||
git diff --name-only
|
||||
exit 1
|
||||
fi
|
||||
chmod -R u+w .
|
||||
- name: Unit tests
|
||||
run: |
|
||||
go install gotest.tools/[email protected]
|
||||
# make the repo read-only, with the exception of coverage output
|
||||
touch coverage.out
|
||||
chmod -R a-w .
|
||||
chmod u+w coverage.out
|
||||
make testcover
|
||||
# ignore changes to codecov.yml
|
||||
if [[ $(git status --porcelain -- . ":(exclude).github/codecov.yml") ]]; then
|
||||
echo "Error: Unit tests should not create or alter files inside the repository. Please use the appropriate testing helpers or otherwise temporary locations."
|
||||
git diff --name-only
|
||||
exit 1
|
||||
fi
|
||||
chmod -R u+w .
|
||||
|
||||
# check if some component stubs are missing
|
||||
- name: "Build profile: minimal"
|
||||
run: |
|
||||
make build BUILD_PROFILE=minimal
|
||||
# check if some component stubs are missing
|
||||
- name: "Build profile: minimal"
|
||||
run: |
|
||||
make build BUILD_PROFILE=minimal
|
||||
|
||||
# check that the freebsd build is not broken
|
||||
- name: "Build target: freebsd"
|
||||
run: |
|
||||
make build BUILD_STATIC=1 BUILD_RE2_WASM=1 BUILD_SQLITE=modernc GOOS=freebsd
|
||||
# check that the freebsd build is not broken
|
||||
- name: "Build target: freebsd"
|
||||
run: |
|
||||
make build BUILD_STATIC=1 BUILD_RE2_WASM=1 BUILD_SQLITE=modernc GOOS=freebsd
|
||||
|
||||
- name: Upload unit coverage to Codecov
|
||||
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
|
||||
with:
|
||||
files: coverage.out
|
||||
flags: unit-linux
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
- name: Upload unit coverage to Codecov
|
||||
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
|
||||
with:
|
||||
files: coverage.out
|
||||
flags: unit-linux
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1
|
||||
with:
|
||||
version: v2.10
|
||||
args: --issues-exit-code=1 --timeout 10m
|
||||
only-new-issues: false
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1
|
||||
with:
|
||||
version: v2.10
|
||||
args: --issues-exit-code=1 --timeout 10m
|
||||
only-new-issues: false
|
||||
|
||||
@@ -66,3 +66,7 @@ msi
|
||||
__pycache__
|
||||
*.py[cod]
|
||||
*.egg-info
|
||||
|
||||
# local agent/tooling dirs (not part of the repo)
|
||||
.agents/
|
||||
.claude/
|
||||
|
||||
@@ -10,3 +10,6 @@
|
||||
[submodule "tests/lib/bats-support"]
|
||||
path = test/lib/bats-support
|
||||
url = https://github.com/bats-core/bats-support.git
|
||||
[submodule "pkg/appsec/challenge/js/fpscanner/fpscanner"]
|
||||
path = pkg/appsec/challenge/js/fpscanner/fpscanner
|
||||
url = https://github.com/antoinevastel/fpscanner
|
||||
|
||||
@@ -337,6 +337,10 @@ linters:
|
||||
# A wildcard `*` can use as a struct name (ex: `*.MethodName`).
|
||||
exclusions:
|
||||
- "HCLogAdapter.SetLevel"
|
||||
# Unmarshal methods must have a pointer receiver by nature; getters on
|
||||
# the same type legitimately use value receivers (same rationale as the
|
||||
# built-in Marshal* exclusions, which don't cover Unmarshal*).
|
||||
- "*.UnmarshalJSON"
|
||||
|
||||
revive:
|
||||
severity: error
|
||||
@@ -548,6 +552,14 @@ linters:
|
||||
path: pkg/acquisition/modules/victorialogs/internal/vlclient/vl_client.go
|
||||
text: 'confusing-naming: Method ''QueryRange'' differs only by capitalization to method ''queryRange'' in the same source file'
|
||||
|
||||
# pkg/dnscache lookups are deliberately detached from the request
|
||||
# context: their results are shared across requests (singleflight +
|
||||
# cache), so one caller's cancellation must not abort the lookup for
|
||||
# the others; a fixed timeout bounds them instead.
|
||||
- linters:
|
||||
- contextcheck
|
||||
text: '(lookupPTR|lookupForward)` should pass the context parameter'
|
||||
|
||||
# tolerate complex functions in tests for now
|
||||
- linters:
|
||||
- maintidx
|
||||
|
||||
@@ -320,6 +320,23 @@ cscli: ## Build cscli
|
||||
crowdsec: ## Build crowdsec
|
||||
@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
|
||||
|
||||
# Regenerate the WAF challenge JS artifacts: the bundled fpscanner source
|
||||
# (fpscanner/bundle.js), the obfuscator runtime (obfuscate/index.wasm.gz),
|
||||
# and the build-time pre-obfuscated initial bundle (initial_bundle.js.gz).
|
||||
# These are committed to the tree so a normal `make build` does NOT rebuild
|
||||
# them — only run this target after intentionally changing challenge.js,
|
||||
# fpscanner sources, the obfuscator wrapper, or the dynamic-module template.
|
||||
#
|
||||
# Requires `javy` (https://github.com/bytecodealliance/javy/releases) on
|
||||
# PATH. The full obfuscation pass is slow (~1 minute) because it runs
|
||||
# javascript-obfuscator's "high-obfuscation" preset over the full bundle;
|
||||
# this is the cost we are paying once at generate time so the runtime
|
||||
# doesn't pay it on every startup.
|
||||
.PHONY: generate-challenge-js
|
||||
generate-challenge-js: ## Regenerate WAF challenge JS bundles (requires javy on PATH)
|
||||
@command -v javy >/dev/null 2>&1 || (echo "Error: javy is not installed. Download it from https://github.com/bytecodealliance/javy/releases and put it on PATH." && exit 1)
|
||||
$(GO) generate ./pkg/appsec/challenge/js/...
|
||||
|
||||
testenv:
|
||||
ifeq ($(TEST_LOCAL_ONLY),)
|
||||
@echo 'NOTE: You need to run "make localstack" in a separate shell, "make localstack-stop" to terminate it; or define the envvar TEST_LOCAL_ONLY to some value.'
|
||||
|
||||
@@ -75,7 +75,9 @@ func expandAlias(args []string) []string {
|
||||
case "lapi":
|
||||
ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine")
|
||||
case "appsec":
|
||||
ret = append(ret, "appsec-engine", "appsec-rule")
|
||||
ret = append(ret, "appsec-engine", "appsec-rule", "appsec-challenge")
|
||||
case "bot-detection":
|
||||
ret = append(ret, "appsec-challenge", "appsec-challenge-infra")
|
||||
default:
|
||||
ret = append(ret, section)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package climetrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
|
||||
"github.com/jedib0t/go-pretty/v6/table"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/core/cstable"
|
||||
)
|
||||
|
||||
// statAppsecChallenge holds two views of the bot-detection counters:
|
||||
//
|
||||
// - Funnel: per-engine aggregates rendered as one row per appsec_engine,
|
||||
// with columns for each lifecycle/outcome bucket. Mirrors the shape
|
||||
// statAppsecEngine uses.
|
||||
// - Reasons: per-engine per-kind per-reason breakdown so operators can see
|
||||
// which RejectSubmission / GrantChallengeCookie strings (and which
|
||||
// bounded protocol/cookie reasons) are firing without dropping into
|
||||
// Prometheus. Rendered as a second table below the funnel.
|
||||
type statAppsecChallenge struct {
|
||||
Funnel map[string]map[string]int `json:"funnel"`
|
||||
Reasons map[string]map[string]map[string]int `json:"reasons"` // engine -> kind -> reason -> count
|
||||
}
|
||||
|
||||
func newStatAppsecChallenge() *statAppsecChallenge {
|
||||
return &statAppsecChallenge{
|
||||
Funnel: map[string]map[string]int{},
|
||||
Reasons: map[string]map[string]map[string]int{},
|
||||
}
|
||||
}
|
||||
|
||||
func (*statAppsecChallenge) Description() (string, string) {
|
||||
return "Bot Detection Metrics",
|
||||
`Measures the challenge lifecycle of the AppSec component.`
|
||||
}
|
||||
|
||||
func (s *statAppsecChallenge) Process(appsecEngine, metric string, val int) {
|
||||
if _, ok := s.Funnel[appsecEngine]; !ok {
|
||||
s.Funnel[appsecEngine] = make(map[string]int)
|
||||
}
|
||||
|
||||
s.Funnel[appsecEngine][metric] += val
|
||||
}
|
||||
|
||||
// ProcessReason records a per-kind/reason datapoint. Empty reasons are
|
||||
// dropped.
|
||||
func (s *statAppsecChallenge) ProcessReason(appsecEngine, kind, reason string, val int) {
|
||||
if reason == "" {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := s.Reasons[appsecEngine]; !ok {
|
||||
s.Reasons[appsecEngine] = map[string]map[string]int{}
|
||||
}
|
||||
|
||||
if _, ok := s.Reasons[appsecEngine][kind]; !ok {
|
||||
s.Reasons[appsecEngine][kind] = map[string]int{}
|
||||
}
|
||||
|
||||
s.Reasons[appsecEngine][kind][reason] += val
|
||||
}
|
||||
|
||||
func (s *statAppsecChallenge) Table(out io.Writer, wantColor string, noUnit bool, showEmpty bool) {
|
||||
t := cstable.New(out, wantColor).Writer
|
||||
t.AppendHeader(table.Row{"Bot Detection", "Requested", "Submitted", "Solved", "Granted", "Protocol Failures", "Submissions Rejected", "Cookies Invalid"})
|
||||
|
||||
keys := []string{"requested", "submitted", "solved", "granted", "rejected_protocol", "rejected_submission", "rejected_cookie"}
|
||||
|
||||
numRows, err := metricsToTable(t, s.Funnel, keys, noUnit)
|
||||
if err != nil {
|
||||
log.Warningf("while collecting appsec challenge stats: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
if numRows == 0 && !showEmpty {
|
||||
return
|
||||
}
|
||||
|
||||
title, _ := s.Description()
|
||||
t.SetTitle(title)
|
||||
fmt.Fprintln(out, t.Render())
|
||||
|
||||
// Detail tables: per-engine breakdown of the user-supplied
|
||||
// (GrantChallengeCookie / RejectSubmission) and bounded-vocabulary
|
||||
// (protocol / cookie) reasons. One table per outcome (accepted /
|
||||
// rejected) so the kind dimension only needs to show up where it
|
||||
// actually varies.
|
||||
s.renderAccepted(out, wantColor, noUnit)
|
||||
s.renderRejected(out, wantColor, noUnit)
|
||||
}
|
||||
|
||||
// rejectedKindDisplay maps a rejected sub-kind to the label shown in the
|
||||
// rejected breakdown table.
|
||||
var rejectedKindDisplay = map[string]string{
|
||||
"protocol": "protocol",
|
||||
"submission": "submission",
|
||||
"cookie": "cookie",
|
||||
}
|
||||
|
||||
// rejectedKindOrder fixes the row order so the rendered table is stable
|
||||
// across runs and groups related kinds together.
|
||||
var rejectedKindOrder = []string{"protocol", "submission", "cookie"}
|
||||
|
||||
// renderAccepted renders the per-engine breakdown of GrantChallengeCookie
|
||||
// reasons. Only kind="granted" carries a reason (kind="solved" has none by
|
||||
// design), so the Kind column is dropped here and the table footer carries
|
||||
// a grand total across all engines and reasons.
|
||||
func (s *statAppsecChallenge) renderAccepted(out io.Writer, wantColor string, noUnit bool) {
|
||||
engines := make([]string, 0, len(s.Reasons))
|
||||
for e, byKind := range s.Reasons {
|
||||
if _, ok := byKind["granted"]; ok {
|
||||
engines = append(engines, e)
|
||||
}
|
||||
}
|
||||
if len(engines) == 0 {
|
||||
return
|
||||
}
|
||||
sort.Strings(engines)
|
||||
|
||||
t := cstable.New(out, wantColor).Writer
|
||||
t.AppendHeader(table.Row{"Appsec Engine", "Reason", "Count"})
|
||||
|
||||
total := int64(0)
|
||||
for _, engine := range engines {
|
||||
counts := s.Reasons[engine]["granted"]
|
||||
reasons := make([]string, 0, len(counts))
|
||||
for r := range counts {
|
||||
reasons = append(reasons, r)
|
||||
}
|
||||
sort.Strings(reasons)
|
||||
for _, r := range reasons {
|
||||
t.AppendRow(table.Row{engine, r, formatNumber(int64(counts[r]), !noUnit)})
|
||||
total += int64(counts[r])
|
||||
}
|
||||
}
|
||||
|
||||
t.AppendFooter(table.Row{"Total", "", formatNumber(total, !noUnit)})
|
||||
t.SetTitle("Bot Detection — Accepted")
|
||||
fmt.Fprintln(out, t.Render())
|
||||
}
|
||||
|
||||
// renderRejected renders the per-engine per-kind per-reason breakdown of
|
||||
// rejected outcomes. Kind matters here (protocol vs submission vs cookie),
|
||||
// and the table footer carries a grand total across all rows.
|
||||
func (s *statAppsecChallenge) renderRejected(out io.Writer, wantColor string, noUnit bool) {
|
||||
engines := make([]string, 0, len(s.Reasons))
|
||||
for e, byKind := range s.Reasons {
|
||||
for _, kind := range rejectedKindOrder {
|
||||
if _, ok := byKind[kind]; ok {
|
||||
engines = append(engines, e)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(engines) == 0 {
|
||||
return
|
||||
}
|
||||
sort.Strings(engines)
|
||||
|
||||
t := cstable.New(out, wantColor).Writer
|
||||
t.AppendHeader(table.Row{"Appsec Engine", "Kind", "Reason", "Count"})
|
||||
|
||||
total := int64(0)
|
||||
for _, engine := range engines {
|
||||
byKind := s.Reasons[engine]
|
||||
for _, kind := range rejectedKindOrder {
|
||||
counts, ok := byKind[kind]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
reasons := make([]string, 0, len(counts))
|
||||
for r := range counts {
|
||||
reasons = append(reasons, r)
|
||||
}
|
||||
sort.Strings(reasons)
|
||||
for _, r := range reasons {
|
||||
t.AppendRow(table.Row{engine, rejectedKindDisplay[kind], r, formatNumber(int64(counts[r]), !noUnit)})
|
||||
total += int64(counts[r])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.AppendFooter(table.Row{"Total", "", "", formatNumber(total, !noUnit)})
|
||||
t.SetTitle("Bot Detection — Rejected")
|
||||
fmt.Fprintln(out, t.Render())
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package climetrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/jedib0t/go-pretty/v6/table"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/core/cstable"
|
||||
)
|
||||
|
||||
// statAppsecChallengeInfra holds the process-global housekeeping counters of
|
||||
// the bot-detection challenge runtime (key rotation, obfuscation, cache
|
||||
// eviction). Unlike the lifecycle funnel these carry no appsec_engine label
|
||||
// — the challenge runtime is a single per-process instance shared by every
|
||||
// engine — so this is a flat metric -> count map rather than per-engine.
|
||||
type statAppsecChallengeInfra map[string]int
|
||||
|
||||
func (statAppsecChallengeInfra) Description() (string, string) {
|
||||
return "Bot Detection Infrastructure Metrics",
|
||||
`Tracks the internal upkeep of the AppSec challenge runtime: signing-key rotation, JS re-obfuscation and cache eviction.`
|
||||
}
|
||||
|
||||
func (s statAppsecChallengeInfra) Process(metric string, val int) {
|
||||
s[metric] += val
|
||||
}
|
||||
|
||||
// infraRow pairs a metric key with the label shown in the table, in a fixed
|
||||
// order so the rendered output is stable across runs.
|
||||
var infraRows = []struct {
|
||||
key string
|
||||
label string
|
||||
}{
|
||||
{"kepoch_generated", "Signing key regenerated"},
|
||||
{"kepoch_evicted", "Signing key evicted"},
|
||||
{"reobfuscation_dynamic", "Re-obfuscation (dynamic module)"},
|
||||
{"reobfuscation_library", "Re-obfuscation (library bundle)"},
|
||||
{"dynamic_module_evicted", "Dynamic module evicted"},
|
||||
}
|
||||
|
||||
func (s statAppsecChallengeInfra) Table(out io.Writer, wantColor string, noUnit bool, showEmpty bool) {
|
||||
t := cstable.New(out, wantColor).Writer
|
||||
t.AppendHeader(table.Row{"Metric", "Count"})
|
||||
|
||||
numRows := 0
|
||||
for _, r := range infraRows {
|
||||
val, ok := s[r.key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
t.AppendRow(table.Row{r.label, formatNumber(int64(val), !noUnit)})
|
||||
numRows++
|
||||
}
|
||||
|
||||
if numRows == 0 && !showEmpty {
|
||||
return
|
||||
}
|
||||
|
||||
title, _ := s.Description()
|
||||
t.SetTitle(title)
|
||||
fmt.Fprintln(out, t.Render())
|
||||
}
|
||||
@@ -27,9 +27,9 @@ func (s statAppsecEngine) Process(appsecEngine, metric string, val int) {
|
||||
|
||||
func (s statAppsecEngine) Table(out io.Writer, wantColor string, noUnit bool, showEmpty bool) {
|
||||
t := cstable.New(out, wantColor).Writer
|
||||
t.AppendHeader(table.Row{"Appsec Engine", "Processed", "Blocked"})
|
||||
t.AppendHeader(table.Row{"Appsec Engine", "Processed", "Blocked", "Ch. Requested", "Ch. Accepted", "Ch. Rejected"})
|
||||
|
||||
keys := []string{"processed", "blocked"}
|
||||
keys := []string{"processed", "blocked", "challenge_requested", "challenge_accepted", "challenge_rejected"}
|
||||
|
||||
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
|
||||
log.Warningf("while collecting appsec stats: %s", err)
|
||||
|
||||
@@ -24,20 +24,22 @@ type metricStore map[string]metricSection
|
||||
|
||||
func NewMetricStore() metricStore {
|
||||
return metricStore{
|
||||
"acquisition": statAcquis{},
|
||||
"alerts": statAlert{},
|
||||
"bouncers": &statBouncer{},
|
||||
"appsec-engine": statAppsecEngine{},
|
||||
"appsec-rule": statAppsecRule{},
|
||||
"decisions": statDecision{},
|
||||
"lapi": statLapi{},
|
||||
"lapi-bouncer": statLapiBouncer{},
|
||||
"lapi-decisions": statLapiDecision{},
|
||||
"lapi-machine": statLapiMachine{},
|
||||
"parsers": statParser{},
|
||||
"scenarios": statBucket{},
|
||||
"stash": statStash{},
|
||||
"whitelists": statWhitelist{},
|
||||
"acquisition": statAcquis{},
|
||||
"alerts": statAlert{},
|
||||
"bouncers": &statBouncer{},
|
||||
"appsec-engine": statAppsecEngine{},
|
||||
"appsec-rule": statAppsecRule{},
|
||||
"appsec-challenge": newStatAppsecChallenge(),
|
||||
"appsec-challenge-infra": statAppsecChallengeInfra{},
|
||||
"decisions": statDecision{},
|
||||
"lapi": statLapi{},
|
||||
"lapi-bouncer": statLapiBouncer{},
|
||||
"lapi-decisions": statLapiDecision{},
|
||||
"lapi-machine": statLapiMachine{},
|
||||
"parsers": statParser{},
|
||||
"scenarios": statBucket{},
|
||||
"stash": statStash{},
|
||||
"whitelists": statWhitelist{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +59,59 @@ func (ms metricStore) Fetch(ctx context.Context, url string, db *database.Client
|
||||
return nil
|
||||
}
|
||||
|
||||
// metricLabels holds the label values extracted from a single Prometheus point.
|
||||
type metricLabels struct {
|
||||
name string
|
||||
source string
|
||||
machine string
|
||||
bouncer string
|
||||
route string
|
||||
method string
|
||||
reason string
|
||||
origin string
|
||||
action string
|
||||
appsecEngine string
|
||||
appsecRule string
|
||||
// kind is carried by the appsec-challenge accepted/rejected counters
|
||||
// to distinguish sub-outcomes; empty for every other metric.
|
||||
kind string
|
||||
// bundle is carried by the challenge re-obfuscation counter
|
||||
// (dynamic vs library); empty for every other metric.
|
||||
bundle string
|
||||
mtype string
|
||||
}
|
||||
|
||||
func extractLabels(p MetricPoint) metricLabels {
|
||||
name, ok := p.Labels["name"]
|
||||
if !ok {
|
||||
log.Debugf("no name in Metric %v", p.Labels)
|
||||
}
|
||||
|
||||
source, ok := p.Labels["source"]
|
||||
if !ok {
|
||||
log.Debugf("no source in Metric %v for %s", p.Labels, p.Name)
|
||||
} else if srctype, ok := p.Labels["type"]; ok {
|
||||
source = srctype + ":" + source
|
||||
}
|
||||
|
||||
return metricLabels{
|
||||
name: name,
|
||||
source: source,
|
||||
machine: p.Labels["machine"],
|
||||
bouncer: p.Labels["bouncer"],
|
||||
route: p.Labels["route"],
|
||||
method: p.Labels["method"],
|
||||
reason: p.Labels["reason"],
|
||||
origin: p.Labels["origin"],
|
||||
action: p.Labels["action"],
|
||||
appsecEngine: p.Labels["appsec_engine"],
|
||||
appsecRule: p.Labels["rule_name"],
|
||||
kind: p.Labels["kind"],
|
||||
bundle: p.Labels["bundle"],
|
||||
mtype: p.Labels["type"],
|
||||
}
|
||||
}
|
||||
|
||||
func (ms metricStore) processPrometheusMetrics(result []MetricPoint) {
|
||||
mAcquis := ms["acquisition"].(statAcquis)
|
||||
mAlert := ms["alerts"].(statAlert)
|
||||
@@ -77,115 +132,157 @@ func (ms metricStore) processPrometheusMetrics(result []MetricPoint) {
|
||||
continue
|
||||
}
|
||||
|
||||
name, ok := p.Labels["name"]
|
||||
if !ok {
|
||||
log.Debugf("no name in Metric %v", p.Labels)
|
||||
}
|
||||
l := extractLabels(p)
|
||||
|
||||
source, ok := p.Labels["source"]
|
||||
if !ok {
|
||||
log.Debugf("no source in Metric %v for %s", p.Labels, p.Name)
|
||||
} else {
|
||||
if srctype, ok := p.Labels["type"]; ok {
|
||||
source = srctype + ":" + source
|
||||
}
|
||||
}
|
||||
ival := int(p.Value)
|
||||
|
||||
machine := p.Labels["machine"]
|
||||
bouncer := p.Labels["bouncer"]
|
||||
|
||||
route := p.Labels["route"]
|
||||
method := p.Labels["method"]
|
||||
|
||||
reason := p.Labels["reason"]
|
||||
origin := p.Labels["origin"]
|
||||
action := p.Labels["action"]
|
||||
|
||||
appsecEngine := p.Labels["appsec_engine"]
|
||||
appsecRule := p.Labels["rule_name"]
|
||||
|
||||
mtype := p.Labels["type"]
|
||||
|
||||
ival := int(p.Value)
|
||||
|
||||
switch p.Name {
|
||||
switch p.Name {
|
||||
//
|
||||
// buckets
|
||||
//
|
||||
case metrics.BucketsInstantiationMetricName:
|
||||
mBucket.Process(name, "instantiation", ival)
|
||||
mBucket.Process(l.name, "instantiation", ival)
|
||||
case metrics.BucketsCurrentCountMetricName:
|
||||
mBucket.Process(name, "curr_count", ival)
|
||||
mBucket.Process(l.name, "curr_count", ival)
|
||||
case metrics.BucketsOverflowMetricName:
|
||||
mBucket.Process(name, "overflow", ival)
|
||||
mBucket.Process(l.name, "overflow", ival)
|
||||
case metrics.BucketPouredMetricName:
|
||||
mBucket.Process(name, "pour", ival)
|
||||
mAcquis.Process(source, "pour", ival)
|
||||
mBucket.Process(l.name, "pour", ival)
|
||||
mAcquis.Process(l.source, "pour", ival)
|
||||
case metrics.BucketsUnderflowMetricName:
|
||||
mBucket.Process(name, "underflow", ival)
|
||||
mBucket.Process(l.name, "underflow", ival)
|
||||
//
|
||||
// parsers
|
||||
//
|
||||
case metrics.GlobalParserHitsMetricName:
|
||||
mAcquis.Process(source, "reads", ival)
|
||||
mAcquis.Process(l.source, "reads", ival)
|
||||
case metrics.GlobalParserHitsOkMetricName:
|
||||
mAcquis.Process(source, "parsed", ival)
|
||||
mAcquis.Process(l.source, "parsed", ival)
|
||||
case metrics.GlobalParserHitsKoMetricName:
|
||||
mAcquis.Process(source, "unparsed", ival)
|
||||
mAcquis.Process(l.source, "unparsed", ival)
|
||||
case metrics.NodesHitsMetricName:
|
||||
mParser.Process(name, "hits", ival)
|
||||
mParser.Process(l.name, "hits", ival)
|
||||
case metrics.NodesHitsOkMetricName:
|
||||
mParser.Process(name, "parsed", ival)
|
||||
mParser.Process(l.name, "parsed", ival)
|
||||
case metrics.NodesHitsKoMetricName:
|
||||
mParser.Process(name, "unparsed", ival)
|
||||
mParser.Process(l.name, "unparsed", ival)
|
||||
//
|
||||
// whitelists
|
||||
//
|
||||
case metrics.NodesWlHitsMetricName:
|
||||
mWhitelist.Process(name, reason, "hits", ival)
|
||||
mWhitelist.Process(l.name, l.reason, "hits", ival)
|
||||
case metrics.NodesWlHitsOkMetricName:
|
||||
mWhitelist.Process(name, reason, "whitelisted", ival)
|
||||
mWhitelist.Process(l.name, l.reason, "whitelisted", ival)
|
||||
// track as well whitelisted lines at acquis level
|
||||
mAcquis.Process(source, "whitelisted", ival)
|
||||
mAcquis.Process(l.source, "whitelisted", ival)
|
||||
//
|
||||
// lapi
|
||||
//
|
||||
case metrics.LapiRouteHitsMetricName:
|
||||
mLapi.Process(route, method, ival)
|
||||
mLapi.Process(l.route, l.method, ival)
|
||||
case metrics.LapiMachineHitsMetricName:
|
||||
mLapiMachine.Process(machine, route, method, ival)
|
||||
mLapiMachine.Process(l.machine, l.route, l.method, ival)
|
||||
case metrics.LapiBouncerHitsMetricName:
|
||||
mLapiBouncer.Process(bouncer, route, method, ival)
|
||||
mLapiBouncer.Process(l.bouncer, l.route, l.method, ival)
|
||||
case metrics.LapiNilDecisionsMetricName, metrics.LapiNonNilDecisionsMetricName:
|
||||
mLapiDecision.Process(bouncer, p.Name, ival)
|
||||
mLapiDecision.Process(l.bouncer, p.Name, ival)
|
||||
//
|
||||
// decisions
|
||||
//
|
||||
case metrics.GlobalActiveDecisionsMetricName:
|
||||
mDecision.Process(reason, origin, action, ival)
|
||||
mDecision.Process(l.reason, l.origin, l.action, ival)
|
||||
case metrics.GlobalAlertsMetricName:
|
||||
mAlert.Process(reason, ival)
|
||||
mAlert.Process(l.reason, ival)
|
||||
//
|
||||
// stash
|
||||
//
|
||||
case metrics.CacheMetricName:
|
||||
mStash.Process(name, mtype, ival)
|
||||
mStash.Process(l.name, l.mtype, ival)
|
||||
//
|
||||
// appsec
|
||||
//
|
||||
case "cs_appsec_reqs_total":
|
||||
mAppsecEngine.Process(appsecEngine, "processed", ival)
|
||||
mAppsecEngine.Process(l.appsecEngine, "processed", ival)
|
||||
case "cs_appsec_block_total":
|
||||
mAppsecEngine.Process(appsecEngine, "blocked", ival)
|
||||
mAppsecEngine.Process(l.appsecEngine, "blocked", ival)
|
||||
case "cs_appsec_rule_hits":
|
||||
mAppsecRule.Process(appsecEngine, appsecRule, "triggered", ival)
|
||||
mAppsecRule.Process(l.appsecEngine, l.appsecRule, "triggered", ival)
|
||||
default:
|
||||
// bot detection / challenge metrics live in their own handler to
|
||||
// keep this switch's complexity in check.
|
||||
if ms.processAppsecChallengeMetric(p.Name, l, ival) {
|
||||
continue
|
||||
}
|
||||
|
||||
log.Debugf("unknown: %+v", p.Name)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processAppsecChallengeMetric handles the bot-detection / challenge lifecycle
|
||||
// and infrastructure metrics. It lives apart from processPrometheusMetrics
|
||||
// because its nested kind/bundle switches would otherwise push that function
|
||||
// over the cyclomatic-complexity limit. Returns false for unrecognized metrics.
|
||||
func (ms metricStore) processAppsecChallengeMetric(name string, l metricLabels, ival int) bool {
|
||||
mAppsecEngine := ms["appsec-engine"].(statAppsecEngine)
|
||||
mAppsecChallenge := ms["appsec-challenge"].(*statAppsecChallenge)
|
||||
mAppsecChallengeInfra := ms["appsec-challenge-infra"].(statAppsecChallengeInfra)
|
||||
|
||||
switch name {
|
||||
//
|
||||
// bot detection / challenge lifecycle
|
||||
//
|
||||
case metrics.AppsecChallengeRequestedMetricName:
|
||||
mAppsecEngine.Process(l.appsecEngine, "challenge_requested", ival)
|
||||
mAppsecChallenge.Process(l.appsecEngine, "requested", ival)
|
||||
case metrics.AppsecChallengeSubmittedMetricName:
|
||||
mAppsecChallenge.Process(l.appsecEngine, "submitted", ival)
|
||||
case metrics.AppsecChallengeAcceptedMetricName:
|
||||
mAppsecEngine.Process(l.appsecEngine, "challenge_accepted", ival)
|
||||
switch l.kind {
|
||||
case "solved":
|
||||
mAppsecChallenge.Process(l.appsecEngine, "solved", ival)
|
||||
case "granted":
|
||||
mAppsecChallenge.Process(l.appsecEngine, "granted", ival)
|
||||
mAppsecChallenge.ProcessReason(l.appsecEngine, "granted", l.reason, ival)
|
||||
}
|
||||
case metrics.AppsecChallengeRejectedMetricName:
|
||||
mAppsecEngine.Process(l.appsecEngine, "challenge_rejected", ival)
|
||||
switch l.kind {
|
||||
case "protocol":
|
||||
mAppsecChallenge.Process(l.appsecEngine, "rejected_protocol", ival)
|
||||
mAppsecChallenge.ProcessReason(l.appsecEngine, "protocol", l.reason, ival)
|
||||
case "submission":
|
||||
mAppsecChallenge.Process(l.appsecEngine, "rejected_submission", ival)
|
||||
mAppsecChallenge.ProcessReason(l.appsecEngine, "submission", l.reason, ival)
|
||||
case "cookie":
|
||||
mAppsecChallenge.Process(l.appsecEngine, "rejected_cookie", ival)
|
||||
mAppsecChallenge.ProcessReason(l.appsecEngine, "cookie", l.reason, ival)
|
||||
}
|
||||
//
|
||||
// bot detection / challenge infrastructure (process-global, no engine label)
|
||||
//
|
||||
case metrics.AppsecChallengeKepochGeneratedMetricName:
|
||||
mAppsecChallengeInfra.Process("kepoch_generated", ival)
|
||||
case metrics.AppsecChallengeKepochEvictedMetricName:
|
||||
mAppsecChallengeInfra.Process("kepoch_evicted", ival)
|
||||
case metrics.AppsecChallengeReobfuscationMetricName:
|
||||
switch l.bundle {
|
||||
case "dynamic":
|
||||
mAppsecChallengeInfra.Process("reobfuscation_dynamic", ival)
|
||||
case "library":
|
||||
mAppsecChallengeInfra.Process("reobfuscation_library", ival)
|
||||
}
|
||||
case metrics.AppsecChallengeDynamicModuleEvictedMetricName:
|
||||
mAppsecChallengeInfra.Process("dynamic_module_evicted", ival)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (ms metricStore) Format(out io.Writer, wantColor string, sections []string, outputFormat string, noUnit bool) error {
|
||||
// copy only the sections we want
|
||||
want := map[string]metricSection{}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/dnscache"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/leakybucket"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/metrics"
|
||||
@@ -31,6 +32,8 @@ func initCrowdsec(ctx context.Context, cConfig *csconfig.Config, hub *cwhub.Hub,
|
||||
return nil, nil, fmt.Errorf("while loading context: %w", err)
|
||||
}
|
||||
|
||||
configureDNSCache(cConfig.Crowdsec.DNSCache)
|
||||
|
||||
err = exprhelpers.GeoIPInit(hub.GetDataDir())
|
||||
if err != nil {
|
||||
// GeoIP databases are not mandatory, do not make crowdsec fail if they are not present
|
||||
@@ -70,6 +73,31 @@ func initCrowdsec(ctx context.Context, cConfig *csconfig.Config, hub *cwhub.Hub,
|
||||
return csParsers, datasources, nil
|
||||
}
|
||||
|
||||
func configureDNSCache(cfg *csconfig.DNSCacheCfg) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
ttl, negTTL time.Duration
|
||||
size int
|
||||
)
|
||||
|
||||
if cfg.TTL != nil {
|
||||
ttl = *cfg.TTL
|
||||
}
|
||||
|
||||
if cfg.NegativeTTL != nil {
|
||||
negTTL = *cfg.NegativeTTL
|
||||
}
|
||||
|
||||
if cfg.Size != nil {
|
||||
size = *cfg.Size
|
||||
}
|
||||
|
||||
dnscache.Configure(ttl, negTTL, size)
|
||||
}
|
||||
|
||||
func startParserRoutines(ctx context.Context, g *errgroup.Group, cConfig *csconfig.Config, parsers *parser.Parsers, stageCollector *parser.StageParseCollector) {
|
||||
for idx := range cConfig.Crowdsec.ParserRoutinesCount {
|
||||
log.WithField("idx", idx).Info("Starting parser routine")
|
||||
|
||||
@@ -31,6 +31,7 @@ require (
|
||||
github.com/crowdsecurity/grokky v0.2.2
|
||||
github.com/crowdsecurity/machineid v1.0.3
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
|
||||
github.com/evanw/esbuild v0.27.2
|
||||
github.com/expr-lang/expr v1.17.8
|
||||
github.com/fatih/color v1.19.0
|
||||
github.com/fsnotify/fsnotify v1.10.1
|
||||
@@ -80,6 +81,7 @@ require (
|
||||
github.com/slack-go/slack v0.23.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tetratelabs/wazero v1.11.0
|
||||
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
|
||||
github.com/wasilibs/go-re2 v1.11.0
|
||||
github.com/xhit/go-simple-mail/v2 v2.16.0
|
||||
@@ -191,6 +193,7 @@ require (
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
|
||||
github.com/magefile/mage v1.17.1 // indirect
|
||||
@@ -226,7 +229,6 @@ require (
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/tetratelabs/wazero v1.11.0 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.2.0 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
|
||||
@@ -157,6 +157,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/evanw/esbuild v0.27.2 h1:3xBEws9y/JosfewXMM2qIyHAi+xRo8hVx475hVkJfNg=
|
||||
github.com/evanw/esbuild v0.27.2/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48=
|
||||
github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM=
|
||||
github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
@@ -734,6 +736,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -13,6 +15,8 @@ import (
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
|
||||
)
|
||||
|
||||
@@ -1348,6 +1352,275 @@ func TestAppsecPhaseScopedHooks(t *testing.T) {
|
||||
runTests(t, tests)
|
||||
}
|
||||
|
||||
func TestAppsecOnChallengeHooks(t *testing.T) {
|
||||
powWorkerURL, err := url.Parse(challenge.ChallengePowWorkerPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []appsecRuleTest{
|
||||
{
|
||||
name: "post_eval issues a challenge when no cookie is present",
|
||||
expected_load_ok: true,
|
||||
post_eval: []appsec.Hook{
|
||||
{Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
RemoteAddr: "1.2.3.4",
|
||||
Method: "GET",
|
||||
URI: "/protected",
|
||||
HTTPRequest: &http.Request{Host: "example.com"},
|
||||
},
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.Equal(t, appsec.ChallengeRemediation, responses[0].Action)
|
||||
require.NotEmpty(t, responses[0].UserHTTPBodyContent)
|
||||
require.Contains(t, responses[0].UserHeaders["Content-Type"], "text/html")
|
||||
// Issuing the challenge page must emit a single "requested" event
|
||||
// on a source distinct from the WAF one.
|
||||
require.Len(t, events, 1)
|
||||
require.Equal(t, pipeline.LOG, events[0].Type)
|
||||
require.Equal(t, appsec.SourceChallenge, events[0].Parsed["source"])
|
||||
require.Equal(t, string(appsec.ChallengeReasonRequested), events[0].Parsed["challenge_event"])
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "on_challenge: no cookie → user hooks skipped (filter would nil-deref)",
|
||||
expected_load_ok: true,
|
||||
// This filter would nil-deref if it ran without a fingerprint. The
|
||||
// dispatcher must skip it when no cookie is present.
|
||||
on_challenge: []appsec.Hook{
|
||||
{Filter: "fingerprint.Bot.MismatchWebGLInWorker", Apply: []string{"DropRequest('bot')"}},
|
||||
},
|
||||
// Must reference SendChallenge() somewhere to force ChallengeRuntime init.
|
||||
post_eval: []appsec.Hook{
|
||||
{Filter: "false", Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
RemoteAddr: "1.2.3.4",
|
||||
Method: "GET",
|
||||
URI: "/protected",
|
||||
HTTPRequest: &http.Request{Host: "example.com"},
|
||||
},
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.False(t, responses[0].InBandInterrupt, "on_challenge must not fire without a fingerprint")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "on_challenge: PoW worker path served, WAF evaluation skipped",
|
||||
expected_load_ok: true,
|
||||
inband_rules: []appsec_rule.CustomRule{
|
||||
{
|
||||
Name: "rule1",
|
||||
Zones: []string{"ARGS"},
|
||||
Variables: []string{"foo"},
|
||||
Match: appsec_rule.Match{Type: "regex", Value: "^toto"},
|
||||
Transform: []string{"lowercase"},
|
||||
},
|
||||
},
|
||||
// Force the challenge runtime to be initialized by referencing SendChallenge()
|
||||
// in a hook that won't match this request.
|
||||
on_challenge: []appsec.Hook{
|
||||
{Filter: "false", Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
RemoteAddr: "1.2.3.4",
|
||||
Method: "GET",
|
||||
URI: challenge.ChallengePowWorkerPath,
|
||||
Args: url.Values{"foo": []string{"toto"}}, // would normally trigger rule1
|
||||
HTTPRequest: &http.Request{Host: "example.com", URL: powWorkerURL},
|
||||
},
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Empty(t, events, "WAF should not have evaluated the infrastructure path")
|
||||
require.Len(t, responses, 1)
|
||||
require.Equal(t, appsec.ChallengeRemediation, responses[0].Action)
|
||||
require.Equal(t, challenge.PowWorkerJS, responses[0].UserHTTPBodyContent)
|
||||
require.Contains(t, responses[0].UserHeaders["Content-Type"], "application/javascript")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "on_challenge: invalid submission returns failed body, no hooks run",
|
||||
expected_load_ok: true,
|
||||
// Unconditional DropRequest in on_challenge must NOT fire on an
|
||||
// invalid submission — the dispatcher returns the failed JSON
|
||||
// body and skips user hooks.
|
||||
on_challenge: []appsec.Hook{
|
||||
{Apply: []string{"DropRequest('should not fire')"}},
|
||||
},
|
||||
// Force ChallengeRuntime init.
|
||||
post_eval: []appsec.Hook{
|
||||
{Filter: "false", Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
RemoteAddr: "1.2.3.4",
|
||||
Method: "POST",
|
||||
URI: challenge.ChallengeSubmitPath,
|
||||
HTTPRequest: func() *http.Request {
|
||||
u, _ := url.Parse(challenge.ChallengeSubmitPath)
|
||||
return &http.Request{Host: "example.com", URL: u, Method: http.MethodPost}
|
||||
}(),
|
||||
},
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.Equal(t, appsec.ChallengeRemediation, responses[0].Action)
|
||||
require.JSONEq(t, `{"status":"failed"}`, responses[0].UserHTTPBodyContent)
|
||||
require.False(t, responses[0].InBandInterrupt, "on_challenge hooks must not run on invalid submission")
|
||||
// A submission attempt emits "submitted", then "failed" (with a reason).
|
||||
require.Len(t, events, 2)
|
||||
require.Equal(t, appsec.SourceChallenge, events[0].Parsed["source"])
|
||||
require.Equal(t, string(appsec.ChallengeReasonSubmitted), events[0].Parsed["challenge_event"])
|
||||
require.Equal(t, string(appsec.ChallengeReasonFailed), events[1].Parsed["challenge_event"])
|
||||
require.NotEmpty(t, events[1].Parsed["challenge_fail_reason"])
|
||||
},
|
||||
},
|
||||
{
|
||||
// Mock LAPI (see testAppSecEngine in appsec_test.go) exposes 5.4.3.2
|
||||
// as an allowlisted IP. SendChallenge() must be a no-op for it: no
|
||||
// challenge HTML, no Set-Cookie, response stays at the default pass
|
||||
// action so the request goes through cleanly.
|
||||
name: "allowlisted IP: post_eval SendChallenge() is suppressed",
|
||||
expected_load_ok: true,
|
||||
post_eval: []appsec.Hook{
|
||||
{Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
ClientIP: "5.4.3.2",
|
||||
RemoteAddr: "5.4.3.2",
|
||||
Method: "GET",
|
||||
URI: "/protected",
|
||||
HTTPRequest: &http.Request{Host: "example.com"},
|
||||
},
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action,
|
||||
"allowlisted IP must not be challenged")
|
||||
require.Equal(t, appsec.AllowRemediation, appsecResponse.Action)
|
||||
require.Empty(t, responses[0].UserHTTPBodyContent, "no challenge HTML must be served")
|
||||
require.Empty(t, responses[0].UserHTTPCookies, "no challenge cookie must be issued")
|
||||
require.False(t, responses[0].InBandInterrupt)
|
||||
},
|
||||
},
|
||||
{
|
||||
// Same allowlisted IP, but inside a /24 CIDR entry (5.4.4.0/24) —
|
||||
// confirms range matches are honored the same way as exact IPs.
|
||||
name: "allowlisted CIDR: post_eval SendChallenge() is suppressed",
|
||||
expected_load_ok: true,
|
||||
post_eval: []appsec.Hook{
|
||||
{Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
ClientIP: "5.4.4.42",
|
||||
RemoteAddr: "5.4.4.42",
|
||||
Method: "GET",
|
||||
URI: "/protected",
|
||||
HTTPRequest: &http.Request{Host: "example.com"},
|
||||
},
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action)
|
||||
require.Equal(t, appsec.AllowRemediation, appsecResponse.Action)
|
||||
require.Empty(t, responses[0].UserHTTPBodyContent)
|
||||
require.Empty(t, responses[0].UserHTTPCookies)
|
||||
},
|
||||
},
|
||||
{
|
||||
// Allowlisted IPs hitting infrastructure paths must not get the PoW
|
||||
// worker JS served either — ProcessOnChallengeRules short-circuits
|
||||
// before the path-based branch, so the request flows through normal
|
||||
// WAF processing.
|
||||
name: "allowlisted IP: PoW worker path is not served",
|
||||
expected_load_ok: true,
|
||||
// Reference SendChallenge() so ChallengeRuntime initializes.
|
||||
on_challenge: []appsec.Hook{
|
||||
{Filter: "false", Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
ClientIP: "5.4.3.2",
|
||||
RemoteAddr: "5.4.3.2",
|
||||
Method: "GET",
|
||||
URI: challenge.ChallengePowWorkerPath,
|
||||
HTTPRequest: &http.Request{Host: "example.com", URL: powWorkerURL},
|
||||
},
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action,
|
||||
"allowlisted IP must not receive the PoW worker JS")
|
||||
require.NotEqual(t, challenge.PowWorkerJS, responses[0].UserHTTPBodyContent)
|
||||
require.Equal(t, appsec.AllowRemediation, appsecResponse.Action)
|
||||
},
|
||||
},
|
||||
{
|
||||
// SendChallenge is restricted to on_challenge + in-band post_eval
|
||||
name: "pre_eval SendChallenge() fails to load",
|
||||
expected_load_ok: false,
|
||||
pre_eval: []appsec.Hook{
|
||||
{Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
RemoteAddr: "1.2.3.4",
|
||||
Method: "GET",
|
||||
URI: "/protected",
|
||||
HTTPRequest: &http.Request{Host: "example.com"},
|
||||
},
|
||||
},
|
||||
{
|
||||
// SendChallenge in an out-of-band post_eval hook is rejected at runtime
|
||||
name: "outofband post_eval SendChallenge() is rejected",
|
||||
expected_load_ok: true,
|
||||
outofband_post_eval: []appsec.Hook{
|
||||
{Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
RemoteAddr: "1.2.3.4",
|
||||
Method: "GET",
|
||||
URI: "/protected",
|
||||
HTTPRequest: &http.Request{Host: "example.com"},
|
||||
},
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action,
|
||||
"out-of-band SendChallenge must not issue a challenge")
|
||||
require.Empty(t, responses[0].UserHTTPBodyContent, "no challenge HTML must be served")
|
||||
},
|
||||
},
|
||||
{
|
||||
// A challenged request is served the challenge and never reaches the
|
||||
// backend;
|
||||
name: "challenge skips out-of-band evaluation",
|
||||
expected_load_ok: true,
|
||||
post_eval: []appsec.Hook{
|
||||
{Apply: []string{"SendChallenge()"}},
|
||||
},
|
||||
outofband_rules: []appsec_rule.CustomRule{
|
||||
{
|
||||
Name: "oob-rule",
|
||||
Zones: []string{"ARGS"},
|
||||
Variables: []string{"foo"},
|
||||
Match: appsec_rule.Match{Type: "regex", Value: "^toto"},
|
||||
Transform: []string{"lowercase"},
|
||||
},
|
||||
},
|
||||
input_request: appsec.ParsedRequest{
|
||||
RemoteAddr: "1.2.3.4",
|
||||
Method: "GET",
|
||||
URI: "/protected",
|
||||
Args: url.Values{"foo": []string{"toto"}}, // would match oob-rule
|
||||
HTTPRequest: &http.Request{Host: "example.com"},
|
||||
},
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.Equal(t, appsec.ChallengeRemediation, responses[0].Action)
|
||||
// Only the challenge "requested" event must be present; the OOB rule
|
||||
// was skipped, so no appsec match event is emitted.
|
||||
require.Len(t, events, 1)
|
||||
require.Equal(t, appsec.SourceChallenge, events[0].Parsed["source"])
|
||||
require.Equal(t, string(appsec.ChallengeReasonRequested), events[0].Parsed["challenge_event"])
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runTests(t, tests)
|
||||
}
|
||||
|
||||
// minimal schema used by the hook_vars-in-event tests: /users only accepts
|
||||
// a POST whose body has a required username field.
|
||||
const testUsersSchema = `openapi: 3.0.0
|
||||
@@ -1602,3 +1875,91 @@ func mustParseURL(raw string) *url.URL {
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// TestAppsecLegitimateBotHooks exercises the IsLegitimateBot/ExemptFromChallenge
|
||||
// helpers through the full runner: a bot definition is loaded from a
|
||||
// legit_bots directory (as AppsecConfig.Build does with the hub data dir),
|
||||
// an in-band rule matches the request, and a pre_eval hook downgrades the
|
||||
// remediation for verified bots only.
|
||||
func TestAppsecLegitimateBotHooks(t *testing.T) {
|
||||
require.NoError(t, exprhelpers.Init(nil))
|
||||
|
||||
datadir := t.TempDir()
|
||||
botsDir := filepath.Join(datadir, "legit_bots")
|
||||
require.NoError(t, os.Mkdir(botsDir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(botsDir, "bots.json"),
|
||||
[]byte(`{"name":"testbot","user_agent":"testbot","ranges":["192.0.2.0/24"]}`), 0o644))
|
||||
require.NoError(t, exprhelpers.FileInit(botsDir, "bots.json", "bots"))
|
||||
|
||||
banRule := appsec_rule.CustomRule{
|
||||
Name: "rule1",
|
||||
Zones: []string{"ARGS"},
|
||||
Variables: []string{"foo"},
|
||||
Match: appsec_rule.Match{Type: "regex", Value: "^toto"},
|
||||
Transform: []string{"lowercase"},
|
||||
}
|
||||
bypassHook := appsec.Hook{
|
||||
Filter: `IsLegitimateBot(req.RemoteAddr, req.UserAgent(), req.URL.Path)`,
|
||||
Apply: []string{`SetRemediation("allow")`},
|
||||
}
|
||||
botRequest := func(remoteAddr string, ua string) appsec.ParsedRequest {
|
||||
return appsec.ParsedRequest{
|
||||
RemoteAddr: remoteAddr,
|
||||
Method: "GET",
|
||||
URI: "/crawl",
|
||||
Args: url.Values{"foo": []string{"toto"}},
|
||||
HTTPRequest: &http.Request{
|
||||
Host: "example.com",
|
||||
RemoteAddr: remoteAddr,
|
||||
URL: mustParseURL("http://example.com/crawl"),
|
||||
Header: http.Header{"User-Agent": []string{ua}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
tests := []appsecRuleTest{
|
||||
{
|
||||
name: "verified bot bypasses the ban",
|
||||
expected_load_ok: true,
|
||||
inband_rules: []appsec_rule.CustomRule{banRule},
|
||||
pre_eval: []appsec.Hook{bypassHook},
|
||||
// the IP is in the declared range; ip:port proves address normalization end to end
|
||||
input_request: botRequest("192.0.2.10:34567", "Mozilla/5.0 (compatible; TestBot/1.0)"),
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.True(t, responses[0].InBandInterrupt, "the rule still matches, only the remediation changes")
|
||||
require.Equal(t, appsec.AllowRemediation, responses[0].Action)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "spoofed UA from a foreign IP is still banned",
|
||||
expected_load_ok: true,
|
||||
inband_rules: []appsec_rule.CustomRule{banRule},
|
||||
pre_eval: []appsec.Hook{bypassHook},
|
||||
input_request: botRequest("203.0.113.9:34567", "Mozilla/5.0 (compatible; TestBot/1.0)"),
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.True(t, responses[0].InBandInterrupt)
|
||||
require.Equal(t, appsec.BanRemediation, responses[0].Action)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ExemptFromChallenge escape hatch short-circuits the checks",
|
||||
expected_load_ok: true,
|
||||
inband_rules: []appsec_rule.CustomRule{banRule},
|
||||
pre_eval: []appsec.Hook{
|
||||
{Apply: []string{"ExemptFromChallenge()"}},
|
||||
bypassHook,
|
||||
},
|
||||
// neither the UA nor the IP matches any definition
|
||||
input_request: botRequest("203.0.113.9:34567", "curl/8.0"),
|
||||
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
|
||||
require.Len(t, responses, 1)
|
||||
require.True(t, responses[0].InBandInterrupt)
|
||||
require.Equal(t, appsec.AllowRemediation, responses[0].Action)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runTests(t, tests)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -129,6 +130,15 @@ func (r *AppsecRunner) Init(datadir string) error {
|
||||
r.logger.Tracef("Loaded inband rules: %+v", r.AppsecInbandEngine.GetRuleGroup().GetRules())
|
||||
r.logger.Tracef("Loaded outband rules: %+v", r.AppsecOutbandEngine.GetRuleGroup().GetRules())
|
||||
|
||||
// Materialize the fingerprint-dump dir under data_dir. Failure doesn't prevent startup.
|
||||
r.AppsecRuntime.FingerprintDumpDir = filepath.Join(datadir, "fingerprint_dumps")
|
||||
if err := os.MkdirAll(r.AppsecRuntime.FingerprintDumpDir, 0o700); err != nil {
|
||||
r.logger.Warnf("could not create fingerprint dump dir %q: %s; DumpFingerprint() will be a no-op", r.AppsecRuntime.FingerprintDumpDir, err)
|
||||
r.AppsecRuntime.FingerprintDumpDir = ""
|
||||
} else if err := os.Chmod(r.AppsecRuntime.FingerprintDumpDir, 0o700); err != nil {
|
||||
r.logger.Warnf("could not chmod fingerprint dump dir %q to 0700: %s", r.AppsecRuntime.FingerprintDumpDir, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -142,10 +152,8 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR
|
||||
}
|
||||
|
||||
defer func() {
|
||||
state.Tx.ProcessLogging()
|
||||
//We don't close the transaction here, as it will reset coraza internal state and break variable tracking
|
||||
|
||||
err := r.AppsecRuntime.ProcessPostEvalRules(state, request)
|
||||
// We don't close the transaction here, as it would reset coraza internal state and break variable tracking.
|
||||
err := r.AppsecRuntime.ProcessPostEvalRules(ctx, state, request)
|
||||
if err != nil {
|
||||
r.logger.Errorf("unable to process PostEval rules: %s", err)
|
||||
}
|
||||
@@ -158,6 +166,13 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR
|
||||
//FIXME: should we abort here ?
|
||||
}
|
||||
|
||||
// User has requested valid challenge, but we did not find a valid cookie
|
||||
// Immediately return, everything has been set already
|
||||
if state.RequireChallenge {
|
||||
r.logger.Debug("serving challenge")
|
||||
return nil
|
||||
}
|
||||
|
||||
if state.DropInfo(request) != nil {
|
||||
r.logger.Debug("drop helper triggered during pre_eval, skipping WAF evaluation")
|
||||
return nil
|
||||
@@ -177,6 +192,11 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR
|
||||
r.logger.Debugf("request body exceeded maximum allowed size but body inspection is disabled, allowing request")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
state.Tx.ProcessLogging()
|
||||
// We don't close the transaction here, as it would reset coraza internal state and break variable tracking.
|
||||
}()
|
||||
|
||||
state.Tx.ProcessConnection(request.ClientIP, 0, "", 0)
|
||||
|
||||
for k, v := range request.Args {
|
||||
@@ -241,14 +261,35 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR
|
||||
func (r *AppsecRunner) ProcessInBandRules(ctx context.Context, state *appsec.AppsecRequestState, request *appsec.ParsedRequest) error {
|
||||
tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID)
|
||||
state.Tx = tx
|
||||
// Even if we have no inband rules, we might have pre-eval or post-eval rules to process
|
||||
// Even if we have no inband rules, we might have pre-eval, post-eval or on_challenge hooks to process
|
||||
if len(r.AppsecRuntime.InBandRules) == 0 &&
|
||||
len(r.AppsecRuntime.CommonHooks.PreEval) == 0 &&
|
||||
len(r.AppsecRuntime.InBandHooks.PreEval) == 0 &&
|
||||
len(r.AppsecRuntime.CommonHooks.PostEval) == 0 &&
|
||||
len(r.AppsecRuntime.InBandHooks.PostEval) == 0 {
|
||||
len(r.AppsecRuntime.InBandHooks.PostEval) == 0 &&
|
||||
len(r.AppsecRuntime.CompiledOnChallenge) == 0 &&
|
||||
r.AppsecRuntime.ChallengeRuntime == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// on_challenge runs before any WAF work: it serves PoW infrastructure paths,
|
||||
// validates submissions, and populates state.Fingerprint from the cookie.
|
||||
if err := r.AppsecRuntime.ProcessOnChallengeRules(ctx, state, request); err != nil {
|
||||
r.logger.Errorf("unable to process OnChallenge rules: %s", err)
|
||||
}
|
||||
|
||||
// Infrastructure paths (PoW worker, challenge submit) already set up the
|
||||
// full response — skip pre_eval, WAF evaluation and post_eval entirely.
|
||||
if state.RequireChallenge {
|
||||
r.logger.Debugf("challenge response set by on_challenge, skipping WAF evaluation")
|
||||
return nil
|
||||
}
|
||||
|
||||
if state.DropInfo(request) != nil {
|
||||
r.logger.Debug("drop helper triggered during on_challenge, skipping WAF evaluation")
|
||||
return nil
|
||||
}
|
||||
|
||||
err := r.processRequest(ctx, state, request)
|
||||
return err
|
||||
}
|
||||
@@ -268,13 +309,8 @@ func (r *AppsecRunner) ProcessOutOfBandRules(ctx context.Context, state *appsec.
|
||||
}
|
||||
|
||||
func (r *AppsecRunner) handleInBandInterrupt(ctx context.Context, state *appsec.AppsecRequestState, request *appsec.ParsedRequest) {
|
||||
if allowed, reason := r.appsecAllowlistsClient.IsAllowlisted(request.ClientIP); allowed {
|
||||
r.logger.Infof("%s is allowlisted by %s, skipping", request.ClientIP, reason)
|
||||
return
|
||||
}
|
||||
|
||||
//create the associated event for crowdsec itself
|
||||
evt, err := EventFromRequest(request, r.Labels, state.Tx.ID())
|
||||
evt, err := appsec.EventFromRequest(request, r.Labels, state.Tx.ID())
|
||||
if err != nil {
|
||||
//let's not interrupt the pipeline for this
|
||||
r.logger.Errorf("unable to create event from request : %s", err)
|
||||
@@ -361,13 +397,7 @@ func copyHookVars(evt *pipeline.Event, state *appsec.AppsecRequestState) {
|
||||
}
|
||||
|
||||
func (r *AppsecRunner) handleOutBandInterrupt(ctx context.Context, state *appsec.AppsecRequestState, request *appsec.ParsedRequest) {
|
||||
|
||||
if allowed, reason := r.appsecAllowlistsClient.IsAllowlisted(request.ClientIP); allowed {
|
||||
r.logger.Infof("%s is allowlisted by %s, skipping", request.ClientIP, reason)
|
||||
return
|
||||
}
|
||||
|
||||
evt, err := EventFromRequest(request, r.Labels, state.Tx.ID())
|
||||
evt, err := appsec.EventFromRequest(request, r.Labels, state.Tx.ID())
|
||||
if err != nil {
|
||||
//let's not interrupt the pipeline for this
|
||||
r.logger.Errorf("unable to create event from request : %s", err)
|
||||
@@ -435,6 +465,19 @@ func (r *AppsecRunner) handleRequest(ctx context.Context, request *appsec.Parsed
|
||||
logger.Debug("Request received in runner")
|
||||
r.AppsecRuntime.ClearResponse(&state)
|
||||
|
||||
// Allowlisted IPs bypass every form of appsec processing (in-band rules,
|
||||
// hooks, challenge issuance, on_challenge cookie validation, out-of-band
|
||||
// rules). We send the default pass response straight back to the bouncer.
|
||||
if r.appsecAllowlistsClient != nil {
|
||||
if allowed, reason := r.appsecAllowlistsClient.IsAllowlisted(request.ClientIP); allowed {
|
||||
logger.Infof("%s is allowlisted by %s, skipping WAF processing", request.ClientIP, reason)
|
||||
// No Clone() needed (unlike the in-band send below): we return
|
||||
// immediately, so no out-of-band phase races this response.
|
||||
request.ResponseChannel <- state.Response
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
request.IsInBand = true
|
||||
request.IsOutBand = false
|
||||
|
||||
@@ -468,8 +511,20 @@ func (r *AppsecRunner) handleRequest(ctx context.Context, request *appsec.Parsed
|
||||
r.logger.Errorf("unable to close inband transaction: %s", err)
|
||||
}
|
||||
|
||||
// send back the result to the HTTP handler for the InBand part
|
||||
request.ResponseChannel <- state.Response
|
||||
// Clone as the out-of-band phase might mutate the response
|
||||
request.ResponseChannel <- state.Response.Clone()
|
||||
|
||||
// A challenge was served, so the request never reaches the backend;
|
||||
if state.RequireChallenge {
|
||||
globalParsingElapsed := time.Since(startGlobalParsing)
|
||||
metrics.AppsecGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(globalParsingElapsed.Seconds())
|
||||
return
|
||||
}
|
||||
|
||||
// Challenge remediation is intentionally a no-op for OOB matches: the inband
|
||||
// response has already been sent to the visitor at this point, so there is
|
||||
// nothing left to challenge. OOB matches still feed alerts/events.
|
||||
// (captcha gets the same treatment for the same reason.)
|
||||
|
||||
//Now let's process the out of band rules
|
||||
|
||||
|
||||
@@ -13,11 +13,11 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/allowlists"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
|
||||
)
|
||||
|
||||
@@ -31,15 +31,17 @@ type appsecRuleTest struct {
|
||||
// schemas registers OpenAPI schemas (ref → YAML content) on the
|
||||
// RequestValidator after Build, so hooks can refer to them via
|
||||
// ValidateRequestWithSchema(ref).
|
||||
schemas map[string]string
|
||||
on_load []appsec.Hook
|
||||
pre_eval []appsec.Hook
|
||||
post_eval []appsec.Hook
|
||||
on_match []appsec.Hook
|
||||
schemas map[string]string
|
||||
on_load []appsec.Hook
|
||||
pre_eval []appsec.Hook
|
||||
post_eval []appsec.Hook
|
||||
on_match []appsec.Hook
|
||||
on_challenge []appsec.Hook
|
||||
// Phase-scoped hooks (dispatched only during the matching phase)
|
||||
inband_on_match []appsec.Hook
|
||||
inband_pre_eval []appsec.Hook
|
||||
inband_post_eval []appsec.Hook
|
||||
inband_on_challenge []appsec.Hook
|
||||
outofband_on_match []appsec.Hook
|
||||
outofband_pre_eval []appsec.Hook
|
||||
outofband_post_eval []appsec.Hook
|
||||
@@ -119,6 +121,7 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
|
||||
PreEval: test.pre_eval,
|
||||
PostEval: test.post_eval,
|
||||
OnMatch: test.on_match,
|
||||
OnChallenge: test.on_challenge,
|
||||
BouncerBlockedHTTPCode: test.BouncerBlockedHTTPCode,
|
||||
UserBlockedHTTPCode: test.UserBlockedHTTPCode,
|
||||
UserPassedHTTPCode: test.UserPassedHTTPCode,
|
||||
@@ -127,11 +130,12 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
|
||||
}
|
||||
|
||||
// Set phase-scoped hooks if any are provided
|
||||
if len(test.inband_on_match) > 0 || len(test.inband_pre_eval) > 0 || len(test.inband_post_eval) > 0 {
|
||||
if len(test.inband_on_match) > 0 || len(test.inband_pre_eval) > 0 || len(test.inband_post_eval) > 0 || len(test.inband_on_challenge) > 0 {
|
||||
appsecCfg.InBand = &appsec.AppsecPhaseConfig{
|
||||
OnMatch: test.inband_on_match,
|
||||
PreEval: test.inband_pre_eval,
|
||||
PostEval: test.inband_post_eval,
|
||||
OnMatch: test.inband_on_match,
|
||||
PreEval: test.inband_pre_eval,
|
||||
PostEval: test.inband_post_eval,
|
||||
OnChallenge: test.inband_on_challenge,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,6 +150,9 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
|
||||
hub := cwhub.Hub{}
|
||||
AppsecRuntime, err := appsecCfg.Build(t.Context(), &hub)
|
||||
if err != nil {
|
||||
if !test.expected_load_ok {
|
||||
return
|
||||
}
|
||||
t.Fatalf("unable to build appsec runtime : %s", err)
|
||||
}
|
||||
for ref, schemaYAML := range test.schemas {
|
||||
@@ -155,10 +162,23 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
|
||||
}
|
||||
AppsecRuntime.InBandRules = []appsec.AppsecCollection{{Rules: inbandRules, NativeRules: nativeInbandRules}}
|
||||
AppsecRuntime.OutOfBandRules = []appsec.AppsecCollection{{Rules: outofbandRules, NativeRules: nativeOutofbandRules}}
|
||||
|
||||
// Hooks using SendChallenge() or on_challenge hooks require the WASM
|
||||
// challenge runtime; mirror what pkg/acquisition/modules/appsec/config.go
|
||||
// does in production. We share a single runtime across tests — building
|
||||
// one is expensive (~15-20s for WASM obfuscator warm-up), and none of the
|
||||
// tests mutate runtime-level state like default difficulty.
|
||||
if AppsecRuntime.NeedWASMVM {
|
||||
AppsecRuntime.ChallengeRuntime = getSharedChallengeRuntime(t)
|
||||
}
|
||||
appsecRunnerUUID := uuid.New().String()
|
||||
//we copy AppsecRutime for each runner
|
||||
wrt := *AppsecRuntime
|
||||
wrt.Logger = logger
|
||||
// mirror production wiring (run.go / config.go) so challenge lifecycle
|
||||
// events are emitted on the same channel the harness collects from.
|
||||
wrt.OutChan = OutChan
|
||||
wrt.Labels = map[string]string{"foo": "bar"}
|
||||
|
||||
mux, urlx, teardown := setupLapi()
|
||||
defer teardown()
|
||||
@@ -242,7 +262,7 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
|
||||
idle.Reset(idleDuration)
|
||||
}
|
||||
|
||||
responses :=[]appsec.AppsecTempResponse{}
|
||||
responses := []appsec.AppsecTempResponse{}
|
||||
events := []pipeline.Event{}
|
||||
|
||||
done := make(chan struct{})
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
@@ -20,6 +21,9 @@ import (
|
||||
"github.com/crowdsecurity/crowdsec/pkg/apiclient/useragent"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/allowlists"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/metrics"
|
||||
)
|
||||
|
||||
@@ -121,6 +125,61 @@ func loadCertPool(caCertPath string, logger log.FieldLogger) (*x509.CertPool, er
|
||||
return caCertPool, nil
|
||||
}
|
||||
|
||||
// expandAppsecConfigEntry resolves a single appsec_config(s) entry into the list
|
||||
// of appsec-config item names to load. A literal entry is returned untouched. An entry containing a glob meta-character
|
||||
// is matched against the installed appsec-configs with the same matcher used
|
||||
// to expand appsec-rule patterns; it errors when no installed config matches.
|
||||
func expandAppsecConfigEntry(entry string, hub *cwhub.Hub) ([]string, error) {
|
||||
if !strings.ContainsAny(entry, "*?") {
|
||||
return []string{entry}, nil
|
||||
}
|
||||
|
||||
var matches []string
|
||||
|
||||
for _, item := range hub.GetInstalledByType(cwhub.APPSEC_CONFIGS, true) {
|
||||
matched, err := exprhelpers.Match(entry, item.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("while matching %q against %q: %w", entry, item.Name, err)
|
||||
}
|
||||
|
||||
if matched.(bool) {
|
||||
matches = append(matches, item.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("no installed appsec-config matches pattern %q", entry)
|
||||
}
|
||||
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
// resolveAppsecConfigEntries expands every appsec_config(s) entry and returns
|
||||
// the de-duplicated list of config names to load, in first-seen order.
|
||||
func resolveAppsecConfigEntries(entries []string, hub *cwhub.Hub) ([]string, error) {
|
||||
seen := make(map[string]struct{})
|
||||
|
||||
var toLoad []string
|
||||
|
||||
for _, entry := range entries {
|
||||
names, err := expandAppsecConfigEntry(entry, hub)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to resolve appsec_config %q: %w", entry, err)
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
if _, ok := seen[name]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
seen[name] = struct{}{}
|
||||
toLoad = append(toLoad, name)
|
||||
}
|
||||
}
|
||||
|
||||
return toLoad, nil
|
||||
}
|
||||
|
||||
func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.Entry, _ metrics.AcquisitionMetricsLevel) error {
|
||||
if w.hub == nil {
|
||||
return errors.New("appsec datasource requires a hub. this is a bug, please report")
|
||||
@@ -177,13 +236,19 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E
|
||||
if err := appsecCfg.LoadByPath(w.config.AppsecConfigPath); err != nil {
|
||||
return fmt.Errorf("unable to load appsec_config: %w", err)
|
||||
}
|
||||
} else if w.config.AppsecConfig != "" {
|
||||
if err := appsecCfg.Load(w.config.AppsecConfig, w.hub); err != nil {
|
||||
return fmt.Errorf("unable to load appsec_config: %w", err)
|
||||
} else if w.config.AppsecConfig != "" || len(w.config.AppsecConfigs) > 0 {
|
||||
entries := w.config.AppsecConfigs
|
||||
if w.config.AppsecConfig != "" {
|
||||
entries = []string{w.config.AppsecConfig}
|
||||
}
|
||||
} else if len(w.config.AppsecConfigs) > 0 {
|
||||
for _, appsecConfig := range w.config.AppsecConfigs {
|
||||
if err := appsecCfg.Load(appsecConfig, w.hub); err != nil {
|
||||
|
||||
names, err := resolveAppsecConfigEntries(entries, w.hub)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
if err := appsecCfg.Load(name, w.hub); err != nil {
|
||||
return fmt.Errorf("unable to load appsec_config: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -199,7 +264,23 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E
|
||||
return fmt.Errorf("unable to build appsec_config: %w", err)
|
||||
}
|
||||
|
||||
if appsecRuntime.NeedWASMVM {
|
||||
logger.Info("Initializing WASM runtime for challenge obfuscation")
|
||||
|
||||
challengeOpts, err := challenge.BuildOptions(appsecCfg.Challenge, appsecCfg.Logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to build challenge options: %w", err)
|
||||
}
|
||||
|
||||
challengeRuntime, err := challenge.NewChallengeRuntime(ctx, challengeOpts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create challenge runtime: %w", err)
|
||||
}
|
||||
appsecRuntime.ChallengeRuntime = challengeRuntime
|
||||
}
|
||||
|
||||
w.AppsecRuntime = appsecRuntime
|
||||
w.AppsecRuntime.Labels = w.config.Labels
|
||||
w.AppsecRuntime.DataDir = w.hub.GetDataDir()
|
||||
|
||||
err = w.AppsecRuntime.ProcessOnLoadRules()
|
||||
@@ -213,14 +294,11 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E
|
||||
|
||||
for nbRoutine := range w.config.Routines {
|
||||
appsecRunnerUUID := uuid.New().String()
|
||||
// we copy AppsecRuntime for each runner
|
||||
wrt := *w.AppsecRuntime
|
||||
wrt.Logger = w.logger.Dup().WithField("runner_uuid", appsecRunnerUUID)
|
||||
runner := AppsecRunner{
|
||||
inChan: w.InChan,
|
||||
UUID: appsecRunnerUUID,
|
||||
logger: w.logger.WithField("runner_uuid", appsecRunnerUUID),
|
||||
AppsecRuntime: &wrt,
|
||||
AppsecRuntime: w.AppsecRuntime,
|
||||
Labels: w.config.Labels,
|
||||
appsecAllowlistsClient: w.appsecAllowlistClient,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
package appsecacquisition
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
)
|
||||
|
||||
// hubWithConfigs builds an in-memory hub holding the given appsec-configs and
|
||||
// marks the listed ones as installed (an installed item is one with a
|
||||
// LocalPath, see cwhub.ItemState.IsInstalled).
|
||||
func hubWithConfigs(t *testing.T, all, installed []string) *cwhub.Hub {
|
||||
t.Helper()
|
||||
|
||||
tempDir := t.TempDir()
|
||||
local := &csconfig.LocalHubCfg{
|
||||
HubDir: filepath.Join(tempDir, "hub"),
|
||||
HubIndexFile: filepath.Join(tempDir, "hub", ".index.json"),
|
||||
InstallDir: filepath.Join(tempDir, "install"),
|
||||
InstallDataDir: filepath.Join(tempDir, "data"),
|
||||
}
|
||||
|
||||
require.NoError(t, os.MkdirAll(local.HubDir, 0o755))
|
||||
require.NoError(t, os.MkdirAll(local.InstallDir, 0o755))
|
||||
require.NoError(t, os.MkdirAll(local.InstallDataDir, 0o755))
|
||||
|
||||
index := `{"appsec-configs": {`
|
||||
for i, name := range all {
|
||||
if i > 0 {
|
||||
index += ","
|
||||
}
|
||||
index += `"` + name + `": {"path": "appsec-configs/` + name + `.yaml", "version": "1.0", "versions": {"1.0": {"digest": "aa"}}}`
|
||||
}
|
||||
index += "}}"
|
||||
|
||||
require.NoError(t, os.WriteFile(local.HubIndexFile, []byte(index), 0o644))
|
||||
|
||||
hub, err := cwhub.NewHub(local, nil)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, hub.Load())
|
||||
|
||||
for _, name := range installed {
|
||||
item := hub.GetItem(cwhub.APPSEC_CONFIGS, name)
|
||||
require.NotNilf(t, item, "appsec-config %q missing from test hub", name)
|
||||
item.State.LocalPath = filepath.Join(local.InstallDir, name+".yaml")
|
||||
}
|
||||
|
||||
return hub
|
||||
}
|
||||
|
||||
func TestExpandAppsecConfigEntry(t *testing.T) {
|
||||
all := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "crowdsecurity/uninstalled", "custom/mine"}
|
||||
installed := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "custom/mine"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
entry string
|
||||
expected []string
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
// Literals pass through untouched, even unknown ones, so the
|
||||
// per-name error still surfaces later from AppsecConfig.Load.
|
||||
name: "literal passthrough",
|
||||
entry: "crowdsecurity/vpatch",
|
||||
expected: []string{"crowdsecurity/vpatch"},
|
||||
},
|
||||
{
|
||||
name: "unknown literal passthrough",
|
||||
entry: "does/not-exist",
|
||||
expected: []string{"does/not-exist"},
|
||||
},
|
||||
{
|
||||
// Sorted (case-insensitive) order from the hub; uninstalled is skipped.
|
||||
name: "wildcard expands installed only",
|
||||
entry: "crowdsecurity/*",
|
||||
expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch"},
|
||||
},
|
||||
{
|
||||
name: "wildcard matches all",
|
||||
entry: "*",
|
||||
expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch", "custom/mine"},
|
||||
},
|
||||
{
|
||||
name: "wildcard with no match errors",
|
||||
entry: "nope/*",
|
||||
expectedErr: `no installed appsec-config matches pattern "nope/*"`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
hub := hubWithConfigs(t, all, installed)
|
||||
|
||||
got, err := expandAppsecConfigEntry(tc.entry, hub)
|
||||
if tc.expectedErr != "" {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tc.expectedErr)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAppsecConfigEntries(t *testing.T) {
|
||||
all := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "crowdsecurity/uninstalled", "custom/mine"}
|
||||
installed := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "custom/mine"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
entries []string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
// "*" already pulls in vpatch; the explicit literal must not load it
|
||||
// a second time. First-seen order from the wildcard expansion wins.
|
||||
name: "wildcard plus overlapping literal dedups",
|
||||
entries: []string{"*", "crowdsecurity/vpatch"},
|
||||
expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch", "custom/mine"},
|
||||
},
|
||||
{
|
||||
name: "same literal twice collapses to one",
|
||||
entries: []string{"crowdsecurity/vpatch", "crowdsecurity/vpatch"},
|
||||
expected: []string{"crowdsecurity/vpatch"},
|
||||
},
|
||||
{
|
||||
name: "unknown literal passes through once",
|
||||
entries: []string{"does/not-exist", "does/not-exist"},
|
||||
expected: []string{"does/not-exist"},
|
||||
},
|
||||
{
|
||||
// Overlapping patterns: generic matches both, must appear once.
|
||||
name: "overlapping patterns dedup",
|
||||
entries: []string{"crowdsecurity/*", "*/generic"},
|
||||
expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
hub := hubWithConfigs(t, all, installed)
|
||||
|
||||
got, err := resolveAppsecConfigEntries(tc.entries, hub)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package appsecacquisition
|
||||
import (
|
||||
"github.com/crowdsecurity/crowdsec/pkg/acquisition/registry"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/acquisition/types"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -14,7 +15,7 @@ var (
|
||||
_ types.LAPIClientAware = (*Source)(nil)
|
||||
)
|
||||
|
||||
const ModuleName = "appsec"
|
||||
const ModuleName = appsec.ModuleName
|
||||
|
||||
//nolint:gochecknoinits
|
||||
func init() {
|
||||
|
||||
@@ -11,6 +11,15 @@ func (*Source) GetMetrics() []prometheus.Collector {
|
||||
metrics.AppsecReqCounter,
|
||||
metrics.AppsecBlockCounter,
|
||||
metrics.AppsecRuleHits,
|
||||
metrics.AppsecFingerprintMismatch,
|
||||
metrics.AppsecChallengeRequested,
|
||||
metrics.AppsecChallengeSubmitted,
|
||||
metrics.AppsecChallengeAccepted,
|
||||
metrics.AppsecChallengeRejected,
|
||||
metrics.AppsecChallengeKepochGenerated,
|
||||
metrics.AppsecChallengeKepochEvicted,
|
||||
metrics.AppsecChallengeReobfuscation,
|
||||
metrics.AppsecChallengeDynamicModuleEvicted,
|
||||
metrics.AppsecValidationOKCounter,
|
||||
metrics.AppsecValidationFailedCounter,
|
||||
metrics.AppsecOutbandParsingHistogram,
|
||||
@@ -24,9 +33,18 @@ func (*Source) GetAggregMetrics() []prometheus.Collector {
|
||||
metrics.AppsecReqCounter,
|
||||
metrics.AppsecBlockCounter,
|
||||
metrics.AppsecRuleHits,
|
||||
metrics.AppsecFingerprintMismatch,
|
||||
metrics.AppsecChallengeRequested,
|
||||
metrics.AppsecChallengeSubmitted,
|
||||
metrics.AppsecChallengeAccepted,
|
||||
metrics.AppsecChallengeRejected,
|
||||
metrics.AppsecChallengeKepochGenerated,
|
||||
metrics.AppsecChallengeKepochEvicted,
|
||||
metrics.AppsecChallengeReobfuscation,
|
||||
metrics.AppsecChallengeDynamicModuleEvicted,
|
||||
metrics.AppsecOutbandParsingHistogram,
|
||||
metrics.AppsecValidationOKCounter,
|
||||
metrics.AppsecValidationFailedCounter,
|
||||
metrics.AppsecOutbandParsingHistogram,
|
||||
metrics.AppsecInbandParsingHistogram,
|
||||
metrics.AppsecGlobalParsingHistogram,
|
||||
}
|
||||
|
||||
@@ -105,6 +105,12 @@ func (w *Source) listenAndServe(ctx context.Context, t *tomb.Tomb) error {
|
||||
w.logger.Errorf("Error shutting down Appsec server: %s", err.Error())
|
||||
}
|
||||
|
||||
if w.AppsecRuntime != nil && w.AppsecRuntime.ChallengeRuntime != nil {
|
||||
if err := w.AppsecRuntime.ChallengeRuntime.Close(ctx); err != nil {
|
||||
w.logger.Errorf("Error closing challenge runtime: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
if w.config.ListenSocket != "" {
|
||||
if err := os.Remove(w.config.ListenSocket); err != nil {
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
@@ -133,6 +139,8 @@ func (w *Source) StreamingAcquisition(ctx context.Context, out chan pipeline.Eve
|
||||
t.Go(func() error {
|
||||
defer trace.ReportPanic()
|
||||
|
||||
w.AppsecRuntime.OutChan = out
|
||||
|
||||
for _, runner := range w.AppsecRunners {
|
||||
runner.outChan = out
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package appsecacquisition
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
)
|
||||
|
||||
// sharedChallengeRuntime lazily creates a single ChallengeRuntime for the
|
||||
// whole package test run. NewChallengeRuntime runs the obfuscator WASM to
|
||||
// generate a challenge JS bundle (~15-20s), so spinning one up per test is
|
||||
// prohibitively slow and unnecessary for integration tests that don't mutate
|
||||
// runtime-level state.
|
||||
var (
|
||||
sharedChallengeRuntimeOnce sync.Once
|
||||
sharedChallengeRuntimeInst *challenge.ChallengeRuntime
|
||||
sharedChallengeRuntimeErr error
|
||||
)
|
||||
|
||||
func getSharedChallengeRuntime(t *testing.T) *challenge.ChallengeRuntime {
|
||||
t.Helper()
|
||||
sharedChallengeRuntimeOnce.Do(func() {
|
||||
sharedChallengeRuntimeInst, sharedChallengeRuntimeErr = challenge.NewChallengeRuntime(t.Context())
|
||||
})
|
||||
require.NoError(t, sharedChallengeRuntimeErr)
|
||||
return sharedChallengeRuntimeInst
|
||||
}
|
||||
@@ -288,37 +288,6 @@ func containsAll(excludedZones []string, matchedZones []string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func EventFromRequest(r *appsec.ParsedRequest, labels map[string]string, txUuid string) (pipeline.Event, error) {
|
||||
evt := pipeline.MakeEvent(false, pipeline.LOG, true)
|
||||
// def needs fixing
|
||||
evt.Stage = "s00-raw"
|
||||
evt.Parsed = map[string]string{
|
||||
"source_ip": r.ClientIP,
|
||||
"target_host": r.Host,
|
||||
"target_uri": r.URI,
|
||||
"method": r.Method,
|
||||
"req_uuid": txUuid,
|
||||
"source": "crowdsec-appsec",
|
||||
"remediation_cmpt_ip": r.RemoteAddrNormalized,
|
||||
// TBD:
|
||||
// http_status
|
||||
// user_agent
|
||||
|
||||
}
|
||||
evt.Line = pipeline.Line{
|
||||
Time: time.Now(),
|
||||
// should we add some info like listen addr/port/path ?
|
||||
Labels: labels,
|
||||
Process: true,
|
||||
Module: ModuleName,
|
||||
Src: ModuleName,
|
||||
Raw: "dummy-appsec-data", // we discard empty Line.Raw items :)
|
||||
}
|
||||
evt.Appsec = pipeline.AppsecEvent{}
|
||||
|
||||
return evt, nil
|
||||
}
|
||||
|
||||
type ruleData struct {
|
||||
ID int
|
||||
Name string
|
||||
|
||||
+777
-37
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,440 @@
|
||||
package appsec
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
)
|
||||
|
||||
// sharedChallengeRuntime lazily creates a single ChallengeRuntime for the
|
||||
// whole package test run. NewChallengeRuntime runs the obfuscator WASM to
|
||||
// generate a challenge JS bundle (~15-20s), so spinning one up per test is
|
||||
// prohibitively slow and unnecessary for logic-only dispatcher tests.
|
||||
var (
|
||||
sharedChallengeRuntimeOnce sync.Once
|
||||
sharedChallengeRuntimeInst *challenge.ChallengeRuntime
|
||||
sharedChallengeRuntimeErr error
|
||||
)
|
||||
|
||||
func getSharedChallengeRuntime(t *testing.T) *challenge.ChallengeRuntime {
|
||||
t.Helper()
|
||||
sharedChallengeRuntimeOnce.Do(func() {
|
||||
// Use context.Background so the runtime survives across tests.
|
||||
// Goroutines it spawns leak for the duration of the test binary; that's
|
||||
// acceptable here since the binary exits after tests complete.
|
||||
sharedChallengeRuntimeInst, sharedChallengeRuntimeErr = challenge.NewChallengeRuntime(t.Context())
|
||||
})
|
||||
require.NoError(t, sharedChallengeRuntimeErr)
|
||||
return sharedChallengeRuntimeInst
|
||||
}
|
||||
|
||||
// newChallengeTestRuntime builds an AppsecRuntimeConfig wired with a real
|
||||
// ChallengeRuntime for dispatcher tests. Provided on_challenge hooks are
|
||||
// compiled via buildHookList, so tests can assert that user expressions run.
|
||||
func newChallengeTestRuntime(t *testing.T, hooks []Hook) *AppsecRuntimeConfig {
|
||||
t.Helper()
|
||||
|
||||
cr := getSharedChallengeRuntime(t)
|
||||
|
||||
compiled, err := buildHookList(t.Context(), hooks, hookOnChallenge, &appsecExprPatcher{})
|
||||
require.NoError(t, err)
|
||||
|
||||
logger := log.NewEntry(log.StandardLogger())
|
||||
|
||||
cfg := &AppsecConfig{
|
||||
Name: "test-challenge",
|
||||
BouncerBlockedHTTPCode: http.StatusForbidden,
|
||||
BouncerPassedHTTPCode: http.StatusOK,
|
||||
UserBlockedHTTPCode: http.StatusForbidden,
|
||||
UserPassedHTTPCode: http.StatusOK,
|
||||
DefaultPassAction: AllowRemediation,
|
||||
DefaultRemediation: BanRemediation,
|
||||
}
|
||||
|
||||
return &AppsecRuntimeConfig{
|
||||
Name: "test-challenge",
|
||||
Config: cfg,
|
||||
Logger: logger,
|
||||
ChallengeRuntime: cr,
|
||||
CompiledOnChallenge: compiled,
|
||||
}
|
||||
}
|
||||
|
||||
func newInBandRequest(method, path string, body []byte) *ParsedRequest {
|
||||
u, _ := url.Parse(path)
|
||||
return &ParsedRequest{
|
||||
HTTPRequest: &http.Request{
|
||||
Method: method,
|
||||
URL: u,
|
||||
Header: http.Header{"User-Agent": []string{"go-test"}},
|
||||
},
|
||||
Body: body,
|
||||
IsInBand: true,
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessOnChallengeRulesNilRuntime(t *testing.T) {
|
||||
rt := &AppsecRuntimeConfig{
|
||||
Logger: log.NewEntry(log.StandardLogger()),
|
||||
Config: &AppsecConfig{},
|
||||
}
|
||||
state := &AppsecRequestState{}
|
||||
req := newInBandRequest(http.MethodGet, "/", nil)
|
||||
|
||||
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
|
||||
assert.False(t, state.RequireChallenge)
|
||||
assert.Nil(t, state.Fingerprint)
|
||||
}
|
||||
|
||||
func TestProcessOnChallengeRulesServesPowWorker(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
req := newInBandRequest(http.MethodGet, challenge.ChallengePowWorkerPath, nil)
|
||||
|
||||
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
|
||||
assert.True(t, state.RequireChallenge)
|
||||
assert.Equal(t, ChallengeRemediation, state.Response.Action)
|
||||
assert.Equal(t, http.StatusOK, state.Response.UserHTTPResponseCode)
|
||||
assert.Equal(t, challenge.PowWorkerJS, state.Response.UserHTTPBodyContent)
|
||||
assert.Contains(t, state.Response.UserHeaders["Content-Type"], "application/javascript")
|
||||
}
|
||||
|
||||
func TestProcessOnChallengeRulesSubmitInvalidBody(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
// Empty body → ValidateChallengeResponse fails; dispatcher returns failed JSON body.
|
||||
req := newInBandRequest(http.MethodPost, challenge.ChallengeSubmitPath, nil)
|
||||
|
||||
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
|
||||
assert.True(t, state.RequireChallenge)
|
||||
assert.JSONEq(t, bodyChallengeFailed, state.Response.UserHTTPBodyContent)
|
||||
assert.Contains(t, state.Response.UserHeaders["Content-Type"], "application/json")
|
||||
// No cookie is issued on failure.
|
||||
assert.Empty(t, state.Response.UserHTTPCookies)
|
||||
}
|
||||
|
||||
func TestProcessOnChallengeRulesNoCookieSkipsUserHooks(t *testing.T) {
|
||||
// A hook that would unconditionally flip PendingAction must NOT run when
|
||||
// there is no fingerprint to inspect — this prevents nil-deref panics on
|
||||
// filters like `fingerprint.Bot.X`.
|
||||
rt := newChallengeTestRuntime(t, []Hook{
|
||||
{Apply: []string{"SetRemediation('allow')"}},
|
||||
})
|
||||
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
req := newInBandRequest(http.MethodGet, "/", nil)
|
||||
|
||||
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
|
||||
assert.Nil(t, state.Fingerprint)
|
||||
assert.Nil(t, state.PendingAction, "user hooks must not run when fingerprint is nil")
|
||||
}
|
||||
|
||||
func TestProcessOnChallengeRulesInvalidCookieSkipsUserHooks(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, []Hook{
|
||||
{Apply: []string{"SetReturnCode(418)"}},
|
||||
})
|
||||
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
// Attach an invalid cookie; ValidCookie fails silently and fingerprint stays nil.
|
||||
u, _ := url.Parse("/")
|
||||
httpReq := &http.Request{
|
||||
Method: http.MethodGet,
|
||||
URL: u,
|
||||
Header: http.Header{
|
||||
"User-Agent": []string{"go-test"},
|
||||
"Cookie": []string{challenge.ChallengeCookieName + "=garbage"},
|
||||
},
|
||||
}
|
||||
req := &ParsedRequest{HTTPRequest: httpReq, IsInBand: true}
|
||||
|
||||
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
|
||||
assert.Nil(t, state.Fingerprint)
|
||||
assert.Nil(t, state.PendingHTTPCode, "user hooks must not run when fingerprint is nil")
|
||||
}
|
||||
|
||||
func TestSendChallengeNoOpWhenStoredDifficultyMeetsTarget(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
// Simulate a client that already proved the runtime-default difficulty.
|
||||
state.Fingerprint = &challenge.FingerprintData{}
|
||||
state.CookiePowDifficulty = rt.ChallengeRuntime.Difficulty()
|
||||
|
||||
req := newInBandRequest(http.MethodGet, "/protected", nil)
|
||||
|
||||
require.NoError(t, rt.SendChallenge(t.Context(), state, req))
|
||||
assert.False(t, state.RequireChallenge)
|
||||
assert.Empty(t, state.Response.UserHTTPBodyContent)
|
||||
assert.NotEqual(t, ChallengeRemediation, state.Response.Action)
|
||||
}
|
||||
|
||||
func TestSendChallengeReIssuesWhenTargetExceedsStored(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
// Client proved medium; this request demands impossible.
|
||||
state.Fingerprint = &challenge.FingerprintData{}
|
||||
state.CookiePowDifficulty = challenge.PowDifficultyMedium
|
||||
impossible := challenge.PowDifficultyImpossible
|
||||
state.ChallengeDifficulty = &impossible
|
||||
|
||||
req := newInBandRequest(http.MethodGet, "/protected", nil)
|
||||
|
||||
require.NoError(t, rt.SendChallenge(t.Context(), state, req))
|
||||
assert.True(t, state.RequireChallenge)
|
||||
assert.Equal(t, ChallengeRemediation, state.Response.Action)
|
||||
assert.Contains(t, state.Response.UserHeaders["Content-Type"], "text/html")
|
||||
assert.NotEmpty(t, state.Response.UserHTTPBodyContent)
|
||||
}
|
||||
|
||||
func TestSendChallengeNoOpWhenTargetLoweredBelowStored(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
// Client proved high; admin lowered the global target.
|
||||
state.Fingerprint = &challenge.FingerprintData{}
|
||||
state.CookiePowDifficulty = challenge.PowDifficultyHigh
|
||||
low := challenge.PowDifficultyLow
|
||||
state.ChallengeDifficulty = &low
|
||||
|
||||
req := newInBandRequest(http.MethodGet, "/protected", nil)
|
||||
|
||||
require.NoError(t, rt.SendChallenge(t.Context(), state, req))
|
||||
assert.False(t, state.RequireChallenge)
|
||||
}
|
||||
|
||||
func TestProcessOnChallengeRulesInvalidSubmissionSkipsUserHooks(t *testing.T) {
|
||||
// A hook that would unconditionally flip PendingAction must NOT run when
|
||||
// the submission is invalid — the failed JSON body is the only response.
|
||||
rt := newChallengeTestRuntime(t, []Hook{
|
||||
{Apply: []string{"SetRemediation('allow')"}},
|
||||
})
|
||||
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
// Empty body → ValidateChallengeResponse fails.
|
||||
req := newInBandRequest(http.MethodPost, challenge.ChallengeSubmitPath, nil)
|
||||
|
||||
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
|
||||
assert.True(t, state.RequireChallenge)
|
||||
assert.JSONEq(t, bodyChallengeFailed, state.Response.UserHTTPBodyContent)
|
||||
assert.Nil(t, state.PendingAction, "user hooks must not run on invalid submission")
|
||||
}
|
||||
|
||||
// TestRejectSubmissionSetsState confirms the helper sets the state field;
|
||||
// downstream ProcessOnChallengeRules submit branch keys off this to refuse
|
||||
// cookie issuance.
|
||||
func TestRejectSubmissionSetsState(t *testing.T) {
|
||||
rt := &AppsecRuntimeConfig{
|
||||
Logger: log.NewEntry(log.StandardLogger()),
|
||||
Config: &AppsecConfig{},
|
||||
}
|
||||
state := &AppsecRequestState{}
|
||||
|
||||
require.NoError(t, rt.RejectSubmission(state, "bot detected"))
|
||||
require.NotNil(t, state.SubmissionRejection)
|
||||
assert.Equal(t, "bot detected", state.SubmissionRejection.Reason)
|
||||
|
||||
// Empty reason gets a default placeholder.
|
||||
state2 := &AppsecRequestState{}
|
||||
require.NoError(t, rt.RejectSubmission(state2, ""))
|
||||
assert.Equal(t, "submission rejected by on_challenge_submit", state2.SubmissionRejection.Reason)
|
||||
}
|
||||
|
||||
// TestGrantChallengeCookieIssuesRedirect confirms the helper mints a
|
||||
// cookie, stamps allowlist fingerprint + bypass state, and produces a
|
||||
// 307 challenge response carrying the Location header and Set-Cookie
|
||||
// back to the visitor. The 307 is necessary because GenerateResponse
|
||||
// only serializes UserCookies on ChallengeRemediation responses.
|
||||
func TestGrantChallengeCookieIssuesRedirect(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
req := newInBandRequest(http.MethodGet, "/protected?a=1", nil)
|
||||
|
||||
require.NoError(t, rt.GrantChallengeCookie(state, req, "Googlebot/2.1", nil))
|
||||
|
||||
require.NotNil(t, state.Fingerprint)
|
||||
assert.True(t, state.Fingerprint.Allowlisted)
|
||||
assert.Equal(t, "Googlebot/2.1", state.Fingerprint.AllowlistReason)
|
||||
assert.True(t, state.ChallengeBypassed, "ChallengeBypassed must be set to suppress later SendChallenge")
|
||||
|
||||
assert.Equal(t, ChallengeRemediation, state.Response.Action, "must produce a challenge-action response so the cookie is serialized")
|
||||
assert.Equal(t, http.StatusTemporaryRedirect, state.Response.UserHTTPResponseCode, "must be a 307 redirect")
|
||||
assert.True(t, state.RequireChallenge, "RequireChallenge must be flagged so the wire response carries the cookie")
|
||||
|
||||
require.Len(t, state.Response.UserHTTPCookies, 1, "allowlist cookie must be appended")
|
||||
require.NotNil(t, state.Response.UserHeaders)
|
||||
require.Contains(t, state.Response.UserHeaders, "Location")
|
||||
assert.Equal(t, []string{"/protected?a=1"}, state.Response.UserHeaders["Location"])
|
||||
assert.Equal(t, challenge.GrantRedirectBody, state.Response.UserHTTPBodyContent, "body must be the redirect fallback page")
|
||||
}
|
||||
|
||||
// TestGrantAllowlistCookieInlineNoRedirect confirms the submit-phase
|
||||
// variant attaches the cookie without producing a redirect response.
|
||||
// on_challenge_submit returns the challenge-submit JSON envelope the
|
||||
// client is already awaiting; a 307 would break the client JS state
|
||||
// machine.
|
||||
func TestGrantAllowlistCookieInlineNoRedirect(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
req := newInBandRequest(http.MethodPost, challenge.ChallengeSubmitPath, nil)
|
||||
|
||||
require.NoError(t, rt.GrantAllowlistCookieInline(state, req, "submit-allowlist", nil))
|
||||
|
||||
require.NotNil(t, state.Fingerprint)
|
||||
assert.True(t, state.Fingerprint.Allowlisted)
|
||||
assert.Equal(t, "submit-allowlist", state.Fingerprint.AllowlistReason)
|
||||
assert.True(t, state.ChallengeBypassed)
|
||||
require.Len(t, state.Response.UserHTTPCookies, 1, "cookie must ride on the existing envelope")
|
||||
assert.NotEqual(t, http.StatusTemporaryRedirect, state.Response.UserHTTPResponseCode, "inline variant must NOT emit a redirect status")
|
||||
assert.NotContains(t, state.Response.UserHeaders, "Location", "inline variant must not set Location")
|
||||
}
|
||||
|
||||
// TestSendChallengeNoOpAfterGrantChallengeCookie covers the bypass guard:
|
||||
// after GrantChallengeCookie sets ChallengeBypassed, a subsequent
|
||||
// SendChallenge call in the same request is a no-op (must not overwrite
|
||||
// the 307 redirect response body with a challenge page).
|
||||
func TestSendChallengeNoOpAfterGrantChallengeCookie(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
req := newInBandRequest(http.MethodGet, "/", nil)
|
||||
require.NoError(t, rt.GrantChallengeCookie(state, req, "ua-pass", nil))
|
||||
require.True(t, state.ChallengeBypassed)
|
||||
|
||||
cookiesBefore := len(state.Response.UserHTTPCookies)
|
||||
bodyBefore := state.Response.UserHTTPBodyContent
|
||||
statusBefore := state.Response.UserHTTPResponseCode
|
||||
|
||||
require.NoError(t, rt.SendChallenge(t.Context(), state, req))
|
||||
|
||||
assert.Len(t, state.Response.UserHTTPCookies, cookiesBefore, "cookie set must be unchanged")
|
||||
assert.Equal(t, bodyBefore, state.Response.UserHTTPBodyContent, "redirect body must not be overwritten")
|
||||
assert.Equal(t, statusBefore, state.Response.UserHTTPResponseCode, "redirect status must not be overwritten")
|
||||
}
|
||||
|
||||
// TestProcessOnChallengeRulesAllowlistCookiePropagatesFlag asserts the
|
||||
// cookie-valid branch of ProcessOnChallengeRules copies the Allowlisted +
|
||||
// AllowlistReason fields from the decoded cookie into state.Fingerprint.
|
||||
func TestProcessOnChallengeRulesAllowlistCookiePropagatesFlag(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
|
||||
// Mint an allowlist cookie via the public helper, then re-attach it to a
|
||||
// regular request and run the dispatcher.
|
||||
mintReq := newInBandRequest(http.MethodGet, "/", nil)
|
||||
ck, err := rt.ChallengeRuntime.SealAllowlistCookie(mintReq.HTTPRequest, "test-bypass", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
u, _ := url.Parse("/protected")
|
||||
getReq := &ParsedRequest{
|
||||
HTTPRequest: &http.Request{
|
||||
Method: http.MethodGet,
|
||||
URL: u,
|
||||
Header: http.Header{
|
||||
"User-Agent": []string{"go-test"},
|
||||
"Cookie": []string{challenge.ChallengeCookieName + "=" + ck.Val},
|
||||
},
|
||||
},
|
||||
IsInBand: true,
|
||||
}
|
||||
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, getReq))
|
||||
require.NotNil(t, state.Fingerprint, "valid allowlist cookie must populate fingerprint")
|
||||
assert.True(t, state.Fingerprint.Allowlisted)
|
||||
assert.Equal(t, "test-bypass", state.Fingerprint.AllowlistReason)
|
||||
assert.True(t, state.ChallengeBypassed,
|
||||
"allowlist cookie must set ChallengeBypassed so later SendChallenge is a no-op")
|
||||
}
|
||||
|
||||
// TestAllowlistCookieRoundtripBypassesSendChallenge covers the actual bug
|
||||
// reported against GrantChallengeCookie: a follow-up request carrying the
|
||||
// minted allowlist cookie must not be re-challenged. Without the
|
||||
// ChallengeBypassed propagation in the cookie-valid branch, SendChallenge
|
||||
// would happily render a challenge page on top of the cookie.
|
||||
func TestAllowlistCookieRoundtripBypassesSendChallenge(t *testing.T) {
|
||||
rt := newChallengeTestRuntime(t, nil)
|
||||
|
||||
mintReq := newInBandRequest(http.MethodGet, "/", nil)
|
||||
ck, err := rt.ChallengeRuntime.SealAllowlistCookie(mintReq.HTTPRequest, "roundtrip", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
u, _ := url.Parse("/protected")
|
||||
replayReq := &ParsedRequest{
|
||||
HTTPRequest: &http.Request{
|
||||
Method: http.MethodGet,
|
||||
URL: u,
|
||||
Header: http.Header{
|
||||
"User-Agent": []string{"go-test"},
|
||||
"Cookie": []string{challenge.ChallengeCookieName + "=" + ck.Val},
|
||||
},
|
||||
},
|
||||
IsInBand: true,
|
||||
}
|
||||
|
||||
state := &AppsecRequestState{}
|
||||
state.ResetResponse(rt.Config)
|
||||
|
||||
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, replayReq))
|
||||
require.NotNil(t, state.Fingerprint)
|
||||
require.True(t, state.ChallengeBypassed, "replayed allowlist cookie must flip ChallengeBypassed")
|
||||
|
||||
// Anything downstream calling SendChallenge must now be a no-op.
|
||||
bodyBefore := state.Response.UserHTTPBodyContent
|
||||
statusBefore := state.Response.UserHTTPResponseCode
|
||||
actionBefore := state.Response.Action
|
||||
|
||||
require.NoError(t, rt.SendChallenge(t.Context(), state, replayReq))
|
||||
|
||||
assert.Equal(t, actionBefore, state.Response.Action, "SendChallenge must not set ChallengeRemediation")
|
||||
assert.NotEqual(t, ChallengeRemediation, state.Response.Action,
|
||||
"a replayed allowlist cookie must not produce a challenge response")
|
||||
assert.Equal(t, bodyBefore, state.Response.UserHTTPBodyContent, "challenge body must not be written")
|
||||
assert.Equal(t, statusBefore, state.Response.UserHTTPResponseCode, "challenge status must not overwrite")
|
||||
assert.False(t, state.RequireChallenge, "RequireChallenge must stay false on bypassed replay")
|
||||
}
|
||||
|
||||
// TestGenerateResponseChallengeNilHeaders guards against a regression where a
|
||||
// `default_remediation: challenge` match reaches GenerateResponse with no
|
||||
// UserHeaders map set (only the internal challenge-serving paths set headers).
|
||||
// Writing the default CSP into that nil map used to panic.
|
||||
func TestGenerateResponseChallengeNilHeaders(t *testing.T) {
|
||||
rt := &AppsecRuntimeConfig{
|
||||
Logger: log.NewEntry(log.StandardLogger()),
|
||||
Config: &AppsecConfig{
|
||||
UserBlockedHTTPCode: http.StatusForbidden,
|
||||
BouncerBlockedHTTPCode: http.StatusForbidden,
|
||||
},
|
||||
}
|
||||
|
||||
resp := AppsecTempResponse{Action: ChallengeRemediation, UserHeaders: nil}
|
||||
|
||||
require.NotPanics(t, func() {
|
||||
_, body := rt.GenerateResponse(resp, rt.Logger)
|
||||
require.NotNil(t, body.UserHeaders)
|
||||
assert.Equal(t, []string{challenge.DefaultChallengeCSP}, body.UserHeaders["Content-Security-Policy"])
|
||||
})
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -326,24 +327,130 @@ func TestBuildNilPhaseConfig(t *testing.T) {
|
||||
assert.Empty(t, rt.OutOfBandHooks.OnMatch)
|
||||
}
|
||||
|
||||
func TestLoadAPISchemaRejectsPathTraversal(t *testing.T) {
|
||||
cases := []string{
|
||||
"../escape.yaml",
|
||||
"../../etc/passwd",
|
||||
"/etc/passwd",
|
||||
func TestBuildOnChallengeTopLevelAndInband(t *testing.T) {
|
||||
cfg := AppsecConfig{
|
||||
Logger: log.NewEntry(log.StandardLogger()),
|
||||
DefaultRemediation: "ban",
|
||||
OnChallenge: []Hook{
|
||||
{Apply: []string{"SetRemediation('allow')"}},
|
||||
},
|
||||
InBand: &AppsecPhaseConfig{
|
||||
OnChallenge: []Hook{
|
||||
{Filter: "IsInBand == true", Apply: []string{"SetReturnCode(418)"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, filename := range cases {
|
||||
t.Run(filename, func(t *testing.T) {
|
||||
rt := &AppsecRuntimeConfig{
|
||||
Logger: log.NewEntry(log.StandardLogger()),
|
||||
DataDir: t.TempDir(),
|
||||
}
|
||||
hub := &cwhub.Hub{}
|
||||
rt, err := cfg.Build(t.Context(), hub)
|
||||
require.NoError(t, err)
|
||||
|
||||
err := rt.LoadAPISchemaWithName("ref", filename)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "must be relative to schemas")
|
||||
})
|
||||
// Both top-level and inband on_challenge hooks end up in CompiledOnChallenge.
|
||||
require.Len(t, rt.CompiledOnChallenge, 2)
|
||||
assert.NotNil(t, rt.CompiledOnChallenge[0].ApplyExpr)
|
||||
assert.NotNil(t, rt.CompiledOnChallenge[1].FilterExpr)
|
||||
|
||||
// Defining on_challenge must force the challenge runtime on.
|
||||
assert.True(t, rt.NeedWASMVM)
|
||||
}
|
||||
|
||||
func TestBuildOnChallengeUnderOutofbandRejected(t *testing.T) {
|
||||
cfg := AppsecConfig{
|
||||
Logger: log.NewEntry(log.StandardLogger()),
|
||||
DefaultRemediation: "ban",
|
||||
OutOfBand: &AppsecPhaseConfig{
|
||||
OnChallenge: []Hook{{Apply: []string{"SetRemediation('allow')"}}},
|
||||
},
|
||||
}
|
||||
|
||||
hub := &cwhub.Hub{}
|
||||
_, err := cfg.Build(t.Context(), hub)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "on_challenge hooks are only valid in-band")
|
||||
}
|
||||
|
||||
func TestLoadByPathOnChallengeYAML(t *testing.T) {
|
||||
cfg := newTestConfig()
|
||||
f := writeTempYAML(t, `
|
||||
name: test-on-challenge
|
||||
on_challenge:
|
||||
- filter: "fingerprint != nil"
|
||||
apply:
|
||||
- SetRemediation('allow')
|
||||
inband:
|
||||
on_challenge:
|
||||
- apply:
|
||||
- SendChallenge()
|
||||
`)
|
||||
|
||||
require.NoError(t, cfg.LoadByPath(f))
|
||||
assert.Len(t, cfg.OnChallenge, 1)
|
||||
require.NotNil(t, cfg.InBand)
|
||||
assert.Len(t, cfg.InBand.OnChallenge, 1)
|
||||
}
|
||||
|
||||
// TestLoadByPathChallengeBlockMergesAcrossFiles confirms that a `challenge:`
|
||||
// block in an appsec-config YAML is parsed onto AppsecConfig.Challenge, and
|
||||
// that loading a second config with a disjoint subset of fields *merges*
|
||||
// rather than replacing — fields set in earlier configs survive when not
|
||||
// touched by later ones, and overlapping fields take the later value.
|
||||
func TestLoadByPathChallengeBlockMergesAcrossFiles(t *testing.T) {
|
||||
cfg := newTestConfig()
|
||||
|
||||
first := writeTempYAML(t, `
|
||||
name: base
|
||||
challenge:
|
||||
master_secret: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
key_rotation_interval: 5m
|
||||
max_live_epochs: 3
|
||||
cookie_ttl: 12h
|
||||
`)
|
||||
require.NoError(t, cfg.LoadByPath(first))
|
||||
require.NotNil(t, cfg.Challenge)
|
||||
require.NotNil(t, cfg.Challenge.MasterSecret)
|
||||
assert.Equal(t, 5*time.Minute, *cfg.Challenge.KeyRotationInterval)
|
||||
assert.Equal(t, 3, *cfg.Challenge.MaxLiveEpochs)
|
||||
assert.Equal(t, 12*time.Hour, *cfg.Challenge.CookieTTL)
|
||||
|
||||
// Second config overrides CookieTTL and adds crypto/spent-set fields,
|
||||
// leaving master_secret / rotation / max_live_epochs from the first
|
||||
// config untouched. This is the multi-config last-wins-per-field
|
||||
// behavior collections rely on.
|
||||
second := writeTempYAML(t, `
|
||||
name: overlay
|
||||
challenge:
|
||||
cookie_ttl: 1h
|
||||
crypto_obfuscation_pool_size: 2
|
||||
spent_set_max_entries: 250000
|
||||
`)
|
||||
require.NoError(t, cfg.LoadByPath(second))
|
||||
require.NotNil(t, cfg.Challenge)
|
||||
assert.Equal(t, 1*time.Hour, *cfg.Challenge.CookieTTL, "later config overrides cookie_ttl")
|
||||
require.NotNil(t, cfg.Challenge.MasterSecret, "master_secret from first config must survive")
|
||||
assert.Equal(t, 5*time.Minute, *cfg.Challenge.KeyRotationInterval)
|
||||
require.NotNil(t, cfg.Challenge.CryptoObfuscationPoolSize)
|
||||
assert.Equal(t, 2, *cfg.Challenge.CryptoObfuscationPoolSize, "new field from second config appears")
|
||||
require.NotNil(t, cfg.Challenge.SpentSetMaxEntries)
|
||||
assert.Equal(t, 250000, *cfg.Challenge.SpentSetMaxEntries)
|
||||
}
|
||||
|
||||
// TestLoadByPathRejectsRemovedLibraryObfuscationKeys confirms that an
|
||||
// appsec-config carrying any of the removed `library_*` obfuscation keys
|
||||
// fails to parse (strict YAML mode). fpscanner is now served unobfuscated,
|
||||
// so these knobs are gone and stale configs must fail loudly rather than
|
||||
// silently doing nothing.
|
||||
func TestLoadByPathRejectsRemovedLibraryObfuscationKeys(t *testing.T) {
|
||||
for _, key := range []string{
|
||||
"library_runtime_obfuscation_enabled: true",
|
||||
"library_obfuscation_pool_size: 2",
|
||||
"library_obfuscation_refresh_interval: 30m",
|
||||
} {
|
||||
cfg := newTestConfig()
|
||||
f := writeTempYAML(t, "name: stale\nchallenge:\n "+key+"\n")
|
||||
err := cfg.LoadByPath(f)
|
||||
require.Error(t, err, "stale key %q must be rejected", key)
|
||||
assert.Contains(t, err.Error(), "library_",
|
||||
"strict YAML parse error must name the unknown field")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -366,3 +473,24 @@ func TestBuildOnLoadStaysOutOfPhaseHooks(t *testing.T) {
|
||||
assert.Empty(t, rt.CommonHooks.PostEval)
|
||||
assert.Empty(t, rt.CommonHooks.OnMatch)
|
||||
}
|
||||
|
||||
func TestLoadAPISchemaRejectsPathTraversal(t *testing.T) {
|
||||
cases := []string{
|
||||
"../escape.yaml",
|
||||
"../../etc/passwd",
|
||||
"/etc/passwd",
|
||||
}
|
||||
|
||||
for _, filename := range cases {
|
||||
t.Run(filename, func(t *testing.T) {
|
||||
rt := &AppsecRuntimeConfig{
|
||||
Logger: log.NewEntry(log.StandardLogger()),
|
||||
DataDir: t.TempDir(),
|
||||
}
|
||||
|
||||
err := rt.LoadAPISchemaWithName("ref", filename)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "must be relative to schemas")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,12 +10,13 @@ import (
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/enrichment"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||
)
|
||||
|
||||
type AppsecCollection struct {
|
||||
Rules []string
|
||||
NativeRules []string
|
||||
Rules []string
|
||||
NativeRules []string
|
||||
}
|
||||
|
||||
const APPSEC_RULE = "appsec-rule"
|
||||
@@ -33,7 +34,7 @@ type AppsecCollectionConfig struct {
|
||||
|
||||
Labels map[string]any `yaml:"labels"` // Labels is K:V list aiming at providing context the overflow
|
||||
|
||||
Data any `yaml:"data"` // Ignore it
|
||||
Data []*enrichment.DataProvider `yaml:"data"`
|
||||
hash string
|
||||
version string
|
||||
}
|
||||
@@ -49,103 +50,148 @@ type RulesDetails struct {
|
||||
// Is using the id is a good idea ? might be too specific to coraza and not easily reusable
|
||||
var AppsecRulesDetails = make(map[int]RulesDetails)
|
||||
|
||||
// matchPattern reports whether appsecRule.Name matches pattern.
|
||||
func matchPattern(pattern, name string, logger *log.Entry) bool {
|
||||
tmpMatch, err := exprhelpers.Match(pattern, name)
|
||||
if err != nil {
|
||||
logger.Errorf("unable to match %s with %s : %s", name, pattern, err)
|
||||
return false
|
||||
}
|
||||
|
||||
matched, ok := tmpMatch.(bool)
|
||||
if !ok {
|
||||
logger.Errorf("unable to match %s with %s : result is not a boolean", name, pattern)
|
||||
return false
|
||||
}
|
||||
|
||||
return matched
|
||||
}
|
||||
|
||||
// loadNativeRulesFromFile reads a single seclang rules file and returns its non-comment, non-empty lines.
|
||||
func loadNativeRulesFromFile(fullPath string, logger *log.Entry) []string {
|
||||
c, err := os.ReadFile(fullPath)
|
||||
if err != nil {
|
||||
logger.Errorf("unable to read file %s : %s", fullPath, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
rules := make([]string, 0)
|
||||
|
||||
for line := range strings.SplitSeq(string(c), "\n") {
|
||||
if strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
rules = append(rules, line)
|
||||
}
|
||||
|
||||
return rules
|
||||
}
|
||||
|
||||
// loadSecLangFilesRules resolves the seclang file globs and appends their rules to appsecCol.
|
||||
func loadSecLangFilesRules(appsecRule AppsecCollectionConfig, hub *cwhub.Hub, logger *log.Entry, appsecCol *AppsecCollection) {
|
||||
for _, rulesFile := range appsecRule.SecLangFilesRules {
|
||||
logger.Debugf("Adding rules from %s", rulesFile)
|
||||
globPattern := filepath.Join(hub.GetDataDir(), rulesFile)
|
||||
|
||||
matches, err := filepath.Glob(globPattern)
|
||||
if err != nil {
|
||||
logger.Errorf("unable to glob %s : %s", rulesFile, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(matches) == 0 {
|
||||
logger.Warnf("no file matched pattern %s", globPattern)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, fullPath := range matches {
|
||||
appsecCol.NativeRules = append(appsecCol.NativeRules, loadNativeRulesFromFile(fullPath, logger)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// registerRuleDetails records the metadata and debug flags for a converted custom rule.
|
||||
func registerRuleDetails(appsecRule AppsecCollectionConfig, ruleName string, rulesId []uint32, logger *log.Entry) {
|
||||
// We only take the first id, as it's the one of the "main" rule
|
||||
if _, ok := AppsecRulesDetails[int(rulesId[0])]; !ok {
|
||||
AppsecRulesDetails[int(rulesId[0])] = RulesDetails{
|
||||
LogLevel: log.InfoLevel,
|
||||
Hash: appsecRule.hash,
|
||||
Version: appsecRule.version,
|
||||
Name: appsecRule.Name,
|
||||
}
|
||||
} else {
|
||||
logger.Warnf("conflicting id %d for rule %s !", rulesId[0], ruleName)
|
||||
}
|
||||
|
||||
for _, id := range rulesId {
|
||||
SetRuleDebug(int(id), appsecRule.Debug)
|
||||
}
|
||||
}
|
||||
|
||||
// loadCustomRules converts the YAML custom rules and appends the resulting seclang rules to appsecCol.
|
||||
func loadCustomRules(appsecRule AppsecCollectionConfig, logger *log.Entry, appsecCol *AppsecCollection) error {
|
||||
for _, rule := range appsecRule.Rules {
|
||||
rule.Severity = appsecRule.Severity
|
||||
|
||||
strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name, appsecRule.Description)
|
||||
if err != nil {
|
||||
logger.Errorf("unable to convert rule %s : %s", appsecRule.Name, err)
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Debugf("Adding rule %s", strRule)
|
||||
appsecCol.Rules = append(appsecCol.Rules, strRule)
|
||||
|
||||
registerRuleDetails(appsecRule, rule.Name, rulesId, logger)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initRuleData initializes the data files referenced by an appsec rule.
|
||||
func initRuleData(appsecRule AppsecCollectionConfig, hub *cwhub.Hub, logger *log.Entry) {
|
||||
for _, appsecRuleData := range appsecRule.Data {
|
||||
if appsecRuleData.DestPath == "" {
|
||||
logger.Errorf("missing dest_path for rule %s : %+v", appsecRule.Name, appsecRuleData)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := exprhelpers.FileInit(hub.GetDataDir(), appsecRuleData.DestPath, appsecRuleData.Type); err != nil {
|
||||
logger.Errorf("unable to initialize data file %s : %s", appsecRuleData.DestPath, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func LoadCollection(pattern string, logger *log.Entry, hub *cwhub.Hub) ([]AppsecCollection, error) {
|
||||
ret := make([]AppsecCollection, 0)
|
||||
|
||||
for _, appsecRule := range appsecRules {
|
||||
tmpMatch, err := exprhelpers.Match(pattern, appsecRule.Name)
|
||||
if err != nil {
|
||||
logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err)
|
||||
continue
|
||||
}
|
||||
|
||||
matched, ok := tmpMatch.(bool)
|
||||
|
||||
if !ok {
|
||||
logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if !matched {
|
||||
if !matchPattern(pattern, appsecRule.Name, logger) {
|
||||
continue
|
||||
}
|
||||
|
||||
appsecCol := AppsecCollection{}
|
||||
|
||||
if appsecRule.SecLangFilesRules != nil {
|
||||
for _, rulesFile := range appsecRule.SecLangFilesRules {
|
||||
logger.Debugf("Adding rules from %s", rulesFile)
|
||||
globPattern := filepath.Join(hub.GetDataDir(), rulesFile)
|
||||
|
||||
matches, err := filepath.Glob(globPattern)
|
||||
|
||||
if err != nil {
|
||||
logger.Errorf("unable to glob %s : %s", rulesFile, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(matches) == 0 {
|
||||
logger.Warnf("no file matched pattern %s", globPattern)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, fullPath := range matches {
|
||||
c, err := os.ReadFile(fullPath)
|
||||
if err != nil {
|
||||
logger.Errorf("unable to read file %s : %s", rulesFile, err)
|
||||
continue
|
||||
}
|
||||
|
||||
for line := range strings.SplitSeq(string(c), "\n") {
|
||||
if strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
appsecCol.NativeRules = append(appsecCol.NativeRules, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
loadSecLangFilesRules(appsecRule, hub, logger, &appsecCol)
|
||||
|
||||
if appsecRule.SecLangRules != nil {
|
||||
logger.Tracef("Adding inline rules %+v", appsecRule.SecLangRules)
|
||||
appsecCol.NativeRules = append(appsecCol.NativeRules, appsecRule.SecLangRules...)
|
||||
}
|
||||
|
||||
if appsecRule.Rules != nil {
|
||||
for _, rule := range appsecRule.Rules {
|
||||
rule.Severity = appsecRule.Severity
|
||||
strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name, appsecRule.Description)
|
||||
if err != nil {
|
||||
logger.Errorf("unable to convert rule %s : %s", appsecRule.Name, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logger.Debugf("Adding rule %s", strRule)
|
||||
appsecCol.Rules = append(appsecCol.Rules, strRule)
|
||||
|
||||
// We only take the first id, as it's the one of the "main" rule
|
||||
if _, ok := AppsecRulesDetails[int(rulesId[0])]; !ok {
|
||||
AppsecRulesDetails[int(rulesId[0])] = RulesDetails{
|
||||
LogLevel: log.InfoLevel,
|
||||
Hash: appsecRule.hash,
|
||||
Version: appsecRule.version,
|
||||
Name: appsecRule.Name,
|
||||
}
|
||||
} else {
|
||||
logger.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name)
|
||||
}
|
||||
|
||||
for _, id := range rulesId {
|
||||
SetRuleDebug(int(id), appsecRule.Debug)
|
||||
}
|
||||
}
|
||||
if err := loadCustomRules(appsecRule, logger, &appsecCol); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
initRuleData(appsecRule, hub, logger)
|
||||
|
||||
ret = append(ret, appsecCol)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,888 @@
|
||||
# WAF Bot Detection / Challenge — Design, Review, Usage
|
||||
|
||||
This document covers the CrowdSec WAF challenge subsystem: how it is built, what
|
||||
reviewers should look for when touching it, and how operators / collection
|
||||
authors are expected to use it.
|
||||
|
||||
The implementation lives under [pkg/appsec/challenge](.) and is wired into the
|
||||
acquisition pipeline by [pkg/acquisition/modules/appsec](../../acquisition/modules/appsec).
|
||||
|
||||
---
|
||||
|
||||
## 1. Design
|
||||
|
||||
### 1.1 What the feature does
|
||||
|
||||
The challenge subsystem lets the WAF interpose a browser-side proof-of-work +
|
||||
device-fingerprint check between a visitor and the protected origin. A visitor
|
||||
who solves the challenge receives a sealed cookie carrying their measured
|
||||
fingerprint; subsequent requests from the same browser can be evaluated by
|
||||
operator-defined hooks (`on_challenge`, `on_challenge_submit`) without
|
||||
re-challenging.
|
||||
|
||||
The challenge runtime is built **lazily**: it only spins up if at least one
|
||||
loaded hook calls `SendChallenge()`, `GrantChallengeCookie()`, or
|
||||
`RejectSubmission()`. This is detected by [`appsecExprPatcher`](../patcher.go)
|
||||
walking the compiled expression AST and setting
|
||||
`AppsecRuntimeConfig.NeedWASMVM`.
|
||||
|
||||
### 1.2 Component map
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────────┐
|
||||
│ Acquisition module (pkg/acquisition/modules/appsec/config.go) │
|
||||
│ - HTTP listener, ParsedRequest plumbing │
|
||||
│ - Loads appsec-config(s), builds AppsecRuntimeConfig │
|
||||
│ - If NeedWASMVM: challenge.BuildOptions → NewChallengeRuntime │
|
||||
└──────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────┐
|
||||
│ pkg/appsec (AppsecConfig / AppsecRuntimeConfig) │
|
||||
│ - YAML schema, hook compilation, request-lifecycle dispatcher │
|
||||
│ - ProcessOnChallengeRules: cookie validation, /submit handling │
|
||||
│ - SendChallenge / GrantChallengeCookie / RejectSubmission helpers │
|
||||
└──────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────┐
|
||||
│ pkg/appsec/challenge (ChallengeRuntime) │
|
||||
│ - KeyRing: HKDF-derived per-epoch keys + long-lived cookie key │
|
||||
│ - Crypto: AES-GCM cookie seal/open, per-challenge + PoW MAC HMACs │
|
||||
│ - Obfuscator: wazero-hosted WASM running javascript-obfuscator │
|
||||
│ - Challenge code: single static build-time obfuscated variant │
|
||||
│ - Dynamic module pool: per-epoch key script, obfuscated variants │
|
||||
│ - Fingerprint mismatch report aggregator │
|
||||
└──────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 1.3 Configuration model
|
||||
|
||||
Challenge tuning lives **inside the appsec-config**. The relevant fields are on [`challenge.Config`](config.go), referenced
|
||||
from `AppsecConfig.Challenge`:
|
||||
|
||||
```go
|
||||
type Config struct {
|
||||
MasterSecret *string
|
||||
KeyRotationInterval *time.Duration
|
||||
MaxLiveEpochs *int
|
||||
CookieTTL *time.Duration
|
||||
CryptoObfuscationPoolSize *int
|
||||
}
|
||||
```
|
||||
|
||||
`challenge.BuildOptions(c *Config) ([]Option, error)` translates the merged
|
||||
config into the option list consumed by `NewChallengeRuntime`.
|
||||
|
||||
### 1.4 Cryptographic core
|
||||
|
||||
**Notation.** These names are used consistently throughout this doc, the wire
|
||||
protocol, and the code. Note the two distinct "nonces": the **challenge nonce
|
||||
`r`** and the **PoW nonce `n`** (the AES-GCM nonce inside the cookie envelope is
|
||||
a third, unrelated value — written `gcm_nonce`).
|
||||
|
||||
| Symbol | Name | On the wire? | Definition |
|
||||
|---|---|---|---|
|
||||
| `master_secret` | master secret | no (config) | long-lived root secret, ≥ 32 B |
|
||||
| `K_epoch` | per-epoch key | no (derived) | `HKDF(master_secret, epoch)`; rotates each interval |
|
||||
| `r` | challenge nonce | yes (`r`) | random 16 B per challenge; identifies it for single-use |
|
||||
| `ts` | timestamp | yes (`ts`) | `time.Now().UnixNano()`; selects the epoch |
|
||||
| `s` | per-challenge secret | **no** | `HMAC(K_epoch, r)`; signs the submission |
|
||||
| `p` | PoW salt | yes (`p`) | random 16 B the client hashes for the PoW |
|
||||
| `n` | PoW nonce | yes (`n`) | PoW solution — makes `SHA256(p‖n)` have ≥ N leading zero bits |
|
||||
| `m` | PoW MAC | yes (`m`) | `HMAC(K_epoch, p‖r‖ts)`; proves `p` is server-issued |
|
||||
| `f` | fingerprint payload | yes (`f`) | obfuscated + base64 device fingerprint |
|
||||
| `encKey` | fingerprint obfuscation key | **no** | `HMAC(s, "fpenc"‖r)` |
|
||||
| `sig` | submission signature | yes (`sig`) | `HMAC(s, r‖ts‖n‖f)` |
|
||||
|
||||
The submit POST therefore carries exactly `f, r, ts, sig, n, p, m` (§1.6); `s`
|
||||
and `encKey` are derived on both ends and never transmitted.
|
||||
|
||||
**At a glance** — every cryptographic operation (all keys are 256-bit; every
|
||||
MAC/signature comparison uses constant-time `hmac.Equal`):
|
||||
|
||||
| Operation | Primitive | Key | Input → output |
|
||||
|---|---|---|---|
|
||||
| Derive operational keys | HKDF-SHA256 (salt `crowdsec-challenge-keyring-v1`) | `master_secret` | info `epoch-sign:<epoch>` → `K_epoch`; info `cookie-master` → master cookie key |
|
||||
| Per-challenge secret `s` | HMAC-SHA256 | `K_epoch` | `r` → `s` |
|
||||
| PoW MAC `m` | HMAC-SHA256 | `K_epoch` | `p‖r‖ts` → `m` |
|
||||
| Proof-of-work | SHA-256 | — | `p‖n` accepted iff ≥ N leading zero bits |
|
||||
| Fingerprint obf. key `encKey` | HMAC-SHA256 | `s` | `"fpenc"‖r` → `encKey` |
|
||||
| Fingerprint payload `f` | repeating-key XOR + base64 | `encKey` | fingerprint JSON → `f` (light obfuscation, **not** confidentiality — §2.6) |
|
||||
| Submission signature `sig` | HMAC-SHA256 | `s` | `r‖ts‖n‖f` → `sig` |
|
||||
| Cookie seal / open | AES-256-GCM | HKDF-derived from master cookie key | proto envelope, AAD = `User-Agent` → sealed cookie |
|
||||
|
||||
The subsections below detail each; the validation order is in §1.6.
|
||||
|
||||
#### Master secret
|
||||
|
||||
- Configured as hex (preferred — even-length, hex digits only) or as a
|
||||
passphrase. Both forms must be at least **32 bytes** decoded — the natural
|
||||
minimum for HMAC-SHA256 with full security margin.
|
||||
- If unconfigured, [`generateRandomSecret`](secret.go) produces a fresh 32-byte
|
||||
secret and the runtime logs a warning. Suitable only for single-instance
|
||||
deployments: a restart invalidates outstanding cookies, and distributed
|
||||
WAFs can't agree on the same value.
|
||||
|
||||
#### KeyRing — per-epoch derivation
|
||||
|
||||
[`pkg/appsec/challenge/keyring.go`](keyring.go) derives two independent key
|
||||
families from the master secret via HKDF-SHA256 (salt
|
||||
`"crowdsec-challenge-keyring-v1"`):
|
||||
|
||||
| Family | HKDF info | Lifetime | Use |
|
||||
|---|---|---|---|
|
||||
| Per-epoch key (`K_epoch`) | `"epoch-sign"` | One rotation interval | Per-challenge secret `s` + PoW MAC `m` |
|
||||
| Master cookie key | `"cookie-master"` | Lifetime of master secret | AES-GCM seal/open of challenge cookies |
|
||||
|
||||
An **epoch** is a time bucket of `rotationInterval` seconds, computed as
|
||||
`now.Unix() / rotation_interval_seconds`. All instances sharing the same
|
||||
master secret + rotation interval derive bit-identical epoch keys with no
|
||||
coordination — the basis for distributed verification of the per-challenge
|
||||
crypto (replay protection is separately stateful; see §2.1).
|
||||
|
||||
The keyring keeps a sliding window of `MaxLiveEpochs` past epochs plus the
|
||||
current epoch plus a one-step clock-skew slack. Submissions signed by any key
|
||||
inside the live window verify; older ones are rejected. Cached entries
|
||||
outside the window are evicted whenever a new epoch is derived.
|
||||
|
||||
Defaults: `keyringDefaultRotation = 5 minutes`, `keyringDefaultMaxLive = 3`,
|
||||
`keyringMinRotation = 30 seconds`.
|
||||
|
||||
#### Cookie envelope (v0)
|
||||
|
||||
[`pkg/appsec/challenge/crypto.go`](crypto.go) seals cookies under the
|
||||
long-lived master cookie key with AES-256-GCM:
|
||||
|
||||
```
|
||||
wire format: version_byte(1) || gcm_nonce(12) || ciphertext
|
||||
plaintext : not_after_be8(8) || flags(1) || reason_len_be(2)
|
||||
|| reason_bytes(0..256)
|
||||
|| protobuf_envelope
|
||||
AAD : user-agent bytes
|
||||
```
|
||||
|
||||
Three things to note:
|
||||
|
||||
1. **Cookie expiry is enforced by the embedded `not_after` timestamp**, not
|
||||
by keyring eviction. This intentional decouples cookie TTL (operator
|
||||
policy, e.g. 12h–24h) from the keyring live window (the window in which an
|
||||
extracted per-epoch key stays usable, e.g. 15m). Long-lived cookies don't
|
||||
widen that signing window.
|
||||
2. **User-Agent is the GCM authentication tag.** A cookie cannot be replayed
|
||||
across UAs — `openCookie` fails authentication on mismatch.
|
||||
3. **Allowlist cookies** set `cookieFlagAllowlisted = 0x01` and carry an
|
||||
operator reason (`MaxAllowlistReasonLen = 256` bytes — bounded to keep the
|
||||
final cookie comfortably under the browser 4 KB limit). Real-submission
|
||||
cookies leave the flag clear and the reason empty.
|
||||
|
||||
#### Per-challenge secret, PoW MAC, and single-use
|
||||
|
||||
Each challenge render carries a fresh 16-byte challenge nonce `r`. From it the
|
||||
client derives the **per-challenge secret** `s`, which is *never transmitted*:
|
||||
|
||||
- `s = HMAC(K_epoch, r)` — the per-challenge secret. Both client and server
|
||||
derive it independently (the client from the `K_epoch` delivered in the
|
||||
obfuscated dynamic module; the server from the keyring). It never appears on
|
||||
the wire.
|
||||
- `m = HMAC(K_epoch, p || r || ts)` — the PoW MAC; authenticates the PoW salt
|
||||
`p` as server-issued and bound to this challenge, so a client can't pick a
|
||||
favourable salt.
|
||||
|
||||
The submission is signed `sig = HMAC(s, r || ts || n || f)`. A valid `sig` proves
|
||||
the submitter knows `K_epoch` — normally by running our obfuscated JS to read it
|
||||
from the dynamic module, though a determined attacker can extract it (§2.6). See
|
||||
[ticket.go](ticket.go) (`deriveChallengeSecret`, `computePowMAC`,
|
||||
`verifyChallenge`).
|
||||
|
||||
`ts` is `time.Now().UnixNano()` formatted as a string; the verifier derives the
|
||||
epoch from `ts` itself and looks up the same key. **Challenge *issuance* is
|
||||
stateless** (any instance with the same master secret can render and verify the
|
||||
crypto), but **submission is single-use**: the validated `r` is burned in an
|
||||
in-memory spent-set ([spent_set.go](spent_set.go)), so a captured submission
|
||||
cannot be replayed against that instance. This is the load-bearing replay
|
||||
control; the per-epoch key obfuscation is only a non-load-bearing speed-bump
|
||||
(§2.6). The spent-set is **per-instance**, so single-use assumes all reverse
|
||||
proxies route to one appsec instance — see §2.1 / §3.4.
|
||||
|
||||
Freshness is doubly gated: keyring liveness (epoch in live window) and a loose
|
||||
backstop of `ticketAgeBackstop = 20 minutes` — which also bounds the spent-set
|
||||
TTL.
|
||||
|
||||
#### Single-use tickets
|
||||
|
||||
[`spent_set.go`](spent_set.go) records consumed challenge nonces (`r`) to
|
||||
enforce single-use. It wraps **gcache** (already used by `pkg/cache`): a bounded
|
||||
LRU (`spent_set_max_entries`, default `spentSetDefaultMaxEntries`) with a per-entry TTL equal to the freshness window,
|
||||
so there is no janitor goroutine or hand-rolled cap. A single mutex makes the
|
||||
`Has`/`SetWithExpire` pair in `checkAndInsert` atomic, so two concurrent
|
||||
identical submissions can't both win. The burn happens **last** in
|
||||
`ValidateChallengeResponse` (after freshness, PoW-MAC, PoW, and `sig` all pass),
|
||||
so the set only ever grows for crypto+PoW-valid submissions — its growth is
|
||||
sig+PoW-gated and TTL-bounded.
|
||||
|
||||
PoW levels (in leading SHA-256 zero bits) live as constants in
|
||||
[`ticket.go`](ticket.go):
|
||||
|
||||
| Level | Bits | Approx work |
|
||||
|---|---|---|
|
||||
| `Disabled` | 0 | nonce `"0"` always valid |
|
||||
| `Low` | 10 | ~1k hashes, 0.2–2 s |
|
||||
| `Medium` (default) | 12 | ~4k hashes, 1–8 s |
|
||||
| `High` | 15 | ~32k hashes, 7–60 s |
|
||||
| `Impossible` | 256 | unsolvable — hard block |
|
||||
|
||||
### 1.5 JS obfuscation pipeline
|
||||
|
||||
The browser receives four logical pieces of JS at every challenge:
|
||||
|
||||
1. **Obfuscated challenge code** — the small crypto/glue bundle (SHA-256, HMAC,
|
||||
PoW orchestration, fingerprint XOR, submission, hook registration). Injected
|
||||
**inline** on the challenge page. A single static build-time obfuscated
|
||||
variant (no runtime pool — see below).
|
||||
2. **Dynamic key module** — a ~30-line script embedding the current epoch's key
|
||||
`K_epoch` as a hex literal, so the client can derive the per-challenge secret
|
||||
`s = HMAC(K_epoch, r)` and sign the submission. Injected inline.
|
||||
3. **fpscanner bundle** — the public `fpscanner` device-fingerprint scanner,
|
||||
served **unobfuscated** as its own file at
|
||||
`/crowdsec-internal/challenge/fpscanner.js` and loaded via a plain
|
||||
`<script src>` tag. It assigns `globalThis.CrowdsecFingerprintScanner`; the
|
||||
challenge code reads that global at use time. Public, non-sensitive code —
|
||||
cacheable.
|
||||
4. **PoW worker** — separate JS served at `/crowdsec-internal/challenge/pow-worker.js`.
|
||||
|
||||
#### The obfuscator itself
|
||||
|
||||
A pre-built `javascript-obfuscator` distribution compiled to WASM lives at
|
||||
[`js/obfuscate/index.wasm.gz`](js/obfuscate) (regeneration is wired through
|
||||
`go:generate` in [`js/generate.go`](js/generate.go)). At runtime
|
||||
[`obfuscator.go`](obfuscator.go) hosts it under [wazero](https://wazero.io):
|
||||
|
||||
- The WASM module is **decompressed and compiled once** at startup
|
||||
(`CompileModule` ~4–5 s); per-call cost is just `InstantiateModule`.
|
||||
- Each obfuscation pass produces byte-distinct output even for the same input
|
||||
(mangled identifier names, random transforms).
|
||||
- A pass on the **challenge code** is cheap (~5 s of CPU) — it is just the small
|
||||
crypto/glue bundle; the bulk of the old library (fpscanner) is no longer
|
||||
obfuscated at all. A pass on the **dynamic module** also takes ~5 s.
|
||||
|
||||
#### Challenge code (single static variant)
|
||||
|
||||
The challenge crypto/glue code is **always** obfuscated, but it is a **single
|
||||
static build-time variant** — there is no runtime pool or refresher for it
|
||||
anymore. The build pipeline splits the two concerns:
|
||||
|
||||
- [`js/cmd/bundle`](js/cmd/bundle) emits two outputs: `fpscanner/fpscanner.js`
|
||||
(the minified, **unobfuscated** fpscanner IIFE, embedded as
|
||||
`challengejs.FPScannerJS` / re-exported as `challenge.FPScannerJS` and served
|
||||
raw at `/crowdsec-internal/challenge/fpscanner.js`) and `challenge_code.js`
|
||||
(the obfuscation input).
|
||||
- [`js/cmd/initialbundle`](js/cmd/initialbundle) substitutes the path
|
||||
placeholders into `challenge_code.js` and obfuscates it into
|
||||
`../initial_bundle.js.gz`, embedded into the binary.
|
||||
|
||||
On startup [`seedCacheFromInitialBundle()`](static_bundle.go) loads that single
|
||||
baked-in variant into the `challengeCode string` field on `ChallengeRuntime`;
|
||||
every visitor receives it inline via [`getChallengeCode()`](static_bundle.go).
|
||||
There is no per-visitor byte variance on the challenge code, and the fpscanner
|
||||
bundle (the public, bulky part) is served as-is.
|
||||
|
||||
#### Dynamic module pool
|
||||
|
||||
[`dynamic_module.go`](dynamic_module.go) renders a small template
|
||||
([`dynamic_module.js.tmpl`](dynamic_module.js.tmpl)) with the current epoch's
|
||||
key + epoch number, obfuscates it, and caches `crypto_obfuscation_pool_size`
|
||||
variants per live epoch.
|
||||
|
||||
Two non-obvious mechanisms keep this fast:
|
||||
|
||||
- **`singleflight` deduplication**: if N concurrent requests arrive right
|
||||
after a rotation, only one runs the obfuscator per variant slot; the others
|
||||
block on its result. Worst-case latency at the rotation boundary is one
|
||||
obfuscation pass, not N serialized passes.
|
||||
- **Pre-warming**: a goroutine fires `1/4 × rotation_interval` before each
|
||||
boundary (capped 1–30 s) and obfuscates the full pool for the upcoming
|
||||
epoch. The first request after rotation finds the cache populated.
|
||||
|
||||
The obfuscator config (in [`js/obfuscate/obfuscate.js`](js/obfuscate))
|
||||
reserves the hook symbol `__CSEC_CHALLENGE_HOOK_v1__` so the challenge code and
|
||||
the dynamic module can find each other by name across independent obfuscation
|
||||
passes. The key material itself is **not** reserved — it's deliberately
|
||||
encoded by the obfuscator's string-array transforms.
|
||||
|
||||
### 1.6 Request lifecycle
|
||||
|
||||
The acquisition module's `appsecHandler` parses every incoming HTTP request
|
||||
into a `ParsedRequest` and pushes it into the runner pool. Each runner calls
|
||||
`AppsecRunner.ProcessInBandRules`, which **first** invokes
|
||||
`AppsecRuntimeConfig.ProcessOnChallengeRules` and only proceeds to WAF
|
||||
evaluation if no challenge response was assembled.
|
||||
|
||||
`ProcessOnChallengeRules` dispatches three cases:
|
||||
|
||||
| Path | Method | Handling |
|
||||
|---|---|---|
|
||||
| `/crowdsec-internal/challenge/pow-worker.js` | GET | Static `PowWorkerJS` body served |
|
||||
| `/crowdsec-internal/challenge/fpscanner.js` | GET | Static (unobfuscated) `FPScannerJS` body served |
|
||||
| `/crowdsec-internal/challenge/submit` | POST | `ValidateChallengeResponse` → single-use burn + fingerprint de-obfuscate → `on_challenge_submit` hooks → `bodyChallengeOK` + Set-Cookie *or* `bodyChallengeRejected` |
|
||||
| anything else | any | Existing `__crowdsec_challenge` cookie validated → `state.Fingerprint` populated → `on_challenge` hooks |
|
||||
|
||||
For a submission, `ValidateChallengeResponse`:
|
||||
|
||||
1. Parses the form (`f, r, ts, sig, n, p, m, d` — obfuscated fingerprint,
|
||||
challenge nonce, timestamp, submission signature, PoW nonce, PoW salt,
|
||||
PoW MAC, and the issued PoW difficulty).
|
||||
2. Calls `verifyChallenge` to gate freshness, derive the epoch from `ts`, look
|
||||
up the per-epoch key, and verify `m == HMAC(K_epoch, p || r || ts || d)`.
|
||||
Binding `d` into the MAC authenticates the difficulty: a client cannot
|
||||
downgrade an escalated per-request challenge without invalidating `m`.
|
||||
3. Rejects outright if the (now-authenticated) difficulty is `Impossible`.
|
||||
4. Verifies the PoW: `hasLeadingZeroBits(SHA-256(p || n), d)` — using the
|
||||
issued per-request difficulty, not the runtime-global default.
|
||||
5. Derives `s = HMAC(K_epoch, r)` (never transmitted) and verifies
|
||||
`sig == HMAC(s, r || ts || n || f)`.
|
||||
6. Burns `r` in the spent-set — **single-use**; a replay is rejected here.
|
||||
7. De-obfuscates `f` (repeating-key XOR keyed by `HMAC(s, "fpenc" || r)`) and
|
||||
JSON-unmarshals into `FingerprintData`.
|
||||
8. Seals a v0 cookie carrying that fingerprint **and the proven difficulty `d`**
|
||||
under the long-lived master cookie key with a fresh `not_after`, so the next
|
||||
request's re-challenge check compares against the work actually performed.
|
||||
|
||||
The challenge page itself (`GetChallengePage`) builds the response by:
|
||||
|
||||
1. Resolving difficulty (per-request override beats runtime default).
|
||||
2. Reading the single static obfuscated challenge code and picking a random
|
||||
dynamic module variant (the unobfuscated fpscanner bundle is referenced by
|
||||
`<script src>`, not inlined).
|
||||
3. Generating `ts`, the challenge nonce `r`, the PoW salt `p`, and the PoW MAC `m`.
|
||||
4. Rendering [`challenge.html.tmpl`](challenge.html.tmpl) with all of the
|
||||
above.
|
||||
|
||||
### 1.7 Hook surface
|
||||
|
||||
Six hook stages, two of which are challenge-specific:
|
||||
|
||||
| Hook | When | Env | Notable helpers |
|
||||
|---|---|---|---|
|
||||
| `on_load` | Engine startup, once | runtime config | `SetChallengeDifficulty` (sets runtime default) |
|
||||
| `pre_eval` | Before WAF | request + state | `SendChallenge`, `GrantChallengeCookie(reason, ttl?)`, `SetChallengeDifficulty` (per-req) |
|
||||
| `post_eval` | After WAF | request + state | same as `pre_eval` |
|
||||
| `on_match` | Per-rule match | event + state | `SetRemediation`, `SetReturnCode` |
|
||||
| `on_challenge` | Valid cookie/submission, fingerprint available | fingerprint + state | `SendChallenge`, `SetChallengeDifficulty`, `EvaluateMismatches`, `fingerprint.*` |
|
||||
| `on_challenge_submit` | POST `/submit` after crypto validation | fingerprint + state | `RejectSubmission`, `GrantChallengeCookie(reason, ttl?)` (inline), `EvaluateMismatches` |
|
||||
|
||||
`on_challenge` is skipped if `state.Fingerprint` is nil (no valid cookie or
|
||||
submission). `on_challenge_submit` is deliberately narrow: only helpers that
|
||||
preserve the challenge-submit JSON envelope shape are exposed — a 307 redirect
|
||||
from this phase would break the client JS state machine.
|
||||
|
||||
### 1.8 Fingerprint mismatch report
|
||||
|
||||
[`fingerprint_mismatch.go`](fingerprint_mismatch.go) aggregates two families
|
||||
of signals into a `MismatchReport`:
|
||||
|
||||
**Library-native** (from `fpscanner`):
|
||||
CDP, WebDriver flavours, Selenium, Playwright, headless Chrome, impossible
|
||||
memory, high CPU count, WebGL/GPU/platform mismatches, bot user-agent —
|
||||
mostly *high* severity. UTC timezone is *medium*; swiftshader and language
|
||||
mismatches are *low*.
|
||||
|
||||
**Custom helpers**:
|
||||
|
||||
- `UAMobileMismatch()` — UA claims mobile but inner viewport ≥ 1024 px
|
||||
(catches desktop UA switchers). Medium.
|
||||
- `AcceptLanguageMismatch(req)` — request `Accept-Language` base differs from
|
||||
`navigator.language` base. Medium.
|
||||
- `TimezoneCountryMismatch(country)` — timezone not valid for GeoIP-derived
|
||||
country. Low (travellers / VPNs legitimately trigger it).
|
||||
|
||||
`EvaluateMismatches()` is the hook-side entry point; it caches the result on
|
||||
`state.LastMismatchReport` so chained calls like
|
||||
`EvaluateMismatches().High() >= 1 && EvaluateMismatches().Has("cdp")` don't
|
||||
recompute. First call also emits a debug log line and bumps per-signal
|
||||
Prometheus counters.
|
||||
|
||||
### 1.9 Allowlist bypass
|
||||
|
||||
`GrantChallengeCookie(reason, ttl?)` is the operator escape hatch for trusted
|
||||
clients (Googlebot, internal probes, paying partners…). It mints a sealed
|
||||
cookie with the `cookieFlagAllowlisted` bit set and the operator reason
|
||||
embedded, stamps a synthetic `Allowlisted` fingerprint on `state.Fingerprint`,
|
||||
and flips `state.ChallengeBypassed` so any later `SendChallenge` in the same
|
||||
request is a no-op.
|
||||
|
||||
Two variants, both exposed under the same name to the operator:
|
||||
|
||||
- In `pre_eval` / `post_eval`: issues HTTP 307 to the same URI with the
|
||||
cookie attached. The visitor bounces back through the WAF with the cookie
|
||||
present and the next iteration short-circuits on the allowlist branch.
|
||||
Required because plain `AllowRemediation` responses don't carry cookies
|
||||
through the bouncer protocol; only `ChallengeRemediation` does.
|
||||
- In `on_challenge_submit`: appends the cookie to the existing
|
||||
`{"status":"ok"}` JSON envelope.
|
||||
|
||||
The optional `ttl` argument is parsed via `time.ParseDuration` and overrides
|
||||
the runtime-global `cookie_ttl` for that single cookie only. Empty / missing
|
||||
falls back to the runtime default; malformed / non-positive surfaces as an
|
||||
expression error so authors see a precise diagnostic.
|
||||
|
||||
### 1.10 Decision logging
|
||||
|
||||
Accept / reject decisions go through `FingerprintData.LogAccepted` /
|
||||
`LogRejected` in [`fingerprint_helpers.go`](fingerprint_helpers.go).
|
||||
Both are nil-safe; the caller picks the log level (Debug for
|
||||
per-request sites, Info for rarer / operator-authored events).
|
||||
|
||||
The asymmetry to remember when reading these helpers: **accept logs
|
||||
always emit the baseline flags** (`is_bot`, `allowlisted`) so a clean
|
||||
acceptance is explicit, while **reject logs emit only positive
|
||||
information** — `"headless": false` is noise on a forensic artefact.
|
||||
`signals` is enumerated by walking `libDetections`, so it cannot
|
||||
drift from `MismatchReport.Reasons()`.
|
||||
|
||||
For submit-phase events the verbosity is operator-controlled via the
|
||||
expr helpers `RejectSubmission(reason, verbosity?)` and
|
||||
`LogAccepted(msg, verbosity?)` (see §3.2); both helpers log at Info,
|
||||
and the verbosity argument tunes only the field-set richness, not the
|
||||
level. `ValidateChallengeResponse` in `challenge.go` is deliberately
|
||||
silent on the success path — the caller in `appsec.go` (and the expr
|
||||
wrappers in `waf_helpers.go`) own the `*log.Entry`, so they own the
|
||||
accept / reject log.
|
||||
|
||||
For the field reference and example output, see §3.1.
|
||||
|
||||
---
|
||||
|
||||
## 2. Review guidelines
|
||||
|
||||
Things to actively look for when reviewing changes that touch this subsystem.
|
||||
|
||||
### 2.1 Invariants that must not break
|
||||
|
||||
- **Stateless *crypto*, stateful *single-use*.** All per-challenge / PoW-MAC /
|
||||
cookie crypto must remain derivable from the master secret + epoch + request data
|
||||
(no node-local randomness baked into verifiable material), so any instance can
|
||||
verify any other's signatures. **However, replay protection is intentionally
|
||||
stateful**: the challenge nonce `r` is burned in an in-memory spent-set
|
||||
([spent_set.go](spent_set.go)). The design assumes **all web servers talk to a
|
||||
single appsec instance** — single-use holds within that instance. A
|
||||
multi-instance fleet without a shared spent-set would not enforce fleet-wide
|
||||
single-use (a captured submission could replay against a *different* instance);
|
||||
that is accepted by assumption. Cookie sealing/validation stays stateless under
|
||||
the master cookie key, so only the brief challenge *handshake* is instance-
|
||||
sticky — the 12h cookie validates on any instance.
|
||||
- **`master_secret` policy.** A misconfigured master_secret must FAIL the
|
||||
config load — it must never silently fall back to a random secret when the
|
||||
operator clearly intended to set one. The "random secret" path is reserved
|
||||
for the *no `master_secret` at all* case, and must always emit the existing
|
||||
warning. [`BuildOptions`](config.go) bubbles `ParseConfiguredSecret` errors
|
||||
for this reason.
|
||||
- **Secret-logging discipline.** `master_secret` is never logged. `k_epoch` /
|
||||
`s` may be logged **only at debug**, behind an `IsLevelEnabled(debug)` guard
|
||||
so they aren't formatted at higher levels (§1.11). A stray `Info`/`Warn`
|
||||
carrying `k_epoch` is a key leak — keep the guard.
|
||||
- **Cookie / keyring decoupling.** Cookie TTL is enforced inside the sealed
|
||||
envelope (`not_after`), not by keyring eviction. Don't tie cookie validity
|
||||
to live-epoch membership — that would force cookie TTL ≤ keyring window,
|
||||
which is the opposite of what the design wants.
|
||||
- **User-Agent AAD.** Removing the UA AAD from `sealCookieV0` / `openCookie`
|
||||
breaks cookie-binding-to-browser. If a change wants per-UA tolerance,
|
||||
introduce a *separate* validation path; don't loosen the GCM tag.
|
||||
- **`NeedWASMVM` detection coverage.** Any new expr helper that needs the
|
||||
challenge runtime must be added to `challengeRuntimeCallees` in
|
||||
[`patcher.go`](../patcher.go). Forgetting this lets a hook reference a
|
||||
helper without spinning the runtime up at all — `nil` deref at request
|
||||
time.
|
||||
- **`state.ChallengeBypassed` honor.** Helpers that produce a challenge
|
||||
response (`SendChallenge`, future variants) must early-return when
|
||||
`ChallengeBypassed` is set. Two `Set-Cookie`s and a redirect with a body
|
||||
is incoherent for the bouncer protocol.
|
||||
- **`on_challenge_submit` narrowness.** The submit-phase env intentionally
|
||||
omits `SendChallenge`, `SetRemediation`, `SetReturnCode`,
|
||||
`SetChallengeDifficulty`, `DropRequest`. Adding any of these here will
|
||||
silently break the client JS state machine. If you need this behaviour,
|
||||
set state and let the *next* request's `pre_eval` act on it.
|
||||
|
||||
### 2.2 Concurrency hazards
|
||||
|
||||
- **Dynamic module pool**: writes go through `singleflight` and a mutex.
|
||||
Don't bypass either. Walking `dynamicModuleCache` without holding
|
||||
`dynamicModuleCacheMu` is a race; mutating it inside a
|
||||
`singleflight.Do` callback is fine.
|
||||
- **Challenge code**: held in a single immutable `challengeCode string` field,
|
||||
seeded once at startup by `seedCacheFromInitialBundle()` and read via
|
||||
`getChallengeCode()`. There is no pool, mutex, or refresher to coordinate —
|
||||
it is set-once and read-only thereafter.
|
||||
- **Pre-warmer lifetime**: the pre-warmer reads from the keyring and writes
|
||||
into the dynamic module cache. It must observe context cancellation. If
|
||||
you add a new goroutine, plumb the same context.
|
||||
- **Obfuscator instantiation**: `Runtime.InstantiateModule` is safe to call
|
||||
concurrently against the same `CompiledModule`; the actual JS execution
|
||||
isn't a concern because each call gets its own module instance.
|
||||
|
||||
### 2.3 Performance and resource shape
|
||||
|
||||
- **Compile-once, instantiate-per-call** is the established WASM pattern.
|
||||
If a change moves WASM compilation onto the request path, that adds ~4–5
|
||||
seconds of CPU per challenge served.
|
||||
- **Pool sizes have CPU cost**, not just memory cost. Each
|
||||
`crypto_obfuscation_pool_size` increment is one extra ~5 s obfuscation
|
||||
pass per epoch rotation. The challenge code is obfuscated once at build
|
||||
time, so it adds no runtime obfuscation cost at all.
|
||||
- **Pre-warm lead time** is `1/4 × rotation_interval`, capped at 30 s and
|
||||
floored at 1 s. Shrinking the rotation interval below ~20 s makes the
|
||||
pre-warmer too tight for the obfuscation pass; the test
|
||||
`TestCryptoObfuscationPoolSize` in
|
||||
[obfuscation_pools_test.go](obfuscation_pools_test.go) is currently
|
||||
flaky around this boundary and is unrelated to most config changes.
|
||||
|
||||
### 2.4 Config and merge semantics
|
||||
|
||||
- **`challenge.Config` fields must stay pointer-typed.** Switching a field
|
||||
to a value type breaks the multi-config merge — `MergeFrom` relies on
|
||||
`nil` to mean "unset, keep dst's value". A `bool` value type would always
|
||||
look set to `false`.
|
||||
- **`LoadByPath` merge order is "last wins" per field.** Tests asserting
|
||||
this live in `pkg/appsec/appsec_config_test.go::TestLoadByPathChallengeBlockMergesAcrossFiles`.
|
||||
Don't switch to first-wins or error-on-conflict without an explicit
|
||||
design decision — collection composability depends on it.
|
||||
- **YAML strict parsing.** [`Source.UnmarshalConfig`](../../acquisition/modules/appsec/config.go)
|
||||
uses `yaml.Strict()`. This is the desired loud-failure surface for stale
|
||||
`challenge_*` keys in acquisition YAML left over from earlier branches.
|
||||
Don't loosen it.
|
||||
|
||||
### 2.5 Test hooks
|
||||
|
||||
Targeted tests live alongside the code:
|
||||
|
||||
- [`config_test.go`](config_test.go) — `MergeFrom`, `BuildOptions`,
|
||||
per-call cookie TTL override.
|
||||
- [`challenge_test.go`](challenge_test.go) — leading-zero-bits PoW check,
|
||||
end-to-end render/submit.
|
||||
- [`keyring_test.go`](keyring_test.go) + [`keyring_integration_test.go`](keyring_integration_test.go)
|
||||
— epoch math, key derivation determinism, liveness window.
|
||||
- [`secret_test.go`](secret_test.go) — hex vs passphrase, min-size guard.
|
||||
- [`initial_bundle_test.go`](initial_bundle_test.go), [`split_bundle_test.go`](split_bundle_test.go)
|
||||
— baked-in bundle integrity. `initial_bundle_test.go` also asserts fpscanner
|
||||
is **not** present in the obfuscated challenge bundle and that `FPScannerJS`
|
||||
exposes the `globalThis.CrowdsecFingerprintScanner` global.
|
||||
- [`obfuscation_pools_test.go`](obfuscation_pools_test.go) — asserts the
|
||||
challenge code is seeded **statically** (single build-time variant, no
|
||||
runtime pool) and exercises the crypto/dynamic-module pool behaviour (note:
|
||||
`TestCryptoObfuscationPoolSize` is flaky; it depends on the JS obfuscator
|
||||
producing byte-distinct output across passes).
|
||||
- [`pkg/appsec/appsec_challenge_test.go`](../appsec_challenge_test.go) —
|
||||
`GrantChallengeCookie` 307 path, inline submit variant,
|
||||
`state.ChallengeBypassed` guard, allowlist flag propagation.
|
||||
- [`pkg/appsec/waf_helpers_test.go`](../waf_helpers_test.go) — TTL
|
||||
argument parsing.
|
||||
|
||||
Run subsets when iterating:
|
||||
|
||||
```
|
||||
go test ./pkg/appsec/challenge/ -run 'TestConfig|TestBuildOptions|TestSealAllowlistCookieTTLOverride'
|
||||
go test ./pkg/appsec/ -run 'TestGrantChallengeCookie|TestProcessOnChallengeRules'
|
||||
```
|
||||
|
||||
The combined `./pkg/appsec/... ./pkg/acquisition/modules/appsec/...` run
|
||||
hits ~10 min wall clock because of the obfuscation passes; expect to time
|
||||
out unless you split by package.
|
||||
|
||||
### 2.6 Things that look wrong but aren't
|
||||
|
||||
- `GrantChallengeCookie` issues a **307** even though it semantically
|
||||
*allows* the visitor. This is because the bouncer protocol only
|
||||
serialises `UserCookies` on a `ChallengeRemediation` response; a plain
|
||||
`AllowRemediation` would drop the `Set-Cookie`. The 307 is the
|
||||
minimal-friction way to deliver the cookie *and* bounce the visitor
|
||||
through the WAF a second time so the allowlist branch hits.
|
||||
- `mintAllowlistCookie` deliberately **overwrites any prior `state.Fingerprint`**
|
||||
set by the cookie-valid branch. Operator-explicit allowlist beats a
|
||||
pre-existing real-submission fingerprint. Filter on `req.Headers` *before*
|
||||
the allowlist rule fires if you want the real signal preserved.
|
||||
- The dynamic module template embeds the per-epoch key as a hex literal in
|
||||
plain JS source before obfuscation. That's intentional — the obfuscator's
|
||||
string-array transforms encode it, and per-epoch rotation bounds the
|
||||
exposure. Note that with per-challenge single-use tickets the obfuscation is
|
||||
**no longer load-bearing**: extracting `K_epoch` lets an attacker self-mint
|
||||
valid submissions *without ever requesting a challenge page* — issuance is
|
||||
stateless, so the server records no issued `r`, and the attacker can pick a
|
||||
fresh `r` each time. Each forged submission still costs one PoW solve and is
|
||||
usable once before the spent-set burns it. The real bound is therefore **one
|
||||
PoW solve per forged submission, for the lifetime of the live epoch window** —
|
||||
no worse than the pre-PR ticket scheme. Raising the JS reverse-engineering bar
|
||||
is tracked separately.
|
||||
- The fingerprint payload `f` is "encrypted" with a repeating-key XOR
|
||||
(`deriveFingerprintObfKey` / `deobfuscateFingerprint`). This is **deliberately
|
||||
light obfuscation, not confidentiality**: a known-plaintext attack recovers
|
||||
only the per-challenge, single-use key `HMAC(s, "fpenc" || r)`, which is
|
||||
one-way-derived (`K_epoch → s → obfKey`) and cannot leak `s`/`K_epoch` or
|
||||
forge `sig`. Transport confidentiality is TLS's job; integrity is `sig`'s.
|
||||
Don't "upgrade" it to WebCrypto AES-GCM — routing `K_epoch`-derived key
|
||||
material through `crypto.subtle.*` creates a trivially-hookable key-
|
||||
interception boundary, which is worse.
|
||||
- Challenge submission is single-use but issuance is not: an attacker spamming
|
||||
GET challenge pages costs nothing extra (issuance is stateless). The
|
||||
spent-set only grows on *valid* submissions (sig+PoW-gated) and entries
|
||||
expire after the freshness window, so it can't be cheaply flooded.
|
||||
|
||||
---
|
||||
|
||||
## 3. Intended usage
|
||||
|
||||
This section is for collection authors and operators shipping bot
|
||||
detection.
|
||||
|
||||
### 3.1 Minimal collection setup
|
||||
|
||||
Install a collection that ships an appsec-config exercising the challenge
|
||||
hooks:
|
||||
|
||||
```sh
|
||||
cscli collections install crowdsecurity/appsec-bot-detection
|
||||
```
|
||||
|
||||
A complete appsec-config for the feature looks like:
|
||||
|
||||
```yaml
|
||||
name: crowdsecurity/appsec-bot-detection
|
||||
default_remediation: ban
|
||||
|
||||
inband_rules:
|
||||
- crowdsecurity/base-config
|
||||
- crowdsecurity/vpatch-*
|
||||
|
||||
challenge:
|
||||
# Required for distributed (multi-WAF) deployments. Hex (preferred) or
|
||||
# passphrase; min 32 bytes / characters. If unset the runtime generates an
|
||||
# ephemeral random secret and logs a warning (single-instance only).
|
||||
master_secret: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
|
||||
# Optional tuning — defaults shown.
|
||||
key_rotation_interval: 5m
|
||||
max_live_epochs: 3
|
||||
cookie_ttl: 12h
|
||||
# max_cookie_size: 4096 # encoded cookie size ceiling (seal + open); bounds envelope allocation
|
||||
crypto_obfuscation_pool_size: 1
|
||||
|
||||
# Hooks define WHEN to challenge and WHO gets allowlisted.
|
||||
inband:
|
||||
pre_eval:
|
||||
# Allowlist trusted reverse-resolved Googlebot. Mint a 24h cookie
|
||||
# explicitly with the optional TTL argument.
|
||||
- filter: "req.Headers.Get('User-Agent') contains 'Googlebot'"
|
||||
apply:
|
||||
- GrantChallengeCookie("Googlebot", "24h")
|
||||
|
||||
# Challenge everything else with a measurable fingerprint signal we want
|
||||
# to inspect on subsequent requests.
|
||||
- filter: "true"
|
||||
apply:
|
||||
- SendChallenge()
|
||||
|
||||
on_challenge:
|
||||
# After a valid submission OR cookie present: escalate to ban if the
|
||||
# fingerprint exhibits strong bot indicators.
|
||||
- filter: "EvaluateMismatches().High() >= 1 || EvaluateMismatches().Has('cdp')"
|
||||
apply:
|
||||
- SetRemediation("ban")
|
||||
|
||||
# Soft escalation: raise difficulty on weak signals.
|
||||
- filter: "EvaluateMismatches().Medium() >= 2"
|
||||
apply:
|
||||
- SetChallengeDifficulty("high")
|
||||
- SendChallenge()
|
||||
|
||||
on_challenge_submit:
|
||||
# Catch automation that solves the PoW but fails atomic fingerprint
|
||||
# consistency checks. Refuse to issue a cookie for these and dump
|
||||
# the full fingerprint detail into the reject log.
|
||||
- filter: "fingerprint.UAMobileMismatch() || fingerprint.AcceptLanguageMismatch(req)"
|
||||
apply:
|
||||
- RejectSubmission("ua_or_lang_mismatch", "verbose")
|
||||
|
||||
# Clean submission: log the acceptance with default-verbosity fields
|
||||
# so operators have an auditable accept event per visitor.
|
||||
- filter: "true"
|
||||
apply:
|
||||
- LogAccepted("challenge submission accepted")
|
||||
```
|
||||
|
||||
#### What the decision logs look like
|
||||
|
||||
Every fingerprint accept / reject above produces a structured log
|
||||
entry via `FingerprintData.LogAccepted` / `LogRejected`. Operators
|
||||
correlate on `fsid` / `source` and can index these directly.
|
||||
|
||||
Examples at the default `"info"` verbosity (what the expr helpers
|
||||
`LogAccepted` / `RejectSubmission` emit when called without a second
|
||||
argument):
|
||||
|
||||
```
|
||||
level=info msg="challenge submission accepted" source=203.0.113.7 bouncer=10.0.0.5 fsid=FS1_abc ua="Mozilla/5.0…" platform=macOS is_bot=false signals="[]" allowlisted=false is_mobile=false
|
||||
level=info msg="on_challenge_submit rejected" source=203.0.113.7 bouncer=10.0.0.5 fsid=FS1_abc reason=ua_or_lang_mismatch ua="Mozilla/5.0…" platform=macOS signals="[cdp]" automation=true
|
||||
level=info msg="granted allowlist challenge cookie via 307 redirect" source=66.249.66.1 bouncer=10.0.0.5 location=/ ua="Googlebot/2.1" allowlisted=true allowlist_reason=Googlebot signals="[]"
|
||||
level=debug msg="valid challenge cookie" source=203.0.113.7 bouncer=10.0.0.5 fsid=FS1_abc ua="Mozilla/5.0…" platform=macOS is_bot=false signals="[]" allowlisted=false
|
||||
```
|
||||
|
||||
The verbosity string (`"minimal"`, `"info"` default, `"verbose"`) is
|
||||
passed as the optional last argument to both `LogAccepted` and
|
||||
`RejectSubmission`. Bad values emit a warning and fall back to
|
||||
`"info"`.
|
||||
|
||||
**Always present (`"minimal"` verbosity):**
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---|---|---|
|
||||
| `source` | string | Real visitor IP (`request.ClientIP`, set by the bouncer via X-Forwarded-For or equivalent). Same field name as `fingerprint mismatch` logs for correlation |
|
||||
| `bouncer` | string | Connection-level peer of the appsec listener (`request.RemoteAddrNormalized`) — the bouncer / gateway that forwarded the request. Useful in multi-WAF setups or when debugging an X-Forwarded-For chain |
|
||||
| `fsid` | string | Per-fingerprint identifier; omitted when empty |
|
||||
| `ua` | string | Browser-reported User-Agent; omitted when empty |
|
||||
| `platform` | string | High-entropy client-hint platform, falling back to `navigator.platform`; omitted when empty |
|
||||
| `is_bot` | bool (accept) / only-if-true (reject) | Library fast-bot verdict |
|
||||
| `signals` | `[]string` | Names of library bot signals that fired (e.g. `["cdp", "headless_screen_resolution"]`); empty on clean fingerprints. Enumerated from `libDetections`, so always in sync with `MismatchReport.Reasons()` |
|
||||
| `allowlisted` | bool (accept) / only-if-true (reject) | True for cookies minted by `GrantChallengeCookie` |
|
||||
| `allowlist_reason` | string | Present only when `allowlisted == true` and the reason is non-empty |
|
||||
| `reason` | string | **Reject-only**, always present. The operator-facing cause (e.g. the string passed to `RejectSubmission`) |
|
||||
|
||||
**`"info"` verbosity adds** `is_mobile` plus the category roll-ups
|
||||
(`automation` / `headless` / `mismatch` / `impossible_device`); the
|
||||
roll-ups only appear when they fired.
|
||||
|
||||
**`"verbose"` verbosity adds** `timezone`, `language`, `cpu_count`,
|
||||
`memory`, `url`, `nonce`, `fp_time`. Zero / empty values are omitted.
|
||||
|
||||
Custom mismatches (`UAMobileMismatch`, `AcceptLanguageMismatch`,
|
||||
`TimezoneCountryMismatch`) are **not** in `signals` — they need
|
||||
request / geo context and are emitted separately on the `fingerprint
|
||||
mismatch` line, with their own `reasons` field. Correlate on `fsid` /
|
||||
`source`.
|
||||
|
||||
### 3.2 YAML field reference
|
||||
|
||||
#### `challenge:` block
|
||||
|
||||
| Key | Type | Default | Notes |
|
||||
|---|---|---|---|
|
||||
| `master_secret` | string | random + warning | Hex (preferred) or passphrase, ≥ 32 bytes |
|
||||
| `key_rotation_interval` | duration | 5m | Min 30 s; must be the same across all instances |
|
||||
| `max_live_epochs` | int | 3 | Past epochs accepted; sized to cover the freshness window |
|
||||
| `cookie_ttl` | duration | 12h | Decoupled from rotation interval; can be long (24h+) |
|
||||
| `max_cookie_size` | int | 4096 | Encoded cookie size ceiling, enforced on seal and open. Bounds the memory allocated from the (attacker-supplied) fingerprint envelope — an over-allocation DoS guard. Matches the per-cookie size browsers guarantee; raise only for non-browser clients that tolerate larger cookies |
|
||||
| `crypto_obfuscation_pool_size` | int | 1 | Variants of dynamic key module per epoch; each costs ~5 s of CPU per rotation |
|
||||
| `log_level` | string | inherits | `debug` / `info` / `warning` (etc.) for the challenge runtime only. ⚠️ **`debug` writes the per-epoch key `k_epoch` to the logs** (key rotation + per challenge) — forgeable signing material; a diagnostic mode only, **not for production** (see §1.10). Unset inherits the appsec logger's level. |
|
||||
|
||||
#### Hook helpers
|
||||
|
||||
| Helper | Available in | Effect |
|
||||
|---|---|---|
|
||||
| `SendChallenge()` | `pre_eval`, `post_eval`, `on_challenge` | Serve a challenge page (no-op if visitor already passed an equal-or-harder PoW, or `ChallengeBypassed`) |
|
||||
| `GrantChallengeCookie(reason, ttl?)` | `pre_eval`, `post_eval` | Mint allowlist cookie + 307 redirect |
|
||||
| `GrantChallengeCookie(reason, ttl?)` | `on_challenge_submit` | Mint allowlist cookie inline on submit response |
|
||||
| `RejectSubmission(reason, verbosity?)` | `on_challenge_submit` | Refuse to issue cookie for an otherwise-valid submission; emits an Info reject log immediately. `verbosity ∈ {minimal, info (default), verbose}` controls how much fingerprint detail rides on the log; bad values warn and fall back to info |
|
||||
| `LogAccepted(msg, verbosity?)` | `on_challenge_submit` | Emit an Info accept log with the current fingerprint. Same `verbosity` vocabulary as `RejectSubmission`. No-op when no fingerprint is present |
|
||||
| `SetChallengeDifficulty(level)` | `on_load` | Set runtime default (`"disabled"`, `"low"`, `"medium"`, `"high"`, `"impossible"`) |
|
||||
| `SetChallengeDifficulty(level)` | `pre_eval`, `post_eval`, `on_challenge` | Per-request override (does NOT change runtime default) |
|
||||
| `EvaluateMismatches()` | `on_challenge`, `on_challenge_submit` | Returns `MismatchReport`; cached per request |
|
||||
| `fingerprint.UAMobileMismatch()` | wherever `fingerprint` is non-nil | Atomic check, no aggregation |
|
||||
| `fingerprint.AcceptLanguageMismatch(req)` | same | Atomic check |
|
||||
| `fingerprint.TimezoneCountryMismatch(country)` | same | Atomic check |
|
||||
|
||||
#### `MismatchReport` API
|
||||
|
||||
```
|
||||
.Count() total signals fired
|
||||
.Empty() true if none fired
|
||||
.High() / .Medium() / .Low() count by severity
|
||||
.BySeverity(s) count for "high" / "medium" / "low"
|
||||
.Has(reason) true if a specific signal fired
|
||||
.Reasons() stable-ordered list of fired reason keys
|
||||
.String() "reason1(sev),reason2(sev)" — handy for logs
|
||||
```
|
||||
|
||||
### 3.3 Multi-config composition
|
||||
|
||||
Multiple appsec-configs can be merged by listing them in acquisition:
|
||||
|
||||
```yaml
|
||||
# acquis.yaml
|
||||
appsec_configs:
|
||||
- crowdsecurity/appsec-default # base rules, no challenge block
|
||||
- crowdsecurity/appsec-bot-detection # challenge block + hooks
|
||||
- mycorp/internal-allowlist # adds an extra pre_eval hook
|
||||
```
|
||||
|
||||
Within the `challenge:` block, **each later config's non-nil fields override
|
||||
earlier ones, field by field.** A collection that only wants to ship a
|
||||
specific `cookie_ttl` can do so without redeclaring the rest:
|
||||
|
||||
```yaml
|
||||
# mycorp/short-cookie-overlay
|
||||
name: mycorp/short-cookie-overlay
|
||||
challenge:
|
||||
cookie_ttl: 30m # overrides whatever the upstream config set
|
||||
```
|
||||
|
||||
Other top-level YAML keys follow the existing appsec-config patterns: rules
|
||||
and hooks are *appended* across configs; scalars like `default_remediation`,
|
||||
HTTP codes, and challenge fields are *overridden*.
|
||||
|
||||
### 3.4 Single-instance vs distributed deployment
|
||||
|
||||
What is stateless vs stateful matters here:
|
||||
|
||||
- **Stateless (works fleet-wide, no coordination):** challenge crypto
|
||||
verification (`sig` / PoW-MAC) and cookie seal/open. Any instance sharing the
|
||||
`master_secret` + `key_rotation_interval` validates any other's challenges and
|
||||
cookies.
|
||||
- **Stateful (per-instance):** single-use replay protection. The spent-set that
|
||||
burns each `r` is **in-memory and local to one appsec instance** — it is *not*
|
||||
shared across instances (no Redis / shared store).
|
||||
|
||||
| Scenario | What to do |
|
||||
|---|---|
|
||||
| Single WAF, dev | Skip `master_secret` entirely. Ephemeral cookies on restart, warning logged. |
|
||||
| Single WAF, prod | Set `master_secret`. Restarts don't invalidate outstanding cookies. Single-use replay protection is fully effective (one local spent-set). |
|
||||
| Multi-WAF, prod | Set the **same** `master_secret` *and* `key_rotation_interval` on every instance — crypto/cookie verification then works without coordination. **But:** because the spent-set is per-instance, a captured submission could be **replayed against a *different* instance**. The design therefore assumes **all reverse proxies route challenge traffic to a single appsec/WAF instance** (or that the front-end pins the challenge handshake to one instance). Running independent instances behind a load balancer means single-use degrades to per-instance — replay across instances is possible until a shared spent-set is added. |
|
||||
|
||||
For multi-instance, also make sure the WAFs' clocks are reasonably aligned —
|
||||
the keyring tolerates one rotation interval of clock skew, no more.
|
||||
|
||||
### 3.5 Tuning checklist
|
||||
|
||||
- **Default-friendly** (low CPU, single-instance): no `challenge:` block, or
|
||||
just `master_secret` + `cookie_ttl`. Acceptable for most deployments.
|
||||
- **High-traffic, wants per-visitor JS variance**: bump
|
||||
`crypto_obfuscation_pool_size` to 2–3. Costs `N × 5 s` of CPU per
|
||||
rotation.
|
||||
- **fpscanner is public, served raw**: the fpscanner bundle at
|
||||
`/crowdsec-internal/challenge/fpscanner.js` is unobfuscated, cacheable public
|
||||
code — there is no runtime knob to obfuscate it, and scrapers keying on that
|
||||
static bundle are not a threat the runtime tries to defeat. The only
|
||||
per-visitor JS variance available is on the dynamic key module
|
||||
(`crypto_obfuscation_pool_size`); the challenge crypto code is a single
|
||||
static build-time obfuscated variant.
|
||||
- **Long-lived cookies for known clients**: set a long `cookie_ttl`
|
||||
(24h, 7 days) — safe because cookie expiry is enforced by `not_after`,
|
||||
not by the much shorter keyring window.
|
||||
- **Aggressive bot detection**: combine `EvaluateMismatches()` thresholds
|
||||
with atomic checks (`fingerprint.UAMobileMismatch`, etc.) and consider
|
||||
`SetChallengeDifficulty("high")` for suspect fingerprints. Reserve
|
||||
`"impossible"` for known-bad: solving it server-side is rejected, so it
|
||||
acts as a soft block via the challenge flow rather than a hard `ban`.
|
||||
|
||||
### 3.6 Operational notes
|
||||
|
||||
- The challenge runtime endpoints (`/crowdsec-internal/challenge/*`) are
|
||||
served on the same listener as the appsec acquisition module. They are
|
||||
not configurable independently.
|
||||
- The default cookie name is `__crowdsec_challenge`. It is `HttpOnly`,
|
||||
`Path=/`, `SameSite=Lax`, and `Secure` when the request was HTTPS.
|
||||
- Operator-issued allowlist cookies carry the reason string in their
|
||||
payload (≤ 256 bytes) — visible in logs and in the cookie value, **not**
|
||||
in the Set-Cookie attributes. Don't put secrets in the reason.
|
||||
- The "impossible" difficulty produces a challenge page the client
|
||||
cannot solve. It functions as a soft block: visitors see a challenge
|
||||
loop instead of a 403, which is often the desired UX for ambiguous
|
||||
cases.
|
||||
|
||||
@@ -0,0 +1,720 @@
|
||||
// Package challenge implements the AppSec WAF challenge mode: a PoW-gated
|
||||
// landing page, a fingerprint collection bundle, and the surrounding key
|
||||
// rotation + cookie machinery. This file holds the runtime orchestration
|
||||
// (lifecycle, HTTP entry points, template rendering, hook plumbing) and
|
||||
// delegates specialized concerns to sibling files in the same package:
|
||||
//
|
||||
// - keyring.go / crypto.go / ticket.go — per-epoch HKDF keys, AES-GCM
|
||||
// cookie seal/unseal, ticket signing
|
||||
// - static_bundle.go — public fpscanner/JS bundle
|
||||
// - dynamic_module.go — sensitive per-epoch sign-key module
|
||||
// - obfuscator.go — wazero wrapper around the JS obfuscator
|
||||
// - fingerprint*.go — fingerprint wire shape + helpers + mismatch report
|
||||
// - config.go / secret.go — YAML config + master-secret handling
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
_ "embed"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/pb"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/cookie"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/logging"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/tetratelabs/wazero"
|
||||
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
// Internal URL paths the challenge runtime intercepts. Bouncers MUST forward
|
||||
// these to the WAF unmodified; they are served by the appsec dispatcher
|
||||
// (pkg/appsec/appsec.go) rather than by the protected origin.
|
||||
const (
|
||||
ChallengeJSPath = "/crowdsec-internal/challenge/challenge.js"
|
||||
ChallengeSubmitPath = "/crowdsec-internal/challenge/submit"
|
||||
ChallengePowWorkerPath = "/crowdsec-internal/challenge/pow-worker.js"
|
||||
ChallengeFPScannerPath = "/crowdsec-internal/challenge/fpscanner.js"
|
||||
)
|
||||
|
||||
// Sentinel errors (reasons) returned by ValidateChallengeResponse.
|
||||
var (
|
||||
ErrChallengeFields = errors.New("missing required fields in challenge response")
|
||||
ErrChallengeTicket = errors.New("invalid ticket in challenge response")
|
||||
ErrChallengeDifficulty = errors.New("challenge difficulty is impossible")
|
||||
ErrChallengePoW = errors.New("invalid proof-of-work in challenge response")
|
||||
ErrChallengeHMAC = errors.New("invalid HMAC in challenge response")
|
||||
ErrChallengePayload = errors.New("invalid challenge response payload")
|
||||
)
|
||||
|
||||
// ChallengeCookieName is the name of the sealed cookie carrying the
|
||||
// successfully-validated fingerprint between requests.
|
||||
const ChallengeCookieName = "__crowdsec_challenge"
|
||||
|
||||
// cryptoObfuscationPoolDefaultSize is how many obfuscations of the per-epoch
|
||||
// key module to keep per live epoch. Each variant embeds the same key
|
||||
// differently (per-visitor byte variance); default 1 keeps prior behavior.
|
||||
const cryptoObfuscationPoolDefaultSize = 1
|
||||
|
||||
// defaultCookieTTL is the default challenge-cookie validity. Decoupled from the
|
||||
// keyring window (enforced by not_after in the envelope), so cookies can
|
||||
// outlive the per-epoch signing window without widening forgery exposure.
|
||||
const defaultCookieTTL = 12 * time.Hour
|
||||
|
||||
// DefaultChallengeCSP is the Content-Security-Policy header used on the
|
||||
// challenge page when the operator hasn't configured a custom one. Allows
|
||||
// inline script/style (the challenge runtime injects both) and blob workers
|
||||
// (the PoW worker is loaded from a blob URL).
|
||||
const DefaultChallengeCSP = "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; worker-src 'self' blob:;"
|
||||
|
||||
//go:embed challenge.html.tmpl
|
||||
var htmlTemplate string
|
||||
|
||||
// grantRedirectBody is the body of the 307 from GrantChallengeCookie — a no-JS
|
||||
// fallback for HTTP clients that don't auto-follow Location. Static, so no
|
||||
// per-request parsing is needed.
|
||||
//
|
||||
//go:embed grant_redirect.html.tmpl
|
||||
var GrantRedirectBody string
|
||||
|
||||
// PowWorkerJS is the JavaScript PoW worker shipped to the browser at
|
||||
// ChallengePowWorkerPath. Served as-is — obfuscation here would only slow
|
||||
// down the per-visitor PoW loop without adding security value.
|
||||
//
|
||||
//go:embed pow-worker.js
|
||||
var PowWorkerJS string
|
||||
|
||||
// ChallengeRuntime is the per-process state for the challenge mode: wazero
|
||||
// instance, signing/cookie keys, obfuscation pools, fingerprint key
|
||||
// rotation. One instance is shared across all appsec runners (see
|
||||
// pkg/acquisition/modules/appsec/config.go). Construct via NewChallengeRuntime.
|
||||
type ChallengeRuntime struct {
|
||||
r wazero.Runtime
|
||||
obfuscatorMod wazero.CompiledModule
|
||||
|
||||
// challengeCode is the build-time-obfuscated challenge crypto/glue code,
|
||||
// decompressed once at startup from initial_bundle.js.gz and injected inline
|
||||
// on every challenge page. Static for the life of the process (no runtime
|
||||
// re-obfuscation or pool); the sensitive per-epoch key module lives in
|
||||
// dynamic_module.go.
|
||||
challengeCode string
|
||||
|
||||
powDifficulty int
|
||||
|
||||
// keys derives per-epoch sign keys (per-challenge secret / PoW MAC HMACs)
|
||||
// and the long-lived master cookie key from the master secret. Routing all
|
||||
// HMAC/AEAD through it makes rotation a bookkeeping change; instances
|
||||
// sharing the secret + rotation interval derive identical keys.
|
||||
keys *KeyRing
|
||||
|
||||
// cryptoPoolSize is how many obfuscations of the per-epoch key module to
|
||||
// keep per live epoch (per-visitor byte variance over the same key);
|
||||
// default 1.
|
||||
cryptoPoolSize int
|
||||
|
||||
// dynamicModuleCache memoizes the obfuscated per-epoch key module variants
|
||||
// (epoch → cryptoPoolSize renderings); currentDynamicModule picks one at
|
||||
// random. dynamicModuleSF coalesces concurrent obfuscations for the same
|
||||
// epoch so a rotation thundering-herd pays at most one obfuscation latency
|
||||
// per slot, not N. The mutex guards only the map, never the obfuscation.
|
||||
dynamicModuleCache map[int64][]string
|
||||
dynamicModuleCacheMu sync.RWMutex
|
||||
dynamicModuleSF singleflight.Group
|
||||
|
||||
// cookieTTL is how long a challenge cookie stays valid. Enforced by the
|
||||
// not_after stamp in the sealed envelope (crypto.go), not keyring eviction,
|
||||
// so it can exceed the per-epoch signing window.
|
||||
cookieTTL time.Duration
|
||||
|
||||
// maxCookieLen is the size ceiling enforced when sealing and opening cookies
|
||||
// (crypto.go). Bounds the allocation derived from the attacker-influenced
|
||||
// fingerprint envelope. Defaults to MaxCookieLen (the browser limit).
|
||||
maxCookieLen int
|
||||
|
||||
// htmlTpl is the challenge HTML template, parsed once and reused so
|
||||
// GetChallengePage doesn't re-parse per request.
|
||||
htmlTpl *template.Template
|
||||
|
||||
// spent burns consumed per-challenge nonces (`r`) to enforce single-use and
|
||||
// eliminate replay (in-memory, single instance — see spent_set.go).
|
||||
spent *spentSet
|
||||
|
||||
// logger is the component logger; its level is set at construction
|
||||
// (WithLogger / BuildOptions). All challenge logging routes through it so
|
||||
// `log_level` controls verbosity independently of the global logger.
|
||||
logger *log.Entry
|
||||
}
|
||||
|
||||
// Option configures a ChallengeRuntime at construction time.
|
||||
type Option func(*runtimeOptions)
|
||||
|
||||
// Field semantics mirror the YAML Config in config.go;
|
||||
type runtimeOptions struct {
|
||||
masterSecret []byte // must be >= minSecretBytes when non-nil
|
||||
rotationInterval time.Duration
|
||||
maxLiveEpochs int
|
||||
cookieTTL time.Duration
|
||||
maxCookieLen int
|
||||
cryptoObfuscationPoolSize int
|
||||
spentSetMaxEntries int
|
||||
logger *log.Entry // nil → default "challenge" sublogger
|
||||
}
|
||||
|
||||
func WithLogger(logger *log.Entry) Option {
|
||||
return func(o *runtimeOptions) {
|
||||
o.logger = logger
|
||||
}
|
||||
}
|
||||
|
||||
// log returns the component logger, defaulting to a standard "challenge"
|
||||
// sublogger when unset (runtimes built directly in tests don't set it).
|
||||
func (c *ChallengeRuntime) log() *log.Entry {
|
||||
if c.logger != nil {
|
||||
return c.logger
|
||||
}
|
||||
return logging.SubLogger(log.StandardLogger(), "challenge", 0)
|
||||
}
|
||||
|
||||
// WithMasterSecret sets the long-lived shared secret (see Config.MasterSecret).
|
||||
func WithMasterSecret(secret []byte) Option {
|
||||
return func(o *runtimeOptions) {
|
||||
o.masterSecret = secret
|
||||
}
|
||||
}
|
||||
|
||||
// WithRotationInterval sets the per-epoch key rotation period (see Config.KeyRotationInterval).
|
||||
func WithRotationInterval(d time.Duration) Option {
|
||||
return func(o *runtimeOptions) {
|
||||
o.rotationInterval = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithMaxLiveEpochs sets how many past epochs the keyring keeps accepting (see Config.MaxLiveEpochs).
|
||||
func WithMaxLiveEpochs(n int) Option {
|
||||
return func(o *runtimeOptions) {
|
||||
o.maxLiveEpochs = n
|
||||
}
|
||||
}
|
||||
|
||||
// WithCookieTTL sets challenge-cookie validity (see Config.CookieTTL); zero/negative is ignored.
|
||||
func WithCookieTTL(ttl time.Duration) Option {
|
||||
return func(o *runtimeOptions) {
|
||||
if ttl > 0 {
|
||||
o.cookieTTL = ttl
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithMaxCookieLen sets the cookie size ceiling (see Config.MaxCookieSize);
|
||||
// zero/negative is ignored, leaving the MaxCookieLen default in effect.
|
||||
func WithMaxCookieLen(n int) Option {
|
||||
return func(o *runtimeOptions) {
|
||||
if n > 0 {
|
||||
o.maxCookieLen = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithCryptoObfuscationPoolSize sets the per-epoch sign-key obfuscation pool size
|
||||
// (see Config.CryptoObfuscationPoolSize); values below 1 are ignored.
|
||||
func WithCryptoObfuscationPoolSize(n int) Option {
|
||||
return func(o *runtimeOptions) {
|
||||
if n >= 1 {
|
||||
o.cryptoObfuscationPoolSize = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithSpentSetMaxEntries caps the replay-protection LRU (see Config.SpentSetMaxEntries); values below 1 are ignored.
|
||||
func WithSpentSetMaxEntries(n int) Option {
|
||||
return func(o *runtimeOptions) {
|
||||
if n >= 1 {
|
||||
o.spentSetMaxEntries = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DifficultyFromLevel resolves a named level ("low", "medium", "high") to
|
||||
// a PoW difficulty in leading zero bits. Case-insensitive.
|
||||
func DifficultyFromLevel(level string) (int, error) {
|
||||
switch strings.ToLower(level) {
|
||||
case "disabled":
|
||||
return PowDifficultyDisabled, nil
|
||||
case "low":
|
||||
return PowDifficultyLow, nil
|
||||
case "medium":
|
||||
return PowDifficultyMedium, nil
|
||||
case "high":
|
||||
return PowDifficultyHigh, nil
|
||||
case "impossible":
|
||||
return PowDifficultyImpossible, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("unknown challenge difficulty %q (expected disabled, low, medium, high, or impossible)", level)
|
||||
}
|
||||
}
|
||||
|
||||
// Difficulty returns the current default PoW difficulty (in leading zero bits).
|
||||
func (c *ChallengeRuntime) Difficulty() int {
|
||||
return c.powDifficulty
|
||||
}
|
||||
|
||||
// SetDifficulty sets the default PoW difficulty from a named level.
|
||||
func (c *ChallengeRuntime) SetDifficulty(level string) error {
|
||||
bits, err := DifficultyFromLevel(level)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.powDifficulty = bits
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewChallengeRuntime builds and starts a ChallengeRuntime: it initializes the
|
||||
// wazero runtime, decompresses the baked-in library bundle, derives the master
|
||||
// secret + keyring, pre-warms the dynamic-module cache, and (if configured)
|
||||
// spawns the background obfuscation refresher. Safe for concurrent use across
|
||||
// all appsec runners. Pass WithXxx options to override defaults.
|
||||
// compileObfuscatorModule decompresses the baked-in obfuscator WASM (once,
|
||||
// process-wide) and compiles it for the given runtime. Pre-compiling lets each
|
||||
// ObfuscateJS call merely instantiate the module instead of re-parsing the WASM
|
||||
// bytes, which would otherwise cost ~4-5s per call.
|
||||
func compileObfuscatorModule(ctx context.Context, r wazero.Runtime) (wazero.CompiledModule, error) {
|
||||
var obfuscatorWasmErr error
|
||||
|
||||
obfuscatorWasmOnce.Do(func() {
|
||||
zr, err := gzip.NewReader(bytes.NewReader(obfuscatorWasmGz))
|
||||
if err != nil {
|
||||
obfuscatorWasmErr = fmt.Errorf("failed to create gzip reader for obfuscator wasm: %w", err)
|
||||
return
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
obfuscatorWasm, err = io.ReadAll(zr)
|
||||
if err != nil {
|
||||
obfuscatorWasmErr = fmt.Errorf("failed to decompress obfuscator wasm: %w", err)
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
if obfuscatorWasmErr != nil {
|
||||
return nil, obfuscatorWasmErr
|
||||
}
|
||||
|
||||
compiledMod, err := r.CompileModule(ctx, obfuscatorWasm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile obfuscator wasm module: %w", err)
|
||||
}
|
||||
|
||||
return compiledMod, nil
|
||||
}
|
||||
|
||||
func NewChallengeRuntime(ctx context.Context, opts ...Option) (*ChallengeRuntime, error) {
|
||||
resolvedOpts := runtimeOptions{}
|
||||
for _, opt := range opts {
|
||||
opt(&resolvedOpts)
|
||||
}
|
||||
|
||||
logger := resolvedOpts.logger
|
||||
if logger == nil {
|
||||
logger = logging.SubLogger(log.StandardLogger(), "challenge", 0)
|
||||
}
|
||||
|
||||
secret := resolvedOpts.masterSecret
|
||||
if secret == nil {
|
||||
var err error
|
||||
secret, err = generateRandomSecret()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logger.Warn("no master secret configured for the WAF challenge runtime; generated an ephemeral random secret. " +
|
||||
"Distributed (multi-WAF) deployments MUST configure a shared master_secret in the appsec config; " +
|
||||
"single-instance deployments will see outstanding challenge cookies invalidated on restart.")
|
||||
} else if len(secret) < minSecretBytes {
|
||||
return nil, fmt.Errorf("master secret is %d bytes; minimum is %d", len(secret), minSecretBytes)
|
||||
}
|
||||
|
||||
rotationInterval := resolvedOpts.rotationInterval
|
||||
if rotationInterval == 0 {
|
||||
rotationInterval = keyringDefaultRotation
|
||||
}
|
||||
|
||||
keys, err := NewKeyRing(secret, rotationInterval, resolvedOpts.maxLiveEpochs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build challenge keyring: %w", err)
|
||||
}
|
||||
keys.logger = logger
|
||||
|
||||
cookieTTL := resolvedOpts.cookieTTL
|
||||
if cookieTTL <= 0 {
|
||||
cookieTTL = defaultCookieTTL
|
||||
}
|
||||
|
||||
maxCookieLen := resolvedOpts.maxCookieLen
|
||||
if maxCookieLen <= 0 {
|
||||
maxCookieLen = MaxCookieLen
|
||||
}
|
||||
|
||||
cryptoPoolSize := resolvedOpts.cryptoObfuscationPoolSize
|
||||
if cryptoPoolSize <= 0 {
|
||||
cryptoPoolSize = cryptoObfuscationPoolDefaultSize
|
||||
}
|
||||
|
||||
spentSetMaxEntries := resolvedOpts.spentSetMaxEntries
|
||||
if spentSetMaxEntries <= 0 {
|
||||
spentSetMaxEntries = spentSetDefaultMaxEntries
|
||||
}
|
||||
|
||||
r := wazero.NewRuntime(ctx)
|
||||
|
||||
// No need to keep the closer around, we can just close the runtime itself when stopping
|
||||
if _, err := wasi_snapshot_preview1.Instantiate(ctx, r); err != nil {
|
||||
return nil, fmt.Errorf("failed to instantiate WASI: %w", err)
|
||||
}
|
||||
|
||||
compiledMod, err := compileObfuscatorModule(ctx, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// We use text/template instead of html/template because the data we send
|
||||
// is pretty much hardcoded and trusted; html/template would escape the JS
|
||||
// we inject. Parsed once here so GetChallengePage doesn't re-parse on
|
||||
// every request.
|
||||
htmlTpl, err := template.New("challenge").Parse(htmlTemplate)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse challenge html template: %w", err)
|
||||
}
|
||||
|
||||
challengeRuntime := &ChallengeRuntime{
|
||||
r: r,
|
||||
obfuscatorMod: compiledMod,
|
||||
powDifficulty: defaultPowDifficulty,
|
||||
keys: keys,
|
||||
cryptoPoolSize: cryptoPoolSize,
|
||||
dynamicModuleCache: make(map[int64][]string),
|
||||
cookieTTL: cookieTTL,
|
||||
maxCookieLen: maxCookieLen,
|
||||
htmlTpl: htmlTpl,
|
||||
spent: newSpentSet(spentSetMaxEntries),
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
// Load the build-time-obfuscated challenge code from the baked-in bundle so
|
||||
// we can serve immediately.
|
||||
if err := challengeRuntime.seedCacheFromInitialBundle(); err != nil {
|
||||
// Initial bundle missing/corrupt (e.g. `go generate` not run): fall back
|
||||
// to obfuscating the challenge code synchronously.
|
||||
logger.Warnf("failed to load baked-in initial challenge bundle (%v); falling back to synchronous generation", err)
|
||||
if err := challengeRuntime.generateAndCacheChallengeJS(ctx); err != nil {
|
||||
return nil, fmt.Errorf("failed to generate initial challenge bundle: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-warm the current epoch's dynamic module so the first GetChallengePage
|
||||
// doesn't pay the ~5s obfuscation cost on the request path.
|
||||
if _, err := challengeRuntime.currentDynamicModule(ctx); err != nil {
|
||||
return nil, fmt.Errorf("warm dynamic key module: %w", err)
|
||||
}
|
||||
|
||||
// The dynamic-module pre-warmer is always on — the per-epoch key must be
|
||||
// re-obfuscated on every rotation (bounded cost, one pass per variant). The
|
||||
// challenge code itself is static (build-time obfuscated), so there is no
|
||||
// library refresher anymore.
|
||||
go challengeRuntime.dynamicModulePreWarmer(ctx)
|
||||
|
||||
logger.WithFields(log.Fields{
|
||||
"rotation_interval": rotationInterval,
|
||||
"cookie_ttl": cookieTTL,
|
||||
"max_cookie_len": maxCookieLen,
|
||||
"pow_difficulty": defaultPowDifficulty,
|
||||
"crypto_pool_size": cryptoPoolSize,
|
||||
}).Info("WAF challenge runtime initialized")
|
||||
|
||||
return challengeRuntime, nil
|
||||
}
|
||||
|
||||
func (c *ChallengeRuntime) Close(ctx context.Context) error {
|
||||
if c == nil || c.r == nil {
|
||||
return nil
|
||||
}
|
||||
return c.r.Close(ctx)
|
||||
}
|
||||
|
||||
// GetChallengePage renders the challenge HTML page with the given PoW difficulty.
|
||||
// If difficulty is 0, the default difficulty is used.
|
||||
func (c *ChallengeRuntime) GetChallengePage(ctx context.Context, userAgent string, difficulty int) (string, error) {
|
||||
_ = userAgent
|
||||
|
||||
if difficulty <= 0 {
|
||||
difficulty = c.powDifficulty
|
||||
}
|
||||
|
||||
challengeCode := c.getChallengeCode()
|
||||
if challengeCode == "" {
|
||||
if err := c.generateAndCacheChallengeJS(ctx); err != nil {
|
||||
return "", fmt.Errorf("failed to generate challenge JS: %w", err)
|
||||
}
|
||||
challengeCode = c.getChallengeCode()
|
||||
if challengeCode == "" {
|
||||
return "", errors.New("challenge JS cache is empty")
|
||||
}
|
||||
}
|
||||
|
||||
// Issuance is stateless (only submission is stateful — see the single-use
|
||||
// burn in ValidateChallengeResponse). `r` seeds the per-challenge secret
|
||||
// `s = HMAC(K_epoch, r)` the client derives from the obfuscated dynamic
|
||||
// module, so `s` never appears in plain HTML; the PoW MAC binds the salt to
|
||||
// `r`+ts so a client can't pick a favorable salt.
|
||||
ts := fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
r, err := generateChallengeNonce()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
powSalt, err := generatePowPrefix()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("generate PoW salt: %w", err)
|
||||
}
|
||||
powMAC := c.computePowMAC(powSalt, r, ts, difficulty)
|
||||
|
||||
if c.log().Logger.IsLevelEnabled(log.DebugLevel) {
|
||||
issEpoch, issKey := c.keys.Current()
|
||||
c.log().WithFields(log.Fields{
|
||||
"r": r,
|
||||
"epoch": issEpoch,
|
||||
"k_epoch": fmt.Sprintf("%x", issKey),
|
||||
"difficulty": difficulty,
|
||||
}).Debug("issued challenge")
|
||||
}
|
||||
|
||||
// The challenge code only carries the hook registration; the dynamic module
|
||||
// carries the per-epoch K, so K never appears in plain HTML.
|
||||
dynamicModule, err := c.currentDynamicModule(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build dynamic key module: %w", err)
|
||||
}
|
||||
|
||||
var renderedPage strings.Builder
|
||||
|
||||
if err := c.htmlTpl.Execute(&renderedPage, map[string]interface{}{
|
||||
"JSChallenge": challengeCode,
|
||||
"DynamicModule": dynamicModule,
|
||||
"FPScannerPath": ChallengeFPScannerPath,
|
||||
"PowDifficulty": difficulty,
|
||||
"PowPrefix": powSalt,
|
||||
"PowMAC": powMAC,
|
||||
"Timestamp": ts,
|
||||
"R": r,
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("render challenge page: %w", err)
|
||||
}
|
||||
return renderedPage.String(), nil
|
||||
}
|
||||
|
||||
// ValidateChallengeResponse parses a submit POST and runs the full chain:
|
||||
// freshness + PoW-salt authenticity + difficulty, PoW solution,
|
||||
// the submission signature `sig` (keyed by the never-transmitted s = HMAC(K_epoch, r)),
|
||||
// a single-use burn of `r` (replay protection),
|
||||
// and fingerprint de-obfuscation. On success it returns the sealed
|
||||
// cookie, decoded FingerprintData, and the proven PoW difficulty; failures
|
||||
// return a generic error so the caller doesn't leak which stage failed.
|
||||
func (c *ChallengeRuntime) ValidateChallengeResponse(request *http.Request, body []byte) (*cookie.AppsecCookie, FingerprintData, int, error) {
|
||||
vars, err := url.ParseQuery(string(body))
|
||||
if err != nil {
|
||||
return nil, FingerprintData{}, 0, fmt.Errorf("%w: %w", ErrChallengePayload, err)
|
||||
}
|
||||
|
||||
encryptedFingerprint := vars.Get("f")
|
||||
clientR := vars.Get("r")
|
||||
clientTS := vars.Get("ts")
|
||||
clientSig := vars.Get("sig")
|
||||
clientNonce := vars.Get("n")
|
||||
clientPowSalt := vars.Get("p")
|
||||
clientPowMAC := vars.Get("m")
|
||||
clientDifficultyStr := vars.Get("d")
|
||||
clientPath := vars.Get("u")
|
||||
|
||||
if encryptedFingerprint == "" || clientR == "" || clientTS == "" || clientSig == "" || clientNonce == "" || clientPowSalt == "" || clientPowMAC == "" || clientDifficultyStr == "" || clientPath == "" {
|
||||
return nil, FingerprintData{}, 0, errors.New("missing required fields in challenge response")
|
||||
}
|
||||
|
||||
// Override the request path so the WAF sees the original URL the client
|
||||
request.URL.Path = clientPath
|
||||
|
||||
// The difficulty the client claims it solved. It is untrusted until the PoW
|
||||
// MAC (which binds it) is verified in verifyChallenge below. Bound to the
|
||||
// valid PoW range so a malformed value can't reach the PoW/seal logic.
|
||||
clientDifficulty, err := strconv.Atoi(clientDifficultyStr)
|
||||
if err != nil || clientDifficulty < PowDifficultyDisabled || clientDifficulty > PowDifficultyImpossible {
|
||||
return nil, FingerprintData{}, 0, errors.New("invalid ticket in challenge response")
|
||||
}
|
||||
|
||||
// Server-issued `r` is a 16-byte nonce in hex (generateChallengeNonce):
|
||||
// exactly 32 hex chars. Reject other shapes early so a K_epoch holder can't
|
||||
// bloat the spent-set with oversized keys, and to keep the key space canonical.
|
||||
if _, err := hex.DecodeString(clientR); err != nil || len(clientR) != 32 {
|
||||
return nil, FingerprintData{}, 0, errors.New("invalid ticket in challenge response")
|
||||
}
|
||||
|
||||
// Verify freshness + PoW-salt/difficulty authenticity and recover the
|
||||
// per-epoch sign key (stateless). Key knowledge is proven by `sig` below.
|
||||
signKey, ok := c.verifyChallenge(clientR, clientTS, clientPowSalt, clientPowMAC, clientDifficulty)
|
||||
if !ok {
|
||||
return nil, FingerprintData{}, 0, errors.New("invalid ticket in challenge response")
|
||||
}
|
||||
|
||||
// Impossible difficulty is a deliberate hard-block: never accept anything.
|
||||
// clientDifficulty is now MAC-authenticated, so this enforces the difficulty
|
||||
// actually issued for this request, not the runtime-global default.
|
||||
if clientDifficulty >= PowDifficultyImpossible {
|
||||
return nil, FingerprintData{}, 0, ErrChallengeDifficulty
|
||||
}
|
||||
|
||||
// Verify proof-of-work: SHA256(powSalt + nonce) must have required leading zero bits
|
||||
powHash := sha256.Sum256([]byte(clientPowSalt + clientNonce))
|
||||
if !hasLeadingZeroBits(powHash[:], clientDifficulty) {
|
||||
return nil, FingerprintData{}, 0, ErrChallengePoW
|
||||
}
|
||||
|
||||
// Verify the submission signature sig = HMAC(s, r||ts||n||f), where the
|
||||
// secret s = HMAC(K_epoch, r) is never transmitted — a valid sig proves the
|
||||
// client derived s from the per-epoch key in the obfuscated dynamic module.
|
||||
s := deriveChallengeSecret(signKey, clientR)
|
||||
|
||||
expectedSig := hmacSHA256Hex([]byte(s), []byte(clientR+clientTS+clientNonce+encryptedFingerprint))
|
||||
if !hmac.Equal([]byte(clientSig), []byte(expectedSig)) {
|
||||
return nil, FingerprintData{}, 0, errors.New("invalid HMAC in challenge response")
|
||||
}
|
||||
|
||||
// Single-use: burn `r` (rejects replays). Done last so the spent-set only
|
||||
// grows on fully-valid submissions.
|
||||
if !c.spent.checkAndInsert(clientR, ticketAgeBackstop) {
|
||||
return nil, FingerprintData{}, 0, errors.New("challenge response already used")
|
||||
}
|
||||
|
||||
obfKey := deriveFingerprintObfKey(s, clientR)
|
||||
|
||||
fingerprint, err := deobfuscateFingerprint(obfKey, encryptedFingerprint)
|
||||
if err != nil {
|
||||
return nil, FingerprintData{}, 0, fmt.Errorf("failed to deobfuscate fingerprint: %w", err)
|
||||
}
|
||||
|
||||
var fpData FingerprintData
|
||||
|
||||
if err := json.Unmarshal([]byte(fingerprint), &fpData); err != nil {
|
||||
return nil, FingerprintData{}, 0, fmt.Errorf("%w: failed to unmarshal fingerprint data: %w", ErrChallengePayload, err)
|
||||
}
|
||||
|
||||
// Debug diagnostic: a validated submission. Guarded so `k_epoch` (forgeable
|
||||
// signing material — DESIGN.md §2.1) is only formatted at debug.
|
||||
if c.log().Logger.IsLevelEnabled(log.DebugLevel) {
|
||||
c.log().WithFields(log.Fields{
|
||||
"r": clientR,
|
||||
"epoch": c.epochForTimestamp(clientTS),
|
||||
"k_epoch": fmt.Sprintf("%x", signKey),
|
||||
"fsid": fpData.FSID,
|
||||
"is_bot": fpData.FastBotDetection,
|
||||
}).Debug("validated submission")
|
||||
}
|
||||
|
||||
// Seal the difficulty the client actually proved (MAC-authenticated above),
|
||||
// so the next request's re-challenge check compares against real work and an
|
||||
// escalated per-request difficulty survives the cookie round-trip.
|
||||
envelope := &pb.ChallengeCookie{
|
||||
Fingerprint: fpData.ToProto(),
|
||||
PowDifficulty: int32(clientDifficulty),
|
||||
}
|
||||
|
||||
// Seal under the long-lived master cookie key. The embedded not_after makes
|
||||
// the server validity window exactly c.cookieTTL (independent of key
|
||||
// rotation); the browser Max-Age below matches so both expire together.
|
||||
notAfter := time.Now().Add(c.cookieTTL).Unix()
|
||||
cookieValue, err := sealCookieV0(envelope, c.keys.MasterCookieKey(), notAfter, 0, "", []byte(request.UserAgent()), c.maxCookieLen)
|
||||
if err != nil {
|
||||
return nil, FingerprintData{}, 0, fmt.Errorf("failed to seal challenge cookie: %w", err)
|
||||
}
|
||||
|
||||
ck := cookie.NewAppsecCookie(ChallengeCookieName).HttpOnly().Path("/").SameSite(cookie.SameSiteLax).ExpiresIn(c.cookieTTL).Value(cookieValue)
|
||||
if request.URL.Scheme == "https" {
|
||||
ck = ck.Secure()
|
||||
}
|
||||
|
||||
return ck, fpData, clientDifficulty, nil
|
||||
}
|
||||
|
||||
// SealAllowlistCookie mints an allowlist-bypass cookie (no fingerprint, with
|
||||
// the operator reason) so GrantChallengeCookie can let trusted bots skip the
|
||||
// challenge UI while still hitting on_challenge rules via fingerprint.Allowlisted.
|
||||
// not_after honors c.cookieTTL unless ttlOverride (>0) is given; reason is
|
||||
// bounded by MaxAllowlistReasonLen (crypto.go).
|
||||
func (c *ChallengeRuntime) SealAllowlistCookie(request *http.Request, reason string, ttlOverride *time.Duration) (*cookie.AppsecCookie, error) {
|
||||
if c == nil {
|
||||
return nil, errors.New("challenge runtime not initialized")
|
||||
}
|
||||
|
||||
ttl := c.cookieTTL
|
||||
if ttlOverride != nil && *ttlOverride > 0 {
|
||||
ttl = *ttlOverride
|
||||
}
|
||||
|
||||
notAfter := time.Now().Add(ttl).Unix()
|
||||
cookieValue, err := sealCookieV0(&pb.ChallengeCookie{}, c.keys.MasterCookieKey(), notAfter, cookieFlagAllowlisted, reason, []byte(request.UserAgent()), c.maxCookieLen)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to seal allowlist cookie: %w", err)
|
||||
}
|
||||
|
||||
ck := cookie.NewAppsecCookie(ChallengeCookieName).HttpOnly().Path("/").SameSite(cookie.SameSiteLax).ExpiresIn(ttl).Value(cookieValue)
|
||||
if request.URL.Scheme == "https" {
|
||||
ck = ck.Secure()
|
||||
}
|
||||
|
||||
return ck, nil
|
||||
}
|
||||
|
||||
// CookieData bundles the decoded fingerprint with cookie-envelope metadata for
|
||||
// re-challenge decisions. Allowlisted/AllowlistReason mark cookies minted by
|
||||
// SealAllowlistCookie; they are zero for real-submission cookies.
|
||||
type CookieData struct {
|
||||
Fingerprint FingerprintData
|
||||
PowDifficulty int
|
||||
Allowlisted bool
|
||||
AllowlistReason string
|
||||
}
|
||||
|
||||
// ValidCookie unseals and validates a challenge cookie: envelope (version,
|
||||
// AES-GCM tag), not_after expiry, and UA-pinning (a stolen cookie is useless to
|
||||
// a different client). On any failure (tampered/expired/UA-mismatch/unknown
|
||||
// version) it returns an error and the caller should treat the request as
|
||||
// cookieless.
|
||||
func (c *ChallengeRuntime) ValidCookie(ck *http.Cookie, userAgent string) (*CookieData, error) {
|
||||
if ck == nil {
|
||||
return nil, errors.New("nil cookie")
|
||||
}
|
||||
|
||||
envelope, err := openCookie(ck.Value, c.keys.MasterCookieKey(), []byte(userAgent), c.maxCookieLen)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid challenge cookie: %w", err)
|
||||
}
|
||||
|
||||
return &CookieData{
|
||||
Fingerprint: fingerprintDataFromProto(envelope.Envelope.GetFingerprint()),
|
||||
PowDifficulty: int(envelope.Envelope.GetPowDifficulty()),
|
||||
Allowlisted: envelope.Allowlisted,
|
||||
AllowlistReason: envelope.AllowlistReason,
|
||||
}, nil
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,346 @@
|
||||
// --- Pure JS SHA-256 (RFC 6234) ---
|
||||
|
||||
const K = [
|
||||
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
|
||||
0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
|
||||
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
|
||||
0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
||||
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147,
|
||||
0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
|
||||
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
|
||||
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
||||
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
|
||||
0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
|
||||
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
|
||||
];
|
||||
|
||||
function sha256(data) {
|
||||
let h0 = 0x6a09e667,
|
||||
h1 = 0xbb67ae85,
|
||||
h2 = 0x3c6ef372,
|
||||
h3 = 0xa54ff53a,
|
||||
h4 = 0x510e527f,
|
||||
h5 = 0x9b05688c,
|
||||
h6 = 0x1f83d9ab,
|
||||
h7 = 0x5be0cd19;
|
||||
|
||||
const len = data.length;
|
||||
const bitLen = len * 8;
|
||||
|
||||
const padLen = 64 - ((len + 9) % 64);
|
||||
const totalLen = len + 1 + (padLen === 64 ? 0 : padLen) + 8;
|
||||
const msg = new Uint8Array(totalLen);
|
||||
msg.set(data);
|
||||
msg[len] = 0x80;
|
||||
const dv = new DataView(msg.buffer);
|
||||
dv.setUint32(totalLen - 4, bitLen, false);
|
||||
|
||||
const w = new Int32Array(64);
|
||||
for (let offset = 0; offset < totalLen; offset += 64) {
|
||||
for (let i = 0; i < 16; i++) {
|
||||
w[i] = dv.getInt32(offset + i * 4, false);
|
||||
}
|
||||
for (let i = 16; i < 64; i++) {
|
||||
const s0 =
|
||||
(((w[i - 15] >>> 7) | (w[i - 15] << 25)) ^
|
||||
((w[i - 15] >>> 18) | (w[i - 15] << 14)) ^
|
||||
(w[i - 15] >>> 3)) |
|
||||
0;
|
||||
const s1 =
|
||||
(((w[i - 2] >>> 17) | (w[i - 2] << 15)) ^
|
||||
((w[i - 2] >>> 19) | (w[i - 2] << 13)) ^
|
||||
(w[i - 2] >>> 10)) |
|
||||
0;
|
||||
w[i] = (w[i - 16] + s0 + w[i - 7] + s1) | 0;
|
||||
}
|
||||
|
||||
let a = h0,
|
||||
b = h1,
|
||||
c = h2,
|
||||
d = h3,
|
||||
e = h4,
|
||||
f = h5,
|
||||
g = h6,
|
||||
h = h7;
|
||||
|
||||
for (let i = 0; i < 64; i++) {
|
||||
const S1 =
|
||||
(((e >>> 6) | (e << 26)) ^
|
||||
((e >>> 11) | (e << 21)) ^
|
||||
((e >>> 25) | (e << 7))) |
|
||||
0;
|
||||
const ch = ((e & f) ^ (~e & g)) | 0;
|
||||
const temp1 = (h + S1 + ch + K[i] + w[i]) | 0;
|
||||
const S0 =
|
||||
(((a >>> 2) | (a << 30)) ^
|
||||
((a >>> 13) | (a << 19)) ^
|
||||
((a >>> 22) | (a << 10))) |
|
||||
0;
|
||||
const maj = ((a & b) ^ (a & c) ^ (b & c)) | 0;
|
||||
const temp2 = (S0 + maj) | 0;
|
||||
|
||||
h = g;
|
||||
g = f;
|
||||
f = e;
|
||||
e = (d + temp1) | 0;
|
||||
d = c;
|
||||
c = b;
|
||||
b = a;
|
||||
a = (temp1 + temp2) | 0;
|
||||
}
|
||||
|
||||
h0 = (h0 + a) | 0;
|
||||
h1 = (h1 + b) | 0;
|
||||
h2 = (h2 + c) | 0;
|
||||
h3 = (h3 + d) | 0;
|
||||
h4 = (h4 + e) | 0;
|
||||
h5 = (h5 + f) | 0;
|
||||
h6 = (h6 + g) | 0;
|
||||
h7 = (h7 + h) | 0;
|
||||
}
|
||||
|
||||
const out = new Uint8Array(32);
|
||||
const ov = new DataView(out.buffer);
|
||||
ov.setUint32(0, h0, false);
|
||||
ov.setUint32(4, h1, false);
|
||||
ov.setUint32(8, h2, false);
|
||||
ov.setUint32(12, h3, false);
|
||||
ov.setUint32(16, h4, false);
|
||||
ov.setUint32(20, h5, false);
|
||||
ov.setUint32(24, h6, false);
|
||||
ov.setUint32(28, h7, false);
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function toHex(bytes) {
|
||||
let hex = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
hex += (bytes[i] >> 4).toString(16) + (bytes[i] & 0xf).toString(16);
|
||||
}
|
||||
return hex;
|
||||
}
|
||||
|
||||
function encode(str) {
|
||||
return new TextEncoder().encode(str);
|
||||
}
|
||||
|
||||
function sha256Hex(str) {
|
||||
return toHex(sha256(encode(str)));
|
||||
}
|
||||
|
||||
// --- HMAC-SHA256 ---
|
||||
|
||||
function hmacSHA256(key, message) {
|
||||
if (key.length > 64) {
|
||||
key = sha256(key);
|
||||
}
|
||||
const paddedKey = new Uint8Array(64);
|
||||
paddedKey.set(key);
|
||||
|
||||
const ipad = new Uint8Array(64);
|
||||
const opad = new Uint8Array(64);
|
||||
for (let i = 0; i < 64; i++) {
|
||||
ipad[i] = paddedKey[i] ^ 0x36;
|
||||
opad[i] = paddedKey[i] ^ 0x5c;
|
||||
}
|
||||
|
||||
const inner = new Uint8Array(64 + message.length);
|
||||
inner.set(ipad);
|
||||
inner.set(message, 64);
|
||||
const innerHash = sha256(inner);
|
||||
|
||||
const outer = new Uint8Array(64 + 32);
|
||||
outer.set(opad);
|
||||
outer.set(innerHash, 64);
|
||||
return sha256(outer);
|
||||
}
|
||||
|
||||
function hmacSHA256Hex(keyStr, msgStr) {
|
||||
return toHex(hmacSHA256(encode(keyStr), encode(msgStr)));
|
||||
}
|
||||
|
||||
// hexToBytes decodes a hex string into a Uint8Array. Used to materialize the
|
||||
// per-epoch HMAC key delivered as hex by the dynamic module.
|
||||
function hexToBytes(hex) {
|
||||
const out = new Uint8Array(hex.length / 2);
|
||||
for (let i = 0; i < out.length; i++) {
|
||||
out[i] = parseInt(hex.substr(i * 2, 2), 16);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function hmacSHA256HexKey(keyHex, msgStr) {
|
||||
return toHex(hmacSHA256(hexToBytes(keyHex), encode(msgStr)));
|
||||
}
|
||||
|
||||
// --- Proof-of-Work (offloaded to Web Worker) ---
|
||||
|
||||
const powWorkerPath = "__CROWDSEC_POW_WORKER_PATH__";
|
||||
|
||||
function solvePoWAsync(prefix, difficulty) {
|
||||
if (difficulty <= 0) {
|
||||
return Promise.resolve("0");
|
||||
}
|
||||
|
||||
try {
|
||||
const worker = new Worker(powWorkerPath);
|
||||
|
||||
return new Promise((resolve) => {
|
||||
worker.onmessage = (e) => {
|
||||
worker.terminate();
|
||||
resolve(e.data);
|
||||
};
|
||||
worker.onerror = () => {
|
||||
worker.terminate();
|
||||
resolve(solvePoWMainThread(prefix, difficulty));
|
||||
};
|
||||
worker.postMessage({ p: prefix, d: difficulty });
|
||||
});
|
||||
} catch (_) {
|
||||
return Promise.resolve(solvePoWMainThread(prefix, difficulty));
|
||||
}
|
||||
}
|
||||
|
||||
function solvePoWMainThread(prefix, difficulty) {
|
||||
let nonce = 0;
|
||||
while (true) {
|
||||
const candidate = prefix + nonce.toString(36);
|
||||
const hash = sha256(encode(candidate));
|
||||
const fullBytes = difficulty >> 3;
|
||||
const remainBits = difficulty & 7;
|
||||
let ok = true;
|
||||
for (let i = 0; i < fullBytes; i++) {
|
||||
if (hash[i] !== 0) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (ok && remainBits > 0) {
|
||||
const mask = (0xff << (8 - remainBits)) & 0xff;
|
||||
if ((hash[fullBytes] & mask) !== 0) ok = false;
|
||||
}
|
||||
if (ok) return nonce.toString(36);
|
||||
nonce++;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Fingerprint encryption (XOR) ---
|
||||
|
||||
function encryptFingerprint(key, fingerprint) {
|
||||
const keyBytes = encode(key);
|
||||
const textBytes = encode(fingerprint);
|
||||
const encrypted = new Uint8Array(textBytes.length);
|
||||
|
||||
for (let i = 0; i < textBytes.length; i++) {
|
||||
encrypted[i] = textBytes[i] ^ keyBytes[i % keyBytes.length];
|
||||
}
|
||||
|
||||
const binaryString = String.fromCharCode(...encrypted);
|
||||
return btoa(binaryString);
|
||||
}
|
||||
|
||||
// --- Non-secret per-request values (plain template injection) ---
|
||||
//
|
||||
// Set on `globalThis` by a plain <script> the server renders before this
|
||||
// module. None are secret:
|
||||
// _ts — server time (forgeable but freshness-windowed)
|
||||
// _powP — random PoW salt (server-issued so clients can't pick easy salts)
|
||||
// _powM — HMAC(_powP, _r, _ts) under the per-epoch key
|
||||
// _powD — PoW difficulty
|
||||
// _r — per-challenge nonce; seeds s = HMAC(K, r)
|
||||
// The secret is the per-epoch K, delivered only by the dynamic key module; the
|
||||
// derived s is never transmitted.
|
||||
|
||||
const ts = typeof _ts !== "undefined" ? _ts : "";
|
||||
const powPrefix = typeof _powP !== "undefined" ? _powP : "";
|
||||
const powMAC = typeof _powM !== "undefined" ? _powM : "";
|
||||
const powDifficulty = typeof _powD !== "undefined" ? _powD : 12;
|
||||
const r = typeof _r !== "undefined" ? _r : "";
|
||||
const submitPath = "__CROWDSEC_SUBMIT_PATH__";
|
||||
|
||||
// --- Challenge status reporting ---
|
||||
|
||||
function reportChallengeStatus(status) {
|
||||
if (typeof window.crowdsecSetChallengeStatus === "function") {
|
||||
window.crowdsecSetChallengeStatus(status);
|
||||
return;
|
||||
}
|
||||
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("crowdsec-challenge-status", {
|
||||
detail: { status },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// --- Main flow ---
|
||||
// The static bundle registers a hook on globalThis; the dynamic key module
|
||||
// (loaded after) calls it with { key, epoch }. Key-dependent crypto lives here;
|
||||
// the static bundle holds no hardcoded K. The hook name survives obfuscation
|
||||
// via `reservedStrings` so both bundles meet on the same global.
|
||||
|
||||
const CSEC_HOOK_NAME = "__CSEC_CHALLENGE_HOOK_v1__";
|
||||
|
||||
async function runChallenge(epochKey) {
|
||||
// Fail closed if the fpscanner bundle didn't load.
|
||||
const Scanner = globalThis.CrowdsecFingerprintScanner;
|
||||
if (typeof Scanner !== "function") {
|
||||
reportChallengeStatus("fail");
|
||||
return;
|
||||
}
|
||||
|
||||
const [nonce, fpResult] = await Promise.all([
|
||||
solvePoWAsync(powPrefix, powDifficulty),
|
||||
new Scanner().collectFingerprint({ encrypt: false }),
|
||||
]);
|
||||
|
||||
// Per-challenge secret s = HMAC(K_epoch, r). Never transmitted; the server
|
||||
// derives the same s from its per-epoch key and the cleartext r. epochKey is
|
||||
// hex-encoded K; HMAC over its raw bytes.
|
||||
const s = hmacSHA256HexKey(epochKey, r);
|
||||
|
||||
// Fingerprint payload obfuscation key, derived from the secret s (NOT a
|
||||
// confidentiality guarantee — light obfuscation only).
|
||||
const encKey = hmacSHA256Hex(s, "fpenc" + r);
|
||||
const f = encryptFingerprint(encKey, JSON.stringify(fpResult));
|
||||
|
||||
// Submission signature binds (r, ts, nonce, f) under the secret s.
|
||||
const sig = hmacSHA256Hex(s, r + ts + nonce + f);
|
||||
|
||||
return fetch(submitPath, {
|
||||
method: "POST",
|
||||
credentials: "same-origin",
|
||||
body: new URLSearchParams({
|
||||
f: f,
|
||||
r: r,
|
||||
ts: ts,
|
||||
sig: sig,
|
||||
n: nonce,
|
||||
p: powPrefix,
|
||||
m: powMAC,
|
||||
d: powDifficulty,
|
||||
u: window.location.pathname,
|
||||
}),
|
||||
})
|
||||
.then((response) => response.json())
|
||||
.then((data) => {
|
||||
const status = typeof data?.status === "string" ? data.status : "fail";
|
||||
reportChallengeStatus(status);
|
||||
})
|
||||
.catch(() => {
|
||||
reportChallengeStatus("fail");
|
||||
});
|
||||
}
|
||||
|
||||
// Register the hook. The dynamic module (loaded after this) reads the same
|
||||
// CSEC_HOOK_NAME string and invokes runChallenge with the per-epoch key.
|
||||
globalThis[CSEC_HOOK_NAME] = function (params) {
|
||||
// Defensive: the dynamic module must pass a non-empty string key.
|
||||
if (!params || typeof params.key !== "string" || params.key.length === 0) {
|
||||
reportChallengeStatus("fail");
|
||||
return;
|
||||
}
|
||||
runChallenge(params.key);
|
||||
};
|
||||
@@ -0,0 +1,704 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHasLeadingZeroBits(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hash []byte
|
||||
bits int
|
||||
expect bool
|
||||
}{
|
||||
{
|
||||
name: "0 bits required",
|
||||
hash: []byte{0xff, 0xff},
|
||||
bits: 0,
|
||||
expect: true,
|
||||
},
|
||||
{
|
||||
name: "8 bits - first byte zero",
|
||||
hash: []byte{0x00, 0xff},
|
||||
bits: 8,
|
||||
expect: true,
|
||||
},
|
||||
{
|
||||
name: "8 bits - first byte not zero",
|
||||
hash: []byte{0x01, 0xff},
|
||||
bits: 8,
|
||||
expect: false,
|
||||
},
|
||||
{
|
||||
name: "16 bits - two zero bytes",
|
||||
hash: []byte{0x00, 0x00, 0xff},
|
||||
bits: 16,
|
||||
expect: true,
|
||||
},
|
||||
{
|
||||
name: "10 bits - partial byte check passes",
|
||||
hash: []byte{0x00, 0x20}, // 0b00000000 0b00100000 → 10 leading zeros
|
||||
bits: 10,
|
||||
expect: true,
|
||||
},
|
||||
{
|
||||
name: "10 bits - partial byte check fails",
|
||||
hash: []byte{0x00, 0x40}, // 0b00000000 0b01000000 → 9 leading zeros
|
||||
bits: 10,
|
||||
expect: false,
|
||||
},
|
||||
{
|
||||
name: "1 bit - first bit zero",
|
||||
hash: []byte{0x7f},
|
||||
bits: 1,
|
||||
expect: true,
|
||||
},
|
||||
{
|
||||
name: "1 bit - first bit set",
|
||||
hash: []byte{0x80},
|
||||
bits: 1,
|
||||
expect: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.expect, hasLeadingZeroBits(tt.hash, tt.bits))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratePowPrefix(t *testing.T) {
|
||||
p1 := mustGeneratePowPrefix(t)
|
||||
p2 := mustGeneratePowPrefix(t)
|
||||
|
||||
assert.Len(t, p1, 32) // 16 bytes = 32 hex chars
|
||||
assert.Len(t, p2, 32)
|
||||
assert.NotEqual(t, p1, p2) // random, should differ
|
||||
}
|
||||
|
||||
func TestDifficultyFromLevel(t *testing.T) {
|
||||
bits, err := DifficultyFromLevel("low")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, PowDifficultyLow, bits)
|
||||
|
||||
bits, err = DifficultyFromLevel("MEDIUM")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, PowDifficultyMedium, bits)
|
||||
|
||||
bits, err = DifficultyFromLevel("High")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, PowDifficultyHigh, bits)
|
||||
|
||||
bits, err = DifficultyFromLevel("disabled")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, PowDifficultyDisabled, bits)
|
||||
|
||||
bits, err = DifficultyFromLevel("IMPOSSIBLE")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, PowDifficultyImpossible, bits)
|
||||
|
||||
_, err = DifficultyFromLevel("extreme")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestSetDifficulty(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(defaultPowDifficulty)
|
||||
|
||||
require.NoError(t, c.SetDifficulty("low"))
|
||||
assert.Equal(t, PowDifficultyLow, c.powDifficulty)
|
||||
|
||||
require.NoError(t, c.SetDifficulty("MEDIUM"))
|
||||
assert.Equal(t, PowDifficultyMedium, c.powDifficulty)
|
||||
|
||||
require.NoError(t, c.SetDifficulty("High"))
|
||||
assert.Equal(t, PowDifficultyHigh, c.powDifficulty)
|
||||
|
||||
require.NoError(t, c.SetDifficulty("impossible"))
|
||||
assert.Equal(t, PowDifficultyImpossible, c.powDifficulty)
|
||||
|
||||
require.Error(t, c.SetDifficulty("extreme"))
|
||||
assert.Equal(t, PowDifficultyImpossible, c.powDifficulty) // unchanged on error
|
||||
}
|
||||
|
||||
func TestDeriveChallengeSecret(t *testing.T) {
|
||||
secret := []byte("0123456789abcdef0123456789abcdef")
|
||||
keys, err := NewKeyRing(secret, time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
c := &ChallengeRuntime{keys: keys}
|
||||
|
||||
ts := fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
epoch := c.epochForTimestamp(ts)
|
||||
signKey, ok := c.keys.SignKey(epoch)
|
||||
require.True(t, ok)
|
||||
|
||||
r1, err := generateChallengeNonce()
|
||||
require.NoError(t, err)
|
||||
r2, err := generateChallengeNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
s1 := deriveChallengeSecret(signKey, r1)
|
||||
s1again := deriveChallengeSecret(signKey, r1)
|
||||
s2 := deriveChallengeSecret(signKey, r2)
|
||||
|
||||
assert.Len(t, s1, 64) // HMAC-SHA256 hex
|
||||
assert.Equal(t, s1, s1again) // deterministic in (signKey, r)
|
||||
assert.NotEqual(t, s1, s2) // distinct per-challenge r → distinct s
|
||||
|
||||
// Matches HMAC-SHA256 under the per-epoch sign key.
|
||||
assert.Equal(t, hmacSHA256Hex(signKey, []byte(r1)), s1)
|
||||
|
||||
// The fingerprint obfuscation key is derived from s and r, distinct from s.
|
||||
obf := deriveFingerprintObfKey(s1, r1)
|
||||
assert.Len(t, obf, 64)
|
||||
assert.NotEqual(t, s1, obf)
|
||||
assert.Equal(t, obf, deriveFingerprintObfKey(s1, r1)) // deterministic
|
||||
}
|
||||
|
||||
func TestVerifyChallenge(t *testing.T) {
|
||||
keys, err := NewKeyRing([]byte("0123456789abcdef0123456789abcdef"), time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
c := &ChallengeRuntime{keys: keys}
|
||||
|
||||
r, ts := freshChallenge(t)
|
||||
salt := mustGeneratePowPrefix(t)
|
||||
const diff = PowDifficultyMedium
|
||||
mac := c.computePowMAC(salt, r, ts, diff)
|
||||
|
||||
// Valid challenge → returns the epoch sign key.
|
||||
signKey, ok := c.verifyChallenge(r, ts, salt, mac, diff)
|
||||
assert.True(t, ok)
|
||||
assert.NotEmpty(t, signKey)
|
||||
|
||||
// Wrong r (MAC bound to a different r).
|
||||
_, ok = c.verifyChallenge("ffffffffffffffffffffffffffffffff", ts, salt, mac, diff)
|
||||
assert.False(t, ok)
|
||||
|
||||
// Malformed timestamp.
|
||||
_, ok = c.verifyChallenge(r, "9999999999999999999", salt, mac, diff)
|
||||
assert.False(t, ok)
|
||||
|
||||
// Forged MAC.
|
||||
_, ok = c.verifyChallenge(r, ts, salt, "forged-mac", diff)
|
||||
assert.False(t, ok)
|
||||
|
||||
// MAC from a different salt.
|
||||
otherSalt := mustGeneratePowPrefix(t)
|
||||
_, ok = c.verifyChallenge(r, ts, otherSalt, mac, diff)
|
||||
assert.False(t, ok)
|
||||
|
||||
// Tampered difficulty: the MAC binds difficulty, so claiming a lower one
|
||||
// than was issued must fail (the per-request escalation bypass).
|
||||
_, ok = c.verifyChallenge(r, ts, salt, mac, PowDifficultyLow)
|
||||
assert.False(t, ok)
|
||||
|
||||
// Expired timestamp (too old).
|
||||
oldTS := fmt.Sprintf("%d", time.Now().Add(-3*ticketAgeBackstop).UnixNano())
|
||||
oldMAC := c.computePowMAC(salt, r, oldTS, diff)
|
||||
_, ok = c.verifyChallenge(r, oldTS, salt, oldMAC, diff)
|
||||
assert.False(t, ok)
|
||||
|
||||
// Cross-secret rejection: a challenge signed by a different master secret
|
||||
// must NOT validate.
|
||||
otherKeys, err := NewKeyRing([]byte("ffffffffffffffffffffffffffffffff"), time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
other := &ChallengeRuntime{keys: otherKeys}
|
||||
otherMAC := other.computePowMAC(salt, r, ts, diff)
|
||||
_, ok = c.verifyChallenge(r, ts, salt, otherMAC, diff)
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
// TestDistributedAgreement asserts that two ChallengeRuntime instances
|
||||
// configured with the same master_secret derive bit-identical per-challenge
|
||||
// secrets and PoW MACs — the property that makes load-balanced deployments
|
||||
// work (challenge issuance is stateless; only the spent-set is local).
|
||||
func TestDistributedAgreement(t *testing.T) {
|
||||
secret := []byte("shared-secret-shared-secret-shar") // 32 bytes
|
||||
keysA, err := NewKeyRing(secret, time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
keysB, err := NewKeyRing(secret, time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
a := &ChallengeRuntime{keys: keysA}
|
||||
b := &ChallengeRuntime{keys: keysB}
|
||||
|
||||
r, ts := freshChallenge(t)
|
||||
salt := "deadbeefcafebabe"
|
||||
|
||||
// Both instances agree on the PoW MAC...
|
||||
assert.Equal(t, a.computePowMAC(salt, r, ts, PowDifficultyMedium), b.computePowMAC(salt, r, ts, PowDifficultyMedium))
|
||||
|
||||
// ...and on the per-challenge secret derived from the epoch for ts.
|
||||
epoch := a.epochForTimestamp(ts)
|
||||
keyA, ok := a.keys.SignKey(epoch)
|
||||
require.True(t, ok)
|
||||
keyB, ok := b.keys.SignKey(epoch)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, deriveChallengeSecret(keyA, r), deriveChallengeSecret(keyB, r))
|
||||
|
||||
// And a challenge issued by A validates against B.
|
||||
signKey, ok := b.verifyChallenge(r, ts, salt, a.computePowMAC(salt, r, ts, PowDifficultyMedium), PowDifficultyMedium)
|
||||
assert.True(t, ok)
|
||||
assert.NotEmpty(t, signKey)
|
||||
}
|
||||
|
||||
// solvePoWGo is a Go implementation of the PoW solver matching the JS client.
|
||||
func solvePoWGo(prefix string, difficulty int) string {
|
||||
for nonce := 0; ; nonce++ {
|
||||
candidate := prefix + formatBase36(nonce)
|
||||
hash := sha256.Sum256([]byte(candidate))
|
||||
if hasLeadingZeroBits(hash[:], difficulty) {
|
||||
return formatBase36(nonce)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func formatBase36(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
|
||||
const digits = "0123456789abcdefghijklmnopqrstuvwxyz"
|
||||
var buf [64]byte
|
||||
i := len(buf)
|
||||
|
||||
for n > 0 {
|
||||
i--
|
||||
buf[i] = digits[n%36]
|
||||
n /= 36
|
||||
}
|
||||
|
||||
return string(buf[i:])
|
||||
}
|
||||
|
||||
func TestPoWVerification(t *testing.T) {
|
||||
prefix := mustGeneratePowPrefix(t)
|
||||
difficulty := 8
|
||||
|
||||
nonce := solvePoWGo(prefix, difficulty)
|
||||
|
||||
powHash := sha256.Sum256([]byte(prefix + nonce))
|
||||
assert.True(t, hasLeadingZeroBits(powHash[:], difficulty))
|
||||
}
|
||||
|
||||
// testSecret is a fixed master secret used by all test helpers below so that
|
||||
// tickets / MACs / cookies issued in one helper validate in another.
|
||||
var testSecret = []byte("test-secret-test-secret-test-sec") // 32 bytes
|
||||
|
||||
// testKeyRing returns a fresh KeyRing built around testSecret. Returned by
|
||||
// value would defeat the cache, so we hand back a pointer and rely on the
|
||||
// caller to discard it after the test.
|
||||
func testKeyRing() *KeyRing {
|
||||
k, err := NewKeyRing(testSecret, time.Minute, 3)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// newTestRuntime returns a minimal ChallengeRuntime suitable for the helpers
|
||||
// in this file (no WASM, no obfuscation cache — those tests construct full
|
||||
// runtimes via NewChallengeRuntime). Includes a spent-set so
|
||||
// ValidateChallengeResponse's single-use burn works.
|
||||
func newTestRuntime() *ChallengeRuntime {
|
||||
return &ChallengeRuntime{keys: testKeyRing(), spent: newSpentSet(spentSetDefaultMaxEntries)}
|
||||
}
|
||||
|
||||
// newTestRuntimeWithDifficulty is a small convenience for the many ValidateChallengeResponse tests.
|
||||
func newTestRuntimeWithDifficulty(d int) *ChallengeRuntime {
|
||||
return &ChallengeRuntime{keys: testKeyRing(), powDifficulty: d, spent: newSpentSet(spentSetDefaultMaxEntries)}
|
||||
}
|
||||
|
||||
// freshChallenge generates a per-request nonce+timestamp pair (matching
|
||||
// GetChallengePage's `r` and `ts`).
|
||||
func freshChallenge(tb testing.TB) (r, ts string) {
|
||||
tb.Helper()
|
||||
ts = fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
var err error
|
||||
r, err = generateChallengeNonce()
|
||||
require.NoError(tb, err)
|
||||
return r, ts
|
||||
}
|
||||
|
||||
// mustGeneratePowPrefix is the test-only wrapper around generatePowPrefix
|
||||
// that aborts the test if crypto/rand fails (which would mean the test
|
||||
// machine is broken in a way that invalidates the run anyway).
|
||||
func mustGeneratePowPrefix(tb testing.TB) string {
|
||||
tb.Helper()
|
||||
p, err := generatePowPrefix()
|
||||
require.NoError(tb, err)
|
||||
return p
|
||||
}
|
||||
|
||||
// buildValidBody constructs a valid challenge POST body with an empty
|
||||
// fingerprint. Existing tests rely on this signature; for tests that need
|
||||
// to assert fingerprint round-trip through the seal/unseal chain, use
|
||||
// buildValidBodyWithFingerprint.
|
||||
func buildValidBody(tb testing.TB, difficulty int, r, ts string) string {
|
||||
return buildValidBodyWithFingerprint(tb, difficulty, r, ts, FingerprintData{})
|
||||
}
|
||||
|
||||
// buildValidBodyWithFingerprint mirrors buildValidBody but lets the caller
|
||||
// supply a populated FingerprintData so tests can assert that the submitted
|
||||
// payload survives the full obfuscate → sign → deobfuscate → unmarshal chain.
|
||||
// It mirrors the client (challenge.js): derive s = HMAC(K_epoch, r), obfuscate
|
||||
// f under HMAC(s, "fpenc"||r), and sign sig = HMAC(s, r||ts||n||f).
|
||||
func buildValidBodyWithFingerprint(tb testing.TB, difficulty int, r, ts string, fp FingerprintData) string {
|
||||
c := newTestRuntime()
|
||||
salt := mustGeneratePowPrefix(tb)
|
||||
powMAC := c.computePowMAC(salt, r, ts, difficulty)
|
||||
nonce := solvePoWGo(salt, difficulty)
|
||||
|
||||
// Derive s exactly as the server will, from the per-epoch sign key for ts.
|
||||
// Mirror computePowMAC's fallback for out-of-window timestamps (used by the
|
||||
// expired-timestamp test) so the body is still constructible; the server
|
||||
// rejects on freshness before the signature is ever checked.
|
||||
epoch := c.epochForTimestamp(ts)
|
||||
signKey, ok := c.keys.SignKey(epoch)
|
||||
if !ok {
|
||||
_, signKey = c.keys.Current()
|
||||
}
|
||||
s := deriveChallengeSecret(signKey, r)
|
||||
|
||||
fpJSON, err := json.Marshal(fp)
|
||||
require.NoError(tb, err)
|
||||
|
||||
obfKey := deriveFingerprintObfKey(s, r)
|
||||
keyBytes := []byte(obfKey)
|
||||
obfuscated := make([]byte, len(fpJSON))
|
||||
for i := range fpJSON {
|
||||
obfuscated[i] = fpJSON[i] ^ keyBytes[i%len(keyBytes)]
|
||||
}
|
||||
f := base64.StdEncoding.EncodeToString(obfuscated)
|
||||
|
||||
sig := hmacSHA256Hex([]byte(s), []byte(r+ts+nonce+f))
|
||||
|
||||
return url.Values{
|
||||
"f": {f},
|
||||
"r": {r},
|
||||
"ts": {ts},
|
||||
"sig": {sig},
|
||||
"n": {nonce},
|
||||
"p": {salt},
|
||||
"m": {powMAC},
|
||||
"d": {strconv.Itoa(difficulty)},
|
||||
"u": {"/"},
|
||||
}.Encode()
|
||||
}
|
||||
|
||||
func TestValidateChallengeResponse(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(8)
|
||||
r, ts := freshChallenge(t)
|
||||
body := buildValidBody(t, c.powDifficulty, r, ts)
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
req.Header.Set("User-Agent", "test-agent")
|
||||
|
||||
ck, fpResult, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, ck)
|
||||
assert.NotNil(t, fpResult)
|
||||
}
|
||||
|
||||
// TestValidateChallengeResponse_Replay asserts single-use: a valid submission
|
||||
// is accepted once, and re-submitting the identical body is rejected because
|
||||
// its per-challenge nonce `r` has been burned.
|
||||
func TestValidateChallengeResponse_Replay(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(8)
|
||||
r, ts := freshChallenge(t)
|
||||
body := buildValidBody(t, c.powDifficulty, r, ts)
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
req.Header.Set("User-Agent", "test-agent")
|
||||
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
require.NoError(t, err)
|
||||
|
||||
// Replay the exact same body — must be rejected as already used.
|
||||
req2, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
req2.Header.Set("User-Agent", "test-agent")
|
||||
_, _, _, err = c.ValidateChallengeResponse(req2, []byte(body))
|
||||
assert.ErrorContains(t, err, "already used")
|
||||
}
|
||||
|
||||
// TestValidateChallengeResponse_FingerprintRoundTrip is the explicit
|
||||
// happy-path guard for the full client→server seal/unseal chain. Other
|
||||
// ValidateChallengeResponse tests only assert that a well-formed submission
|
||||
// is accepted; this one verifies that the FingerprintData the browser
|
||||
// submits is the same FingerprintData the server gets back, then survives
|
||||
// the cookie Seal → Open round-trip too. Regression guard for any future
|
||||
// change to encryption, HMAC keying, or proto conversion.
|
||||
func TestValidateChallengeResponse_FingerprintRoundTrip(t *testing.T) {
|
||||
c := &ChallengeRuntime{keys: testKeyRing(), powDifficulty: 8, cookieTTL: time.Hour, spent: newSpentSet(spentSetDefaultMaxEntries)}
|
||||
|
||||
submitted := FingerprintData{
|
||||
FSID: "fsid-abcdef",
|
||||
Nonce: "client-nonce-xyz",
|
||||
Time: 1700000000,
|
||||
URL: "https://example.com/protected",
|
||||
FastBotDetection: FlexBool(false),
|
||||
}
|
||||
|
||||
r, ts := freshChallenge(t)
|
||||
body := buildValidBodyWithFingerprint(t, c.powDifficulty, r, ts, submitted)
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("User-Agent", "test-agent")
|
||||
|
||||
ck, decoded, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ck)
|
||||
|
||||
// The fingerprint returned by ValidateChallengeResponse must match what
|
||||
// the browser submitted (XOR'd, base64'd, then decrypted server-side).
|
||||
assert.Equal(t, submitted.FSID, decoded.FSID)
|
||||
assert.Equal(t, submitted.Nonce, decoded.Nonce)
|
||||
assert.Equal(t, submitted.Time, decoded.Time)
|
||||
assert.Equal(t, submitted.URL, decoded.URL)
|
||||
|
||||
// The sealed cookie must round-trip through ValidCookie and yield the
|
||||
// same fingerprint contents — guards the proto conversion + AES-GCM
|
||||
// seal/unseal chain.
|
||||
parsed, err := http.ParseSetCookie(ck.String())
|
||||
require.NoError(t, err)
|
||||
|
||||
cd, err := c.ValidCookie(parsed, "test-agent")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cd)
|
||||
assert.Equal(t, submitted.FSID, cd.Fingerprint.FSID)
|
||||
assert.Equal(t, submitted.URL, cd.Fingerprint.URL)
|
||||
assert.Equal(t, c.powDifficulty, cd.PowDifficulty)
|
||||
}
|
||||
|
||||
func TestValidateChallengeResponse_MultipleConcurrentClients(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(8)
|
||||
|
||||
for range 20 {
|
||||
r, ts := freshChallenge(t)
|
||||
body := buildValidBody(t, c.powDifficulty, r, ts)
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
req.Header.Set("User-Agent", "test-agent")
|
||||
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateChallengeResponse_InvalidPoW(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(8)
|
||||
r, ts := freshChallenge(t)
|
||||
salt := mustGeneratePowPrefix(t)
|
||||
powMAC := c.computePowMAC(salt, r, ts, 8)
|
||||
|
||||
body := url.Values{
|
||||
"f": {"dGVzdA=="},
|
||||
"r": {r},
|
||||
"ts": {ts},
|
||||
"sig": {"deadbeef"},
|
||||
"n": {"invalid-nonce"},
|
||||
"p": {salt},
|
||||
"m": {powMAC},
|
||||
"d": {"8"},
|
||||
"u": {"/"},
|
||||
}.Encode()
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
assert.ErrorContains(t, err, "invalid proof-of-work")
|
||||
}
|
||||
|
||||
func TestValidateChallengeResponse_ImpossibleDifficulty(t *testing.T) {
|
||||
// A submission that would otherwise pass at low difficulty is rejected
|
||||
// outright when the runtime is set to impossible.
|
||||
c := newTestRuntimeWithDifficulty(PowDifficultyImpossible)
|
||||
r, ts := freshChallenge(t)
|
||||
salt := mustGeneratePowPrefix(t)
|
||||
// A ticket issued (and MAC-bound) at impossible difficulty: the hard-block
|
||||
// fires before the unsolvable PoW is ever checked, so a dummy nonce is fine.
|
||||
powMAC := c.computePowMAC(salt, r, ts, PowDifficultyImpossible)
|
||||
body := url.Values{
|
||||
"f": {"dGVzdA=="},
|
||||
"r": {r},
|
||||
"ts": {ts},
|
||||
"sig": {"deadbeef"},
|
||||
"n": {"0"},
|
||||
"p": {salt},
|
||||
"m": {powMAC},
|
||||
"d": {strconv.Itoa(PowDifficultyImpossible)},
|
||||
"u": {"/"},
|
||||
}.Encode()
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
req.Header.Set("User-Agent", "test-agent")
|
||||
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
assert.ErrorContains(t, err, "impossible")
|
||||
}
|
||||
|
||||
func TestValidateChallengeResponse_ExpiredTimestamp(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(8)
|
||||
r, _ := freshChallenge(t)
|
||||
oldTS := fmt.Sprintf("%d", time.Now().Add(-3*ticketAgeBackstop).UnixNano())
|
||||
body := buildValidBody(t, c.powDifficulty, r, oldTS)
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
req.Header.Set("User-Agent", "test-agent")
|
||||
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
assert.ErrorContains(t, err, "invalid ticket")
|
||||
}
|
||||
|
||||
// TestValidateChallengeResponse_TamperedR sends a different `r` than the one
|
||||
// the PoW MAC was issued for, so verifyChallenge's MAC check fails.
|
||||
func TestValidateChallengeResponse_TamperedR(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(8)
|
||||
r, ts := freshChallenge(t)
|
||||
salt := mustGeneratePowPrefix(t)
|
||||
powMAC := c.computePowMAC(salt, r, ts, 8) // bound to r, not to the tampered value
|
||||
|
||||
body := url.Values{
|
||||
"f": {"dGVzdA=="},
|
||||
"r": {"ffffffffffffffffffffffffffffffff"},
|
||||
"ts": {ts},
|
||||
"sig": {"deadbeef"},
|
||||
"n": {"0"},
|
||||
"p": {salt},
|
||||
"m": {powMAC},
|
||||
"d": {"8"},
|
||||
"u": {"/"},
|
||||
}.Encode()
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
assert.ErrorContains(t, err, "invalid ticket")
|
||||
}
|
||||
|
||||
func TestValidateChallengeResponse_ForgedMAC(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(8)
|
||||
r, ts := freshChallenge(t)
|
||||
salt := mustGeneratePowPrefix(t)
|
||||
|
||||
body := url.Values{
|
||||
"f": {"dGVzdA=="},
|
||||
"r": {r},
|
||||
"ts": {ts},
|
||||
"sig": {"deadbeef"},
|
||||
"n": {"0"},
|
||||
"p": {salt},
|
||||
"m": {"forged-mac-value"},
|
||||
"d": {"8"},
|
||||
"u": {"/"},
|
||||
}.Encode()
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
assert.ErrorContains(t, err, "invalid ticket")
|
||||
}
|
||||
|
||||
// TestValidateChallengeResponse_TamperedDifficulty submits a lower difficulty
|
||||
// than the MAC was issued for; verifyChallenge's MAC check fails because the
|
||||
// MAC binds the difficulty (the per-request escalation-bypass guard).
|
||||
func TestValidateChallengeResponse_TamperedDifficulty(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(8)
|
||||
r, ts := freshChallenge(t)
|
||||
salt := mustGeneratePowPrefix(t)
|
||||
powMAC := c.computePowMAC(salt, r, ts, PowDifficultyHigh) // issued at high
|
||||
nonce := solvePoWGo(salt, 8)
|
||||
|
||||
body := url.Values{
|
||||
"f": {"dGVzdA=="},
|
||||
"r": {r},
|
||||
"ts": {ts},
|
||||
"sig": {"deadbeef"},
|
||||
"n": {nonce},
|
||||
"p": {salt},
|
||||
"m": {powMAC},
|
||||
"d": {"8"}, // client claims a lower difficulty than was bound
|
||||
"u": {"/"},
|
||||
}.Encode()
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
assert.ErrorContains(t, err, "invalid ticket")
|
||||
}
|
||||
|
||||
func TestValidateChallengeResponse_InvalidSig(t *testing.T) {
|
||||
c := newTestRuntimeWithDifficulty(8)
|
||||
r, ts := freshChallenge(t)
|
||||
salt := mustGeneratePowPrefix(t)
|
||||
powMAC := c.computePowMAC(salt, r, ts, 8)
|
||||
nonce := solvePoWGo(salt, c.powDifficulty)
|
||||
|
||||
body := url.Values{
|
||||
"f": {"dGVzdA=="},
|
||||
"r": {r},
|
||||
"ts": {ts},
|
||||
"sig": {"0000000000000000000000000000000000000000000000000000000000000000"},
|
||||
"n": {nonce},
|
||||
"p": {salt},
|
||||
"m": {powMAC},
|
||||
"d": {"8"},
|
||||
"u": {"/"},
|
||||
}.Encode()
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
assert.ErrorContains(t, err, "invalid HMAC")
|
||||
}
|
||||
|
||||
func TestValidateChallengeResponse_MissingFields(t *testing.T) {
|
||||
c := &ChallengeRuntime{}
|
||||
|
||||
body := url.Values{
|
||||
"f": {"dGVzdA=="},
|
||||
"r": {"abcd"},
|
||||
}.Encode()
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
assert.ErrorContains(t, err, "missing required fields")
|
||||
}
|
||||
|
||||
func TestValidateChallengeResponse_MalformedR(t *testing.T) {
|
||||
c := &ChallengeRuntime{}
|
||||
|
||||
// All fields present (passes the missing-field guard) but `r` is not a
|
||||
// well-formed 32-char hex nonce, so the shape check rejects it before any
|
||||
// spent-set burn. Error is the generic "invalid ticket" so the stage stays
|
||||
// indistinguishable from a verifyChallenge failure.
|
||||
for name, badR := range map[string]string{
|
||||
"too short": strings.Repeat("a", 31),
|
||||
"too long": strings.Repeat("a", 33),
|
||||
"right len non-hex": strings.Repeat("z", 32),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body := url.Values{
|
||||
"f": {"dGVzdA=="},
|
||||
"r": {badR},
|
||||
"ts": {"1"},
|
||||
"sig": {"x"},
|
||||
"n": {"x"},
|
||||
"p": {"x"},
|
||||
"m": {"x"},
|
||||
"d": {"12"},
|
||||
"u": {"/"},
|
||||
}.Encode()
|
||||
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
assert.ErrorContains(t, err, "invalid ticket")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
// config.go defines the YAML-facing challenge configuration. Multiple
|
||||
// appsec-config files can each contribute a disjoint subset; MergeFrom
|
||||
// composes them without one wiping the others. Once merged, BuildOptions
|
||||
// translates the Config into the runtime's functional-option list
|
||||
// (NewChallengeRuntime(...)).
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/logging"
|
||||
)
|
||||
|
||||
// Config carries the YAML-configurable challenge runtime settings.
|
||||
type Config struct {
|
||||
// MasterSecret is the long-lived secret all per-epoch HMAC keys and the
|
||||
// cookie-sealing AES key derive from. In a distributed deployment every
|
||||
// instance MUST share the same value to sign/verify each other's
|
||||
// challenges. If unset, the runtime generates an ephemeral random secret
|
||||
// at startup — fine for a single instance, but restarts then invalidate
|
||||
// outstanding cookies.
|
||||
MasterSecret *string `yaml:"master_secret"`
|
||||
|
||||
// KeyRotationInterval is the per-epoch key advance period. All instances
|
||||
// in a distributed setup MUST agree on it to derive identical keys.
|
||||
// Defaults to 5m.
|
||||
KeyRotationInterval *time.Duration `yaml:"key_rotation_interval"`
|
||||
|
||||
// MaxLiveEpochs is how many past epochs (besides the current one) the
|
||||
// keyring keeps accepting, so in-flight submissions aren't invalidated at
|
||||
// a rotation boundary. Bounds ticket-forgery exposure to
|
||||
// MaxLiveEpochs × KeyRotationInterval. Defaults to 3.
|
||||
MaxLiveEpochs *int `yaml:"max_live_epochs"`
|
||||
|
||||
// CookieTTL is how long a successful-challenge cookie stays valid.
|
||||
// Decoupled from the keyring window (enforced by a not_after stamp inside
|
||||
// the sealed cookie, not key eviction) so cookies can be long-lived while
|
||||
// per-epoch keys rotate tightly. Defaults to 12h.
|
||||
CookieTTL *time.Duration `yaml:"cookie_ttl"`
|
||||
|
||||
// MaxCookieSize caps the challenge cookie's encoded size, enforced on both
|
||||
// seal and open. It bounds the memory allocated from the (attacker-supplied)
|
||||
// fingerprint envelope, closing an over-allocation DoS. Defaults to
|
||||
// MaxCookieLen (4096, the per-cookie size browsers guarantee); raise it only
|
||||
// if a non-browser client tolerates larger cookies.
|
||||
MaxCookieSize *int `yaml:"max_cookie_size"`
|
||||
|
||||
// CryptoObfuscationPoolSize is how many obfuscations of the per-epoch
|
||||
// sign-key module to keep per live epoch. Each variant embeds the same key
|
||||
// with different byte layout, giving per-visitor variance. Defaults to 1.
|
||||
CryptoObfuscationPoolSize *int `yaml:"crypto_obfuscation_pool_size"`
|
||||
|
||||
// SpentSetMaxEntries caps the replay-protection LRU. A deep DoS backstop;
|
||||
// steady-state stays far below it. Defaults to spentSetDefaultMaxEntries.
|
||||
SpentSetMaxEntries *int `yaml:"spent_set_max_entries"`
|
||||
|
||||
// LogLevel sets the challenge runtime's own log verbosity, independent of
|
||||
// the global level. Note: `panic` is not supported — logrus.PanicLevel is 0,
|
||||
// which SubLogger (pkg/logging/sublogger.go) treats as "inherit the parent level".
|
||||
LogLevel *log.Level `yaml:"log_level,omitempty"`
|
||||
}
|
||||
|
||||
// MergeFrom overlays the non-nil fields of other onto c, field by field, so
|
||||
// multiple appsec-configs can each contribute a disjoint subset (last non-nil
|
||||
// wins). A nil receiver or argument is a no-op.
|
||||
func (c *Config) MergeFrom(other *Config) {
|
||||
if c == nil || other == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if other.MasterSecret != nil {
|
||||
c.MasterSecret = other.MasterSecret
|
||||
}
|
||||
if other.KeyRotationInterval != nil {
|
||||
c.KeyRotationInterval = other.KeyRotationInterval
|
||||
}
|
||||
if other.MaxLiveEpochs != nil {
|
||||
c.MaxLiveEpochs = other.MaxLiveEpochs
|
||||
}
|
||||
if other.CookieTTL != nil {
|
||||
c.CookieTTL = other.CookieTTL
|
||||
}
|
||||
if other.MaxCookieSize != nil {
|
||||
c.MaxCookieSize = other.MaxCookieSize
|
||||
}
|
||||
if other.CryptoObfuscationPoolSize != nil {
|
||||
c.CryptoObfuscationPoolSize = other.CryptoObfuscationPoolSize
|
||||
}
|
||||
if other.SpentSetMaxEntries != nil {
|
||||
c.SpentSetMaxEntries = other.SpentSetMaxEntries
|
||||
}
|
||||
if other.LogLevel != nil {
|
||||
c.LogLevel = other.LogLevel
|
||||
}
|
||||
}
|
||||
|
||||
// BuildOptions translates a (possibly nil) merged Config into the WithXxx
|
||||
// Option list for NewChallengeRuntime; unset fields are omitted so the runtime
|
||||
// uses its built-in defaults. Returns an error if MasterSecret is set but
|
||||
// invalid. parent (may be nil) is the logger the "challenge" sublogger derives
|
||||
// from, at the configured log_level or parent's level.
|
||||
func BuildOptions(c *Config, parent *log.Entry) ([]Option, error) {
|
||||
// Always give the runtime its own component sublogger.
|
||||
base := log.StandardLogger()
|
||||
if parent != nil {
|
||||
base = parent.Logger
|
||||
}
|
||||
|
||||
var lvl log.Level
|
||||
if c != nil && c.LogLevel != nil {
|
||||
lvl = *c.LogLevel
|
||||
}
|
||||
|
||||
opts := []Option{WithLogger(logging.SubLogger(base, "challenge", lvl))}
|
||||
|
||||
if c == nil {
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
if c.MasterSecret != nil && *c.MasterSecret != "" {
|
||||
secret, err := ParseConfiguredSecret(*c.MasterSecret)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid challenge master_secret: %w", err)
|
||||
}
|
||||
opts = append(opts, WithMasterSecret(secret))
|
||||
}
|
||||
if c.KeyRotationInterval != nil {
|
||||
opts = append(opts, WithRotationInterval(*c.KeyRotationInterval))
|
||||
}
|
||||
if c.MaxLiveEpochs != nil && *c.MaxLiveEpochs > 0 {
|
||||
opts = append(opts, WithMaxLiveEpochs(*c.MaxLiveEpochs))
|
||||
}
|
||||
if c.CookieTTL != nil {
|
||||
opts = append(opts, WithCookieTTL(*c.CookieTTL))
|
||||
}
|
||||
if c.MaxCookieSize != nil && *c.MaxCookieSize > 0 {
|
||||
opts = append(opts, WithMaxCookieLen(*c.MaxCookieSize))
|
||||
}
|
||||
if c.CryptoObfuscationPoolSize != nil && *c.CryptoObfuscationPoolSize > 0 {
|
||||
opts = append(opts, WithCryptoObfuscationPoolSize(*c.CryptoObfuscationPoolSize))
|
||||
}
|
||||
if c.SpentSetMaxEntries != nil && *c.SpentSetMaxEntries > 0 {
|
||||
opts = append(opts, WithSpentSetMaxEntries(*c.SpentSetMaxEntries))
|
||||
}
|
||||
|
||||
return opts, nil
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestConfigMergeFromNilReceiverAndOther covers the two no-op edge cases:
|
||||
// merging into a nil receiver and merging a nil source. Both must not panic
|
||||
// and must not change anything.
|
||||
func TestConfigMergeFromNilReceiverAndOther(t *testing.T) {
|
||||
var nilCfg *Config
|
||||
require.NotPanics(t, func() { nilCfg.MergeFrom(&Config{CookieTTL: new(time.Hour)}) })
|
||||
|
||||
dst := &Config{CookieTTL: new(time.Hour)}
|
||||
dst.MergeFrom(nil)
|
||||
require.NotNil(t, dst.CookieTTL)
|
||||
assert.Equal(t, time.Hour, *dst.CookieTTL, "nil source must leave existing fields untouched")
|
||||
}
|
||||
|
||||
// TestConfigMergeFromOverlaysOnlyNonNilFields confirms the per-field "last
|
||||
// wins" semantics: each set field in `other` overrides `c`, but unset fields
|
||||
// leave `c`'s existing values intact. This is what lets multiple appsec-
|
||||
// configs each contribute a disjoint subset without one wiping the others.
|
||||
func TestConfigMergeFromOverlaysOnlyNonNilFields(t *testing.T) {
|
||||
dst := &Config{
|
||||
MasterSecret: new("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"),
|
||||
KeyRotationInterval: new(5 * time.Minute),
|
||||
MaxLiveEpochs: new(3),
|
||||
CookieTTL: new(12 * time.Hour),
|
||||
CryptoObfuscationPoolSize: new(1),
|
||||
}
|
||||
|
||||
src := &Config{
|
||||
// Override CookieTTL and CryptoObfuscationPoolSize; leave others
|
||||
// nil so the dst values must survive.
|
||||
CookieTTL: new(1 * time.Hour),
|
||||
CryptoObfuscationPoolSize: new(4),
|
||||
// New fields not present on dst.
|
||||
SpentSetMaxEntries: new(500_000),
|
||||
MaxCookieSize: new(8192),
|
||||
}
|
||||
|
||||
dst.MergeFrom(src)
|
||||
|
||||
// Overridden fields take src values.
|
||||
assert.Equal(t, 1*time.Hour, *dst.CookieTTL)
|
||||
assert.Equal(t, 4, *dst.CryptoObfuscationPoolSize)
|
||||
|
||||
// Untouched-by-src fields keep dst values.
|
||||
require.NotNil(t, dst.MasterSecret)
|
||||
assert.Equal(t, "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", *dst.MasterSecret)
|
||||
assert.Equal(t, 5*time.Minute, *dst.KeyRotationInterval)
|
||||
assert.Equal(t, 3, *dst.MaxLiveEpochs)
|
||||
|
||||
// New-from-src fields appear on dst.
|
||||
require.NotNil(t, dst.SpentSetMaxEntries)
|
||||
assert.Equal(t, 500_000, *dst.SpentSetMaxEntries)
|
||||
assert.Equal(t, 8192, *dst.MaxCookieSize)
|
||||
}
|
||||
|
||||
// TestBuildOptionsNilOrEmptyConfig confirms a nil or fully-empty Config emits
|
||||
// only the always-present component-logger option; the runtime is otherwise
|
||||
// constructed with its built-in defaults.
|
||||
func TestBuildOptionsNilOrEmptyConfig(t *testing.T) {
|
||||
opts, err := BuildOptions(nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, opts, 1, "nil config still emits the component-logger option")
|
||||
|
||||
opts, err = BuildOptions(&Config{}, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, opts, 1, "empty config still emits the component-logger option")
|
||||
}
|
||||
|
||||
// TestBuildOptionsTranslatesFieldsToRuntimeBehavior wires a populated Config
|
||||
// through BuildOptions and NewChallengeRuntime, then asserts the runtime's
|
||||
// observable fields match the configured values. This is the integration
|
||||
// guard for "appsec-config values reach the running challenge engine".
|
||||
func TestBuildOptionsTranslatesFieldsToRuntimeBehavior(t *testing.T) {
|
||||
cfg := &Config{
|
||||
MasterSecret: new("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"),
|
||||
KeyRotationInterval: new(3 * time.Minute),
|
||||
MaxLiveEpochs: new(4),
|
||||
CookieTTL: new(2 * time.Hour),
|
||||
CryptoObfuscationPoolSize: new(2),
|
||||
SpentSetMaxEntries: new(250_000),
|
||||
MaxCookieSize: new(8192),
|
||||
}
|
||||
|
||||
opts, err := BuildOptions(cfg, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, opts, 8, "every populated field + the component logger must emit an option")
|
||||
|
||||
rt, err := NewChallengeRuntime(t.Context(), opts...)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 2*time.Hour, rt.cookieTTL, "CookieTTL must reach the runtime")
|
||||
assert.Equal(t, 2, rt.cryptoPoolSize, "CryptoObfuscationPoolSize must reach the runtime")
|
||||
assert.Equal(t, 250_000, rt.spent.maxEntries, "SpentSetMaxEntries must reach the runtime")
|
||||
assert.Equal(t, 8192, rt.maxCookieLen, "MaxCookieSize must reach the runtime")
|
||||
}
|
||||
|
||||
// TestBuildOptionsInvalidMasterSecret confirms a malformed master_secret
|
||||
// surfaces as a configuration error rather than silently falling back to a
|
||||
// random secret — that would be a footgun in distributed deployments.
|
||||
func TestBuildOptionsInvalidMasterSecret(t *testing.T) {
|
||||
cfg := &Config{
|
||||
MasterSecret: new("too-short"),
|
||||
}
|
||||
_, err := BuildOptions(cfg, nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "master_secret")
|
||||
}
|
||||
|
||||
// TestSealAllowlistCookieTTLOverride exercises the per-call TTL plumbing on
|
||||
// SealAllowlistCookie: when ttlOverride is non-nil and positive, the
|
||||
// generated cookie's Max-Age reflects the override rather than the
|
||||
// runtime-global cookie_ttl. Nil falls back to the runtime default.
|
||||
func TestSealAllowlistCookieTTLOverride(t *testing.T) {
|
||||
rt, err := NewChallengeRuntime(t.Context(),
|
||||
WithMasterSecret([]byte("0123456789abcdef0123456789abcdef")),
|
||||
WithCookieTTL(12*time.Hour),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
req := &http.Request{
|
||||
Header: http.Header{"User-Agent": []string{"test-agent"}},
|
||||
URL: mustURL("https://example.test/protected"),
|
||||
}
|
||||
|
||||
// Override → short TTL.
|
||||
override := 30 * time.Minute
|
||||
before := time.Now().Unix()
|
||||
ck, err := rt.SealAllowlistCookie(req, "explicit-short", &override)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ck)
|
||||
assert.InDelta(t, before+int64(override.Seconds()), ck.Expiration, 2,
|
||||
"override TTL must drive the cookie expiration")
|
||||
|
||||
// No override → runtime default (12h).
|
||||
before = time.Now().Unix()
|
||||
ck, err = rt.SealAllowlistCookie(req, "fallback", nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ck)
|
||||
assert.InDelta(t, before+int64((12*time.Hour).Seconds()), ck.Expiration, 2,
|
||||
"nil override must fall back to runtime cookie_ttl")
|
||||
|
||||
// Zero/negative override → treated as "no override", uses runtime default.
|
||||
before = time.Now().Unix()
|
||||
zero := time.Duration(0)
|
||||
ck, err = rt.SealAllowlistCookie(req, "zero-fallback", &zero)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ck)
|
||||
assert.InDelta(t, before+int64((12*time.Hour).Seconds()), ck.Expiration, 2,
|
||||
"non-positive override must fall back to runtime cookie_ttl")
|
||||
}
|
||||
|
||||
func mustURL(s string) *url.URL {
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
// crypto.go implements AES-GCM cookie sealing/unsealing for the challenge
|
||||
// cookie envelope: a versioned, integrity-protected blob carrying the
|
||||
// fingerprint proto bytes plus a not_after expiration timestamp. The
|
||||
// encryption key comes from the keyring (cookie-master derivation, long
|
||||
// lifetime); rotation is handled by the keyring, not by this file. All
|
||||
// helpers here are pure functions over byte slices — no IO, no globals.
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/hkdf"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/pb"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCookieMalformed = errors.New("malformed cookie")
|
||||
ErrCookieSignature = errors.New("invalid cookie signature")
|
||||
ErrCookiePayload = errors.New("invalid cookie payload")
|
||||
ErrCookieExpired = errors.New("cookie expired")
|
||||
ErrCookieVersion = errors.New("unknown cookie version")
|
||||
ErrAllowlistReasonSize = errors.New("allowlist reason exceeds maximum length")
|
||||
ErrCookieTooLarge = errors.New("cookie exceeds maximum size")
|
||||
)
|
||||
|
||||
const hkdfInfo = "crowdsec-challenge-cookie"
|
||||
|
||||
// MaxAllowlistReasonLen caps the reason string operators pass to
|
||||
// GrantChallengeCookie. The reason travels inside every Set-Cookie + Cookie
|
||||
// header round-trip until the cookie expires; bounding it keeps the cookie
|
||||
// well under the 4 KB browser limit even with the AES-GCM tag + base64
|
||||
// expansion.
|
||||
const MaxAllowlistReasonLen = 256
|
||||
|
||||
// MaxCookieLen is the DEFAULT per-cookie size (RFC 6265 §6.1: 4096 bytes).
|
||||
// Can be configured via Config.MaxCookieSize and we reject anything bigger.
|
||||
const MaxCookieLen = 4096
|
||||
|
||||
// Cookie wire format. A single version byte at offset 0 lets us evolve the
|
||||
// format without flag-day-style cookie invalidation. New formats add a new
|
||||
// case in openCookie's switch.
|
||||
//
|
||||
// v0: cookieVersionV0 || nonce(12) || ciphertext
|
||||
//
|
||||
// Sealed under KeyRing.MasterCookieKey() — a long-lived AES-GCM key
|
||||
// derived from master_secret with no epoch dependency. AAD is the
|
||||
// User-Agent.
|
||||
//
|
||||
// Plaintext layout (inside the AEAD envelope):
|
||||
//
|
||||
// not_after_be8 (8 bytes BE, unix seconds)
|
||||
// flags_byte (1 byte; bit 0 = allowlisted, bits 1-7 reserved)
|
||||
// reason_len_be (2 bytes BE, length of reason_bytes)
|
||||
// reason_bytes (0..MaxAllowlistReasonLen UTF-8 bytes)
|
||||
// protobuf(envelope)
|
||||
//
|
||||
// All four header fields are inside the AEAD envelope, so any
|
||||
// tampering with them invalidates the GCM tag.
|
||||
//
|
||||
// Decoupling the cookie key from the epoch keyring is what lets us
|
||||
// have e.g. 12h cookies while keeping ticket-signing keys rotating
|
||||
// every few minutes — the two security windows are independent.
|
||||
const cookieVersionV0 byte = 0x00
|
||||
|
||||
// cookiePlaintextFixedHeaderLen is the size of the fixed-layout portion of
|
||||
// the plaintext header that precedes the protobuf envelope:
|
||||
//
|
||||
// not_after_be8 (8) + flags_byte (1) + reason_len_be (2) = 11
|
||||
//
|
||||
// followed by reason_bytes (variable, 0..MaxAllowlistReasonLen).
|
||||
const cookiePlaintextFixedHeaderLen = 8 + 1 + 2
|
||||
|
||||
// cookieFlagAllowlisted marks a cookie minted by GrantChallengeCookie
|
||||
// (operator allowlist bypass) rather than by a real challenge submission.
|
||||
const cookieFlagAllowlisted byte = 0x01
|
||||
|
||||
func deriveKey(secret []byte) ([]byte, error) {
|
||||
h := hkdf.New(sha256.New, secret, nil, []byte(hkdfInfo))
|
||||
key := make([]byte, 32) // AES-256
|
||||
|
||||
if _, err := h.Read(key); err != nil {
|
||||
return nil, fmt.Errorf("failed to derive key: %w", err)
|
||||
}
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// sealCookieV0 produces a v0 cookie sealed under the long-lived master
|
||||
// cookie key. notAfter is the unix-seconds expiration; flags carries the
|
||||
// allowlist bit set by GrantChallengeCookie (0 for normal cookies);
|
||||
// reason is the operator-supplied allowlist reason (empty for normal
|
||||
// cookies). All three are prepended to the marshaled proto BEFORE
|
||||
// encryption so they are both confidential (not observable from the wire)
|
||||
// and authenticated (any tamper attempt invalidates the GCM tag).
|
||||
//
|
||||
// Returns ErrAllowlistReasonSize if reason exceeds MaxAllowlistReasonLen.
|
||||
func sealCookieV0(envelope *pb.ChallengeCookie, masterCookieKey []byte, notAfter int64, flags byte, reason string, aad []byte, maxCookieLen int) (string, error) {
|
||||
if maxCookieLen <= 0 {
|
||||
maxCookieLen = MaxCookieLen
|
||||
}
|
||||
|
||||
if len(reason) > MaxAllowlistReasonLen {
|
||||
return "", fmt.Errorf("%w: %d > %d", ErrAllowlistReasonSize, len(reason), MaxAllowlistReasonLen)
|
||||
}
|
||||
|
||||
key, err := deriveKey(masterCookieKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create cipher: %w", err)
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create GCM: %w", err)
|
||||
}
|
||||
|
||||
// Reject an over-limit envelope before marshaling it.
|
||||
maxPlaintext := maxCookieLen/4*3 - 1 - gcm.NonceSize() - gcm.Overhead()
|
||||
if plaintextLen := cookiePlaintextFixedHeaderLen + len(reason) + proto.Size(envelope); plaintextLen > maxPlaintext {
|
||||
return "", fmt.Errorf("%w: plaintext=%d > %d", ErrCookieTooLarge, plaintextLen, maxPlaintext)
|
||||
}
|
||||
|
||||
envelopeBytes, err := proto.Marshal(envelope)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal challenge cookie proto: %w", err)
|
||||
}
|
||||
|
||||
nonce := make([]byte, gcm.NonceSize())
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return "", fmt.Errorf("failed to generate nonce: %w", err)
|
||||
}
|
||||
|
||||
// Build the plaintext: not_after_be8 || flags || reason_len_be || reason || envelope
|
||||
plaintext := make([]byte, 0, cookiePlaintextFixedHeaderLen+len(reason)+len(envelopeBytes))
|
||||
|
||||
var notAfterBytes [8]byte
|
||||
binary.BigEndian.PutUint64(notAfterBytes[:], uint64(notAfter))
|
||||
plaintext = append(plaintext, notAfterBytes[:]...)
|
||||
|
||||
plaintext = append(plaintext, flags)
|
||||
|
||||
var reasonLenBytes [2]byte
|
||||
binary.BigEndian.PutUint16(reasonLenBytes[:], uint16(len(reason)))
|
||||
plaintext = append(plaintext, reasonLenBytes[:]...)
|
||||
|
||||
plaintext = append(plaintext, []byte(reason)...)
|
||||
plaintext = append(plaintext, envelopeBytes...)
|
||||
|
||||
ciphertext := gcm.Seal(nonce, nonce, plaintext, aad)
|
||||
|
||||
// Wire form: version || nonce || ciphertext
|
||||
out := make([]byte, 0, 1+len(ciphertext))
|
||||
out = append(out, cookieVersionV0)
|
||||
out = append(out, ciphertext...)
|
||||
|
||||
return base64.RawURLEncoding.EncodeToString(out), nil
|
||||
}
|
||||
|
||||
// CookieEnvelope bundles the proto payload with the header fields that
|
||||
// openCookie pulls out of the AEAD-sealed plaintext: allowlist marker,
|
||||
// allowlist reason, expiration. Returned by openCookie so callers can
|
||||
// route allowlist cookies differently from real-submission ones without
|
||||
// re-parsing the plaintext.
|
||||
type CookieEnvelope struct {
|
||||
Envelope *pb.ChallengeCookie
|
||||
Allowlisted bool
|
||||
AllowlistReason string
|
||||
NotAfter int64
|
||||
}
|
||||
|
||||
// openCookie decodes a sealed cookie, dispatching on the version byte.
|
||||
// Unknown versions are rejected with ErrCookieVersion. Expired cookies
|
||||
// (notAfter <= now) are rejected with ErrCookieExpired.
|
||||
func openCookie(encoded string, masterCookieKey []byte, aad []byte, maxCookieLen int) (*CookieEnvelope, error) {
|
||||
if maxCookieLen <= 0 {
|
||||
maxCookieLen = MaxCookieLen
|
||||
}
|
||||
|
||||
if len(encoded) > maxCookieLen {
|
||||
return nil, fmt.Errorf("%w: %d > %d", ErrCookieTooLarge, len(encoded), maxCookieLen)
|
||||
}
|
||||
|
||||
raw, err := base64.RawURLEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to decode: %w", ErrCookieMalformed, err)
|
||||
}
|
||||
if len(raw) < 1 {
|
||||
return nil, fmt.Errorf("%w: empty cookie", ErrCookieMalformed)
|
||||
}
|
||||
|
||||
switch raw[0] {
|
||||
case cookieVersionV0:
|
||||
return openCookieV0Bytes(raw[1:], masterCookieKey, aad, time.Now())
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: 0x%02x", ErrCookieVersion, raw[0])
|
||||
}
|
||||
}
|
||||
|
||||
func openCookieV0Bytes(body []byte, masterCookieKey []byte, aad []byte, now time.Time) (*CookieEnvelope, error) {
|
||||
key, err := deriveKey(masterCookieKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create cipher: %w", err)
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create GCM: %w", err)
|
||||
}
|
||||
|
||||
nonceSize := gcm.NonceSize()
|
||||
if len(body) < nonceSize {
|
||||
return nil, fmt.Errorf("%w: ciphertext too short", ErrCookieMalformed)
|
||||
}
|
||||
|
||||
nonce, ciphertext := body[:nonceSize], body[nonceSize:]
|
||||
|
||||
plaintext, err := gcm.Open(nil, nonce, ciphertext, aad)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrCookieSignature, err)
|
||||
}
|
||||
|
||||
if len(plaintext) < cookiePlaintextFixedHeaderLen {
|
||||
return nil, fmt.Errorf("%w: plaintext shorter than fixed header", ErrCookieMalformed)
|
||||
}
|
||||
|
||||
notAfter := int64(binary.BigEndian.Uint64(plaintext[:8]))
|
||||
flags := plaintext[8]
|
||||
reasonLen := int(binary.BigEndian.Uint16(plaintext[9:11]))
|
||||
|
||||
if reasonLen > MaxAllowlistReasonLen {
|
||||
return nil, fmt.Errorf("%w: reason_len=%d", ErrCookieMalformed, reasonLen)
|
||||
}
|
||||
|
||||
if len(plaintext) < cookiePlaintextFixedHeaderLen+reasonLen {
|
||||
return nil, fmt.Errorf("%w: plaintext shorter than declared reason_len", ErrCookieMalformed)
|
||||
}
|
||||
|
||||
if notAfter <= now.Unix() {
|
||||
return nil, fmt.Errorf("%w: not_after=%d now=%d", ErrCookieExpired, notAfter, now.Unix())
|
||||
}
|
||||
|
||||
reasonStart := cookiePlaintextFixedHeaderLen
|
||||
reasonEnd := reasonStart + reasonLen
|
||||
reason := string(plaintext[reasonStart:reasonEnd])
|
||||
|
||||
envelope := &pb.ChallengeCookie{}
|
||||
if err := proto.Unmarshal(plaintext[reasonEnd:], envelope); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrCookiePayload, err)
|
||||
}
|
||||
|
||||
return &CookieEnvelope{
|
||||
Envelope: envelope,
|
||||
Allowlisted: flags&cookieFlagAllowlisted != 0,
|
||||
AllowlistReason: reason,
|
||||
NotAfter: notAfter,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
// Package challenge dynamic_module.go handles the **sensitive** per-epoch
|
||||
// sign-key module — the small JS that embeds the current epoch's HMAC
|
||||
// key (~30 lines, expanded to ~10-30KB after obfuscation). This is the
|
||||
// actual cryptographic material protected by the challenge runtime;
|
||||
// obfuscation here is doing real work, not cosmetic byte variance.
|
||||
//
|
||||
// The module is rebuilt whenever the keyring advances to a new epoch
|
||||
// (default cadence: 5 minutes via keyringDefaultRotation). To give
|
||||
// per-visitor variance in how the same key is embedded, the runtime
|
||||
// keeps a small pool of cryptoPoolSize variants per epoch — each
|
||||
// variant is an independent obfuscation of the same input. Default
|
||||
// pool size is 1, preserving the historical single-variant-per-epoch
|
||||
// behavior; operators that want per-visitor variance for the
|
||||
// sensitive code can raise WithCryptoObfuscationPoolSize.
|
||||
//
|
||||
// For the **non-sensitive** challenge code (the build-time-obfuscated
|
||||
// crypto/glue served inline) and the separately-served, unobfuscated
|
||||
// fpscanner bundle, see static_bundle.go.
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/metrics"
|
||||
)
|
||||
|
||||
//go:embed dynamic_module.js.tmpl
|
||||
var dynamicModuleTemplate string
|
||||
|
||||
// preWarmLeadTime returns how far before the next rotation boundary we want
|
||||
// to start obfuscating the upcoming epoch's dynamic module. Capped at 30s
|
||||
// (no point starting much earlier — the module is unused until the
|
||||
// boundary), floored at 1s so very small rotation intervals don't make the
|
||||
// ticker degenerate into a tight loop.
|
||||
func (c *ChallengeRuntime) preWarmLeadTime() time.Duration {
|
||||
return max(min(c.keys.rotationInterval/4, 30*time.Second), time.Second)
|
||||
}
|
||||
|
||||
// dynamicModulePreWarmer runs in the background and obfuscates the next
|
||||
// epoch's dynamic key module (all cryptoPoolSize variants) shortly before
|
||||
// the rotation boundary, so the first request after a rotation finds the
|
||||
// module already cached and pays no obfuscation latency on the
|
||||
// request-serving path.
|
||||
//
|
||||
// The ticker goroutine calls dynamicModuleForEpoch synchronously. The
|
||||
// singleflight inside that call coalesces a real request that arrives
|
||||
// slightly after the rotation, and time.Ticker drops ticks while the
|
||||
// receiver is busy — so successive triggers during a single obfuscation
|
||||
// are naturally absorbed.
|
||||
func (c *ChallengeRuntime) dynamicModulePreWarmer(ctx context.Context) {
|
||||
leadTime := c.preWarmLeadTime()
|
||||
tick := time.NewTicker(leadTime)
|
||||
defer tick.Stop()
|
||||
|
||||
intervalSecs := int64(c.keys.rotationInterval / time.Second)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
now := c.keys.now()
|
||||
current := c.keys.CurrentEpoch()
|
||||
nextBoundary := time.Unix((current+1)*intervalSecs, 0)
|
||||
|
||||
// Skip ticks that are not yet inside the lead-time window
|
||||
// before the next boundary — pre-warming a far-future epoch
|
||||
// would waste CPU on a module that might never be served.
|
||||
if nextBoundary.Sub(now) > leadTime {
|
||||
continue
|
||||
}
|
||||
|
||||
nextEpoch := current + 1
|
||||
c.log().WithFields(log.Fields{
|
||||
"epoch": nextEpoch,
|
||||
"current_epoch": current,
|
||||
"seconds_to_boundary": int64(nextBoundary.Sub(now).Seconds()),
|
||||
"pool_size": c.cryptoPoolSize,
|
||||
}).Debug("pre-warming dynamic key module for upcoming epoch")
|
||||
|
||||
if _, err := c.dynamicModuleForEpoch(ctx, nextEpoch); err != nil {
|
||||
c.log().WithError(err).WithField("epoch", nextEpoch).
|
||||
Warn("pre-warm of next epoch failed; first request after rotation will pay obfuscation cost")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dynamicModuleForEpoch returns the cryptoPoolSize obfuscated variants of the
|
||||
// per-epoch key module, deriving the key from the keyring on demand. Results
|
||||
// are cached; concurrent calls for the same epoch coalesce via singleflight so
|
||||
// only one batch obfuscation runs under a thundering-herd at a rotation
|
||||
// boundary. The cache mutex guards only the map, never the obfuscation.
|
||||
func (c *ChallengeRuntime) dynamicModuleForEpoch(ctx context.Context, epoch int64) ([]string, error) {
|
||||
// Fast path: cached.
|
||||
c.dynamicModuleCacheMu.RLock()
|
||||
if cached, ok := c.dynamicModuleCache[epoch]; ok {
|
||||
c.dynamicModuleCacheMu.RUnlock()
|
||||
return cached, nil
|
||||
}
|
||||
c.dynamicModuleCacheMu.RUnlock()
|
||||
|
||||
// Slow path: deduplicate concurrent obfuscation calls for the same
|
||||
// epoch. Only one goroutine runs the obfuscator batch; the others
|
||||
// block on the singleflight result. The key is the epoch itself,
|
||||
// formatted as a string.
|
||||
key := strconv.FormatInt(epoch, 10)
|
||||
v, err, _ := c.dynamicModuleSF.Do(key, func() (interface{}, error) {
|
||||
// Re-check the cache inside the singleflight callback in case
|
||||
// another goroutine populated it between the fast-path read and
|
||||
// our entry into Do.
|
||||
c.dynamicModuleCacheMu.RLock()
|
||||
if cached, ok := c.dynamicModuleCache[epoch]; ok {
|
||||
c.dynamicModuleCacheMu.RUnlock()
|
||||
return cached, nil
|
||||
}
|
||||
c.dynamicModuleCacheMu.RUnlock()
|
||||
|
||||
signKey, ok := c.keys.SignKey(epoch)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("epoch %d is outside the keyring live window", epoch)
|
||||
}
|
||||
|
||||
tmpl, err := template.New("dynamic-module").Parse(dynamicModuleTemplate)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse dynamic module template: %w", err)
|
||||
}
|
||||
var rendered strings.Builder
|
||||
if err := tmpl.Execute(&rendered, map[string]interface{}{
|
||||
"Key": hex.EncodeToString(signKey),
|
||||
"Epoch": epoch,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("render dynamic module template: %w", err)
|
||||
}
|
||||
input := rendered.String()
|
||||
inputSize := len(input)
|
||||
|
||||
poolSize := max(c.cryptoPoolSize, 1)
|
||||
|
||||
variants := make([]string, 0, poolSize)
|
||||
batchStart := time.Now()
|
||||
for i := range poolSize {
|
||||
obfuscateStart := time.Now()
|
||||
obfuscated, err := c.ObfuscateJS(ctx, input)
|
||||
obfuscateDuration := time.Since(obfuscateStart)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("obfuscate dynamic module variant %d/%d: %w", i+1, poolSize, err)
|
||||
}
|
||||
if obfuscated == "" {
|
||||
return nil, fmt.Errorf("obfuscator produced empty dynamic module output (variant %d/%d)", i+1, poolSize)
|
||||
}
|
||||
|
||||
metrics.AppsecChallengeReobfuscation.WithLabelValues("dynamic").Inc()
|
||||
|
||||
c.log().WithFields(log.Fields{
|
||||
"epoch": epoch,
|
||||
"variant": i,
|
||||
"input_bytes": inputSize,
|
||||
"output_bytes": len(obfuscated),
|
||||
"duration_ms": obfuscateDuration.Milliseconds(),
|
||||
}).Debug("obfuscated dynamic key module variant")
|
||||
|
||||
variants = append(variants, obfuscated)
|
||||
}
|
||||
|
||||
c.log().WithFields(log.Fields{
|
||||
"epoch": epoch,
|
||||
"variants": poolSize,
|
||||
"batch_ms": time.Since(batchStart).Milliseconds(),
|
||||
"input_bytes": inputSize,
|
||||
}).Debug("obfuscated dynamic key module batch for new epoch")
|
||||
|
||||
c.dynamicModuleCacheMu.Lock()
|
||||
// Prune any cached modules whose epoch has fallen out of the
|
||||
// live window before inserting the new one.
|
||||
live := make(map[int64]bool, len(c.dynamicModuleCache))
|
||||
for _, e := range c.keys.LiveEpochs() {
|
||||
live[e] = true
|
||||
}
|
||||
evicted := 0
|
||||
for e := range c.dynamicModuleCache {
|
||||
if !live[e] {
|
||||
delete(c.dynamicModuleCache, e)
|
||||
evicted++
|
||||
}
|
||||
}
|
||||
c.dynamicModuleCache[epoch] = variants
|
||||
c.dynamicModuleCacheMu.Unlock()
|
||||
|
||||
if evicted > 0 {
|
||||
metrics.AppsecChallengeDynamicModuleEvicted.Add(float64(evicted))
|
||||
}
|
||||
|
||||
return variants, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return v.([]string), nil
|
||||
}
|
||||
|
||||
// currentDynamicModule returns one of the obfuscated dynamic-module
|
||||
// variants for the keyring's current epoch, selected uniformly at
|
||||
// random. When the pool size is 1 (default), every render returns the
|
||||
// same variant; when raised via WithCryptoObfuscationPoolSize, each
|
||||
// render picks one of the N variants of the same epoch key.
|
||||
func (c *ChallengeRuntime) currentDynamicModule(ctx context.Context) (string, error) {
|
||||
epoch, _ := c.keys.Current()
|
||||
variants, err := c.dynamicModuleForEpoch(ctx, epoch)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(variants) == 0 {
|
||||
return "", fmt.Errorf("dynamic module pool is empty for epoch %d", epoch)
|
||||
}
|
||||
if len(variants) == 1 {
|
||||
return variants[0], nil
|
||||
}
|
||||
return variants[rand.IntN(len(variants))], nil
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// dynamic_module.js.tmpl — the per-epoch module, obfuscated at runtime on each
|
||||
// keyring rotation. Small by design (the heavy fpscanner/crypto lives in the
|
||||
// pre-obfuscated static bundle). {{.Key}} = hex per-epoch key, {{.Epoch}} =
|
||||
// epoch id. The hook name must match CSEC_HOOK_NAME in challenge.js and is kept
|
||||
// literal via reservedStrings in obfuscate.js; the key string is deliberately
|
||||
// left for the obfuscator's string-array transform to encode.
|
||||
(function () {
|
||||
var hookName = "__CSEC_CHALLENGE_HOOK_v1__";
|
||||
var key = "{{.Key}}";
|
||||
var epoch = {{.Epoch}};
|
||||
|
||||
// The static bundle registers the hook synchronously at module load. If
|
||||
// for some reason it is not present yet, retry on a microtask.
|
||||
function invoke() {
|
||||
var hook = globalThis[hookName];
|
||||
if (typeof hook === "function") {
|
||||
hook({ key: key, epoch: epoch });
|
||||
return;
|
||||
}
|
||||
setTimeout(invoke, 10);
|
||||
}
|
||||
invoke();
|
||||
})();
|
||||
@@ -0,0 +1,721 @@
|
||||
// fingerprint.go holds the wire-shape structs the JS fingerprint bundle
|
||||
// returns: FlexInt / FlexBool tolerant primitives and the nested
|
||||
// FingerprintData / fingerprintBotAlias structures that mirror the JSON
|
||||
// payload one-to-one. Handwritten accessors live in fingerprint_helpers.go,
|
||||
// proto ↔ struct conversion in fingerprint_proto.go, and mismatch detection
|
||||
// methods in fingerprint_mismatch.go.
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// FlexInt handles JSON values that may be either a number or a string.
|
||||
// The fingerprint JS library returns a string (error message) instead of
|
||||
// a number when it fails to collect a value.
|
||||
type FlexInt int
|
||||
|
||||
// FlexBool handles JSON values that may be either a boolean or a string.
|
||||
type FlexBool bool
|
||||
|
||||
func (fb *FlexBool) UnmarshalJSON(data []byte) error {
|
||||
var b bool
|
||||
if err := json.Unmarshal(data, &b); err == nil {
|
||||
*fb = FlexBool(b)
|
||||
return nil
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*fb = FlexBool(s == "true" || s == "1")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fb FlexBool) Bool() bool {
|
||||
return bool(fb)
|
||||
}
|
||||
|
||||
func (fi *FlexInt) UnmarshalJSON(data []byte) error {
|
||||
// Try number first (most common case).
|
||||
var n int
|
||||
if err := json.Unmarshal(data, &n); err == nil {
|
||||
*fi = FlexInt(n)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fall back to string → parse as int, or default to 0.
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
parsed, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
*fi = 0
|
||||
return nil //nolint:nilerr // tolerant by design: non-numeric strings (JS-side error messages) default to 0
|
||||
}
|
||||
|
||||
*fi = FlexInt(parsed)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fi FlexInt) Int() int {
|
||||
return int(fi)
|
||||
}
|
||||
|
||||
/*
|
||||
{
|
||||
"signals": {
|
||||
"automation": {
|
||||
"webdriver": false,
|
||||
"webdriverWritable": false,
|
||||
"selenium": false,
|
||||
"cdp": true,
|
||||
"playwright": false,
|
||||
"navigatorPropertyDescriptors": "00000"
|
||||
},
|
||||
"device": {
|
||||
"cpuCount": 14,
|
||||
"memory": 8,
|
||||
"platform": "MacIntel",
|
||||
"screenResolution": {
|
||||
"width": 2304,
|
||||
"height": 1296,
|
||||
"pixelDepth": 30,
|
||||
"colorDepth": 30,
|
||||
"availableWidth": 2304,
|
||||
"availableHeight": 1265,
|
||||
"innerWidth": 1444,
|
||||
"innerHeight": 1144,
|
||||
"hasMultipleDisplays": true
|
||||
},
|
||||
"multimediaDevices": {
|
||||
"speakers": 1,
|
||||
"microphones": 1,
|
||||
"webcams": 1
|
||||
},
|
||||
"mediaQueries": {
|
||||
"prefersColorScheme": "dark",
|
||||
"prefersReducedMotion": false,
|
||||
"prefersReducedTransparency": false,
|
||||
"colorGamut": "p3",
|
||||
"pointer": "fine",
|
||||
"anyPointer": "fine",
|
||||
"hover": true,
|
||||
"anyHover": true,
|
||||
"colorDepth": 10
|
||||
}
|
||||
},
|
||||
"browser": {
|
||||
"userAgent": "Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_15_7)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/144.0.0.0+Safari/537.36",
|
||||
"features": {
|
||||
"bitmask": "1000111100",
|
||||
"chrome": true,
|
||||
"brave": false,
|
||||
"applePaySupport": false,
|
||||
"opera": false,
|
||||
"serial": true,
|
||||
"attachShadow": true,
|
||||
"caches": true,
|
||||
"webAssembly": true,
|
||||
"buffer": false,
|
||||
"showModalDialog": false
|
||||
},
|
||||
"plugins": {
|
||||
"isValidPluginArray": true,
|
||||
"pluginCount": 5,
|
||||
"pluginNamesHash": "-2cdc5c8b",
|
||||
"pluginConsistency1": true,
|
||||
"pluginOverflow": false
|
||||
},
|
||||
"extensions": {
|
||||
"bitmask": "00000000",
|
||||
"extensions": []
|
||||
},
|
||||
"highEntropyValues": {
|
||||
"architecture": "arm",
|
||||
"bitness": "64",
|
||||
"brands": [
|
||||
{
|
||||
"brand": "Not(A:Brand",
|
||||
"version": "8"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "144"
|
||||
},
|
||||
{
|
||||
"brand": "Google+Chrome",
|
||||
"version": "144"
|
||||
}
|
||||
],
|
||||
"mobile": false,
|
||||
"model": "",
|
||||
"platform": "macOS",
|
||||
"platformVersion": "26.2.0",
|
||||
"uaFullVersion": "144.0.7559.133"
|
||||
},
|
||||
"etsl": 33,
|
||||
"maths": "67d0b556",
|
||||
"toSourceError": {
|
||||
"toSourceError": "TypeError:+Cannot+read+properties+of+null+(reading+usdfsh)",
|
||||
"hasToSource": false
|
||||
}
|
||||
},
|
||||
"graphics": {
|
||||
"webGL": {
|
||||
"vendor": "Google+Inc.+(Apple)",
|
||||
"renderer": "ANGLE+(Apple,+ANGLE+Metal+Renderer:+Apple+M4+Max,+Unspecified+Version)"
|
||||
},
|
||||
"webgpu": {
|
||||
"vendor": "apple",
|
||||
"architecture": "metal-3",
|
||||
"device": "",
|
||||
"description": ""
|
||||
},
|
||||
"canvas": {
|
||||
"hasModifiedCanvas": false,
|
||||
"canvasFingerprint": "-421c84e0"
|
||||
}
|
||||
},
|
||||
"codecs": {
|
||||
"audioCanPlayTypeHash": "688c7345",
|
||||
"videoCanPlayTypeHash": "-126cde82",
|
||||
"audioMediaSourceHash": "-3cbc04a4",
|
||||
"videoMediaSourceHash": "-48c15d34",
|
||||
"rtcAudioCapabilitiesHash": "26a15cc5",
|
||||
"rtcVideoCapabilitiesHash": "4f24a817",
|
||||
"hasMediaSource": true
|
||||
},
|
||||
"locale": {
|
||||
"internationalization": {
|
||||
"timezone": "Europe/Paris",
|
||||
"localeLanguage": "en-US"
|
||||
},
|
||||
"languages": {
|
||||
"languages": [
|
||||
"en",
|
||||
"en-US",
|
||||
"fr-FR",
|
||||
"fr"
|
||||
],
|
||||
"language": "en"
|
||||
}
|
||||
},
|
||||
"contexts": {
|
||||
"iframe": {
|
||||
"webdriver": false,
|
||||
"userAgent": "Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_15_7)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/144.0.0.0+Safari/537.36",
|
||||
"platform": "MacIntel",
|
||||
"memory": 8,
|
||||
"cpuCount": 14,
|
||||
"language": "en"
|
||||
},
|
||||
"webWorker": {
|
||||
"vendor": "Google+Inc.+(Apple)",
|
||||
"renderer": "ANGLE+(Apple,+ANGLE+Metal+Renderer:+Apple+M4+Max,+Unspecified+Version)",
|
||||
"userAgent": "Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_15_7)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/144.0.0.0+Safari/537.36",
|
||||
"language": "en",
|
||||
"platform": "MacIntel",
|
||||
"memory": 8,
|
||||
"cpuCount": 14
|
||||
}
|
||||
}
|
||||
},
|
||||
"fsid": "FS1_00001000000000_00010h02ba_2304x1296c14m08b10011h-5db59_f1000111100e00000000p1100h-7ce12_0h06653c_1h-36e6b_en4h720e_0100h-156f8",
|
||||
"nonce": "jb8t2jfm839",
|
||||
"time": 1770669806462,
|
||||
"url": "http://localhost/",
|
||||
"fastBotDetection": true,
|
||||
"fastBotDetectionDetails": {
|
||||
"headlessChromeScreenResolution": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasWebdriver": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasWebdriverWritable": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasSeleniumProperty": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasCDP": {
|
||||
"detected": true,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasPlaywright": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasImpossibleDeviceMemory": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasHighCPUCount": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasMissingChromeObject": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasWebdriverIframe": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasWebdriverWorker": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasMismatchWebGLInWorker": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasMismatchPlatformIframe": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasMismatchPlatformWorker": {
|
||||
"detected": false,
|
||||
"severity": "high"
|
||||
},
|
||||
"hasSwiftshaderRenderer": {
|
||||
"detected": false,
|
||||
"severity": "low"
|
||||
},
|
||||
"hasUTCTimezone": {
|
||||
"detected": false,
|
||||
"severity": "medium"
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
// FingerprintData is the deserialized payload produced by the JS
|
||||
// fingerprint bundle and (after a successful challenge) carried inside the
|
||||
// sealed challenge cookie. It is the value exposed to rule authors via the
|
||||
// `fingerprint` variable in expr environments. The struct mirrors the JSON
|
||||
// wire shape one-to-one; FlexInt/FlexBool primitives tolerate the bundle's
|
||||
// occasional "error string instead of value" outputs without aborting the
|
||||
// whole submission.
|
||||
type FingerprintData struct {
|
||||
Signals fingerprintSignals `json:"signals"`
|
||||
FSID string `json:"fsid"`
|
||||
Nonce string `json:"nonce"`
|
||||
Time int64 `json:"time"`
|
||||
URL string `json:"url"`
|
||||
FastBotDetection FlexBool `json:"fastBotDetection"`
|
||||
FastBotDetectionDetails fingerprintFastBotDetectionDetails `json:"fastBotDetectionDetails"`
|
||||
Bot fingerprintBotAlias `json:"-"`
|
||||
|
||||
// Allowlisted is true on cookies minted by GrantChallengeCookie (operator
|
||||
// bypass for trusted bots like Googlebot) — these cookies never went
|
||||
// through a real challenge submission and carry no measured signals.
|
||||
// AllowlistReason is the operator-supplied free-form string identifying
|
||||
// why the bypass was granted, exposed to on_challenge expressions so
|
||||
// per-route policy can distinguish bypass categories.
|
||||
//
|
||||
// Both fields live in the cookie's AEAD plaintext header (see
|
||||
// crypto.go), NOT in the protobuf envelope. They are populated by
|
||||
// ValidCookie / GrantChallengeCookie and zero for normal cookies.
|
||||
Allowlisted bool `json:"-"`
|
||||
AllowlistReason string `json:"-"`
|
||||
}
|
||||
|
||||
type fingerprintSignals struct {
|
||||
Automation fingerprintAutomation `json:"automation"`
|
||||
Device fingerprintDevice `json:"device"`
|
||||
Browser fingerprintBrowser `json:"browser"`
|
||||
Graphics fingerprintGraphics `json:"graphics"`
|
||||
Codecs fingerprintCodecs `json:"codecs"`
|
||||
Locale fingerprintLocale `json:"locale"`
|
||||
Contexts fingerprintContexts `json:"contexts"`
|
||||
}
|
||||
|
||||
type fingerprintAutomation struct {
|
||||
Webdriver FlexBool `json:"webdriver"`
|
||||
WebdriverWritable FlexBool `json:"webdriverWritable"`
|
||||
Selenium FlexBool `json:"selenium"`
|
||||
CDP FlexBool `json:"cdp"`
|
||||
Playwright FlexBool `json:"playwright"`
|
||||
NavigatorPropertyDescriptors string `json:"navigatorPropertyDescriptors"`
|
||||
}
|
||||
|
||||
type fingerprintDevice struct {
|
||||
CPUCount FlexInt `json:"cpuCount"`
|
||||
Memory FlexInt `json:"memory"`
|
||||
Platform string `json:"platform"`
|
||||
ScreenResolution fingerprintScreenResolution `json:"screenResolution"`
|
||||
MultimediaDevices fingerprintMultimediaDevices `json:"multimediaDevices"`
|
||||
MediaQueries fingerprintDeviceMediaQueries `json:"mediaQueries"`
|
||||
}
|
||||
|
||||
type fingerprintScreenResolution struct {
|
||||
Width FlexInt `json:"width"`
|
||||
Height FlexInt `json:"height"`
|
||||
PixelDepth FlexInt `json:"pixelDepth"`
|
||||
ColorDepth FlexInt `json:"colorDepth"`
|
||||
AvailableWidth FlexInt `json:"availableWidth"`
|
||||
AvailableHeight FlexInt `json:"availableHeight"`
|
||||
InnerWidth FlexInt `json:"innerWidth"`
|
||||
InnerHeight FlexInt `json:"innerHeight"`
|
||||
HasMultipleDisplays FlexBool `json:"hasMultipleDisplays"`
|
||||
}
|
||||
|
||||
type fingerprintMultimediaDevices struct {
|
||||
Speakers FlexInt `json:"speakers"`
|
||||
Microphones FlexInt `json:"microphones"`
|
||||
Webcams FlexInt `json:"webcams"`
|
||||
}
|
||||
|
||||
type fingerprintDeviceMediaQueries struct {
|
||||
PrefersColorScheme string `json:"prefersColorScheme"`
|
||||
PrefersReducedMotion FlexBool `json:"prefersReducedMotion"`
|
||||
PrefersReducedTransparency FlexBool `json:"prefersReducedTransparency"`
|
||||
ColorGamut string `json:"colorGamut"`
|
||||
Pointer string `json:"pointer"`
|
||||
AnyPointer string `json:"anyPointer"`
|
||||
Hover FlexBool `json:"hover"`
|
||||
AnyHover FlexBool `json:"anyHover"`
|
||||
ColorDepth FlexInt `json:"colorDepth"`
|
||||
}
|
||||
|
||||
type fingerprintBrowser struct {
|
||||
UserAgent string `json:"userAgent"`
|
||||
Features fingerprintBrowserFeatures `json:"features"`
|
||||
Plugins fingerprintBrowserPlugins `json:"plugins"`
|
||||
Extensions fingerprintBrowserExtensions `json:"extensions"`
|
||||
HighEntropyValues fingerprintBrowserHighEntropyValues `json:"highEntropyValues"`
|
||||
ETSL FlexInt `json:"etsl"`
|
||||
Maths string `json:"maths"`
|
||||
ToSourceError fingerprintBrowserToSourceError `json:"toSourceError"`
|
||||
}
|
||||
|
||||
type fingerprintBrowserFeatures struct {
|
||||
Bitmask string `json:"bitmask"`
|
||||
Chrome FlexBool `json:"chrome"`
|
||||
Brave FlexBool `json:"brave"`
|
||||
ApplePaySupport FlexBool `json:"applePaySupport"`
|
||||
Opera FlexBool `json:"opera"`
|
||||
Serial FlexBool `json:"serial"`
|
||||
AttachShadow FlexBool `json:"attachShadow"`
|
||||
Caches FlexBool `json:"caches"`
|
||||
WebAssembly FlexBool `json:"webAssembly"`
|
||||
Buffer FlexBool `json:"buffer"`
|
||||
ShowModalDialog FlexBool `json:"showModalDialog"`
|
||||
Safari FlexBool `json:"safari"`
|
||||
WebkitPrefixedFunction FlexBool `json:"webkitPrefixedFunction"`
|
||||
MozPrefixedFunction FlexBool `json:"mozPrefixedFunction"`
|
||||
USB FlexBool `json:"usb"`
|
||||
BrowserCapture FlexBool `json:"browserCapture"`
|
||||
PaymentRequestUpdateEvent FlexBool `json:"paymentRequestUpdateEvent"`
|
||||
PressureObserver FlexBool `json:"pressureObserver"`
|
||||
AudioSession FlexBool `json:"audioSession"`
|
||||
SelectAudioOutput FlexBool `json:"selectAudioOutput"`
|
||||
BarcodeDetector FlexBool `json:"barcodeDetector"`
|
||||
Battery FlexBool `json:"battery"`
|
||||
DevicePosture FlexBool `json:"devicePosture"`
|
||||
DocumentPictureInPicture FlexBool `json:"documentPictureInPicture"`
|
||||
EyeDropper FlexBool `json:"eyeDropper"`
|
||||
EditContext FlexBool `json:"editContext"`
|
||||
FencedFrame FlexBool `json:"fencedFrame"`
|
||||
Sanitizer FlexBool `json:"sanitizer"`
|
||||
OTPCredential FlexBool `json:"otpCredential"`
|
||||
}
|
||||
|
||||
type fingerprintBrowserPlugins struct {
|
||||
IsValidPluginArray FlexBool `json:"isValidPluginArray"`
|
||||
PluginCount FlexInt `json:"pluginCount"`
|
||||
PluginNamesHash string `json:"pluginNamesHash"`
|
||||
PluginConsistency1 FlexBool `json:"pluginConsistency1"`
|
||||
PluginOverflow FlexBool `json:"pluginOverflow"`
|
||||
}
|
||||
|
||||
type fingerprintBrowserExtensions struct {
|
||||
Bitmask string `json:"bitmask"`
|
||||
Extensions []string `json:"extensions"`
|
||||
}
|
||||
|
||||
type fingerprintBrowserHighEntropyValues struct {
|
||||
Architecture string `json:"architecture"`
|
||||
Bitness string `json:"bitness"`
|
||||
Brands FlexBrandVersions `json:"brands"`
|
||||
Mobile FlexBool `json:"mobile"`
|
||||
Model string `json:"model"`
|
||||
Platform string `json:"platform"`
|
||||
PlatformVersion string `json:"platformVersion"`
|
||||
UAFullVersion string `json:"uaFullVersion"`
|
||||
}
|
||||
|
||||
type fingerprintBrandVersion struct {
|
||||
Brand string `json:"brand"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// FlexBrandVersions handles a JSON value that should be []fingerprintBrandVersion
|
||||
// but may be a string (error) from the fingerprint library.
|
||||
type FlexBrandVersions []fingerprintBrandVersion
|
||||
|
||||
func (f *FlexBrandVersions) UnmarshalJSON(data []byte) error {
|
||||
var arr []fingerprintBrandVersion
|
||||
if err := json.Unmarshal(data, &arr); err == nil {
|
||||
*f = arr
|
||||
return nil
|
||||
}
|
||||
|
||||
// Probably a string error — ignore it, leave empty.
|
||||
*f = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type fingerprintBrowserToSourceError struct {
|
||||
ToSourceError string `json:"toSourceError"`
|
||||
HasToSource FlexBool `json:"hasToSource"`
|
||||
}
|
||||
|
||||
type fingerprintGraphics struct {
|
||||
WebGL fingerprintGraphicsWebGL `json:"webGL"`
|
||||
WebGPU fingerprintGraphicsWebGPU `json:"webgpu"`
|
||||
Canvas fingerprintGraphicsCanvas `json:"canvas"`
|
||||
}
|
||||
|
||||
type fingerprintGraphicsWebGL struct {
|
||||
Vendor string `json:"vendor"`
|
||||
Renderer string `json:"renderer"`
|
||||
}
|
||||
|
||||
type fingerprintGraphicsWebGPU struct {
|
||||
Vendor string `json:"vendor"`
|
||||
Architecture string `json:"architecture"`
|
||||
Device string `json:"device"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
type fingerprintGraphicsCanvas struct {
|
||||
HasModifiedCanvas FlexBool `json:"hasModifiedCanvas"`
|
||||
CanvasFingerprint string `json:"canvasFingerprint"`
|
||||
}
|
||||
|
||||
type fingerprintCodecs struct {
|
||||
AudioCanPlayTypeHash string `json:"audioCanPlayTypeHash"`
|
||||
VideoCanPlayTypeHash string `json:"videoCanPlayTypeHash"`
|
||||
AudioMediaSourceHash string `json:"audioMediaSourceHash"`
|
||||
VideoMediaSourceHash string `json:"videoMediaSourceHash"`
|
||||
RTCAudioCapabilitiesHash string `json:"rtcAudioCapabilitiesHash"`
|
||||
RTCVideoCapabilitiesHash string `json:"rtcVideoCapabilitiesHash"`
|
||||
HasMediaSource FlexBool `json:"hasMediaSource"`
|
||||
}
|
||||
|
||||
type fingerprintLocale struct {
|
||||
Internationalization fingerprintLocaleInternationalization `json:"internationalization"`
|
||||
Languages fingerprintLocaleLanguages `json:"languages"`
|
||||
}
|
||||
|
||||
type fingerprintLocaleInternationalization struct {
|
||||
Timezone string `json:"timezone"`
|
||||
LocaleLanguage string `json:"localeLanguage"`
|
||||
}
|
||||
|
||||
type fingerprintLocaleLanguages struct {
|
||||
Languages []string `json:"languages"`
|
||||
Language string `json:"language"`
|
||||
}
|
||||
|
||||
type fingerprintContexts struct {
|
||||
Iframe fingerprintIframeContext `json:"iframe"`
|
||||
WebWorker fingerprintWebWorkerContext `json:"webWorker"`
|
||||
}
|
||||
|
||||
type fingerprintIframeContext struct {
|
||||
Webdriver FlexBool `json:"webdriver"`
|
||||
UserAgent string `json:"userAgent"`
|
||||
Platform string `json:"platform"`
|
||||
Memory FlexInt `json:"memory"`
|
||||
CPUCount FlexInt `json:"cpuCount"`
|
||||
Language string `json:"language"`
|
||||
}
|
||||
|
||||
type fingerprintWebWorkerContext struct {
|
||||
Vendor string `json:"vendor"`
|
||||
Renderer string `json:"renderer"`
|
||||
UserAgent string `json:"userAgent"`
|
||||
Language string `json:"language"`
|
||||
Platform string `json:"platform"`
|
||||
Memory FlexInt `json:"memory"`
|
||||
CPUCount FlexInt `json:"cpuCount"`
|
||||
}
|
||||
|
||||
type fingerprintFastBotDetectionDetails struct {
|
||||
HeadlessChromeScreenResolution fingerprintDetectionResult `json:"headlessChromeScreenResolution"`
|
||||
HasWebdriver fingerprintDetectionResult `json:"hasWebdriver"`
|
||||
HasWebdriverWritable fingerprintDetectionResult `json:"hasWebdriverWritable"`
|
||||
HasSeleniumProperty fingerprintDetectionResult `json:"hasSeleniumProperty"`
|
||||
HasCDP fingerprintDetectionResult `json:"hasCDP"`
|
||||
HasPlaywright fingerprintDetectionResult `json:"hasPlaywright"`
|
||||
HasImpossibleDeviceMemory fingerprintDetectionResult `json:"hasImpossibleDeviceMemory"`
|
||||
HasHighCPUCount fingerprintDetectionResult `json:"hasHighCPUCount"`
|
||||
HasMissingChromeObject fingerprintDetectionResult `json:"hasMissingChromeObject"`
|
||||
HasWebdriverIframe fingerprintDetectionResult `json:"hasWebdriverIframe"`
|
||||
HasWebdriverWorker fingerprintDetectionResult `json:"hasWebdriverWorker"`
|
||||
HasMismatchWebGLInWorker fingerprintDetectionResult `json:"hasMismatchWebGLInWorker"`
|
||||
HasMismatchPlatformIframe fingerprintDetectionResult `json:"hasMismatchPlatformIframe"`
|
||||
HasMismatchPlatformWorker fingerprintDetectionResult `json:"hasMismatchPlatformWorker"`
|
||||
HasSwiftshaderRenderer fingerprintDetectionResult `json:"hasSwiftshaderRenderer"`
|
||||
HasUTCTimezone fingerprintDetectionResult `json:"hasUTCTimezone"`
|
||||
HasMismatchLanguages fingerprintDetectionResult `json:"hasMismatchLanguages"`
|
||||
HasInconsistentEtsl fingerprintDetectionResult `json:"hasInconsistentEtsl"`
|
||||
HasBotUserAgent fingerprintDetectionResult `json:"hasBotUserAgent"`
|
||||
HasGPUMismatch fingerprintDetectionResult `json:"hasGPUMismatch"`
|
||||
HasPlatformMismatch fingerprintDetectionResult `json:"hasPlatformMismatch"`
|
||||
}
|
||||
|
||||
type fingerprintDetectionResult struct {
|
||||
Detected FlexBool `json:"detected"`
|
||||
}
|
||||
|
||||
type fingerprintBotAlias struct {
|
||||
HeadlessChromeScreenResolution bool `json:"headlessChromeScreenResolution"`
|
||||
Webdriver bool `json:"webdriver"`
|
||||
WebdriverWritable bool `json:"webdriverWritable"`
|
||||
Selenium bool `json:"selenium"`
|
||||
CDP bool `json:"cdp"`
|
||||
Playwright bool `json:"playwright"`
|
||||
ImpossibleDeviceMemory bool `json:"impossibleDeviceMemory"`
|
||||
HighCPUCount bool `json:"highCPUCount"`
|
||||
MissingChromeObject bool `json:"missingChromeObject"`
|
||||
WebdriverIframe bool `json:"webdriverIframe"`
|
||||
WebdriverWorker bool `json:"webdriverWorker"`
|
||||
MismatchWebGLInWorker bool `json:"mismatchWebGLInWorker"`
|
||||
MismatchPlatformIframe bool `json:"mismatchPlatformIframe"`
|
||||
MismatchPlatformWorker bool `json:"mismatchPlatformWorker"`
|
||||
SwiftshaderRenderer bool `json:"swiftshaderRenderer"`
|
||||
UTCTimezone bool `json:"utcTimezone"`
|
||||
MismatchLanguages bool `json:"mismatchLanguages"`
|
||||
InconsistentEtsl bool `json:"inconsistentEtsl"`
|
||||
BotUserAgent bool `json:"botUserAgent"`
|
||||
GPUMismatch bool `json:"gpuMismatch"`
|
||||
PlatformMismatch bool `json:"platformMismatch"`
|
||||
AnyDetected bool `json:"anyDetected"`
|
||||
DetectedCount FlexInt `json:"detectedCount"`
|
||||
}
|
||||
|
||||
func (f *FingerprintData) UnmarshalJSON(data []byte) error {
|
||||
type rawFingerprintData FingerprintData
|
||||
|
||||
var raw rawFingerprintData
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*f = FingerprintData(raw)
|
||||
f.Bot = newFingerprintBotAlias(f.FastBotDetectionDetails)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newFingerprintBotAlias(details fingerprintFastBotDetectionDetails) fingerprintBotAlias {
|
||||
alias := fingerprintBotAlias{
|
||||
HeadlessChromeScreenResolution: bool(details.HeadlessChromeScreenResolution.Detected),
|
||||
Webdriver: bool(details.HasWebdriver.Detected),
|
||||
WebdriverWritable: bool(details.HasWebdriverWritable.Detected),
|
||||
Selenium: bool(details.HasSeleniumProperty.Detected),
|
||||
CDP: bool(details.HasCDP.Detected),
|
||||
Playwright: bool(details.HasPlaywright.Detected),
|
||||
ImpossibleDeviceMemory: bool(details.HasImpossibleDeviceMemory.Detected),
|
||||
HighCPUCount: bool(details.HasHighCPUCount.Detected),
|
||||
MissingChromeObject: bool(details.HasMissingChromeObject.Detected),
|
||||
WebdriverIframe: bool(details.HasWebdriverIframe.Detected),
|
||||
WebdriverWorker: bool(details.HasWebdriverWorker.Detected),
|
||||
MismatchWebGLInWorker: bool(details.HasMismatchWebGLInWorker.Detected),
|
||||
MismatchPlatformIframe: bool(details.HasMismatchPlatformIframe.Detected),
|
||||
MismatchPlatformWorker: bool(details.HasMismatchPlatformWorker.Detected),
|
||||
SwiftshaderRenderer: bool(details.HasSwiftshaderRenderer.Detected),
|
||||
UTCTimezone: bool(details.HasUTCTimezone.Detected),
|
||||
MismatchLanguages: bool(details.HasMismatchLanguages.Detected),
|
||||
InconsistentEtsl: bool(details.HasInconsistentEtsl.Detected),
|
||||
BotUserAgent: bool(details.HasBotUserAgent.Detected),
|
||||
GPUMismatch: bool(details.HasGPUMismatch.Detected),
|
||||
PlatformMismatch: bool(details.HasPlatformMismatch.Detected),
|
||||
}
|
||||
|
||||
if alias.HeadlessChromeScreenResolution {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.Webdriver {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.WebdriverWritable {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.Selenium {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.CDP {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.Playwright {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.ImpossibleDeviceMemory {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.HighCPUCount {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.MissingChromeObject {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.WebdriverIframe {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.WebdriverWorker {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.MismatchWebGLInWorker {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.MismatchPlatformIframe {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.MismatchPlatformWorker {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.SwiftshaderRenderer {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.UTCTimezone {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.MismatchLanguages {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.InconsistentEtsl {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.BotUserAgent {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.GPUMismatch {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
if alias.PlatformMismatch {
|
||||
alias.DetectedCount++
|
||||
}
|
||||
|
||||
alias.AnyDetected = alias.DetectedCount > 0
|
||||
|
||||
return alias
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Handwritten helpers on *FingerprintData. Kept in a dedicated file (not
|
||||
// fingerprint.go) so a future fpscanner library bump — which rewrites the
|
||||
// data-shape structs — doesn't clobber these methods.
|
||||
|
||||
// IsBot returns the library's fast-bot verdict as a native bool, so rules can write
|
||||
// `fingerprint.IsBot()` instead of `fingerprint.FastBotDetection.Bool() == true`.
|
||||
func (f *FingerprintData) IsBot() bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return bool(f.FastBotDetection)
|
||||
}
|
||||
|
||||
// BotSignalCount returns the number of fast-bot-detection signals that fired.
|
||||
func (f *FingerprintData) BotSignalCount() int {
|
||||
if f == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return f.Bot.DetectedCount.Int()
|
||||
}
|
||||
|
||||
// HasBotSignal returns true if any fast-bot-detection signal fired.
|
||||
func (f *FingerprintData) HasBotSignal() bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return f.Bot.AnyDetected
|
||||
}
|
||||
|
||||
// HasAutomationSignal returns true if any automation-framework signal fired
|
||||
// (webdriver, selenium, CDP, playwright, bot user-agent regex).
|
||||
func (f *FingerprintData) HasAutomationSignal() bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
b := f.Bot
|
||||
|
||||
return b.Webdriver ||
|
||||
b.WebdriverWritable ||
|
||||
b.Selenium ||
|
||||
b.CDP ||
|
||||
b.Playwright ||
|
||||
b.BotUserAgent
|
||||
}
|
||||
|
||||
// HasHeadlessSignal returns true if any headless-browser signal fired. Also
|
||||
// folds in inconsistent-etsl, which fires when the TLS-level `etsl` integer
|
||||
// disagrees with the claimed browser family — characteristic of patched /
|
||||
// forged headless environments.
|
||||
func (f *FingerprintData) HasHeadlessSignal() bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
b := f.Bot
|
||||
|
||||
return b.HeadlessChromeScreenResolution ||
|
||||
b.MissingChromeObject ||
|
||||
b.SwiftshaderRenderer ||
|
||||
b.InconsistentEtsl
|
||||
}
|
||||
|
||||
// HasMismatchSignal returns true if any cross-context or cross-API mismatch
|
||||
// signal fired (iframe/worker webdriver, platform, WebGL in worker, UA-vs-
|
||||
// navigator platform, GPU, languages).
|
||||
func (f *FingerprintData) HasMismatchSignal() bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
b := f.Bot
|
||||
|
||||
return b.MismatchWebGLInWorker ||
|
||||
b.MismatchPlatformIframe ||
|
||||
b.MismatchPlatformWorker ||
|
||||
b.WebdriverIframe ||
|
||||
b.WebdriverWorker ||
|
||||
b.PlatformMismatch ||
|
||||
b.GPUMismatch ||
|
||||
b.MismatchLanguages
|
||||
}
|
||||
|
||||
// HasImpossibleDeviceSignal returns true if the reported device specs are outside
|
||||
// plausible bounds (memory / CPU count).
|
||||
func (f *FingerprintData) HasImpossibleDeviceSignal() bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
b := f.Bot
|
||||
|
||||
return b.ImpossibleDeviceMemory || b.HighCPUCount
|
||||
}
|
||||
|
||||
// UserAgent returns the user-agent reported by the browser.
|
||||
func (f *FingerprintData) UserAgent() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return f.Signals.Browser.UserAgent
|
||||
}
|
||||
|
||||
// Platform returns the browser-reported platform, preferring the high-entropy
|
||||
// client-hint value and falling back to navigator.platform.
|
||||
func (f *FingerprintData) Platform() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
if p := f.Signals.Browser.HighEntropyValues.Platform; p != "" {
|
||||
return p
|
||||
}
|
||||
|
||||
return f.Signals.Device.Platform
|
||||
}
|
||||
|
||||
// Timezone returns the browser-reported IANA timezone.
|
||||
func (f *FingerprintData) Timezone() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return f.Signals.Locale.Internationalization.Timezone
|
||||
}
|
||||
|
||||
// Language returns the browser's primary language.
|
||||
func (f *FingerprintData) Language() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return f.Signals.Locale.Languages.Language
|
||||
}
|
||||
|
||||
// IsMobile returns true if the browser advertises a mobile form factor
|
||||
// (via UA client hints).
|
||||
func (f *FingerprintData) IsMobile() bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return bool(f.Signals.Browser.HighEntropyValues.Mobile)
|
||||
}
|
||||
|
||||
// CPUCount returns navigator.hardwareConcurrency as a native int.
|
||||
func (f *FingerprintData) CPUCount() int {
|
||||
if f == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return f.Signals.Device.CPUCount.Int()
|
||||
}
|
||||
|
||||
// Memory returns navigator.deviceMemory as a native int.
|
||||
func (f *FingerprintData) Memory() int {
|
||||
if f == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return f.Signals.Device.Memory.Int()
|
||||
}
|
||||
|
||||
// BotSignals returns the names of every library-native bot-detection signal
|
||||
// that fired on this fingerprint (e.g. "webdriver", "cdp",
|
||||
// "headless_screen_resolution", "platform_mismatch"). The slice is empty
|
||||
// when nothing fired. Ordering matches libDetections and is therefore
|
||||
// stable across calls, which keeps log output deterministic.
|
||||
//
|
||||
// Only signals carried directly by f.Bot are returned: the custom
|
||||
// CrowdSec mismatches in customDetections need request/geo context that
|
||||
// is not on FingerprintData and are surfaced separately via
|
||||
// MismatchReport.
|
||||
func (f *FingerprintData) BotSignals() []string {
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
out := make([]string, 0, len(libDetections))
|
||||
|
||||
for _, d := range libDetections {
|
||||
if d.Read(f.Bot) {
|
||||
out = append(out, d.Key)
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// FingerprintLogVerbosity controls how much of a fingerprint is included
|
||||
// in accept/reject log entries.
|
||||
type FingerprintLogVerbosity int
|
||||
|
||||
const (
|
||||
// FingerprintLogMinimal: source, fsid, ua, platform, is_bot, signals,
|
||||
// allowlisted (and allowlist_reason when allowlisted). This is the
|
||||
// default when verbosity is omitted.
|
||||
FingerprintLogMinimal FingerprintLogVerbosity = iota
|
||||
// FingerprintLogInfo: minimal + is_mobile and the category roll-ups
|
||||
// (automation/headless/mismatch/impossible_device) when they fired.
|
||||
// This is the default tier surfaced to expr helpers as the string
|
||||
// "info".
|
||||
FingerprintLogInfo
|
||||
// FingerprintLogVerbose: info + timezone, language, cpu_count,
|
||||
// memory, url, nonce, fp_time.
|
||||
FingerprintLogVerbose
|
||||
)
|
||||
|
||||
// resolveVerbosity returns the verbosity to use, defaulting to
|
||||
// FingerprintLogMinimal when the variadic argument is omitted.
|
||||
func resolveVerbosity(verbosity []FingerprintLogVerbosity) FingerprintLogVerbosity {
|
||||
if len(verbosity) == 0 {
|
||||
return FingerprintLogMinimal
|
||||
}
|
||||
|
||||
return verbosity[0]
|
||||
}
|
||||
|
||||
// LogAccepted emits a single structured log line for a fingerprint we
|
||||
// are accepting (valid cookie, successful submission, allowlist grant).
|
||||
//
|
||||
// clientIP is the real visitor address (typically request.ClientIP, set
|
||||
// by the bouncer via X-Forwarded-For or equivalent) and is logged as
|
||||
// "source". bouncerIP is the connection-level peer of the appsec listener
|
||||
// (typically request.RemoteAddrNormalized) and is logged as "bouncer".
|
||||
// Both are needed: operators correlate visitor behavior on "source", but
|
||||
// "bouncer" is what they use to debug which gateway forwarded the request
|
||||
// (multi-WAF setups, misconfigured X-Forwarded-For chains, etc.).
|
||||
//
|
||||
// The caller picks the level — typically Debug for per-request sites and
|
||||
// Info for rarer events. verbosity defaults to FingerprintLogMinimal.
|
||||
func (f *FingerprintData) LogAccepted(logger *log.Entry, level log.Level, clientIP, bouncerIP, msg string, verbosity ...FingerprintLogVerbosity) {
|
||||
if f == nil || logger == nil {
|
||||
return
|
||||
}
|
||||
|
||||
emitFingerprintLog(logger.WithFields(f.logFieldsAccept(clientIP, bouncerIP, resolveVerbosity(verbosity))), level, msg)
|
||||
}
|
||||
|
||||
// LogRejected emits a single structured log line for a fingerprint we
|
||||
// are rejecting.
|
||||
//
|
||||
// reason is the operator-facing rejection cause and is always included
|
||||
// regardless of verbosity. clientIP is logged as "source", bouncerIP as
|
||||
// "bouncer" — see LogAccepted for the rationale. Only positive
|
||||
// information is included — negative facts ("not headless", "0 mismatches")
|
||||
// are omitted because they are noise on a reject log.
|
||||
func (f *FingerprintData) LogRejected(logger *log.Entry, level log.Level, clientIP, bouncerIP, reason, msg string, verbosity ...FingerprintLogVerbosity) {
|
||||
if f == nil || logger == nil {
|
||||
return
|
||||
}
|
||||
|
||||
emitFingerprintLog(logger.WithFields(f.logFieldsReject(clientIP, bouncerIP, reason, resolveVerbosity(verbosity))), level, msg)
|
||||
}
|
||||
|
||||
// emitFingerprintLog dispatches a message to the right logrus method for
|
||||
// the requested level. Unknown levels fall back to Info so a typo can
|
||||
// never silently drop the log.
|
||||
func emitFingerprintLog(entry *log.Entry, level log.Level, msg string) {
|
||||
switch level {
|
||||
case log.PanicLevel, log.FatalLevel, log.ErrorLevel:
|
||||
entry.Error(msg)
|
||||
case log.WarnLevel:
|
||||
entry.Warn(msg)
|
||||
case log.DebugLevel:
|
||||
entry.Debug(msg)
|
||||
case log.TraceLevel:
|
||||
entry.Trace(msg)
|
||||
default:
|
||||
entry.Info(msg)
|
||||
}
|
||||
}
|
||||
|
||||
// logFieldsAccept builds the structured field set for an accept log.
|
||||
// Empty-string fields are omitted so partial fingerprints stay compact.
|
||||
func (f *FingerprintData) logFieldsAccept(clientIP, bouncerIP string, v FingerprintLogVerbosity) log.Fields {
|
||||
fields := log.Fields{
|
||||
"is_bot": f.IsBot(),
|
||||
"signals": f.BotSignals(),
|
||||
"allowlisted": f.Allowlisted,
|
||||
}
|
||||
|
||||
addCommonFingerprintFields(fields, f, clientIP, bouncerIP)
|
||||
|
||||
if f.Allowlisted && f.AllowlistReason != "" {
|
||||
fields["allowlist_reason"] = f.AllowlistReason
|
||||
}
|
||||
|
||||
if v >= FingerprintLogInfo {
|
||||
fields["is_mobile"] = f.IsMobile()
|
||||
}
|
||||
|
||||
if v >= FingerprintLogVerbose {
|
||||
addVerboseFingerprintFields(fields, f)
|
||||
}
|
||||
|
||||
return fields
|
||||
}
|
||||
|
||||
// logFieldsReject builds the structured field set for a reject log. Only
|
||||
// positive signals/flags are included; negative facts are dropped.
|
||||
func (f *FingerprintData) logFieldsReject(clientIP, bouncerIP, reason string, v FingerprintLogVerbosity) log.Fields {
|
||||
fields := log.Fields{
|
||||
"reason": reason,
|
||||
"signals": f.BotSignals(),
|
||||
}
|
||||
|
||||
addCommonFingerprintFields(fields, f, clientIP, bouncerIP)
|
||||
|
||||
if f.IsBot() {
|
||||
fields["is_bot"] = true
|
||||
}
|
||||
|
||||
if f.Allowlisted {
|
||||
fields["allowlisted"] = true
|
||||
|
||||
if f.AllowlistReason != "" {
|
||||
fields["allowlist_reason"] = f.AllowlistReason
|
||||
}
|
||||
}
|
||||
|
||||
if v >= FingerprintLogInfo {
|
||||
if f.IsMobile() {
|
||||
fields["is_mobile"] = true
|
||||
}
|
||||
|
||||
if f.HasAutomationSignal() {
|
||||
fields["automation"] = true
|
||||
}
|
||||
|
||||
if f.HasHeadlessSignal() {
|
||||
fields["headless"] = true
|
||||
}
|
||||
|
||||
if f.HasMismatchSignal() {
|
||||
fields["mismatch"] = true
|
||||
}
|
||||
|
||||
if f.HasImpossibleDeviceSignal() {
|
||||
fields["impossible_device"] = true
|
||||
}
|
||||
}
|
||||
|
||||
if v >= FingerprintLogVerbose {
|
||||
addVerboseFingerprintFields(fields, f)
|
||||
}
|
||||
|
||||
return fields
|
||||
}
|
||||
|
||||
// addCommonFingerprintFields adds the identity fields shared by accept
|
||||
// and reject logs. Empty strings are omitted so partial / synthetic
|
||||
// fingerprints (e.g. allowlist cookies with no measured signals) don't
|
||||
// log noisy empty keys.
|
||||
func addCommonFingerprintFields(fields log.Fields, f *FingerprintData, clientIP, bouncerIP string) {
|
||||
if clientIP != "" {
|
||||
fields["source"] = clientIP
|
||||
}
|
||||
|
||||
if bouncerIP != "" {
|
||||
fields["bouncer"] = bouncerIP
|
||||
}
|
||||
|
||||
if f.FSID != "" {
|
||||
fields["fsid"] = f.FSID
|
||||
}
|
||||
|
||||
if ua := f.UserAgent(); ua != "" {
|
||||
fields["ua"] = ua
|
||||
}
|
||||
|
||||
if p := f.Platform(); p != "" {
|
||||
fields["platform"] = p
|
||||
}
|
||||
}
|
||||
|
||||
// addVerboseFingerprintFields adds the verbose-only fields shared by
|
||||
// accept and reject logs. Zero / empty values are omitted.
|
||||
func addVerboseFingerprintFields(fields log.Fields, f *FingerprintData) {
|
||||
if tz := f.Timezone(); tz != "" {
|
||||
fields["timezone"] = tz
|
||||
}
|
||||
|
||||
if lang := f.Language(); lang != "" {
|
||||
fields["language"] = lang
|
||||
}
|
||||
|
||||
if cpu := f.CPUCount(); cpu != 0 {
|
||||
fields["cpu_count"] = cpu
|
||||
}
|
||||
|
||||
if mem := f.Memory(); mem != 0 {
|
||||
fields["memory"] = mem
|
||||
}
|
||||
|
||||
if f.URL != "" {
|
||||
fields["url"] = f.URL
|
||||
}
|
||||
|
||||
if f.Nonce != "" {
|
||||
fields["nonce"] = f.Nonce
|
||||
}
|
||||
|
||||
if f.Time != 0 {
|
||||
fields["fp_time"] = f.Time
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const sampleFingerprintJSON = `{
|
||||
"signals": {
|
||||
"device": {
|
||||
"cpuCount": 14,
|
||||
"memory": 8,
|
||||
"platform": "MacIntel"
|
||||
},
|
||||
"browser": {
|
||||
"userAgent": "Mozilla/5.0 Chrome",
|
||||
"highEntropyValues": {
|
||||
"platform": "macOS",
|
||||
"mobile": false
|
||||
}
|
||||
},
|
||||
"locale": {
|
||||
"internationalization": {"timezone": "Europe/Paris"},
|
||||
"languages": {"language": "en", "languages": ["en","fr"]}
|
||||
}
|
||||
},
|
||||
"fsid": "FS1_abc",
|
||||
"nonce": "n1",
|
||||
"time": 1770669806462,
|
||||
"url": "http://localhost/",
|
||||
"fastBotDetection": true,
|
||||
"fastBotDetectionDetails": {
|
||||
"hasCDP": {"detected": true, "severity": "high"},
|
||||
"headlessChromeScreenResolution": {"detected": false, "severity": "high"},
|
||||
"hasWebdriver": {"detected": false, "severity": "high"},
|
||||
"hasWebdriverWritable": {"detected": false, "severity": "high"},
|
||||
"hasSeleniumProperty": {"detected": false, "severity": "high"},
|
||||
"hasPlaywright": {"detected": false, "severity": "high"},
|
||||
"hasImpossibleDeviceMemory":{"detected": false, "severity": "high"},
|
||||
"hasHighCPUCount": {"detected": false, "severity": "high"},
|
||||
"hasMissingChromeObject": {"detected": false, "severity": "high"},
|
||||
"hasWebdriverIframe": {"detected": false, "severity": "high"},
|
||||
"hasWebdriverWorker": {"detected": false, "severity": "high"},
|
||||
"hasMismatchWebGLInWorker": {"detected": false, "severity": "high"},
|
||||
"hasMismatchPlatformIframe":{"detected": false, "severity": "high"},
|
||||
"hasMismatchPlatformWorker":{"detected": false, "severity": "high"},
|
||||
"hasSwiftshaderRenderer": {"detected": false, "severity": "low"},
|
||||
"hasUTCTimezone": {"detected": false, "severity": "medium"}
|
||||
}
|
||||
}`
|
||||
|
||||
func TestFingerprintHelpers_Getters(t *testing.T) {
|
||||
var fp FingerprintData
|
||||
require.NoError(t, json.Unmarshal([]byte(sampleFingerprintJSON), &fp))
|
||||
|
||||
assert.True(t, fp.IsBot())
|
||||
assert.True(t, fp.HasBotSignal())
|
||||
assert.Equal(t, 1, fp.BotSignalCount())
|
||||
|
||||
// only CDP is set in the sample, so only HasAutomationSignal should flip
|
||||
assert.True(t, fp.HasAutomationSignal())
|
||||
assert.False(t, fp.HasHeadlessSignal())
|
||||
assert.False(t, fp.HasMismatchSignal())
|
||||
assert.False(t, fp.HasImpossibleDeviceSignal())
|
||||
|
||||
assert.Equal(t, "Mozilla/5.0 Chrome", fp.UserAgent())
|
||||
assert.Equal(t, "macOS", fp.Platform()) // prefers high-entropy value
|
||||
assert.Equal(t, "Europe/Paris", fp.Timezone())
|
||||
assert.Equal(t, "en", fp.Language())
|
||||
assert.False(t, fp.IsMobile())
|
||||
assert.Equal(t, 14, fp.CPUCount())
|
||||
assert.Equal(t, 8, fp.Memory())
|
||||
}
|
||||
|
||||
func TestFingerprintHelpers_PlatformFallback(t *testing.T) {
|
||||
// no high-entropy platform -> falls back to navigator.platform
|
||||
fp := FingerprintData{}
|
||||
fp.Signals.Device.Platform = "Linux x86_64"
|
||||
assert.Equal(t, "Linux x86_64", fp.Platform())
|
||||
}
|
||||
|
||||
func TestFingerprintHelpers_NilReceiver(t *testing.T) {
|
||||
var fp *FingerprintData
|
||||
|
||||
assert.False(t, fp.IsBot())
|
||||
assert.False(t, fp.HasBotSignal())
|
||||
assert.Equal(t, 0, fp.BotSignalCount())
|
||||
assert.False(t, fp.HasAutomationSignal())
|
||||
assert.False(t, fp.HasHeadlessSignal())
|
||||
assert.False(t, fp.HasMismatchSignal())
|
||||
assert.False(t, fp.HasImpossibleDeviceSignal())
|
||||
assert.Empty(t, fp.UserAgent())
|
||||
assert.Empty(t, fp.Platform())
|
||||
assert.Empty(t, fp.Timezone())
|
||||
assert.Empty(t, fp.Language())
|
||||
assert.False(t, fp.IsMobile())
|
||||
assert.Equal(t, 0, fp.CPUCount())
|
||||
assert.Equal(t, 0, fp.Memory())
|
||||
}
|
||||
|
||||
func TestFingerprintHelpers_CategoryAggregates(t *testing.T) {
|
||||
const (
|
||||
catAutomation = "automation"
|
||||
catHeadless = "headless"
|
||||
catMismatch = "mismatch"
|
||||
catImpossibleDevice = "impossible_device"
|
||||
)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
set func(*fingerprintBotAlias)
|
||||
want string
|
||||
}{
|
||||
{"webdriver", func(b *fingerprintBotAlias) { b.Webdriver = true }, catAutomation},
|
||||
{"webdriverWritable", func(b *fingerprintBotAlias) { b.WebdriverWritable = true }, catAutomation},
|
||||
{"selenium", func(b *fingerprintBotAlias) { b.Selenium = true }, catAutomation},
|
||||
{"cdp", func(b *fingerprintBotAlias) { b.CDP = true }, catAutomation},
|
||||
{"playwright", func(b *fingerprintBotAlias) { b.Playwright = true }, catAutomation},
|
||||
|
||||
{"headlessChromeScreenResolution", func(b *fingerprintBotAlias) { b.HeadlessChromeScreenResolution = true }, catHeadless},
|
||||
{"missingChromeObject", func(b *fingerprintBotAlias) { b.MissingChromeObject = true }, catHeadless},
|
||||
{"swiftshaderRenderer", func(b *fingerprintBotAlias) { b.SwiftshaderRenderer = true }, catHeadless},
|
||||
|
||||
{"mismatchWebGLInWorker", func(b *fingerprintBotAlias) { b.MismatchWebGLInWorker = true }, catMismatch},
|
||||
{"mismatchPlatformIframe", func(b *fingerprintBotAlias) { b.MismatchPlatformIframe = true }, catMismatch},
|
||||
{"mismatchPlatformWorker", func(b *fingerprintBotAlias) { b.MismatchPlatformWorker = true }, catMismatch},
|
||||
{"webdriverIframe", func(b *fingerprintBotAlias) { b.WebdriverIframe = true }, catMismatch},
|
||||
{"webdriverWorker", func(b *fingerprintBotAlias) { b.WebdriverWorker = true }, catMismatch},
|
||||
|
||||
{"impossibleDeviceMemory", func(b *fingerprintBotAlias) { b.ImpossibleDeviceMemory = true }, catImpossibleDevice},
|
||||
{"highCPUCount", func(b *fingerprintBotAlias) { b.HighCPUCount = true }, catImpossibleDevice},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fp := &FingerprintData{}
|
||||
tc.set(&fp.Bot)
|
||||
|
||||
got := map[string]bool{
|
||||
catAutomation: fp.HasAutomationSignal(),
|
||||
catHeadless: fp.HasHeadlessSignal(),
|
||||
catMismatch: fp.HasMismatchSignal(),
|
||||
catImpossibleDevice: fp.HasImpossibleDeviceSignal(),
|
||||
}
|
||||
|
||||
for cat, fired := range got {
|
||||
if cat == tc.want {
|
||||
assert.True(t, fired, "expected category %q to fire", cat)
|
||||
} else {
|
||||
assert.False(t, fired, "expected category %q to stay false", cat)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
// fingerprint_mismatch.go implements the *atomic* mismatch checks: small,
|
||||
// composable predicates on FingerprintData (UA-vs-viewport, UA-language vs
|
||||
// Accept-Language, timezone vs GeoIP country, etc.) that rules can call
|
||||
// directly. The aggregate report and the lookup tables that map these
|
||||
// signals to stable reason keys + severities live in
|
||||
// fingerprint_mismatch_data.go.
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/text/language"
|
||||
)
|
||||
|
||||
// UAMobileMismatch reports whether the fingerprint's user-agent claims a
|
||||
// mobile form factor while the reported inner viewport width is implausibly
|
||||
// wide for a real device. Catches the "UA switcher set to Android while
|
||||
// actually on a desktop" pattern.
|
||||
//
|
||||
// Returns false (no signal) when either the UA doesn't claim mobile or the
|
||||
// inner width is zero/missing.
|
||||
func (f *FingerprintData) UAMobileMismatch() bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
ua := f.UserAgent()
|
||||
if ua == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
if !mobileUARegex.MatchString(ua) {
|
||||
return false
|
||||
}
|
||||
|
||||
innerWidth := f.Signals.Device.ScreenResolution.InnerWidth.Int()
|
||||
if innerWidth <= 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
return innerWidth >= mobileViewportThreshold
|
||||
}
|
||||
|
||||
// AcceptLanguageMismatch reports whether the request's Accept-Language
|
||||
// header disagrees with the fingerprint's navigator.language. Both are
|
||||
// derived from the same browser preference, so on a real browser they
|
||||
// should agree at the base-language level.
|
||||
//
|
||||
// Returns false when either side is empty or unparseable — never a false
|
||||
// positive on missing data.
|
||||
//
|
||||
// Deliberately different from the library's MismatchLanguages detection,
|
||||
// which only compares navigator.languages[0] to navigator.language inside
|
||||
// the browser; this helper brings the HTTP header into the comparison.
|
||||
func (f *FingerprintData) AcceptLanguageMismatch(req *http.Request) bool {
|
||||
if f == nil || req == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
fpLang := f.Language()
|
||||
if fpLang == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
header := req.Header.Get("Accept-Language")
|
||||
if header == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
tags, _, err := language.ParseAcceptLanguage(header)
|
||||
if err != nil || len(tags) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
headerBase, _ := tags[0].Base()
|
||||
|
||||
fpTag, err := language.Parse(fpLang)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
fpBase, _ := fpTag.Base()
|
||||
|
||||
return headerBase.String() != fpBase.String()
|
||||
}
|
||||
|
||||
// TimezoneCountryMismatch reports whether the fingerprint's timezone
|
||||
// disagrees with the client's country of origin (typically resolved via
|
||||
// IPToCountry from the client IP).
|
||||
//
|
||||
// SOFT SIGNAL. Travelers whose OS timezone hasn't auto-adjusted and VPN
|
||||
// users will legitimately trigger this. Rule authors should combine it
|
||||
// with other signals (High()/Medium() counts, or reason-specific checks)
|
||||
// before using it alone to ban.
|
||||
//
|
||||
// Returns false when either the timezone is unknown to our IANA table or
|
||||
// the country is empty.
|
||||
func (f *FingerprintData) TimezoneCountryMismatch(country string) bool {
|
||||
if f == nil || country == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
tz := f.Timezone()
|
||||
if tz == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
countries := lookupTimezoneCountries(tz)
|
||||
if len(countries) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
return !slices.Contains(countries, country)
|
||||
}
|
||||
|
||||
// MismatchSignal is a single fired reason/severity pair on the report.
|
||||
type MismatchSignal struct {
|
||||
Reason string
|
||||
Severity string
|
||||
}
|
||||
|
||||
// MismatchReport aggregates the fired mismatch signals for a single
|
||||
// fingerprint evaluation. Returned by ComputeMismatchReport; typically
|
||||
// accessed through the cached EvaluateMismatches closure in the
|
||||
// on_challenge expr env.
|
||||
type MismatchReport struct {
|
||||
Signals []MismatchSignal
|
||||
}
|
||||
|
||||
// Count returns the total number of fired signals.
|
||||
func (r *MismatchReport) Count() int {
|
||||
if r == nil {
|
||||
return 0
|
||||
}
|
||||
return len(r.Signals)
|
||||
}
|
||||
|
||||
// Empty reports whether no signal fired.
|
||||
func (r *MismatchReport) Empty() bool {
|
||||
return r.Count() == 0
|
||||
}
|
||||
|
||||
// Has reports whether a signal with the given reason key is present.
|
||||
func (r *MismatchReport) Has(reason string) bool {
|
||||
if r == nil {
|
||||
return false
|
||||
}
|
||||
for _, s := range r.Signals {
|
||||
if s.Reason == reason {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Reasons returns the stable-ordered list of fired reason keys.
|
||||
func (r *MismatchReport) Reasons() []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, len(r.Signals))
|
||||
for i, s := range r.Signals {
|
||||
out[i] = s.Reason
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// High returns the count of signals tagged with the "high" severity.
|
||||
func (r *MismatchReport) High() int {
|
||||
return r.BySeverity(SeverityHigh)
|
||||
}
|
||||
|
||||
// Medium returns the count of signals tagged with the "medium" severity.
|
||||
func (r *MismatchReport) Medium() int {
|
||||
return r.BySeverity(SeverityMedium)
|
||||
}
|
||||
|
||||
// Low returns the count of signals tagged with the "low" severity.
|
||||
func (r *MismatchReport) Low() int {
|
||||
return r.BySeverity(SeverityLow)
|
||||
}
|
||||
|
||||
// BySeverity returns the count of signals at the requested severity level.
|
||||
func (r *MismatchReport) BySeverity(sev string) int {
|
||||
if r == nil {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, s := range r.Signals {
|
||||
if s.Severity == sev {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// String renders the report as "reason1(sev),reason2(sev)" for log lines.
|
||||
func (r *MismatchReport) String() string {
|
||||
if r == nil || len(r.Signals) == 0 {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, s := range r.Signals {
|
||||
if i > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(s.Reason)
|
||||
b.WriteByte('(')
|
||||
b.WriteString(s.Severity)
|
||||
b.WriteByte(')')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// ComputeMismatchReport walks the library-native bot alias plus the
|
||||
// CrowdSec custom helpers and returns every fired signal in a stable
|
||||
// order. It is the pure computation backing EvaluateMismatches; callers
|
||||
// wanting caching + observability should use the env closure registered
|
||||
// in GetOnChallengeEnv instead.
|
||||
//
|
||||
// `country` is the ISO-3166 alpha-2 code of the client's geolocated
|
||||
// country, or "" when unknown. Pass `exprhelpers.IPToCountry(...)`'s
|
||||
// return value (or equivalent) — resolved once at the call site so this
|
||||
// method stays free of side-effects.
|
||||
//
|
||||
// Iteration order is libDetections first (in the order declared there),
|
||||
// then customDetections. Reason / severity / accessor binding for every
|
||||
// check lives in one place — fingerprint_mismatch_data.go.
|
||||
func (f *FingerprintData) ComputeMismatchReport(req *http.Request, country string) *MismatchReport {
|
||||
report := &MismatchReport{}
|
||||
|
||||
if f == nil {
|
||||
return report
|
||||
}
|
||||
|
||||
for _, d := range libDetections {
|
||||
if d.Read(f.Bot) {
|
||||
report.Signals = append(report.Signals, MismatchSignal{
|
||||
Reason: d.Key,
|
||||
Severity: d.Severity,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for _, d := range customDetections {
|
||||
if d.Fire(f, req, country) {
|
||||
report.Signals = append(report.Signals, MismatchSignal{
|
||||
Reason: d.Key,
|
||||
Severity: d.Severity,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return report
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
// fingerprint_mismatch_data.go is the static lookup data backing the
|
||||
// mismatch report: stable reason keys, severity labels, and the slice that
|
||||
// binds each fpscanner-native signal to the bool it reads from the
|
||||
// FingerprintData wire shape. Atomic detection methods live in
|
||||
// fingerprint_mismatch.go; the aggregate report builder lives in
|
||||
// fingerprint_helpers.go.
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// Severity labels. Mirror the fpscanner library's fastBotDetectionDetails[*].severity
|
||||
// strings so the vocabulary is consistent end-to-end.
|
||||
const (
|
||||
SeverityHigh = "high"
|
||||
SeverityMedium = "medium"
|
||||
SeverityLow = "low"
|
||||
)
|
||||
|
||||
// Stable reason keys. One per entry in libDetections / customDetections
|
||||
// below. Kept as exported constants so rule authors can
|
||||
// `Has("platform_mismatch")` without typos.
|
||||
const (
|
||||
ReasonCDP = "cdp"
|
||||
ReasonWebdriver = "webdriver"
|
||||
ReasonWebdriverWritable = "webdriver_writable"
|
||||
ReasonSelenium = "selenium"
|
||||
ReasonPlaywright = "playwright"
|
||||
ReasonWebdriverIframe = "webdriver_iframe"
|
||||
ReasonWebdriverWorker = "webdriver_worker"
|
||||
ReasonHeadlessScreenResolution = "headless_screen_resolution"
|
||||
ReasonMissingChromeObject = "missing_chrome_object"
|
||||
ReasonImpossibleMemory = "impossible_memory"
|
||||
ReasonHighCPUCount = "high_cpu_count"
|
||||
ReasonMismatchWebGLWorker = "mismatch_webgl_worker"
|
||||
ReasonMismatchPlatformIframe = "mismatch_platform_iframe"
|
||||
ReasonMismatchPlatformWorker = "mismatch_platform_worker"
|
||||
ReasonPlatformMismatch = "platform_mismatch"
|
||||
ReasonGPUMismatch = "gpu_mismatch"
|
||||
ReasonBotUserAgent = "bot_user_agent"
|
||||
ReasonInconsistentEtsl = "inconsistent_etsl"
|
||||
ReasonUAMobile = "ua_mobile"
|
||||
ReasonUTCTimezone = "utc_timezone"
|
||||
ReasonAcceptLanguage = "accept_language"
|
||||
ReasonSwiftshaderRenderer = "swiftshader_renderer"
|
||||
ReasonMismatchLanguages = "mismatch_languages"
|
||||
ReasonTimezoneCountry = "timezone_country"
|
||||
)
|
||||
|
||||
// libDetection binds an fpscanner-native signal to the bot-alias bool that
|
||||
// carries its `detected` flag, together with our severity label for it.
|
||||
//
|
||||
// Note: the fpscanner library ships each check with its own severity in the
|
||||
// JSON payload, but `fingerprintDetectionResult` only unmarshals `detected`
|
||||
// today, so the severity values below mirror the library defaults. The day
|
||||
// we extend the struct with a Severity field, this column can be dropped
|
||||
// and read from the payload instead.
|
||||
type libDetection struct {
|
||||
Key string
|
||||
Severity string
|
||||
Read func(fingerprintBotAlias) bool
|
||||
}
|
||||
|
||||
// libDetections is the single source of truth for every library-native bot
|
||||
// signal we surface: the reason key, its severity, and how to read its
|
||||
// `detected` value from the bot alias. Append-only and ordering-stable so
|
||||
// `MismatchReport.Reasons()` has a deterministic output.
|
||||
var libDetections = []libDetection{
|
||||
{ReasonCDP, SeverityHigh, func(b fingerprintBotAlias) bool { return b.CDP }},
|
||||
{ReasonWebdriver, SeverityHigh, func(b fingerprintBotAlias) bool { return b.Webdriver }},
|
||||
{ReasonWebdriverWritable, SeverityHigh, func(b fingerprintBotAlias) bool { return b.WebdriverWritable }},
|
||||
{ReasonSelenium, SeverityHigh, func(b fingerprintBotAlias) bool { return b.Selenium }},
|
||||
{ReasonPlaywright, SeverityHigh, func(b fingerprintBotAlias) bool { return b.Playwright }},
|
||||
{ReasonWebdriverIframe, SeverityHigh, func(b fingerprintBotAlias) bool { return b.WebdriverIframe }},
|
||||
{ReasonWebdriverWorker, SeverityHigh, func(b fingerprintBotAlias) bool { return b.WebdriverWorker }},
|
||||
{ReasonHeadlessScreenResolution, SeverityHigh, func(b fingerprintBotAlias) bool { return b.HeadlessChromeScreenResolution }},
|
||||
{ReasonMissingChromeObject, SeverityHigh, func(b fingerprintBotAlias) bool { return b.MissingChromeObject }},
|
||||
{ReasonImpossibleMemory, SeverityHigh, func(b fingerprintBotAlias) bool { return b.ImpossibleDeviceMemory }},
|
||||
{ReasonHighCPUCount, SeverityHigh, func(b fingerprintBotAlias) bool { return b.HighCPUCount }},
|
||||
{ReasonMismatchWebGLWorker, SeverityHigh, func(b fingerprintBotAlias) bool { return b.MismatchWebGLInWorker }},
|
||||
{ReasonMismatchPlatformIframe, SeverityHigh, func(b fingerprintBotAlias) bool { return b.MismatchPlatformIframe }},
|
||||
{ReasonMismatchPlatformWorker, SeverityHigh, func(b fingerprintBotAlias) bool { return b.MismatchPlatformWorker }},
|
||||
{ReasonPlatformMismatch, SeverityHigh, func(b fingerprintBotAlias) bool { return b.PlatformMismatch }},
|
||||
{ReasonGPUMismatch, SeverityHigh, func(b fingerprintBotAlias) bool { return b.GPUMismatch }},
|
||||
{ReasonBotUserAgent, SeverityHigh, func(b fingerprintBotAlias) bool { return b.BotUserAgent }},
|
||||
{ReasonInconsistentEtsl, SeverityHigh, func(b fingerprintBotAlias) bool { return b.InconsistentEtsl }},
|
||||
{ReasonUTCTimezone, SeverityMedium, func(b fingerprintBotAlias) bool { return b.UTCTimezone }},
|
||||
{ReasonSwiftshaderRenderer, SeverityLow, func(b fingerprintBotAlias) bool { return b.SwiftshaderRenderer }},
|
||||
{ReasonMismatchLanguages, SeverityLow, func(b fingerprintBotAlias) bool { return b.MismatchLanguages }},
|
||||
}
|
||||
|
||||
// customDetection binds a CrowdSec-authored mismatch helper (which may need
|
||||
// request or geoip data) to its reason key and severity. The Fire function
|
||||
// wraps whichever method-on-fp is the actual check — unified signature so
|
||||
// customDetections can be iterated like libDetections.
|
||||
type customDetection struct {
|
||||
Key string
|
||||
Severity string
|
||||
Fire func(fp *FingerprintData, req *http.Request, country string) bool
|
||||
}
|
||||
|
||||
// customDetections lists every check we run on top of the library's own.
|
||||
// Ordering is stable and comes after libDetections in the final report.
|
||||
var customDetections = []customDetection{
|
||||
{
|
||||
Key: ReasonUAMobile,
|
||||
Severity: SeverityMedium,
|
||||
Fire: func(fp *FingerprintData, _ *http.Request, _ string) bool { return fp.UAMobileMismatch() },
|
||||
},
|
||||
{
|
||||
Key: ReasonAcceptLanguage,
|
||||
Severity: SeverityMedium,
|
||||
Fire: func(fp *FingerprintData, req *http.Request, _ string) bool { return fp.AcceptLanguageMismatch(req) },
|
||||
},
|
||||
{
|
||||
Key: ReasonTimezoneCountry,
|
||||
Severity: SeverityLow,
|
||||
Fire: func(fp *FingerprintData, _ *http.Request, country string) bool { return fp.TimezoneCountryMismatch(country) },
|
||||
},
|
||||
}
|
||||
|
||||
// severityByReason is derived once at init from the two detection tables
|
||||
// above. It exists so callers that only need the severity for a reason key
|
||||
// (metric labels, tests, SeverityFor) don't have to walk the tables.
|
||||
var severityByReason = buildSeverityByReason()
|
||||
|
||||
func buildSeverityByReason() map[string]string {
|
||||
m := make(map[string]string, len(libDetections)+len(customDetections))
|
||||
for _, d := range libDetections {
|
||||
m[d.Key] = d.Severity
|
||||
}
|
||||
for _, d := range customDetections {
|
||||
m[d.Key] = d.Severity
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// SeverityFor returns the severity tagged to a reason key, or "" for
|
||||
// unknown reasons.
|
||||
func SeverityFor(reason string) string {
|
||||
return severityByReason[reason]
|
||||
}
|
||||
|
||||
// KnownReasons returns the full set of reason keys the aggregator may emit.
|
||||
// Order is not guaranteed; callers that need deterministic ordering should
|
||||
// walk libDetections / customDetections directly.
|
||||
func KnownReasons() []string {
|
||||
out := make([]string, 0, len(severityByReason))
|
||||
for k := range severityByReason {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mobileUARegex matches UAs that claim a PHONE form factor. Deliberately
|
||||
// does not match bare "Android", because per Google's UA convention the
|
||||
// `Mobile` / `Mobi` token is what phones add and tablets omit. Matching
|
||||
// Android alone would FP on every Android tablet in landscape and on
|
||||
// Samsung DeX / Android desktop mode (UA keeps "Android", viewport is
|
||||
// ≥1000px).
|
||||
var mobileUARegex = regexp.MustCompile(`(?i)Mobi|iPhone|iPod`)
|
||||
|
||||
// mobileViewportThreshold is the innerWidth cutoff above which we treat a
|
||||
// mobile-claiming UA as inconsistent. Real phones report CSS pixel widths
|
||||
// well below this (typical phones: 360-430). Tablets run wider but don't
|
||||
// match the UA regex above, so this threshold only applies to phone-UAs.
|
||||
const mobileViewportThreshold = 1000
|
||||
@@ -0,0 +1,334 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// androidOnLinuxSample is the user-provided fingerprint with a spoofed
|
||||
// Android/Firefox UA on a real Chromium-on-Linux environment. It's kept
|
||||
// verbatim as a permanent regression fixture.
|
||||
const androidOnLinuxSample = `{
|
||||
"signals": {
|
||||
"automation": {"webdriver": false, "cdp": true},
|
||||
"device": {
|
||||
"cpuCount": 16,
|
||||
"memory": 8,
|
||||
"platform": "Linux x86_64",
|
||||
"screenResolution": {
|
||||
"width": 1920, "height": 1080,
|
||||
"innerWidth": 1920, "innerHeight": 993
|
||||
}
|
||||
},
|
||||
"browser": {
|
||||
"userAgent": "Mozilla/5.0 (Android 4.4; Mobile; rv:70.0) Gecko/70.0 Firefox/70.0",
|
||||
"features": {"chrome": true},
|
||||
"highEntropyValues": {"platform": "", "mobile": false}
|
||||
},
|
||||
"locale": {
|
||||
"internationalization": {"timezone": "Europe/Paris"},
|
||||
"languages": {"language": "en-US", "languages": ["en-US"]}
|
||||
}
|
||||
},
|
||||
"fsid": "FS_SAMPLE",
|
||||
"nonce": "n",
|
||||
"time": 1776669341379,
|
||||
"url": "https://example/",
|
||||
"fastBotDetection": true,
|
||||
"fastBotDetectionDetails": {
|
||||
"hasCDP": {"detected": true, "severity": "high"},
|
||||
"hasInconsistentEtsl": {"detected": true, "severity": "high"},
|
||||
"hasPlatformMismatch": {"detected": true, "severity": "high"},
|
||||
"headlessChromeScreenResolution": {"detected": false, "severity": "high"},
|
||||
"hasWebdriver": {"detected": false, "severity": "high"},
|
||||
"hasWebdriverWritable": {"detected": false, "severity": "high"},
|
||||
"hasSeleniumProperty": {"detected": false, "severity": "high"},
|
||||
"hasPlaywright": {"detected": false, "severity": "high"},
|
||||
"hasImpossibleDeviceMemory":{"detected": false, "severity": "high"},
|
||||
"hasHighCPUCount": {"detected": false, "severity": "high"},
|
||||
"hasMissingChromeObject": {"detected": false, "severity": "high"},
|
||||
"hasWebdriverIframe": {"detected": false, "severity": "high"},
|
||||
"hasWebdriverWorker": {"detected": false, "severity": "high"},
|
||||
"hasMismatchWebGLInWorker": {"detected": false, "severity": "high"},
|
||||
"hasMismatchPlatformIframe":{"detected": false, "severity": "high"},
|
||||
"hasMismatchPlatformWorker":{"detected": false, "severity": "high"},
|
||||
"hasSwiftshaderRenderer": {"detected": false, "severity": "low"},
|
||||
"hasUTCTimezone": {"detected": false, "severity": "medium"},
|
||||
"hasMismatchLanguages": {"detected": false, "severity": "low"},
|
||||
"hasBotUserAgent": {"detected": false, "severity": "high"},
|
||||
"hasGPUMismatch": {"detected": false, "severity": "high"}
|
||||
}
|
||||
}`
|
||||
|
||||
func mustUnmarshal(t *testing.T, raw string) *FingerprintData {
|
||||
t.Helper()
|
||||
fp := &FingerprintData{}
|
||||
require.NoError(t, json.Unmarshal([]byte(raw), fp))
|
||||
return fp
|
||||
}
|
||||
|
||||
func TestUAMobileMismatch(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
ua string
|
||||
innerWidth int
|
||||
want bool
|
||||
}{
|
||||
{"android phone UA desktop viewport", "Mozilla/5.0 (Android 4.4; Mobile; rv:70.0)", 1920, true},
|
||||
{"iphone UA huge viewport", "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0)", 1440, true},
|
||||
{"real android phone small viewport", "Mozilla/5.0 (Linux; Android 14; Pixel 7) Mobile", 412, false},
|
||||
{"desktop UA desktop viewport", "Mozilla/5.0 (X11; Linux x86_64) Chrome/120", 1920, false},
|
||||
{"mobile UA missing innerWidth", "Mozilla/5.0 (Android 10; Mobile)", 0, false},
|
||||
{"empty UA", "", 1920, false},
|
||||
// Regression: a real Android tablet UA lacks the "Mobile" token by
|
||||
// Google's convention. Its landscape viewport is wider than 1000px.
|
||||
// Must NOT fire — previously a FP when the regex matched bare "Android".
|
||||
{"android tablet landscape", "Mozilla/5.0 (Linux; Android 13; SM-X710) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36", 1280, false},
|
||||
// Regression: Samsung DeX / Android desktop mode keeps "Android" in
|
||||
// the UA but the user is on a desktop-sized screen. Must NOT fire.
|
||||
{"samsung dex desktop viewport", "Mozilla/5.0 (Linux; Android 13; SM-S908U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36", 1920, false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fp := &FingerprintData{}
|
||||
fp.Signals.Browser.UserAgent = tc.ua
|
||||
fp.Signals.Device.ScreenResolution.InnerWidth = FlexInt(tc.innerWidth)
|
||||
assert.Equal(t, tc.want, fp.UAMobileMismatch())
|
||||
})
|
||||
}
|
||||
|
||||
var nilFP *FingerprintData
|
||||
assert.False(t, nilFP.UAMobileMismatch())
|
||||
}
|
||||
|
||||
func TestAcceptLanguageMismatch(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
header string
|
||||
fpLang string
|
||||
want bool
|
||||
}{
|
||||
{"match en simple", "en-US,fr;q=0.8", "en", false},
|
||||
{"match fr simple", "fr-FR,fr;q=0.9,en;q=0.5", "fr", false},
|
||||
{"mismatch de vs en", "de-DE,de;q=0.9", "en", true},
|
||||
{"mismatch zh vs en", "zh-CN", "en-US", true},
|
||||
{"missing header", "", "en", false},
|
||||
{"missing fp lang", "en-US", "", false},
|
||||
{"garbage header", "@@@", "en", false},
|
||||
{"garbage fp lang", "en-US", "@@@", false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fp := &FingerprintData{}
|
||||
fp.Signals.Locale.Languages.Language = tc.fpLang
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
if tc.header != "" {
|
||||
req.Header.Set("Accept-Language", tc.header)
|
||||
}
|
||||
|
||||
assert.Equal(t, tc.want, fp.AcceptLanguageMismatch(req))
|
||||
})
|
||||
}
|
||||
|
||||
var nilFP *FingerprintData
|
||||
assert.False(t, nilFP.AcceptLanguageMismatch(nil))
|
||||
}
|
||||
|
||||
func TestTimezoneCountryMismatch(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
tz string
|
||||
country string
|
||||
want bool
|
||||
}{
|
||||
{"paris in france", "Europe/Paris", "FR", false},
|
||||
{"paris for monaco too", "Europe/Paris", "MC", false},
|
||||
{"paris claimed from US", "Europe/Paris", "US", true},
|
||||
{"new york in us", "America/New_York", "US", false},
|
||||
{"new york claimed from fr", "America/New_York", "FR", true},
|
||||
{"alias calcutta", "Asia/Calcutta", "IN", false},
|
||||
{"alias kiev", "Europe/Kiev", "UA", false},
|
||||
{"unknown tz", "Made/Up", "FR", false},
|
||||
{"empty country", "Europe/Paris", "", false},
|
||||
{"empty tz", "", "FR", false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fp := &FingerprintData{}
|
||||
fp.Signals.Locale.Internationalization.Timezone = tc.tz
|
||||
assert.Equal(t, tc.want, fp.TimezoneCountryMismatch(tc.country))
|
||||
})
|
||||
}
|
||||
|
||||
var nilFP *FingerprintData
|
||||
assert.False(t, nilFP.TimezoneCountryMismatch("FR"))
|
||||
}
|
||||
|
||||
func TestMismatchReport_Helpers(t *testing.T) {
|
||||
r := &MismatchReport{Signals: []MismatchSignal{
|
||||
{Reason: ReasonCDP, Severity: SeverityHigh},
|
||||
{Reason: ReasonPlatformMismatch, Severity: SeverityHigh},
|
||||
{Reason: ReasonAcceptLanguage, Severity: SeverityMedium},
|
||||
{Reason: ReasonTimezoneCountry, Severity: SeverityLow},
|
||||
}}
|
||||
|
||||
assert.Equal(t, 4, r.Count())
|
||||
assert.False(t, r.Empty())
|
||||
assert.Equal(t, 2, r.High())
|
||||
assert.Equal(t, 1, r.Medium())
|
||||
assert.Equal(t, 1, r.Low())
|
||||
assert.Equal(t, 2, r.BySeverity(SeverityHigh))
|
||||
assert.True(t, r.Has(ReasonCDP))
|
||||
assert.False(t, r.Has(ReasonBotUserAgent))
|
||||
assert.Equal(t, []string{ReasonCDP, ReasonPlatformMismatch, ReasonAcceptLanguage, ReasonTimezoneCountry}, r.Reasons())
|
||||
assert.Equal(t,
|
||||
"cdp(high),platform_mismatch(high),accept_language(medium),timezone_country(low)",
|
||||
r.String(),
|
||||
)
|
||||
}
|
||||
|
||||
func TestMismatchReport_NilReceiver(t *testing.T) {
|
||||
var r *MismatchReport
|
||||
assert.Equal(t, 0, r.Count())
|
||||
assert.True(t, r.Empty())
|
||||
assert.False(t, r.Has(ReasonCDP))
|
||||
assert.Nil(t, r.Reasons())
|
||||
assert.Equal(t, 0, r.High())
|
||||
assert.Equal(t, 0, r.Medium())
|
||||
assert.Equal(t, 0, r.Low())
|
||||
assert.Empty(t, r.String())
|
||||
}
|
||||
|
||||
func TestComputeMismatchReport_AndroidOnLinuxSample(t *testing.T) {
|
||||
fp := mustUnmarshal(t, androidOnLinuxSample)
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Request originates from France; timezone is also Europe/Paris, so
|
||||
// timezone_country should NOT fire. The Android-UA + 1920 viewport
|
||||
// should fire ua_mobile, and the library-native detections fire cdp,
|
||||
// inconsistent_etsl, platform_mismatch.
|
||||
report := fp.ComputeMismatchReport(req, "FR")
|
||||
|
||||
assert.Equal(t, []string{
|
||||
ReasonCDP,
|
||||
ReasonPlatformMismatch,
|
||||
ReasonInconsistentEtsl,
|
||||
ReasonUAMobile,
|
||||
}, report.Reasons())
|
||||
assert.Equal(t, 3, report.High())
|
||||
assert.Equal(t, 1, report.Medium(), "ua_mobile is medium severity")
|
||||
assert.Equal(t, 0, report.Low())
|
||||
}
|
||||
|
||||
func TestComputeMismatchReport_BaselineConsistent(t *testing.T) {
|
||||
// Minimal consistent fingerprint — no library detections, consistent
|
||||
// UA/platform, en Accept-Language, FR timezone, from FR.
|
||||
raw := `{
|
||||
"signals": {
|
||||
"device": {"platform": "MacIntel"},
|
||||
"browser": {
|
||||
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120",
|
||||
"highEntropyValues": {"platform": "macOS", "mobile": false}
|
||||
},
|
||||
"locale": {
|
||||
"internationalization": {"timezone": "Europe/Paris"},
|
||||
"languages": {"language": "en"}
|
||||
}
|
||||
},
|
||||
"fsid": "FS_OK", "nonce": "n", "time": 1, "url": "http://x/",
|
||||
"fastBotDetection": false,
|
||||
"fastBotDetectionDetails": {}
|
||||
}`
|
||||
fp := mustUnmarshal(t, raw)
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
report := fp.ComputeMismatchReport(req, "FR")
|
||||
assert.True(t, report.Empty())
|
||||
}
|
||||
|
||||
func TestComputeMismatchReport_AcceptLanguageOnly(t *testing.T) {
|
||||
raw := `{
|
||||
"signals": {
|
||||
"device": {"platform": "MacIntel"},
|
||||
"browser": {
|
||||
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X) Chrome/120",
|
||||
"highEntropyValues": {"platform": "macOS"}
|
||||
},
|
||||
"locale": {
|
||||
"internationalization": {"timezone": "Europe/Paris"},
|
||||
"languages": {"language": "en"}
|
||||
}
|
||||
},
|
||||
"fsid": "X", "nonce": "n", "time": 1, "url": "http://x/",
|
||||
"fastBotDetection": false,
|
||||
"fastBotDetectionDetails": {}
|
||||
}`
|
||||
fp := mustUnmarshal(t, raw)
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9")
|
||||
|
||||
report := fp.ComputeMismatchReport(req, "FR")
|
||||
assert.Equal(t, []string{ReasonAcceptLanguage}, report.Reasons())
|
||||
assert.Equal(t, 0, report.High())
|
||||
assert.Equal(t, 1, report.Medium())
|
||||
assert.Equal(t, 0, report.Low())
|
||||
}
|
||||
|
||||
func TestComputeMismatchReport_TimezoneCountryOnly(t *testing.T) {
|
||||
raw := `{
|
||||
"signals": {
|
||||
"device": {"platform": "MacIntel"},
|
||||
"browser": {
|
||||
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X) Chrome/120",
|
||||
"highEntropyValues": {"platform": "macOS"}
|
||||
},
|
||||
"locale": {
|
||||
"internationalization": {"timezone": "Europe/Paris"},
|
||||
"languages": {"language": "en"}
|
||||
}
|
||||
},
|
||||
"fsid": "X", "nonce": "n", "time": 1, "url": "http://x/",
|
||||
"fastBotDetection": false,
|
||||
"fastBotDetectionDetails": {}
|
||||
}`
|
||||
fp := mustUnmarshal(t, raw)
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Accept-Language", "en-US")
|
||||
|
||||
// Europe/Paris but client IP geolocated in the US.
|
||||
report := fp.ComputeMismatchReport(req, "US")
|
||||
assert.Equal(t, []string{ReasonTimezoneCountry}, report.Reasons())
|
||||
assert.Equal(t, 0, report.High())
|
||||
assert.Equal(t, 0, report.Medium())
|
||||
assert.Equal(t, 1, report.Low())
|
||||
}
|
||||
|
||||
func TestComputeMismatchReport_NilFingerprint(t *testing.T) {
|
||||
report := ((*FingerprintData)(nil)).ComputeMismatchReport(nil, "")
|
||||
assert.True(t, report.Empty())
|
||||
}
|
||||
|
||||
func TestSeverityFor(t *testing.T) {
|
||||
assert.Equal(t, SeverityHigh, SeverityFor(ReasonCDP))
|
||||
assert.Equal(t, SeverityMedium, SeverityFor(ReasonAcceptLanguage))
|
||||
assert.Equal(t, SeverityLow, SeverityFor(ReasonTimezoneCountry))
|
||||
assert.Empty(t, SeverityFor("not_a_reason"))
|
||||
}
|
||||
@@ -0,0 +1,568 @@
|
||||
// fingerprint_proto.go bridges the protobuf-encoded fingerprint payload
|
||||
// (pb.FingerprintData, the wire format used inside the encrypted cookie) and
|
||||
// the FingerprintData struct used at runtime. Unexported helpers only; the
|
||||
// conversion is mechanical field-by-field and intentionally verbose to keep
|
||||
// it auditable. One helper per signal section, in the same order on both the
|
||||
// from-proto and to-proto sides.
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/pb"
|
||||
)
|
||||
|
||||
func fingerprintDataFromProto(p *pb.FingerprintData) FingerprintData {
|
||||
f := FingerprintData{
|
||||
FSID: p.GetFsid(),
|
||||
Nonce: p.GetNonce(),
|
||||
Time: p.GetTime(),
|
||||
URL: p.GetUrl(),
|
||||
FastBotDetection: FlexBool(p.GetFastBotDetection()),
|
||||
}
|
||||
|
||||
if s := p.GetSignals(); s != nil {
|
||||
f.Signals = signalsFromProto(s)
|
||||
}
|
||||
|
||||
if det := p.GetFastBotDetectionDetails(); det != nil {
|
||||
f.FastBotDetectionDetails = fastBotDetectionDetailsFromProto(det)
|
||||
}
|
||||
|
||||
f.Bot = newFingerprintBotAlias(f.FastBotDetectionDetails)
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func signalsFromProto(s *pb.FingerprintSignals) fingerprintSignals {
|
||||
var out fingerprintSignals
|
||||
|
||||
if a := s.GetAutomation(); a != nil {
|
||||
out.Automation = automationFromProto(a)
|
||||
}
|
||||
if d := s.GetDevice(); d != nil {
|
||||
out.Device = deviceFromProto(d)
|
||||
}
|
||||
if b := s.GetBrowser(); b != nil {
|
||||
out.Browser = browserFromProto(b)
|
||||
}
|
||||
if g := s.GetGraphics(); g != nil {
|
||||
out.Graphics = graphicsFromProto(g)
|
||||
}
|
||||
if c := s.GetCodecs(); c != nil {
|
||||
out.Codecs = codecsFromProto(c)
|
||||
}
|
||||
if l := s.GetLocale(); l != nil {
|
||||
out.Locale = localeFromProto(l)
|
||||
}
|
||||
if c := s.GetContexts(); c != nil {
|
||||
out.Contexts = contextsFromProto(c)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func automationFromProto(a *pb.FingerprintAutomation) fingerprintAutomation {
|
||||
return fingerprintAutomation{
|
||||
Webdriver: FlexBool(a.GetWebdriver()),
|
||||
WebdriverWritable: FlexBool(a.GetWebdriverWritable()),
|
||||
Selenium: FlexBool(a.GetSelenium()),
|
||||
CDP: FlexBool(a.GetCdp()),
|
||||
Playwright: FlexBool(a.GetPlaywright()),
|
||||
NavigatorPropertyDescriptors: a.GetNavigatorPropertyDescriptors(),
|
||||
}
|
||||
}
|
||||
|
||||
func deviceFromProto(d *pb.FingerprintDevice) fingerprintDevice {
|
||||
out := fingerprintDevice{
|
||||
CPUCount: FlexInt(d.GetCpuCount()),
|
||||
Memory: FlexInt(d.GetMemory()),
|
||||
Platform: d.GetPlatform(),
|
||||
}
|
||||
if sr := d.GetScreenResolution(); sr != nil {
|
||||
out.ScreenResolution = fingerprintScreenResolution{
|
||||
Width: FlexInt(sr.GetWidth()),
|
||||
Height: FlexInt(sr.GetHeight()),
|
||||
PixelDepth: FlexInt(sr.GetPixelDepth()),
|
||||
ColorDepth: FlexInt(sr.GetColorDepth()),
|
||||
AvailableWidth: FlexInt(sr.GetAvailableWidth()),
|
||||
AvailableHeight: FlexInt(sr.GetAvailableHeight()),
|
||||
InnerWidth: FlexInt(sr.GetInnerWidth()),
|
||||
InnerHeight: FlexInt(sr.GetInnerHeight()),
|
||||
HasMultipleDisplays: FlexBool(sr.GetHasMultipleDisplays()),
|
||||
}
|
||||
}
|
||||
if mm := d.GetMultimediaDevices(); mm != nil {
|
||||
out.MultimediaDevices = fingerprintMultimediaDevices{
|
||||
Speakers: FlexInt(mm.GetSpeakers()),
|
||||
Microphones: FlexInt(mm.GetMicrophones()),
|
||||
Webcams: FlexInt(mm.GetWebcams()),
|
||||
}
|
||||
}
|
||||
if mq := d.GetMediaQueries(); mq != nil {
|
||||
out.MediaQueries = fingerprintDeviceMediaQueries{
|
||||
PrefersColorScheme: mq.GetPrefersColorScheme(),
|
||||
PrefersReducedMotion: FlexBool(mq.GetPrefersReducedMotion()),
|
||||
PrefersReducedTransparency: FlexBool(mq.GetPrefersReducedTransparency()),
|
||||
ColorGamut: mq.GetColorGamut(),
|
||||
Pointer: mq.GetPointer(),
|
||||
AnyPointer: mq.GetAnyPointer(),
|
||||
Hover: FlexBool(mq.GetHover()),
|
||||
AnyHover: FlexBool(mq.GetAnyHover()),
|
||||
ColorDepth: FlexInt(mq.GetColorDepth()),
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func browserFromProto(b *pb.FingerprintBrowser) fingerprintBrowser {
|
||||
out := fingerprintBrowser{
|
||||
UserAgent: b.GetUserAgent(),
|
||||
ETSL: FlexInt(b.GetEtsl()),
|
||||
Maths: b.GetMaths(),
|
||||
}
|
||||
if ft := b.GetFeatures(); ft != nil {
|
||||
out.Features = fingerprintBrowserFeatures{
|
||||
Bitmask: ft.GetBitmask(),
|
||||
Chrome: FlexBool(ft.GetChrome()),
|
||||
Brave: FlexBool(ft.GetBrave()),
|
||||
ApplePaySupport: FlexBool(ft.GetApplePaySupport()),
|
||||
Opera: FlexBool(ft.GetOpera()),
|
||||
Serial: FlexBool(ft.GetSerial()),
|
||||
AttachShadow: FlexBool(ft.GetAttachShadow()),
|
||||
Caches: FlexBool(ft.GetCaches()),
|
||||
WebAssembly: FlexBool(ft.GetWebAssembly()),
|
||||
Buffer: FlexBool(ft.GetBuffer()),
|
||||
ShowModalDialog: FlexBool(ft.GetShowModalDialog()),
|
||||
Safari: FlexBool(ft.GetSafari()),
|
||||
WebkitPrefixedFunction: FlexBool(ft.GetWebkitPrefixedFunction()),
|
||||
MozPrefixedFunction: FlexBool(ft.GetMozPrefixedFunction()),
|
||||
USB: FlexBool(ft.GetUsb()),
|
||||
BrowserCapture: FlexBool(ft.GetBrowserCapture()),
|
||||
PaymentRequestUpdateEvent: FlexBool(ft.GetPaymentRequestUpdateEvent()),
|
||||
PressureObserver: FlexBool(ft.GetPressureObserver()),
|
||||
AudioSession: FlexBool(ft.GetAudioSession()),
|
||||
SelectAudioOutput: FlexBool(ft.GetSelectAudioOutput()),
|
||||
BarcodeDetector: FlexBool(ft.GetBarcodeDetector()),
|
||||
Battery: FlexBool(ft.GetBattery()),
|
||||
DevicePosture: FlexBool(ft.GetDevicePosture()),
|
||||
DocumentPictureInPicture: FlexBool(ft.GetDocumentPictureInPicture()),
|
||||
EyeDropper: FlexBool(ft.GetEyeDropper()),
|
||||
EditContext: FlexBool(ft.GetEditContext()),
|
||||
FencedFrame: FlexBool(ft.GetFencedFrame()),
|
||||
Sanitizer: FlexBool(ft.GetSanitizer()),
|
||||
OTPCredential: FlexBool(ft.GetOtpCredential()),
|
||||
}
|
||||
}
|
||||
if pl := b.GetPlugins(); pl != nil {
|
||||
out.Plugins = fingerprintBrowserPlugins{
|
||||
IsValidPluginArray: FlexBool(pl.GetIsValidPluginArray()),
|
||||
PluginCount: FlexInt(pl.GetPluginCount()),
|
||||
PluginNamesHash: pl.GetPluginNamesHash(),
|
||||
PluginConsistency1: FlexBool(pl.GetPluginConsistency1()),
|
||||
PluginOverflow: FlexBool(pl.GetPluginOverflow()),
|
||||
}
|
||||
}
|
||||
if ext := b.GetExtensions(); ext != nil {
|
||||
out.Extensions = fingerprintBrowserExtensions{
|
||||
Bitmask: ext.GetBitmask(),
|
||||
Extensions: ext.GetExtensions(),
|
||||
}
|
||||
}
|
||||
if hev := b.GetHighEntropyValues(); hev != nil {
|
||||
out.HighEntropyValues = fingerprintBrowserHighEntropyValues{
|
||||
Architecture: hev.GetArchitecture(),
|
||||
Bitness: hev.GetBitness(),
|
||||
Mobile: FlexBool(hev.GetMobile()),
|
||||
Model: hev.GetModel(),
|
||||
Platform: hev.GetPlatform(),
|
||||
PlatformVersion: hev.GetPlatformVersion(),
|
||||
UAFullVersion: hev.GetUaFullVersion(),
|
||||
}
|
||||
for _, bv := range hev.GetBrands() {
|
||||
out.HighEntropyValues.Brands = append(out.HighEntropyValues.Brands, fingerprintBrandVersion{
|
||||
Brand: bv.GetBrand(),
|
||||
Version: bv.GetVersion(),
|
||||
})
|
||||
}
|
||||
}
|
||||
if tse := b.GetToSourceError(); tse != nil {
|
||||
out.ToSourceError = fingerprintBrowserToSourceError{
|
||||
ToSourceError: tse.GetToSourceError(),
|
||||
HasToSource: FlexBool(tse.GetHasToSource()),
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func graphicsFromProto(g *pb.FingerprintGraphics) fingerprintGraphics {
|
||||
var out fingerprintGraphics
|
||||
|
||||
if gl := g.GetWebGL(); gl != nil {
|
||||
out.WebGL = fingerprintGraphicsWebGL{
|
||||
Vendor: gl.GetVendor(),
|
||||
Renderer: gl.GetRenderer(),
|
||||
}
|
||||
}
|
||||
if gpu := g.GetWebgpu(); gpu != nil {
|
||||
out.WebGPU = fingerprintGraphicsWebGPU{
|
||||
Vendor: gpu.GetVendor(),
|
||||
Architecture: gpu.GetArchitecture(),
|
||||
Device: gpu.GetDevice(),
|
||||
Description: gpu.GetDescription(),
|
||||
}
|
||||
}
|
||||
if cv := g.GetCanvas(); cv != nil {
|
||||
out.Canvas = fingerprintGraphicsCanvas{
|
||||
HasModifiedCanvas: FlexBool(cv.GetHasModifiedCanvas()),
|
||||
CanvasFingerprint: cv.GetCanvasFingerprint(),
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func codecsFromProto(c *pb.FingerprintCodecs) fingerprintCodecs {
|
||||
return fingerprintCodecs{
|
||||
AudioCanPlayTypeHash: c.GetAudioCanPlayTypeHash(),
|
||||
VideoCanPlayTypeHash: c.GetVideoCanPlayTypeHash(),
|
||||
AudioMediaSourceHash: c.GetAudioMediaSourceHash(),
|
||||
VideoMediaSourceHash: c.GetVideoMediaSourceHash(),
|
||||
RTCAudioCapabilitiesHash: c.GetRtcAudioCapabilitiesHash(),
|
||||
RTCVideoCapabilitiesHash: c.GetRtcVideoCapabilitiesHash(),
|
||||
HasMediaSource: FlexBool(c.GetHasMediaSource()),
|
||||
}
|
||||
}
|
||||
|
||||
func localeFromProto(l *pb.FingerprintLocale) fingerprintLocale {
|
||||
var out fingerprintLocale
|
||||
|
||||
if i18n := l.GetInternationalization(); i18n != nil {
|
||||
out.Internationalization = fingerprintLocaleInternationalization{
|
||||
Timezone: i18n.GetTimezone(),
|
||||
LocaleLanguage: i18n.GetLocaleLanguage(),
|
||||
}
|
||||
}
|
||||
if lang := l.GetLanguages(); lang != nil {
|
||||
out.Languages = fingerprintLocaleLanguages{
|
||||
Languages: lang.GetLanguages(),
|
||||
Language: lang.GetLanguage(),
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func contextsFromProto(c *pb.FingerprintContexts) fingerprintContexts {
|
||||
var out fingerprintContexts
|
||||
|
||||
if iframe := c.GetIframe(); iframe != nil {
|
||||
out.Iframe = fingerprintIframeContext{
|
||||
Webdriver: FlexBool(iframe.GetWebdriver()),
|
||||
UserAgent: iframe.GetUserAgent(),
|
||||
Platform: iframe.GetPlatform(),
|
||||
Memory: FlexInt(iframe.GetMemory()),
|
||||
CPUCount: FlexInt(iframe.GetCpuCount()),
|
||||
Language: iframe.GetLanguage(),
|
||||
}
|
||||
}
|
||||
if ww := c.GetWebWorker(); ww != nil {
|
||||
out.WebWorker = fingerprintWebWorkerContext{
|
||||
Vendor: ww.GetVendor(),
|
||||
Renderer: ww.GetRenderer(),
|
||||
UserAgent: ww.GetUserAgent(),
|
||||
Language: ww.GetLanguage(),
|
||||
Platform: ww.GetPlatform(),
|
||||
Memory: FlexInt(ww.GetMemory()),
|
||||
CPUCount: FlexInt(ww.GetCpuCount()),
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func fastBotDetectionDetailsFromProto(det *pb.FingerprintFastBotDetectionDetails) fingerprintFastBotDetectionDetails {
|
||||
return fingerprintFastBotDetectionDetails{
|
||||
HeadlessChromeScreenResolution: fingerprintDetectionResult{Detected: FlexBool(det.GetHeadlessChromeScreenResolution().GetDetected())},
|
||||
HasWebdriver: fingerprintDetectionResult{Detected: FlexBool(det.GetHasWebdriver().GetDetected())},
|
||||
HasWebdriverWritable: fingerprintDetectionResult{Detected: FlexBool(det.GetHasWebdriverWritable().GetDetected())},
|
||||
HasSeleniumProperty: fingerprintDetectionResult{Detected: FlexBool(det.GetHasSeleniumProperty().GetDetected())},
|
||||
HasCDP: fingerprintDetectionResult{Detected: FlexBool(det.GetHasCDP().GetDetected())},
|
||||
HasPlaywright: fingerprintDetectionResult{Detected: FlexBool(det.GetHasPlaywright().GetDetected())},
|
||||
HasImpossibleDeviceMemory: fingerprintDetectionResult{Detected: FlexBool(det.GetHasImpossibleDeviceMemory().GetDetected())},
|
||||
HasHighCPUCount: fingerprintDetectionResult{Detected: FlexBool(det.GetHasHighCPUCount().GetDetected())},
|
||||
HasMissingChromeObject: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMissingChromeObject().GetDetected())},
|
||||
HasWebdriverIframe: fingerprintDetectionResult{Detected: FlexBool(det.GetHasWebdriverIframe().GetDetected())},
|
||||
HasWebdriverWorker: fingerprintDetectionResult{Detected: FlexBool(det.GetHasWebdriverWorker().GetDetected())},
|
||||
HasMismatchWebGLInWorker: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMismatchWebGLInWorker().GetDetected())},
|
||||
HasMismatchPlatformIframe: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMismatchPlatformIframe().GetDetected())},
|
||||
HasMismatchPlatformWorker: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMismatchPlatformWorker().GetDetected())},
|
||||
HasSwiftshaderRenderer: fingerprintDetectionResult{Detected: FlexBool(det.GetHasSwiftshaderRenderer().GetDetected())},
|
||||
HasUTCTimezone: fingerprintDetectionResult{Detected: FlexBool(det.GetHasUTCTimezone().GetDetected())},
|
||||
HasMismatchLanguages: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMismatchLanguages().GetDetected())},
|
||||
HasInconsistentEtsl: fingerprintDetectionResult{Detected: FlexBool(det.GetHasInconsistentEtsl().GetDetected())},
|
||||
HasBotUserAgent: fingerprintDetectionResult{Detected: FlexBool(det.GetHasBotUserAgent().GetDetected())},
|
||||
HasGPUMismatch: fingerprintDetectionResult{Detected: FlexBool(det.GetHasGPUMismatch().GetDetected())},
|
||||
HasPlatformMismatch: fingerprintDetectionResult{Detected: FlexBool(det.GetHasPlatformMismatch().GetDetected())},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FingerprintData) ToProto() *pb.FingerprintData {
|
||||
return &pb.FingerprintData{
|
||||
Signals: &pb.FingerprintSignals{
|
||||
Automation: f.Signals.Automation.toProto(),
|
||||
Device: f.Signals.Device.toProto(),
|
||||
Browser: f.Signals.Browser.toProto(),
|
||||
Graphics: f.Signals.Graphics.toProto(),
|
||||
Codecs: f.Signals.Codecs.toProto(),
|
||||
Locale: f.Signals.Locale.toProto(),
|
||||
Contexts: f.Signals.Contexts.toProto(),
|
||||
},
|
||||
Fsid: f.FSID,
|
||||
Nonce: f.Nonce,
|
||||
Time: f.Time,
|
||||
Url: f.URL,
|
||||
FastBotDetection: bool(f.FastBotDetection),
|
||||
FastBotDetectionDetails: f.FastBotDetectionDetails.toProto(),
|
||||
Bot: f.Bot.toProto(),
|
||||
}
|
||||
}
|
||||
|
||||
func (a fingerprintAutomation) toProto() *pb.FingerprintAutomation {
|
||||
return &pb.FingerprintAutomation{
|
||||
Webdriver: bool(a.Webdriver),
|
||||
WebdriverWritable: bool(a.WebdriverWritable),
|
||||
Selenium: bool(a.Selenium),
|
||||
Cdp: bool(a.CDP),
|
||||
Playwright: bool(a.Playwright),
|
||||
NavigatorPropertyDescriptors: a.NavigatorPropertyDescriptors,
|
||||
}
|
||||
}
|
||||
|
||||
func (d fingerprintDevice) toProto() *pb.FingerprintDevice {
|
||||
return &pb.FingerprintDevice{
|
||||
CpuCount: int32(d.CPUCount),
|
||||
Memory: int32(d.Memory),
|
||||
Platform: d.Platform,
|
||||
ScreenResolution: &pb.FingerprintScreenResolution{
|
||||
Width: int32(d.ScreenResolution.Width),
|
||||
Height: int32(d.ScreenResolution.Height),
|
||||
PixelDepth: int32(d.ScreenResolution.PixelDepth),
|
||||
ColorDepth: int32(d.ScreenResolution.ColorDepth),
|
||||
AvailableWidth: int32(d.ScreenResolution.AvailableWidth),
|
||||
AvailableHeight: int32(d.ScreenResolution.AvailableHeight),
|
||||
InnerWidth: int32(d.ScreenResolution.InnerWidth),
|
||||
InnerHeight: int32(d.ScreenResolution.InnerHeight),
|
||||
HasMultipleDisplays: bool(d.ScreenResolution.HasMultipleDisplays),
|
||||
},
|
||||
MultimediaDevices: &pb.FingerprintMultimediaDevices{
|
||||
Speakers: int32(d.MultimediaDevices.Speakers),
|
||||
Microphones: int32(d.MultimediaDevices.Microphones),
|
||||
Webcams: int32(d.MultimediaDevices.Webcams),
|
||||
},
|
||||
MediaQueries: &pb.FingerprintDeviceMediaQueries{
|
||||
PrefersColorScheme: d.MediaQueries.PrefersColorScheme,
|
||||
PrefersReducedMotion: bool(d.MediaQueries.PrefersReducedMotion),
|
||||
PrefersReducedTransparency: bool(d.MediaQueries.PrefersReducedTransparency),
|
||||
ColorGamut: d.MediaQueries.ColorGamut,
|
||||
Pointer: d.MediaQueries.Pointer,
|
||||
AnyPointer: d.MediaQueries.AnyPointer,
|
||||
Hover: bool(d.MediaQueries.Hover),
|
||||
AnyHover: bool(d.MediaQueries.AnyHover),
|
||||
ColorDepth: int32(d.MediaQueries.ColorDepth),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (b fingerprintBrowser) toProto() *pb.FingerprintBrowser {
|
||||
brands := make([]*pb.FingerprintBrandVersion, len(b.HighEntropyValues.Brands))
|
||||
for i, bv := range b.HighEntropyValues.Brands {
|
||||
brands[i] = &pb.FingerprintBrandVersion{
|
||||
Brand: bv.Brand,
|
||||
Version: bv.Version,
|
||||
}
|
||||
}
|
||||
|
||||
return &pb.FingerprintBrowser{
|
||||
UserAgent: b.UserAgent,
|
||||
Features: &pb.FingerprintBrowserFeatures{
|
||||
Bitmask: b.Features.Bitmask,
|
||||
Chrome: bool(b.Features.Chrome),
|
||||
Brave: bool(b.Features.Brave),
|
||||
ApplePaySupport: bool(b.Features.ApplePaySupport),
|
||||
Opera: bool(b.Features.Opera),
|
||||
Serial: bool(b.Features.Serial),
|
||||
AttachShadow: bool(b.Features.AttachShadow),
|
||||
Caches: bool(b.Features.Caches),
|
||||
WebAssembly: bool(b.Features.WebAssembly),
|
||||
Buffer: bool(b.Features.Buffer),
|
||||
ShowModalDialog: bool(b.Features.ShowModalDialog),
|
||||
Safari: bool(b.Features.Safari),
|
||||
WebkitPrefixedFunction: bool(b.Features.WebkitPrefixedFunction),
|
||||
MozPrefixedFunction: bool(b.Features.MozPrefixedFunction),
|
||||
Usb: bool(b.Features.USB),
|
||||
BrowserCapture: bool(b.Features.BrowserCapture),
|
||||
PaymentRequestUpdateEvent: bool(b.Features.PaymentRequestUpdateEvent),
|
||||
PressureObserver: bool(b.Features.PressureObserver),
|
||||
AudioSession: bool(b.Features.AudioSession),
|
||||
SelectAudioOutput: bool(b.Features.SelectAudioOutput),
|
||||
BarcodeDetector: bool(b.Features.BarcodeDetector),
|
||||
Battery: bool(b.Features.Battery),
|
||||
DevicePosture: bool(b.Features.DevicePosture),
|
||||
DocumentPictureInPicture: bool(b.Features.DocumentPictureInPicture),
|
||||
EyeDropper: bool(b.Features.EyeDropper),
|
||||
EditContext: bool(b.Features.EditContext),
|
||||
FencedFrame: bool(b.Features.FencedFrame),
|
||||
Sanitizer: bool(b.Features.Sanitizer),
|
||||
OtpCredential: bool(b.Features.OTPCredential),
|
||||
},
|
||||
Plugins: &pb.FingerprintBrowserPlugins{
|
||||
IsValidPluginArray: bool(b.Plugins.IsValidPluginArray),
|
||||
PluginCount: int32(b.Plugins.PluginCount),
|
||||
PluginNamesHash: b.Plugins.PluginNamesHash,
|
||||
PluginConsistency1: bool(b.Plugins.PluginConsistency1),
|
||||
PluginOverflow: bool(b.Plugins.PluginOverflow),
|
||||
},
|
||||
Extensions: &pb.FingerprintBrowserExtensions{
|
||||
Bitmask: b.Extensions.Bitmask,
|
||||
Extensions: b.Extensions.Extensions,
|
||||
},
|
||||
HighEntropyValues: &pb.FingerprintBrowserHighEntropyValues{
|
||||
Architecture: b.HighEntropyValues.Architecture,
|
||||
Bitness: b.HighEntropyValues.Bitness,
|
||||
Brands: brands,
|
||||
Mobile: bool(b.HighEntropyValues.Mobile),
|
||||
Model: b.HighEntropyValues.Model,
|
||||
Platform: b.HighEntropyValues.Platform,
|
||||
PlatformVersion: b.HighEntropyValues.PlatformVersion,
|
||||
UaFullVersion: b.HighEntropyValues.UAFullVersion,
|
||||
},
|
||||
Etsl: int32(b.ETSL),
|
||||
Maths: b.Maths,
|
||||
ToSourceError: &pb.FingerprintBrowserToSourceError{
|
||||
ToSourceError: b.ToSourceError.ToSourceError,
|
||||
HasToSource: bool(b.ToSourceError.HasToSource),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (g fingerprintGraphics) toProto() *pb.FingerprintGraphics {
|
||||
return &pb.FingerprintGraphics{
|
||||
WebGL: &pb.FingerprintGraphicsWebGL{
|
||||
Vendor: g.WebGL.Vendor,
|
||||
Renderer: g.WebGL.Renderer,
|
||||
},
|
||||
Webgpu: &pb.FingerprintGraphicsWebGPU{
|
||||
Vendor: g.WebGPU.Vendor,
|
||||
Architecture: g.WebGPU.Architecture,
|
||||
Device: g.WebGPU.Device,
|
||||
Description: g.WebGPU.Description,
|
||||
},
|
||||
Canvas: &pb.FingerprintGraphicsCanvas{
|
||||
HasModifiedCanvas: bool(g.Canvas.HasModifiedCanvas),
|
||||
CanvasFingerprint: g.Canvas.CanvasFingerprint,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c fingerprintCodecs) toProto() *pb.FingerprintCodecs {
|
||||
return &pb.FingerprintCodecs{
|
||||
AudioCanPlayTypeHash: c.AudioCanPlayTypeHash,
|
||||
VideoCanPlayTypeHash: c.VideoCanPlayTypeHash,
|
||||
AudioMediaSourceHash: c.AudioMediaSourceHash,
|
||||
VideoMediaSourceHash: c.VideoMediaSourceHash,
|
||||
RtcAudioCapabilitiesHash: c.RTCAudioCapabilitiesHash,
|
||||
RtcVideoCapabilitiesHash: c.RTCVideoCapabilitiesHash,
|
||||
HasMediaSource: bool(c.HasMediaSource),
|
||||
}
|
||||
}
|
||||
|
||||
func (l fingerprintLocale) toProto() *pb.FingerprintLocale {
|
||||
return &pb.FingerprintLocale{
|
||||
Internationalization: &pb.FingerprintLocaleInternationalization{
|
||||
Timezone: l.Internationalization.Timezone,
|
||||
LocaleLanguage: l.Internationalization.LocaleLanguage,
|
||||
},
|
||||
Languages: &pb.FingerprintLocaleLanguages{
|
||||
Languages: l.Languages.Languages,
|
||||
Language: l.Languages.Language,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c fingerprintContexts) toProto() *pb.FingerprintContexts {
|
||||
return &pb.FingerprintContexts{
|
||||
Iframe: &pb.FingerprintIframeContext{
|
||||
Webdriver: bool(c.Iframe.Webdriver),
|
||||
UserAgent: c.Iframe.UserAgent,
|
||||
Platform: c.Iframe.Platform,
|
||||
Memory: int32(c.Iframe.Memory),
|
||||
CpuCount: int32(c.Iframe.CPUCount),
|
||||
Language: c.Iframe.Language,
|
||||
},
|
||||
WebWorker: &pb.FingerprintWebWorkerContext{
|
||||
Vendor: c.WebWorker.Vendor,
|
||||
Renderer: c.WebWorker.Renderer,
|
||||
UserAgent: c.WebWorker.UserAgent,
|
||||
Language: c.WebWorker.Language,
|
||||
Platform: c.WebWorker.Platform,
|
||||
Memory: int32(c.WebWorker.Memory),
|
||||
CpuCount: int32(c.WebWorker.CPUCount),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d fingerprintFastBotDetectionDetails) toProto() *pb.FingerprintFastBotDetectionDetails {
|
||||
return &pb.FingerprintFastBotDetectionDetails{
|
||||
HeadlessChromeScreenResolution: &pb.FingerprintDetectionResult{Detected: bool(d.HeadlessChromeScreenResolution.Detected)},
|
||||
HasWebdriver: &pb.FingerprintDetectionResult{Detected: bool(d.HasWebdriver.Detected)},
|
||||
HasWebdriverWritable: &pb.FingerprintDetectionResult{Detected: bool(d.HasWebdriverWritable.Detected)},
|
||||
HasSeleniumProperty: &pb.FingerprintDetectionResult{Detected: bool(d.HasSeleniumProperty.Detected)},
|
||||
HasCDP: &pb.FingerprintDetectionResult{Detected: bool(d.HasCDP.Detected)},
|
||||
HasPlaywright: &pb.FingerprintDetectionResult{Detected: bool(d.HasPlaywright.Detected)},
|
||||
HasImpossibleDeviceMemory: &pb.FingerprintDetectionResult{Detected: bool(d.HasImpossibleDeviceMemory.Detected)},
|
||||
HasHighCPUCount: &pb.FingerprintDetectionResult{Detected: bool(d.HasHighCPUCount.Detected)},
|
||||
HasMissingChromeObject: &pb.FingerprintDetectionResult{Detected: bool(d.HasMissingChromeObject.Detected)},
|
||||
HasWebdriverIframe: &pb.FingerprintDetectionResult{Detected: bool(d.HasWebdriverIframe.Detected)},
|
||||
HasWebdriverWorker: &pb.FingerprintDetectionResult{Detected: bool(d.HasWebdriverWorker.Detected)},
|
||||
HasMismatchWebGLInWorker: &pb.FingerprintDetectionResult{Detected: bool(d.HasMismatchWebGLInWorker.Detected)},
|
||||
HasMismatchPlatformIframe: &pb.FingerprintDetectionResult{Detected: bool(d.HasMismatchPlatformIframe.Detected)},
|
||||
HasMismatchPlatformWorker: &pb.FingerprintDetectionResult{Detected: bool(d.HasMismatchPlatformWorker.Detected)},
|
||||
HasSwiftshaderRenderer: &pb.FingerprintDetectionResult{Detected: bool(d.HasSwiftshaderRenderer.Detected)},
|
||||
HasUTCTimezone: &pb.FingerprintDetectionResult{Detected: bool(d.HasUTCTimezone.Detected)},
|
||||
HasMismatchLanguages: &pb.FingerprintDetectionResult{Detected: bool(d.HasMismatchLanguages.Detected)},
|
||||
HasInconsistentEtsl: &pb.FingerprintDetectionResult{Detected: bool(d.HasInconsistentEtsl.Detected)},
|
||||
HasBotUserAgent: &pb.FingerprintDetectionResult{Detected: bool(d.HasBotUserAgent.Detected)},
|
||||
HasGPUMismatch: &pb.FingerprintDetectionResult{Detected: bool(d.HasGPUMismatch.Detected)},
|
||||
HasPlatformMismatch: &pb.FingerprintDetectionResult{Detected: bool(d.HasPlatformMismatch.Detected)},
|
||||
}
|
||||
}
|
||||
|
||||
func (b fingerprintBotAlias) toProto() *pb.FingerprintBotAlias {
|
||||
return &pb.FingerprintBotAlias{
|
||||
HeadlessChromeScreenResolution: b.HeadlessChromeScreenResolution,
|
||||
Webdriver: b.Webdriver,
|
||||
WebdriverWritable: b.WebdriverWritable,
|
||||
Selenium: b.Selenium,
|
||||
Cdp: b.CDP,
|
||||
Playwright: b.Playwright,
|
||||
ImpossibleDeviceMemory: b.ImpossibleDeviceMemory,
|
||||
HighCPUCount: b.HighCPUCount,
|
||||
MissingChromeObject: b.MissingChromeObject,
|
||||
WebdriverIframe: b.WebdriverIframe,
|
||||
WebdriverWorker: b.WebdriverWorker,
|
||||
MismatchWebGLInWorker: b.MismatchWebGLInWorker,
|
||||
MismatchPlatformIframe: b.MismatchPlatformIframe,
|
||||
MismatchPlatformWorker: b.MismatchPlatformWorker,
|
||||
SwiftshaderRenderer: b.SwiftshaderRenderer,
|
||||
UtcTimezone: b.UTCTimezone,
|
||||
MismatchLanguages: b.MismatchLanguages,
|
||||
InconsistentEtsl: b.InconsistentEtsl,
|
||||
BotUserAgent: b.BotUserAgent,
|
||||
GpuMismatch: b.GPUMismatch,
|
||||
PlatformMismatch: b.PlatformMismatch,
|
||||
AnyDetected: b.AnyDetected,
|
||||
DetectedCount: int32(b.DetectedCount),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Redirecting…</title>
|
||||
<meta name="robots" content="noindex,nofollow">
|
||||
</head>
|
||||
<body>
|
||||
<p>You are being redirected. If your browser does not redirect automatically, reload this page.</p>
|
||||
</body>
|
||||
</html>
|
||||
Binary file not shown.
@@ -0,0 +1,84 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
challengejs "github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/js"
|
||||
)
|
||||
|
||||
// TestNewChallengeRuntimeStartupBudget ensures NewChallengeRuntime returns
|
||||
// quickly (seeded from the baked-in bundle), without paying the synchronous
|
||||
// obfuscation it would otherwise pay.
|
||||
func TestNewChallengeRuntimeStartupBudget(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("timing is not reliable on windows CI (wazero/WASM obfuscation runs ~2.5x slower)")
|
||||
}
|
||||
|
||||
const budget = 10 * time.Second
|
||||
|
||||
start := time.Now()
|
||||
rt, err := NewChallengeRuntime(t.Context())
|
||||
dur := time.Since(start)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, rt)
|
||||
t.Logf("NewChallengeRuntime returned in %s", dur)
|
||||
|
||||
if dur > budget {
|
||||
t.Fatalf("NewChallengeRuntime took %s, exceeds budget of %s; baked-in initial bundle should make this near-instant", dur, budget)
|
||||
}
|
||||
|
||||
// The challenge code must be loaded so the very first request can be served.
|
||||
require.NotEmpty(t, rt.getChallengeCode(), "challenge code should be seeded from the baked-in initial bundle")
|
||||
}
|
||||
|
||||
// TestSeedCacheFromInitialBundle directly exercises the seeding helper and
|
||||
// asserts the bundle decompresses to non-empty obfuscated JS that holds the
|
||||
// challenge crypto code — and crucially NOT the public fpscanner (which is now
|
||||
// served separately and unobfuscated).
|
||||
func TestSeedCacheFromInitialBundle(t *testing.T) {
|
||||
rt := &ChallengeRuntime{}
|
||||
|
||||
require.NoError(t, rt.seedCacheFromInitialBundle())
|
||||
|
||||
code := rt.getChallengeCode()
|
||||
require.NotEmpty(t, code)
|
||||
|
||||
// The obfuscated challenge code inflates well beyond its tiny source.
|
||||
t.Logf("seeded challenge code size: %d bytes", len(code))
|
||||
require.Greater(t, len(code), 50_000,
|
||||
"seeded challenge code is suspiciously small; was it actually obfuscated?")
|
||||
|
||||
// Path placeholders must have been substituted before obfuscation.
|
||||
require.NotContains(t, code, "__CROWDSEC_SUBMIT_PATH__",
|
||||
"submit path placeholder still present; substitution missed at build time")
|
||||
require.NotContains(t, code, "__CROWDSEC_POW_WORKER_PATH__",
|
||||
"pow worker path placeholder still present; substitution missed at build time")
|
||||
|
||||
// The fpscanner sources must NOT be baked into the obfuscated challenge
|
||||
// code anymore — they live in the separately-served fpscanner bundle.
|
||||
// fpscanner-only detection strings are a reliable tell.
|
||||
require.NotContains(t, strings.ToLower(code), "swiftshader",
|
||||
"fpscanner code leaked into the obfuscated challenge bundle (it should be served separately)")
|
||||
require.NotContains(t, strings.ToLower(code), "playwright",
|
||||
"fpscanner code leaked into the obfuscated challenge bundle (it should be served separately)")
|
||||
}
|
||||
|
||||
// TestFPScannerBundleServedRaw asserts the public fpscanner bundle is the
|
||||
// unobfuscated, separately-served asset: it exposes the scanner on globalThis
|
||||
// and still contains the readable fpscanner detection logic.
|
||||
func TestFPScannerBundleServedRaw(t *testing.T) {
|
||||
require.NotEmpty(t, challengejs.FPScannerJS)
|
||||
require.Equal(t, challengejs.FPScannerJS, FPScannerJS,
|
||||
"challenge.FPScannerJS must re-export the js subpackage bundle verbatim")
|
||||
|
||||
require.Contains(t, FPScannerJS, "globalThis.CrowdsecFingerprintScanner",
|
||||
"served fpscanner bundle must expose the scanner on globalThis for the challenge code")
|
||||
require.Contains(t, strings.ToLower(FPScannerJS), "playwright",
|
||||
"served fpscanner bundle is minified but NOT obfuscated; detection strings should remain readable")
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
# WAF challenge JS pipeline
|
||||
|
||||
This directory holds the JavaScript sources that the WAF challenge runtime
|
||||
serves to clients, and the build-time tools that turn them into the
|
||||
artifacts embedded in the crowdsec binary.
|
||||
|
||||
End users running `make build` do **not** need any of the tools below —
|
||||
the generated artifacts are committed to the repository and pulled in via
|
||||
`//go:embed`. The pipeline is only exercised when someone intentionally
|
||||
changes the JS sources.
|
||||
|
||||
## What gets generated
|
||||
|
||||
Three pipeline steps, all driven by `go:generate` directives in
|
||||
`generate.go`:
|
||||
|
||||
| Step | Tool used | Output | Used at runtime by |
|
||||
|---|---|---|---|
|
||||
| `cmd/bundle` | esbuild (Go module) | `fpscanner/fpscanner.js` **and** `challenge_code.js` | `fpscanner.js` is served as-is; `challenge_code.js` is the obfuscator input |
|
||||
| `cmd/obfuscate` | esbuild + `javy` | `obfuscate/index.wasm.gz` | `wazero` to obfuscate the dynamic key module per epoch |
|
||||
| `cmd/initialbundle` | `wazero` running the obfuscator wasm | `../initial_bundle.js.gz` | Seeded into the challenge cache at startup so the service is ready instantly |
|
||||
|
||||
The client JS is split into **two** bundles:
|
||||
|
||||
- **`fpscanner/fpscanner.js`** — the public fingerprint scanner
|
||||
(`fpscanner/src/*`, entered via `fpscanner/global.js`), built as a minified
|
||||
IIFE that assigns `globalThis.CrowdsecFingerprintScanner`. It is **served
|
||||
unobfuscated** at `ChallengeFPScannerPath` via a plain `<script src>` tag
|
||||
(cacheable across challenge pages). fpscanner is public open-source code, so
|
||||
there is nothing to hide and no reason to spend CPU obfuscating it.
|
||||
- **`challenge_code.js`** — the crypto/glue (`../challenge.js`: SHA-256, HMAC,
|
||||
PoW, fingerprint obfuscation, submission, and the hook registration). This is
|
||||
the **obfuscation input**: `cmd/initialbundle` substitutes the internal-path
|
||||
placeholders and obfuscates it into `../initial_bundle.js.gz`, which the
|
||||
runtime injects inline on the challenge page. The challenge code reads the
|
||||
scanner from `globalThis.CrowdsecFingerprintScanner` at use time.
|
||||
|
||||
There is no runtime re-obfuscation of the challenge code — the build-time
|
||||
`initial_bundle.js.gz` is the single static variant served. Only the sensitive
|
||||
per-epoch key module is re-obfuscated at runtime (see `../dynamic_module.go`).
|
||||
|
||||
`obfuscate/obfuscate.js` is a small driver around the vendored
|
||||
`obfuscate/javascript-obfuscator.js` (≈1.6 MB, kept in tree so we don't
|
||||
pull from npm at build time). The driver reads source code from stdin,
|
||||
runs `JavaScriptObfuscator.obfuscate(...)` with the `high-obfuscation`
|
||||
preset (plus a `reservedStrings` entry for the `__CSEC_CHALLENGE_HOOK_v1__`
|
||||
sentinel that bridges the static and dynamic bundles), and writes the
|
||||
obfuscated code to stdout. `javy` then compiles that driver to a WASM
|
||||
module.
|
||||
|
||||
## How to regenerate
|
||||
|
||||
```sh
|
||||
make generate-challenge-js
|
||||
```
|
||||
|
||||
Or directly:
|
||||
|
||||
```sh
|
||||
go generate ./pkg/appsec/challenge/js/...
|
||||
```
|
||||
|
||||
Both invocations require `javy` (the Bytecode Alliance JS-to-WASM
|
||||
compiler) to be on `PATH`. There is no apt/brew package; install via
|
||||
the upstream releases:
|
||||
|
||||
```sh
|
||||
curl -sL -o javy.gz \
|
||||
https://github.com/bytecodealliance/javy/releases/download/v8.1.1/javy-x86_64-linux-v8.1.1.gz
|
||||
gunzip javy.gz && chmod +x javy && mv javy /usr/local/bin/
|
||||
```
|
||||
|
||||
(Substitute the platform suffix for macOS/Windows.) `javy` is
|
||||
**build-time only** — it is not a runtime dependency of crowdsec.
|
||||
|
||||
Regeneration is fast now: only the small `challenge_code.js` (~4 KB) goes
|
||||
through the obfuscator's `high-obfuscation` preset; the bulk of the client
|
||||
JS (fpscanner, ~33 KB) is served raw and is never obfuscated.
|
||||
|
||||
> `javy` is only needed by `cmd/obfuscate` (it rebuilds `index.wasm.gz`). If
|
||||
> you only changed `challenge.js` or the fpscanner sources, you can skip javy
|
||||
> and regenerate just the two bundles:
|
||||
>
|
||||
> ```sh
|
||||
> cd pkg/appsec/challenge/js && go run ./cmd/bundle && go run ./cmd/initialbundle
|
||||
> ```
|
||||
|
||||
## When to regenerate
|
||||
|
||||
Run `make generate-challenge-js` (or the two-step command above) after
|
||||
changing any of:
|
||||
|
||||
- `pkg/appsec/challenge/challenge.js`
|
||||
- `pkg/appsec/challenge/dynamic_module.js.tmpl`
|
||||
- `pkg/appsec/challenge/js/fpscanner/global.js`
|
||||
- `pkg/appsec/challenge/js/fpscanner/src/**/*.ts`
|
||||
- `pkg/appsec/challenge/js/obfuscate/obfuscate.js`
|
||||
- The vendored `pkg/appsec/challenge/js/obfuscate/javascript-obfuscator.js`
|
||||
(only when intentionally upgrading)
|
||||
|
||||
The pipeline output bytes (`fpscanner/fpscanner.js`, `challenge_code.js`,
|
||||
`index.wasm.gz`, `initial_bundle.js.gz`) are committed so a clean `git clone`
|
||||
builds and runs without javy.
|
||||
|
||||
## Sentinel survival
|
||||
|
||||
The `__CSEC_CHALLENGE_HOOK_v1__` literal must appear verbatim in both
|
||||
the obfuscated challenge code (`initial_bundle.js.gz`) and any dynamic key
|
||||
module generated at runtime, so the two bundles meet at the same `globalThis`
|
||||
key after independent obfuscation passes. The
|
||||
`reservedStrings: ["__CSEC_CHALLENGE_HOOK_v1__"]` option in
|
||||
`obfuscate/obfuscate.js` is what guarantees this. If you change the
|
||||
sentinel string, change it in **all three** places:
|
||||
|
||||
- `obfuscate/obfuscate.js` — `reservedStrings`
|
||||
- `pkg/appsec/challenge/challenge.js` — `CSEC_HOOK_NAME`
|
||||
- `pkg/appsec/challenge/dynamic_module.js.tmpl` — `hookName`
|
||||
|
||||
The Go-side regression test
|
||||
`TestSplitBundle_HookSentinelInBakedBundle` will fire on a mismatch.
|
||||
@@ -0,0 +1 @@
|
||||
var G=[1116352408,1899447441,3049323471,3921009573,961987163,1508970993,2453635748,2870763221,3624381080,310598401,607225278,1426881987,1925078388,2162078206,2614888103,3248222580,3835390401,4022224774,264347078,604807628,770255983,1249150122,1555081692,1996064986,2554220882,2821834349,2952996808,3210313671,3336571891,3584528711,113926993,338241895,666307205,773529912,1294757372,1396182291,1695183700,1986661051,2177026350,2456956037,2730485921,2820302411,3259730800,3345764771,3516065817,3600352804,4094571909,275423344,430227734,506948616,659060556,883997877,958139571,1322822218,1537002063,1747873779,1955562222,2024104815,2227730452,2361852424,2428436474,2756734187,3204031479,3329325298];function S(e){let n=1779033703,t=3144134277,s=1013904242,r=2773480762,f=1359893119,a=2600822924,i=528734635,c=1541459225,h=e.length,F=h*8,T=64-(h+9)%64,U=h+1+(T===64?0:T)+8,A=new Uint8Array(U);A.set(e),A[h]=128;let v=new DataView(A.buffer);v.setUint32(U-4,F,!1);let x=new Int32Array(64);for(let C=0;C<U;C+=64){for(let o=0;o<16;o++)x[o]=v.getInt32(C+o*4,!1);for(let o=16;o<64;o++){let E=(x[o-15]>>>7|x[o-15]<<25)^(x[o-15]>>>18|x[o-15]<<14)^x[o-15]>>>3|0,O=(x[o-2]>>>17|x[o-2]<<15)^(x[o-2]>>>19|x[o-2]<<13)^x[o-2]>>>10|0;x[o]=x[o-16]+E+x[o-7]+O|0}let l=n,w=t,g=s,P=r,d=f,p=a,_=i,H=c;for(let o=0;o<64;o++){let E=(d>>>6|d<<26)^(d>>>11|d<<21)^(d>>>25|d<<7)|0,O=d&p^~d&_|0,k=H+E+O+G[o]+x[o]|0,N=(l>>>2|l<<30)^(l>>>13|l<<19)^(l>>>22|l<<10)|0,j=l&w^l&g^w&g|0,V=N+j|0;H=_,_=p,p=d,d=P+k|0,P=g,g=w,w=l,l=k+V|0}n=n+l|0,t=t+w|0,s=s+g|0,r=r+P|0,f=f+d|0,a=a+p|0,i=i+_|0,c=c+H|0}let W=new Uint8Array(32),u=new DataView(W.buffer);return u.setUint32(0,n,!1),u.setUint32(4,t,!1),u.setUint32(8,s,!1),u.setUint32(12,r,!1),u.setUint32(16,f,!1),u.setUint32(20,a,!1),u.setUint32(24,i,!1),u.setUint32(28,c,!1),W}function R(e){let n="";for(let t=0;t<e.length;t++)n+=(e[t]>>4).toString(16)+(e[t]&15).toString(16);return n}function b(e){return new TextEncoder().encode(e)}function I(e,n){e.length>64&&(e=S(e));let t=new Uint8Array(64);t.set(e);let s=new Uint8Array(64),r=new Uint8Array(64);for(let c=0;c<64;c++)s[c]=t[c]^54,r[c]=t[c]^92;let f=new Uint8Array(64+n.length);f.set(s),f.set(n,64);let a=S(f),i=new Uint8Array(96);return i.set(r),i.set(a,64),S(i)}function D(e,n){return R(I(b(e),b(n)))}function J(e){let n=new Uint8Array(e.length/2);for(let t=0;t<n.length;t++)n[t]=parseInt(e.substr(t*2,2),16);return n}function q(e,n){return R(I(J(e),b(n)))}var z="__CROWDSEC_POW_WORKER_PATH__";function Q(e,n){if(n<=0)return Promise.resolve("0");try{let t=new Worker(z);return new Promise(s=>{t.onmessage=r=>{t.terminate(),s(r.data)},t.onerror=()=>{t.terminate(),s(K(e,n))},t.postMessage({p:e,d:n})})}catch{return Promise.resolve(K(e,n))}}function K(e,n){let t=0;for(;;){let s=e+t.toString(36),r=S(b(s)),f=n>>3,a=n&7,i=!0;for(let c=0;c<f;c++)if(r[c]!==0){i=!1;break}if(i&&a>0){let c=255<<8-a&255;(r[f]&c)!==0&&(i=!1)}if(i)return t.toString(36);t++}}function X(e,n){let t=b(e),s=b(n),r=new Uint8Array(s.length);for(let a=0;a<s.length;a++)r[a]=s[a]^t[a%t.length];let f=String.fromCharCode(...r);return btoa(f)}var M=typeof _ts<"u"?_ts:"",B=typeof _powP<"u"?_powP:"",Y=typeof _powM<"u"?_powM:"",L=typeof _powD<"u"?_powD:12,y=typeof _r<"u"?_r:"",Z="__CROWDSEC_SUBMIT_PATH__";function m(e){if(typeof window.crowdsecSetChallengeStatus=="function"){window.crowdsecSetChallengeStatus(e);return}window.dispatchEvent(new CustomEvent("crowdsec-challenge-status",{detail:{status:e}}))}var $="__CSEC_CHALLENGE_HOOK_v1__";async function ee(e){let n=globalThis.CrowdsecFingerprintScanner;if(typeof n!="function"){m("fail");return}let[t,s]=await Promise.all([Q(B,L),new n().collectFingerprint({encrypt:!1})]),r=q(e,y),f=D(r,"fpenc"+y),a=X(f,JSON.stringify(s)),i=D(r,y+M+t+a);return fetch(Z,{method:"POST",credentials:"same-origin",body:new URLSearchParams({f:a,r:y,ts:M,sig:i,n:t,p:B,m:Y,d:L,u:window.location.pathname})}).then(c=>c.json()).then(c=>{let h=typeof c?.status=="string"?c.status:"fail";m(h)}).catch(()=>{m("fail")})}globalThis[$]=function(e){if(!e||typeof e.key!="string"||e.key.length===0){m("fail");return}ee(e.key)};
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
esbuildapi "github.com/evanw/esbuild/pkg/api"
|
||||
)
|
||||
|
||||
// The challenge JS is shipped as two separate bundles:
|
||||
//
|
||||
// - fpscanner.js — the public fingerprint scanner, served UNOBFUSCATED via a
|
||||
// plain <script src> tag (cacheable). Built as a minified IIFE that assigns
|
||||
// globalThis.CrowdsecFingerprintScanner (see fpscanner/global.js).
|
||||
// - challenge_code.js — the crypto/glue (SHA-256, HMAC, PoW, submission, hook
|
||||
// registration). This is the obfuscation input consumed by ./cmd/initialbundle.
|
||||
type bundleTarget struct {
|
||||
entryPoint string
|
||||
outFile string
|
||||
format esbuildapi.Format
|
||||
}
|
||||
|
||||
var targets = []bundleTarget{
|
||||
{entryPoint: "fpscanner/global.js", outFile: "fpscanner/fpscanner.js", format: esbuildapi.FormatIIFE},
|
||||
{entryPoint: "../challenge.js", outFile: "challenge_code.js", format: esbuildapi.FormatESModule},
|
||||
}
|
||||
|
||||
func main() {
|
||||
for _, t := range targets {
|
||||
if err := build(t); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "bundle %s: %v\n", t.outFile, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func build(t bundleTarget) error {
|
||||
result := esbuildapi.Build(esbuildapi.BuildOptions{
|
||||
EntryPoints: []string{t.entryPoint},
|
||||
Bundle: true,
|
||||
Write: false,
|
||||
Format: t.format,
|
||||
Platform: esbuildapi.PlatformBrowser,
|
||||
Target: esbuildapi.ES2022,
|
||||
MinifyWhitespace: true,
|
||||
MinifyIdentifiers: true,
|
||||
MinifySyntax: true,
|
||||
Sourcemap: esbuildapi.SourceMapNone,
|
||||
})
|
||||
|
||||
if len(result.Errors) > 0 {
|
||||
for _, msg := range result.Errors {
|
||||
fmt.Fprintln(os.Stderr, msg.Text)
|
||||
}
|
||||
return fmt.Errorf("esbuild failed with %d error(s)", len(result.Errors))
|
||||
}
|
||||
|
||||
if len(result.OutputFiles) == 0 {
|
||||
return errors.New("esbuild returned no output files")
|
||||
}
|
||||
|
||||
if err := os.WriteFile(t.outFile, result.OutputFiles[0].Contents, 0o644); err != nil {
|
||||
return fmt.Errorf("write %s: %w", t.outFile, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// initialbundle is a build-time tool that produces a pre-obfuscated initial
|
||||
// challenge bundle, embedded into the Go binary. This eliminates the ~1 minute
|
||||
// of synchronous obfuscation that NewChallengeRuntime would otherwise have to
|
||||
// pay at startup before the service is ready to serve challenges.
|
||||
//
|
||||
// Pipeline:
|
||||
//
|
||||
// challenge_code.js ──substitute placeholders──▶ source JS
|
||||
// obfuscate/index.wasm.gz ──decompress──▶ obfuscator WASM
|
||||
// wazero(obfuscator).Run(stdin=source) ──▶ obfuscated JS
|
||||
// gzip ──▶ ../initial_bundle.js.gz
|
||||
//
|
||||
// The output `initial_bundle.js.gz` is committed and embedded via go:embed in
|
||||
// pkg/appsec/challenge/challenge.go.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/tetratelabs/wazero"
|
||||
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
|
||||
)
|
||||
|
||||
// These must match the ChallengeSubmitPath / ChallengePowWorkerPath constants
|
||||
// in pkg/appsec/challenge/challenge.go. Kept as plain strings here to avoid an
|
||||
// import cycle (pkg/appsec/challenge depends on this js subpackage).
|
||||
const (
|
||||
submitPath = "/crowdsec-internal/challenge/submit"
|
||||
powWorkerPath = "/crowdsec-internal/challenge/pow-worker.js"
|
||||
)
|
||||
|
||||
const (
|
||||
// Inputs (relative to pkg/appsec/challenge/js where this tool runs).
|
||||
challengeCodePath = "challenge_code.js"
|
||||
obfuscatorWasmGz = "obfuscate/index.wasm.gz"
|
||||
// Output (relative to pkg/appsec/challenge/js).
|
||||
outputPath = "../initial_bundle.js.gz"
|
||||
)
|
||||
|
||||
func main() {
|
||||
start := time.Now()
|
||||
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "initialbundle: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "initialbundle: wrote %s in %s\n", outputPath, time.Since(start).Round(time.Second))
|
||||
}
|
||||
|
||||
func run() error {
|
||||
source, err := buildSourceBundle()
|
||||
if err != nil {
|
||||
return fmt.Errorf("build source bundle: %w", err)
|
||||
}
|
||||
|
||||
wasmBytes, err := loadObfuscatorWasm()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load obfuscator wasm: %w", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
r := wazero.NewRuntime(ctx)
|
||||
defer r.Close(ctx)
|
||||
|
||||
if _, err := wasi_snapshot_preview1.Instantiate(ctx, r); err != nil {
|
||||
return fmt.Errorf("instantiate wasi: %w", err)
|
||||
}
|
||||
|
||||
compiled, err := r.CompileModule(ctx, wasmBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compile obfuscator wasm: %w", err)
|
||||
}
|
||||
|
||||
obfuscated, err := obfuscate(ctx, r, compiled, source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("obfuscate: %w", err)
|
||||
}
|
||||
|
||||
if obfuscated == "" {
|
||||
return errors.New("obfuscator produced empty output")
|
||||
}
|
||||
|
||||
if err := writeGzip(outputPath, obfuscated); err != nil {
|
||||
return fmt.Errorf("write %s: %w", outputPath, err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "initialbundle: input %d bytes -> obfuscated %d bytes -> compressed file written\n",
|
||||
len(source), len(obfuscated))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildSourceBundle() (string, error) {
|
||||
raw, err := os.ReadFile(challengeCodePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s: %w", challengeCodePath, err)
|
||||
}
|
||||
|
||||
// Same substitution as ChallengeRuntime.buildChallengeBundle in challenge.go.
|
||||
source := strings.NewReplacer(
|
||||
"__CROWDSEC_SUBMIT_PATH__", submitPath,
|
||||
"__CROWDSEC_POW_WORKER_PATH__", powWorkerPath,
|
||||
).Replace(string(raw))
|
||||
|
||||
return source, nil
|
||||
}
|
||||
|
||||
func loadObfuscatorWasm() ([]byte, error) {
|
||||
f, err := os.Open(obfuscatorWasmGz)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gzip reader: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
return io.ReadAll(gz)
|
||||
}
|
||||
|
||||
func obfuscate(ctx context.Context, r wazero.Runtime, compiled wazero.CompiledModule, source string) (string, error) {
|
||||
stdin := bytes.NewReader([]byte(source))
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
cfg := wazero.NewModuleConfig().
|
||||
WithStdin(stdin).
|
||||
WithStdout(&stdout).
|
||||
WithStderr(&stderr)
|
||||
|
||||
mod, err := r.InstantiateModule(ctx, compiled, cfg)
|
||||
if err != nil {
|
||||
if stderr.Len() > 0 {
|
||||
return "", fmt.Errorf("wasm runtime error: %v | stderr: %s", err, stderr.String())
|
||||
}
|
||||
return "", fmt.Errorf("wasm instantiation: %w", err)
|
||||
}
|
||||
mod.Close(ctx)
|
||||
|
||||
return stdout.String(), nil
|
||||
}
|
||||
|
||||
func writeGzip(path string, data string) error {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
zw, err := gzip.NewWriterLevel(f, gzip.BestCompression)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gzip writer: %w", err)
|
||||
}
|
||||
zw.Name = "initial_bundle.js"
|
||||
zw.ModTime = time.Unix(0, 0)
|
||||
|
||||
if _, err := zw.Write([]byte(data)); err != nil {
|
||||
zw.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
return zw.Close()
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
esbuildapi "github.com/evanw/esbuild/pkg/api"
|
||||
)
|
||||
|
||||
const (
|
||||
entryPoint = "obfuscate/obfuscate.js"
|
||||
bundleFile = "obfuscate/bundle.js"
|
||||
wasmFile = "obfuscate/index.wasm"
|
||||
wasmGzFile = "obfuscate/index.wasm.gz"
|
||||
)
|
||||
|
||||
func main() {
|
||||
bundleData, err := buildObfuscatorBundle()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to build obfuscator bundle: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// javy output is not byte-reproducible, so rebuilding from unchanged
|
||||
// sources would always dirty index.wasm.gz. The inputs (bundle content,
|
||||
// javy version) are recorded in the gzip header comment; skip the rebuild
|
||||
// when they match the existing file.
|
||||
buildKey, err := computeBuildKey(bundleData)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to compute build key: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if currentBuildKey() == buildKey {
|
||||
fmt.Fprintf(os.Stderr, "obfuscate: %s is up to date, skipping javy build\n", wasmGzFile)
|
||||
return
|
||||
}
|
||||
|
||||
if err := os.WriteFile(bundleFile, bundleData, 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to write %s: %v\n", bundleFile, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := buildWASM(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to build obfuscator wasm: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := gzipWASM(buildKey); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to compress obfuscator wasm: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func buildObfuscatorBundle() ([]byte, error) {
|
||||
result := esbuildapi.Build(esbuildapi.BuildOptions{
|
||||
EntryPoints: []string{entryPoint},
|
||||
Bundle: true,
|
||||
Write: false,
|
||||
Format: esbuildapi.FormatIIFE,
|
||||
Platform: esbuildapi.PlatformBrowser,
|
||||
Target: esbuildapi.ES2022,
|
||||
Sourcemap: esbuildapi.SourceMapNone,
|
||||
// JS obfuscator expects window/global to be available, but in WASM we don't have them, so we alias them to globalThis
|
||||
Banner: map[string]string{"js": "var self=globalThis; var window=globalThis; var global=globalThis;"},
|
||||
})
|
||||
|
||||
if len(result.Errors) > 0 {
|
||||
return nil, fmt.Errorf("esbuild failed with %d error(s): %s", len(result.Errors), result.Errors[0].Text)
|
||||
}
|
||||
if len(result.OutputFiles) == 0 {
|
||||
return nil, errors.New("esbuild returned no output files")
|
||||
}
|
||||
|
||||
return result.OutputFiles[0].Contents, nil
|
||||
}
|
||||
|
||||
func computeBuildKey(bundleData []byte) (string, error) {
|
||||
out, err := exec.CommandContext(context.Background(), "javy", "--version").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to run 'javy --version' (is javy in PATH?): %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("bundle-sha256=%x %s", sha256.Sum256(bundleData), strings.TrimSpace(string(out))), nil
|
||||
}
|
||||
|
||||
// currentBuildKey returns the build key recorded in the existing wasm.gz
|
||||
// header, or "" if the file is missing or unreadable.
|
||||
func currentBuildKey() string {
|
||||
f, err := os.Open(wasmGzFile)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
zr, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
return zr.Comment
|
||||
}
|
||||
|
||||
func buildWASM() error {
|
||||
cmd := exec.CommandContext(context.Background(), "javy", "build", "-o", wasmFile, bundleFile)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("javy build failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func gzipWASM(buildKey string) error {
|
||||
wasmData, err := os.ReadFile(wasmFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read %s: %w", wasmFile, err)
|
||||
}
|
||||
|
||||
f, err := os.Create(wasmGzFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create %s: %w", wasmGzFile, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
zw, err := gzip.NewWriterLevel(f, gzip.BestCompression)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create gzip writer: %w", err)
|
||||
}
|
||||
zw.Name = "index.wasm"
|
||||
zw.ModTime = time.Unix(0, 0)
|
||||
zw.Comment = buildKey
|
||||
|
||||
if _, err := zw.Write(wasmData); err != nil {
|
||||
zw.Close()
|
||||
return fmt.Errorf("failed to write gzip data: %w", err)
|
||||
}
|
||||
|
||||
if err := zw.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close gzip writer: %w", err)
|
||||
}
|
||||
|
||||
os.Remove(wasmFile)
|
||||
os.Remove(bundleFile)
|
||||
|
||||
return nil
|
||||
}
|
||||
+1
Submodule pkg/appsec/challenge/js/fpscanner/fpscanner added at bc6d53f220
File diff suppressed because one or more lines are too long
@@ -0,0 +1,5 @@
|
||||
// Entry point for the fpscanner bundle: exposes the scanner on globalThis.
|
||||
// Built by ./cmd/bundle into ./fpscanner.js (minified IIFE, served as-is).
|
||||
import FingerprintScanner from "./fpscanner/src/index.ts";
|
||||
|
||||
globalThis.CrowdsecFingerprintScanner = FingerprintScanner;
|
||||
@@ -0,0 +1,5 @@
|
||||
package js
|
||||
|
||||
//go:generate go run ./cmd/bundle
|
||||
//go:generate go run ./cmd/obfuscate
|
||||
//go:generate go run ./cmd/initialbundle
|
||||
@@ -0,0 +1,27 @@
|
||||
// This package contains all JavaScript-related code for the WAF challenge and fingerprinting.
|
||||
// fpscanner is https://github.com/antoinevastel/fpscanner/
|
||||
// obfuscate is a custom wrapper around javascript-obfuscator.
|
||||
// It uses a vendored bundle of javascript-obfuscator to avoid any dependency on npm at build time.
|
||||
// The goal is to build a WASM module that can dynamically obfuscate the challenge code at runtime.
|
||||
//
|
||||
// The client JS is split into two bundles (built by ./cmd/bundle):
|
||||
// - FPScannerJS: the public fingerprint scanner, served UNOBFUSCATED via its
|
||||
// own <script src> tag (cacheable across challenge pages).
|
||||
// - ChallengeCode: the crypto/glue, obfuscated at build time (./cmd/initialbundle)
|
||||
// into ../initial_bundle.js.gz and injected inline on the challenge page.
|
||||
|
||||
package js
|
||||
|
||||
import _ "embed"
|
||||
|
||||
// FPScannerJS is the standalone, unobfuscated fpscanner bundle served as-is.
|
||||
//
|
||||
//go:embed fpscanner/fpscanner.js
|
||||
var FPScannerJS string
|
||||
|
||||
// ChallengeCode is the minified (not-yet-obfuscated) challenge crypto/glue code.
|
||||
// It still carries the __CROWDSEC_*_PATH__ placeholders; the runtime substitutes
|
||||
// them before obfuscation (see buildChallengeBundle in static_bundle.go).
|
||||
//
|
||||
//go:embed challenge_code.js
|
||||
var ChallengeCode string
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
@@ -0,0 +1,56 @@
|
||||
// Downloaded from https://cdn.jsdelivr.net/npm/javascript-obfuscator/dist/index.browser.js
|
||||
import JavaScriptObfuscator from "./javascript-obfuscator.js";
|
||||
|
||||
function readStdin() {
|
||||
const buffer = new Uint8Array(1024);
|
||||
const input = [];
|
||||
while (true) {
|
||||
const bytesRead = Javy.IO.readSync(0, buffer);
|
||||
if (bytesRead === 0) break;
|
||||
const chunk = buffer.subarray(0, bytesRead);
|
||||
for (let i = 0; i < bytesRead; i++) {
|
||||
input.push(chunk[i]);
|
||||
}
|
||||
}
|
||||
return new TextDecoder().decode(new Uint8Array(input));
|
||||
}
|
||||
|
||||
function writeStdout(str) {
|
||||
const buffer = new TextEncoder().encode(str);
|
||||
Javy.IO.writeSync(1, buffer);
|
||||
}
|
||||
|
||||
try {
|
||||
const sourceCode = readStdin();
|
||||
|
||||
// Spread the high-obfuscation preset, then add reservedStrings so the
|
||||
// sentinel that bridges the static bundle and the per-epoch dynamic key
|
||||
// module (CSEC_CHALLENGE_HOOK_v1) survives the string-array transform
|
||||
// identically in both bundles. Without this, the two independently-
|
||||
// obfuscated artifacts wouldn't agree on the globalThis symbol they
|
||||
// need to meet at.
|
||||
//
|
||||
// disableConsoleOutput is forced off because fpscanner's CDP detection
|
||||
// (signals/cdp.ts) relies on `console.log(err)` triggering DevTools'
|
||||
// eager access of `err.stack`, which in turn fires our overridden
|
||||
// `Error.prepareStackTrace`. The preset's default `true` rewrites the
|
||||
// call into a no-op stub and the detection silently always reports
|
||||
// false. Leave it off so the side-channel works.
|
||||
const opts = Object.assign(
|
||||
{},
|
||||
JavaScriptObfuscator.getOptionsByPreset("high-obfuscation"),
|
||||
{
|
||||
reservedStrings: ["__CSEC_CHALLENGE_HOOK_v1__"],
|
||||
disableConsoleOutput: false,
|
||||
},
|
||||
);
|
||||
|
||||
const result = JavaScriptObfuscator.obfuscate(sourceCode, opts);
|
||||
|
||||
writeStdout(result.getObfuscatedCode());
|
||||
} catch (e) {
|
||||
const msg = (e && e.message) ? e.message : String(e);
|
||||
const buffer = new TextEncoder().encode(`obfuscate: ${msg}\n`);
|
||||
Javy.IO.writeSync(2, buffer);
|
||||
throw e;
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
// keyring.go derives and rotates the per-epoch HMAC signing keys (and the
|
||||
// long-lived AES cookie-sealing key) from the master secret using HKDF.
|
||||
// Epoch advances on a fixed cadence (keyringDefaultRotation, 5m); past
|
||||
// epochs stay valid for the configured live window so in-flight tickets
|
||||
// don't get invalidated at the boundary. The cookie key is derived once and
|
||||
// reused — cookie expiration is enforced by an explicit not_after stamp
|
||||
// inside the sealed envelope (see crypto.go), not by key rotation.
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/crypto/hkdf"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/metrics"
|
||||
)
|
||||
|
||||
// keyringHKDFSalt and the per-context info strings define the HKDF
|
||||
// derivations used to turn the long-lived master secret into operational
|
||||
// keys. They are part of the wire protocol — changing them silently would
|
||||
// invalidate every outstanding ticket and cookie across a fleet upgrade.
|
||||
// Bump the version suffix when intentionally rotating.
|
||||
//
|
||||
// Two independent derivation contexts are used:
|
||||
//
|
||||
// - "epoch-sign": per-epoch HMAC key for ticket / PoW MAC signing.
|
||||
// Rotates with the keyring (tight forgery window).
|
||||
// - "cookie-master": single long-lived AES key for cookie sealing.
|
||||
// Lifetime equals master_secret's lifetime; cookie
|
||||
// expiration is enforced by an explicit not_after
|
||||
// timestamp inside the sealed envelope (see crypto.go).
|
||||
//
|
||||
// The two contexts produce cryptographically independent keys: leaking
|
||||
// one tells the attacker nothing about the other.
|
||||
const (
|
||||
keyringHKDFSalt = "crowdsec-challenge-keyring-v1"
|
||||
keyringInfoSign = "epoch-sign"
|
||||
keyringInfoMasterCookie = "cookie-master"
|
||||
keyringDerivedKeyBytes = 32 // 256-bit keys throughout
|
||||
keyringClockSkew = 1 // accept currentEpoch + 1
|
||||
keyringDefaultMaxLive = 3 // currentEpoch and 2 prior
|
||||
keyringDefaultRotation = 5 * time.Minute
|
||||
keyringMinRotation = 30 * time.Second
|
||||
)
|
||||
|
||||
// KeyRing produces keys deterministically from a shared master secret. Two
|
||||
// instances configured with the same masterSecret derive bit-identical
|
||||
// keys — the property that lets distributed (multi-WAF) deployments sign
|
||||
// and verify each other's challenges and cookies without coordination.
|
||||
//
|
||||
// Two key families with different lifetimes:
|
||||
//
|
||||
// - Per-epoch signing key (rotates on rotationInterval). Used for ticket
|
||||
// HMAC and PoW MAC. Live window:
|
||||
// [currentEpoch - maxLive + 1 ... currentEpoch + clockSkew]
|
||||
// Epochs outside the window are rejected, bounding ticket forgery
|
||||
// exposure to maxLive * rotationInterval.
|
||||
//
|
||||
// - Long-lived master cookie key (no rotation). Used for AES-GCM cookie
|
||||
// sealing. Cookie expiration is enforced by an explicit not_after
|
||||
// timestamp inside the sealed envelope, NOT by key eviction — so
|
||||
// cookie TTL can be much larger than the ticket window without
|
||||
// widening ticket forgery exposure.
|
||||
type KeyRing struct {
|
||||
masterSecret []byte
|
||||
rotationInterval time.Duration
|
||||
maxLive int
|
||||
clockSkew int
|
||||
|
||||
// masterCookieKey is the long-lived cookie-sealing key; derived once
|
||||
// at construction. Pointer-receiver methods read it without locking
|
||||
// because it's never mutated after NewKeyRing returns.
|
||||
masterCookieKey []byte
|
||||
|
||||
now func() time.Time // overridable for tests
|
||||
|
||||
logger *log.Entry
|
||||
|
||||
mu sync.RWMutex
|
||||
cache map[int64][]byte // epoch -> per-epoch sign key
|
||||
}
|
||||
|
||||
// log returns the component logger (never nil; see the logger field).
|
||||
func (k *KeyRing) log() *log.Entry {
|
||||
return k.logger
|
||||
}
|
||||
|
||||
// NewKeyRing constructs a KeyRing. masterSecret must be at least minSecretBytes
|
||||
// long (callers should already have validated this via WithMasterSecret); the
|
||||
// rotation interval must be at least keyringMinRotation. maxLive defaults to
|
||||
// keyringDefaultMaxLive when zero.
|
||||
func NewKeyRing(masterSecret []byte, rotationInterval time.Duration, maxLive int) (*KeyRing, error) {
|
||||
if len(masterSecret) < minSecretBytes {
|
||||
return nil, fmt.Errorf("keyring master secret is %d bytes; minimum is %d", len(masterSecret), minSecretBytes)
|
||||
}
|
||||
if rotationInterval < keyringMinRotation {
|
||||
return nil, fmt.Errorf("keyring rotation interval %s is below the floor %s", rotationInterval, keyringMinRotation)
|
||||
}
|
||||
if maxLive <= 0 {
|
||||
maxLive = keyringDefaultMaxLive
|
||||
}
|
||||
|
||||
return &KeyRing{
|
||||
masterSecret: masterSecret,
|
||||
rotationInterval: rotationInterval,
|
||||
maxLive: maxLive,
|
||||
clockSkew: keyringClockSkew,
|
||||
masterCookieKey: deriveMasterCookieKey(masterSecret),
|
||||
logger: log.StandardLogger().WithField("module", "challenge"),
|
||||
now: time.Now,
|
||||
cache: make(map[int64][]byte),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CurrentEpoch returns the epoch identifier for the current wall-clock time.
|
||||
// Equal across all instances with synchronized clocks.
|
||||
func (k *KeyRing) CurrentEpoch() int64 {
|
||||
return k.now().Unix() / int64(k.rotationInterval/time.Second)
|
||||
}
|
||||
|
||||
// Current returns the epoch and signing key that should be used to sign new
|
||||
// outbound material right now.
|
||||
func (k *KeyRing) Current() (int64, []byte) {
|
||||
epoch := k.CurrentEpoch()
|
||||
return epoch, k.deriveOrCache(epoch)
|
||||
}
|
||||
|
||||
// SignKey returns the HMAC key for an epoch if it's within the live window;
|
||||
// returns (nil, false) otherwise.
|
||||
func (k *KeyRing) SignKey(epoch int64) ([]byte, bool) {
|
||||
if !k.isLive(epoch) {
|
||||
return nil, false
|
||||
}
|
||||
return k.deriveOrCache(epoch), true
|
||||
}
|
||||
|
||||
// MasterCookieKey returns the long-lived AES-key-input for cookie sealing.
|
||||
// Does not depend on epoch; same value for the lifetime of the master
|
||||
// secret. Cookie expiration is enforced via an explicit not_after timestamp
|
||||
// inside the sealed envelope (see crypto.go).
|
||||
func (k *KeyRing) MasterCookieKey() []byte {
|
||||
return k.masterCookieKey
|
||||
}
|
||||
|
||||
// LiveEpochs returns every epoch currently in the live window, oldest first.
|
||||
// Used by callers that need to enumerate the acceptable ticket-signing epochs.
|
||||
func (k *KeyRing) LiveEpochs() []int64 {
|
||||
current := k.CurrentEpoch()
|
||||
out := make([]int64, 0, k.maxLive+k.clockSkew)
|
||||
for e := current - int64(k.maxLive-1); e <= current+int64(k.clockSkew); e++ {
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (k *KeyRing) isLive(epoch int64) bool {
|
||||
current := k.CurrentEpoch()
|
||||
return epoch >= current-int64(k.maxLive-1) && epoch <= current+int64(k.clockSkew)
|
||||
}
|
||||
|
||||
// deriveOrCache returns the cached per-epoch signing key or derives and
|
||||
// stores it. Pure function of (masterSecret, epoch) so two instances
|
||||
// always agree.
|
||||
func (k *KeyRing) deriveOrCache(epoch int64) []byte {
|
||||
k.mu.RLock()
|
||||
if cached, ok := k.cache[epoch]; ok {
|
||||
k.mu.RUnlock()
|
||||
return cached
|
||||
}
|
||||
k.mu.RUnlock()
|
||||
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
|
||||
// Re-check after acquiring the write lock.
|
||||
if cached, ok := k.cache[epoch]; ok {
|
||||
return cached
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
derived := deriveEpochKey(k.masterSecret, epoch, keyringInfoSign)
|
||||
derivationDuration := time.Since(start)
|
||||
|
||||
// Bound the cache so it can't grow without limit if the clock jumps.
|
||||
// Anything outside the live window is safe to evict.
|
||||
evicted := 0
|
||||
for cachedEpoch := range k.cache {
|
||||
if !k.isLive(cachedEpoch) {
|
||||
delete(k.cache, cachedEpoch)
|
||||
evicted++
|
||||
}
|
||||
}
|
||||
|
||||
k.cache[epoch] = derived
|
||||
|
||||
metrics.AppsecChallengeKepochGenerated.Inc()
|
||||
if evicted > 0 {
|
||||
metrics.AppsecChallengeKepochEvicted.Add(float64(evicted))
|
||||
}
|
||||
|
||||
k.log().WithFields(log.Fields{
|
||||
"epoch": epoch,
|
||||
"current_epoch": k.CurrentEpoch(),
|
||||
"derivation_us": derivationDuration.Microseconds(),
|
||||
"cache_evicted": evicted,
|
||||
"cache_size": len(k.cache),
|
||||
}).Debug("derived per-epoch key")
|
||||
|
||||
return derived
|
||||
}
|
||||
|
||||
// deriveEpochKey performs HKDF-SHA256 with a stable salt and a context-
|
||||
// specific info value: <context bytes> || ':' || <epoch_be8>. Two instances
|
||||
// with the same masterSecret always derive the same bytes for the same epoch
|
||||
// and context — this is what makes distributed agreement work.
|
||||
//
|
||||
// The colon separator prevents context-prefix collisions (e.g. avoids "epoch"
|
||||
// + "sign" colliding with "epochs" + "ign").
|
||||
func deriveEpochKey(masterSecret []byte, epoch int64, context string) []byte {
|
||||
info := make([]byte, 0, len(context)+1+8)
|
||||
info = append(info, context...)
|
||||
info = append(info, ':')
|
||||
var epochBytes [8]byte
|
||||
binary.BigEndian.PutUint64(epochBytes[:], uint64(epoch))
|
||||
info = append(info, epochBytes[:]...)
|
||||
|
||||
return hkdfExtract(masterSecret, info)
|
||||
}
|
||||
|
||||
// deriveMasterCookieKey performs HKDF-SHA256 with no epoch component, so
|
||||
// the output depends only on the master secret. Identical across all
|
||||
// instances sharing the master, stable for the lifetime of the master.
|
||||
// Cookie expiration is enforced separately by a not_after timestamp
|
||||
// inside the sealed envelope (see crypto.go).
|
||||
func deriveMasterCookieKey(masterSecret []byte) []byte {
|
||||
return hkdfExtract(masterSecret, []byte(keyringInfoMasterCookie))
|
||||
}
|
||||
|
||||
// hkdfExtract is the shared HKDF-SHA256 call used by both derivation helpers
|
||||
// (fixed keyringHKDFSalt, caller-supplied info). It panics on a Read error
|
||||
// because a 32-byte SHA-256 output is infallible (HKDF-Expand only fails above
|
||||
// 255*HashLen bytes); an error here means the crypto stack itself is broken,
|
||||
// not a recoverable condition, so threading an error through every call site
|
||||
// would be handling an impossible case.
|
||||
func hkdfExtract(masterSecret []byte, info []byte) []byte {
|
||||
r := hkdf.New(sha256.New, masterSecret, []byte(keyringHKDFSalt), info)
|
||||
|
||||
out := make([]byte, keyringDerivedKeyBytes)
|
||||
if _, err := r.Read(out); err != nil {
|
||||
panic(fmt.Sprintf("keyring: hkdf read failed (should be unreachable): %v", err))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,474 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestCookieV0_RoundTrip is the basic positive case: seal under the master
|
||||
// cookie key with an explicit not_after one hour in the future, open
|
||||
// immediately, fingerprint comes back.
|
||||
func TestCookieV0_RoundTrip(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
envelope := &pb.ChallengeCookie{PowDifficulty: 12}
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
|
||||
encoded, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(12), got.Envelope.GetPowDifficulty())
|
||||
}
|
||||
|
||||
// TestCookieV0_ExpiredRejected confirms the not_after timestamp embedded
|
||||
// inside the encrypted envelope is enforced on open.
|
||||
func TestCookieV0_ExpiredRejected(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
|
||||
// notAfter is one hour in the past — the cookie was already expired
|
||||
// when issued (this is what a stale cookie sent by a slow client would
|
||||
// look like on the server).
|
||||
pastNotAfter := time.Now().Add(-time.Hour).Unix()
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), pastNotAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrCookieExpired,
|
||||
"expired cookie must produce ErrCookieExpired, got %v", err)
|
||||
}
|
||||
|
||||
// TestCookieV0_TamperedExpirationRejected is the security-critical test for
|
||||
// the design choice of putting not_after INSIDE the encrypted envelope. Any
|
||||
// flip of a byte in the ciphertext (including bytes covering the not_after
|
||||
// header) must invalidate the AEAD tag, not just shift the apparent
|
||||
// expiration time.
|
||||
func TestCookieV0_TamperedExpirationRejected(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
raw, err := base64.RawURLEncoding.DecodeString(encoded)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wire layout: version(1) || nonce(12) || ciphertext(...)
|
||||
// The first byte of ciphertext corresponds to the first byte of
|
||||
// plaintext, which is the high byte of not_after_be8. Flipping it
|
||||
// MUST invalidate the AEAD tag.
|
||||
tampered := append([]byte(nil), raw...)
|
||||
tampered[1+12] ^= 0x80 // flip MSB of not_after's high byte
|
||||
tamperedEncoded := base64.RawURLEncoding.EncodeToString(tampered)
|
||||
|
||||
_, err = openCookie(tamperedEncoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrCookieSignature,
|
||||
"tampering with the not_after region must produce ErrCookieSignature (AEAD tag failure)")
|
||||
}
|
||||
|
||||
// TestCookieV0_UAMismatchRejected confirms the User-Agent AAD binding.
|
||||
func TestCookieV0_UAMismatchRejected(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua-A"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua-B"), MaxCookieLen)
|
||||
assert.ErrorIs(t, err, ErrCookieSignature)
|
||||
}
|
||||
|
||||
// TestCookieV0_SurvivesArbitraryRotation is the headline test for the v0
|
||||
// design: a cookie sealed under the long-lived master cookie key remains
|
||||
// valid across as many ticket-signing-key rotations as the keyring goes
|
||||
// through. The previous design tied cookie validity to the same live
|
||||
// window as ticket signing; this one decouples them completely.
|
||||
//
|
||||
// We advance the keyring's clock (forcing it to derive a stream of new
|
||||
// epoch signing keys) and confirm that openCookie — which depends only
|
||||
// on the master cookie key, not on any epoch — keeps working.
|
||||
func TestCookieV0_SurvivesArbitraryRotation(t *testing.T) {
|
||||
t0 := time.Now()
|
||||
keys := newTestKeyRing(t, testSecret, time.Minute, t0)
|
||||
|
||||
// notAfter is well in the future from real wall-clock time, so the
|
||||
// in-envelope expiration check (which uses real time.Now()) won't
|
||||
// reject the cookie.
|
||||
notAfter := t0.Add(24 * time.Hour).Unix()
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{PowDifficulty: 9}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Roll the keyring's clock forward — this triggers per-epoch sign-key
|
||||
// derivations and (with maxLive=3) evictions. None of that affects
|
||||
// the master cookie key, so the cookie keeps validating.
|
||||
for _, advance := range []time.Duration{
|
||||
1 * time.Minute,
|
||||
10 * time.Minute,
|
||||
1 * time.Hour,
|
||||
6 * time.Hour,
|
||||
23 * time.Hour,
|
||||
} {
|
||||
keys.now = func() time.Time { return t0.Add(advance) }
|
||||
|
||||
// Touching keys.Current() forces a derivation under the new
|
||||
// keyring time, simulating real rotation activity.
|
||||
_, _ = keys.Current()
|
||||
|
||||
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err,
|
||||
"cookie should still validate after %s of keyring rotation; got %v", advance, err)
|
||||
assert.Equal(t, int32(9), got.Envelope.GetPowDifficulty())
|
||||
}
|
||||
}
|
||||
|
||||
// TestCookieV0_ExpiryEnforcedAgainstWallClock is the counterpart to the
|
||||
// rotation test: cookie expiration uses real wall-clock time (not the
|
||||
// keyring's overridable now), so we seal a cookie with notAfter already
|
||||
// in the past and confirm rejection.
|
||||
func TestCookieV0_ExpiryEnforcedAgainstWallClock(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
|
||||
pastNotAfter := time.Now().Add(-time.Minute).Unix()
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), pastNotAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrCookieExpired)
|
||||
}
|
||||
|
||||
// TestCookieV0_MasterSecretChangeInvalidates is the expected counter-property
|
||||
// to SurvivesArbitraryRotation: if the operator rotates the master secret
|
||||
// (a manual config change), every outstanding cookie is invalidated.
|
||||
func TestCookieV0_MasterSecretChangeInvalidates(t *testing.T) {
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
|
||||
keysA, err := NewKeyRing(bytes.Repeat([]byte{0xaa}, 32), time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
keysB, err := NewKeyRing(bytes.Repeat([]byte{0xbb}, 32), time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keysA.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = openCookie(encoded, keysB.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
assert.ErrorIs(t, err, ErrCookieSignature,
|
||||
"cookie sealed under secret A must not decrypt under secret B")
|
||||
}
|
||||
|
||||
// TestCookieV0_DistributedSetup_TwoInstances confirms cookies issued by
|
||||
// instance A validate against instance B as long as both share the same
|
||||
// master secret. This is the multi-WAF deployment invariant.
|
||||
func TestCookieV0_DistributedSetup_TwoInstances(t *testing.T) {
|
||||
secret := bytes.Repeat([]byte{0xcc}, 32)
|
||||
a, err := NewKeyRing(secret, time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
b, err := NewKeyRing(secret, time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, bytes.Equal(a.MasterCookieKey(), b.MasterCookieKey()),
|
||||
"two instances with the same master_secret must derive the same master cookie key")
|
||||
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{PowDifficulty: 14}, a.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := openCookie(encoded, b.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err, "cookie issued by A must validate against B (same master_secret)")
|
||||
assert.Equal(t, int32(14), got.Envelope.GetPowDifficulty())
|
||||
}
|
||||
|
||||
// TestCookie_UnknownVersionRejected makes the version-byte dispatch
|
||||
// explicit: a cookie with an unknown leading byte is rejected with the
|
||||
// dedicated ErrCookieVersion sentinel. This is the extension point for
|
||||
// any future cookie schema bump.
|
||||
func TestCookie_UnknownVersionRejected(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
|
||||
// Forge: version byte 0xFE followed by random-looking junk.
|
||||
raw := append([]byte{0xFE}, []byte("0123456789abcdefghijklmnop")...)
|
||||
encoded := base64.RawURLEncoding.EncodeToString(raw)
|
||||
|
||||
_, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrCookieVersion, "unknown version byte must produce ErrCookieVersion")
|
||||
}
|
||||
|
||||
// TestRotation_TicketSurvives_WithinLiveWindow exercises the most important
|
||||
// guarantee for in-flight requests: a ticket signed under epoch N validates
|
||||
// after the keyring rolls to epoch N+1, as long as N is still within the
|
||||
// live window. Without this, every key rotation would invalidate every
|
||||
// challenge currently in flight.
|
||||
//
|
||||
// verifyChallenge uses time.Since on the wall clock (not the keyring's
|
||||
// overridable now), so we anchor t0 at the real current time and only
|
||||
// override the keyring's clock — small enough offsets stay inside the
|
||||
// ticketAgeBackstop freshness check.
|
||||
func TestRotation_TicketSurvives_WithinLiveWindow(t *testing.T) {
|
||||
t0 := time.Now()
|
||||
keys := newTestKeyRing(t, testSecret, time.Minute, t0)
|
||||
|
||||
c := &ChallengeRuntime{keys: keys}
|
||||
tsStr := strconvI64(t0.UnixNano())
|
||||
r, err := generateChallengeNonce()
|
||||
require.NoError(t, err)
|
||||
salt := mustGeneratePowPrefix(t)
|
||||
mac := c.computePowMAC(salt, r, tsStr, PowDifficultyMedium)
|
||||
|
||||
// Roll the keyring clock forward one rotation interval — the challenge's
|
||||
// epoch is now "previous" but still in the live window.
|
||||
keys.now = func() time.Time { return t0.Add(70 * time.Second) }
|
||||
|
||||
_, ok := c.verifyChallenge(r, tsStr, salt, mac, PowDifficultyMedium)
|
||||
assert.True(t, ok, "challenge from previous epoch must still validate within the live window")
|
||||
}
|
||||
|
||||
// TestRotation_TicketRejected_OutOfWindow proves the negative side of the
|
||||
// previous test: once the keyring's view of time has advanced enough to
|
||||
// evict the challenge's epoch, validation fails. Anchored at real-time t0 so
|
||||
// verifyChallenge's own freshness check (time.Since) doesn't conflate with
|
||||
// the keyring's eviction logic.
|
||||
func TestRotation_TicketRejected_OutOfWindow(t *testing.T) {
|
||||
t0 := time.Now()
|
||||
keys := newTestKeyRing(t, testSecret, time.Minute, t0)
|
||||
|
||||
c := &ChallengeRuntime{keys: keys}
|
||||
tsStr := strconvI64(t0.UnixNano())
|
||||
r, err := generateChallengeNonce()
|
||||
require.NoError(t, err)
|
||||
salt := mustGeneratePowPrefix(t)
|
||||
mac := c.computePowMAC(salt, r, tsStr, PowDifficultyMedium)
|
||||
|
||||
// Jump the keyring's clock past the live window (maxLive=3 + skew=1 =
|
||||
// any epoch >4 minutes ahead in the keyring's view evicts the original).
|
||||
keys.now = func() time.Time { return t0.Add(5 * time.Minute) }
|
||||
|
||||
_, ok := c.verifyChallenge(r, tsStr, salt, mac, PowDifficultyMedium)
|
||||
assert.False(t, ok, "challenge from an evicted epoch must not validate")
|
||||
}
|
||||
|
||||
// TestEndToEnd_ValidateChallengeResponse walks the full validation path
|
||||
// (matches challenge, opens cookie). Regression guard for keyring +
|
||||
// cookie-v0 wired together.
|
||||
func TestEndToEnd_ValidateChallengeResponse(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
c := &ChallengeRuntime{keys: keys, powDifficulty: 8, cookieTTL: time.Hour, spent: newSpentSet(spentSetDefaultMaxEntries)}
|
||||
|
||||
r, ts := freshChallenge(t)
|
||||
body := buildValidBody(t, c.powDifficulty, r, ts)
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("User-Agent", "test-agent")
|
||||
|
||||
ck, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ck)
|
||||
|
||||
// Round-trip the cookie through the public ValidCookie API to confirm
|
||||
// the full Seal → parse Set-Cookie → Open path is correct.
|
||||
parsed, err := http.ParseSetCookie(ck.String())
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := c.ValidCookie(parsed, "test-agent")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 8, got.PowDifficulty)
|
||||
}
|
||||
|
||||
// TestCookieV0_BrowserTTLMatchesServerTTL guards the gotcha mentioned in
|
||||
// the v0 design discussion: if the browser drops the cookie before the
|
||||
// server stops accepting it (or vice versa), users get unexpected
|
||||
// re-challenges. The Set-Cookie Max-Age MUST match the server-side
|
||||
// cookieTTL.
|
||||
func TestCookieV0_BrowserTTLMatchesServerTTL(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
c := &ChallengeRuntime{keys: keys, powDifficulty: 8, cookieTTL: 90 * time.Minute, spent: newSpentSet(spentSetDefaultMaxEntries)}
|
||||
|
||||
r, ts := freshChallenge(t)
|
||||
body := buildValidBody(t, c.powDifficulty, r, ts)
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("User-Agent", "test-agent")
|
||||
|
||||
ck, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
|
||||
require.NoError(t, err)
|
||||
|
||||
parsed, err := http.ParseSetCookie(ck.String())
|
||||
require.NoError(t, err)
|
||||
|
||||
// MaxAge is a count of seconds; compare to our cookieTTL.
|
||||
// The cookie helper computes MaxAge as
|
||||
// int(time.Until(Expires).Seconds())
|
||||
// where Expires was set from time.Now()+TTL on a slightly earlier
|
||||
// time.Now() reading. Sub-second elapsed time between the two reads
|
||||
// can shave a full second off after truncation, so we allow ±2s.
|
||||
expected := int((90 * time.Minute).Seconds())
|
||||
assert.InDelta(t, expected, parsed.MaxAge, 2,
|
||||
"browser-side Max-Age (%d) should be within 2s of server-side cookieTTL (%d)",
|
||||
parsed.MaxAge, expected)
|
||||
}
|
||||
|
||||
func strconvI64(n int64) string {
|
||||
return strconv.FormatInt(n, 10)
|
||||
}
|
||||
|
||||
// TestCookieV0_AllowlistRoundTrip seals an allowlist cookie (Allowlisted bit
|
||||
// + reason) and confirms the open path returns both fields.
|
||||
func TestCookieV0_AllowlistRoundTrip(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
reason := "Googlebot/2.1 (compatible)"
|
||||
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, cookieFlagAllowlisted, reason, []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, got.Allowlisted, "allowlist flag should round-trip true")
|
||||
assert.Equal(t, reason, got.AllowlistReason, "reason should round-trip verbatim")
|
||||
}
|
||||
|
||||
// TestCookieV0_AllowlistFlagDefaultsFalse confirms a regular (non-allowlist)
|
||||
// cookie reports Allowlisted=false and empty reason after open.
|
||||
func TestCookieV0_AllowlistFlagDefaultsFalse(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{PowDifficulty: 12}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, got.Allowlisted, "default flag must be false")
|
||||
assert.Empty(t, got.AllowlistReason)
|
||||
}
|
||||
|
||||
// TestCookieV0_AllowlistReasonTooLong asserts the seal path rejects reasons
|
||||
// larger than MaxAllowlistReasonLen with the typed error.
|
||||
func TestCookieV0_AllowlistReasonTooLong(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
oversize := bytes.Repeat([]byte("x"), MaxAllowlistReasonLen+1)
|
||||
|
||||
_, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, cookieFlagAllowlisted, string(oversize), []byte("ua"), MaxCookieLen)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrAllowlistReasonSize)
|
||||
}
|
||||
|
||||
// TestCookieV0_SealEnvelopeTooLarge confirms the seal path rejects an envelope
|
||||
// (built from the client-submitted fingerprint) that would produce an
|
||||
// over-limit cookie, before allocating for it. Without this bound a crafted
|
||||
// submission forces a large allocation for a cookie the browser would refuse.
|
||||
func TestCookieV0_SealEnvelopeTooLarge(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
|
||||
// A fingerprint whose marshaled size blows past the plaintext budget.
|
||||
envelope := &pb.ChallengeCookie{
|
||||
Fingerprint: &pb.FingerprintData{Fsid: strings.Repeat("x", MaxCookieLen)},
|
||||
}
|
||||
|
||||
_, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrCookieTooLarge)
|
||||
}
|
||||
|
||||
// TestCookieV0_OpenValueTooLarge confirms the open path rejects an over-limit
|
||||
// cookie value before base64-decoding or AEAD-opening it.
|
||||
func TestCookieV0_OpenValueTooLarge(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
|
||||
oversized := strings.Repeat("A", MaxCookieLen+1)
|
||||
|
||||
_, err := openCookie(oversized, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrCookieTooLarge)
|
||||
}
|
||||
|
||||
// TestCookieV0_RealCookieUnderLimit guards against the cap being set too low:
|
||||
// a normal sealed cookie must stay under MaxCookieLen and still round-trip.
|
||||
func TestCookieV0_RealCookieUnderLimit(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
envelope := &pb.ChallengeCookie{
|
||||
Fingerprint: &pb.FingerprintData{Fsid: "abc123", Url: "https://example.com/"},
|
||||
PowDifficulty: 12,
|
||||
}
|
||||
|
||||
encoded, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(encoded), MaxCookieLen)
|
||||
|
||||
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestCookieV0_ConfigurableLimit confirms the seal/open ceiling honors a
|
||||
// caller-supplied bound (Config.MaxCookieSize → runtime.maxCookieLen) and that
|
||||
// a non-positive bound falls back to the MaxCookieLen default.
|
||||
func TestCookieV0_ConfigurableLimit(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
|
||||
// A ~4 KB fingerprint: seals to a cookie that exceeds the 4096 default but
|
||||
// fits a raised 8192 ceiling.
|
||||
envelope := &pb.ChallengeCookie{
|
||||
Fingerprint: &pb.FingerprintData{Fsid: strings.Repeat("x", 4000)},
|
||||
}
|
||||
|
||||
// A tight limit rejects it on seal, before allocating.
|
||||
_, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), 1024)
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, ErrCookieTooLarge)
|
||||
|
||||
// A raised limit accepts the same envelope.
|
||||
encoded, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), 8192)
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, len(encoded), MaxCookieLen, "sanity: this cookie exceeds the default ceiling")
|
||||
|
||||
// The default ceiling (0 → MaxCookieLen) then rejects it on open...
|
||||
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), 0)
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, ErrCookieTooLarge)
|
||||
|
||||
// ...while the matching raised ceiling opens it successfully.
|
||||
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), 8192)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, got)
|
||||
}
|
||||
|
||||
// TestCookieV0_TamperedFlagsRejected confirms the flags byte is INSIDE the
|
||||
// AEAD envelope: any modification invalidates the GCM tag.
|
||||
func TestCookieV0_TamperedFlagsRejected(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
notAfter := time.Now().Add(time.Hour).Unix()
|
||||
|
||||
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
|
||||
require.NoError(t, err)
|
||||
|
||||
raw, err := base64.RawURLEncoding.DecodeString(encoded)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wire layout: version(1) || nonce(12) || ciphertext(...). The 9th
|
||||
// byte of ciphertext (offset 1+12+8 = 21) covers the flags_byte of
|
||||
// the plaintext. Flipping any bit there should fail the AEAD tag.
|
||||
tampered := append([]byte(nil), raw...)
|
||||
tampered[1+12+8] ^= 0x01
|
||||
tamperedEncoded := base64.RawURLEncoding.EncodeToString(tampered)
|
||||
|
||||
_, err = openCookie(tamperedEncoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrCookieSignature)
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newTestKeyRing(t testing.TB, secret []byte, interval time.Duration, fixed time.Time) *KeyRing {
|
||||
t.Helper()
|
||||
if secret == nil {
|
||||
secret = bytes.Repeat([]byte{0xab}, 32)
|
||||
}
|
||||
if interval == 0 {
|
||||
interval = time.Minute
|
||||
}
|
||||
k, err := NewKeyRing(secret, interval, 3)
|
||||
require.NoError(t, err)
|
||||
if !fixed.IsZero() {
|
||||
k.now = func() time.Time { return fixed }
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
func TestNewKeyRing_RejectsShortSecret(t *testing.T) {
|
||||
_, err := NewKeyRing([]byte("too-short"), time.Minute, 3)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestNewKeyRing_RejectsTinyInterval(t *testing.T) {
|
||||
_, err := NewKeyRing(bytes.Repeat([]byte{1}, 32), time.Second, 3)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestKeyRing_DeterministicAcrossInstances(t *testing.T) {
|
||||
// Two KeyRings with the same secret + same interval, evaluated at the
|
||||
// same wall-clock moment, MUST produce bit-identical keys for the same
|
||||
// epoch. Regression guard for distributed (multi-WAF) deployments.
|
||||
secret := bytes.Repeat([]byte{0x42}, 32)
|
||||
fixed := time.Date(2026, 5, 7, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
a := newTestKeyRing(t, secret, time.Minute, fixed)
|
||||
b := newTestKeyRing(t, secret, time.Minute, fixed)
|
||||
|
||||
epochA, signA := a.Current()
|
||||
epochB, signB := b.Current()
|
||||
assert.Equal(t, epochA, epochB)
|
||||
assert.True(t, bytes.Equal(signA, signB), "sign keys diverge across instances")
|
||||
|
||||
// The master cookie key is independent of epoch and must also agree
|
||||
// across instances configured with the same master secret.
|
||||
assert.True(t, bytes.Equal(a.MasterCookieKey(), b.MasterCookieKey()),
|
||||
"master cookie keys diverge across instances")
|
||||
|
||||
// Sign and master cookie keys derive from the same secret with
|
||||
// different HKDF info strings, so they MUST differ. If they didn't,
|
||||
// a leaked sign key would also forge cookies (and vice versa).
|
||||
assert.False(t, bytes.Equal(signA, a.MasterCookieKey()),
|
||||
"sign and master cookie keys must differ — domain separation is the whole point")
|
||||
}
|
||||
|
||||
func TestKeyRing_RotatesAtIntervalBoundary(t *testing.T) {
|
||||
secret := bytes.Repeat([]byte{0x99}, 32)
|
||||
t0 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
k := newTestKeyRing(t, secret, time.Minute, t0)
|
||||
e0, key0 := k.Current()
|
||||
|
||||
// Cross the 60-second boundary — current epoch must increment.
|
||||
k.now = func() time.Time { return t0.Add(61 * time.Second) }
|
||||
e1, key1 := k.Current()
|
||||
|
||||
assert.Equal(t, e0+1, e1)
|
||||
assert.False(t, bytes.Equal(key0, key1), "key did not change across rotation")
|
||||
|
||||
// And the previous epoch's key is still derivable while in the live window.
|
||||
prev, ok := k.SignKey(e0)
|
||||
require.True(t, ok)
|
||||
assert.True(t, bytes.Equal(key0, prev), "previous epoch key changed when re-derived")
|
||||
}
|
||||
|
||||
func TestKeyRing_LiveWindow(t *testing.T) {
|
||||
t0 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
k := newTestKeyRing(t, nil, time.Minute, t0)
|
||||
|
||||
// maxLive=3 + clockSkew=1 → live window is current-2 .. current+1.
|
||||
current, _ := k.Current()
|
||||
|
||||
// In window.
|
||||
for _, e := range []int64{current - 2, current - 1, current, current + 1} {
|
||||
_, ok := k.SignKey(e)
|
||||
assert.True(t, ok, "epoch %d should be live", e)
|
||||
}
|
||||
|
||||
// Out of window.
|
||||
for _, e := range []int64{current - 3, current + 2} {
|
||||
_, ok := k.SignKey(e)
|
||||
assert.False(t, ok, "epoch %d should NOT be live", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyRing_LiveEpochsOrdered(t *testing.T) {
|
||||
t0 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
k := newTestKeyRing(t, nil, time.Minute, t0)
|
||||
current, _ := k.Current()
|
||||
|
||||
live := k.LiveEpochs()
|
||||
require.Len(t, live, 4) // 3 past/current + 1 future skew
|
||||
|
||||
// Ordered ascending, contiguous.
|
||||
for i, e := range live {
|
||||
assert.Equal(t, current-2+int64(i), e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyRing_CacheEvictsStale(t *testing.T) {
|
||||
t0 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
k := newTestKeyRing(t, nil, time.Minute, t0)
|
||||
|
||||
// Touch the current epoch to seed the cache.
|
||||
startEpoch, _ := k.Current()
|
||||
|
||||
// Jump far into the future so startEpoch falls out of the live window;
|
||||
// the next derivation must evict it.
|
||||
k.now = func() time.Time { return t0.Add(time.Hour) }
|
||||
_, _ = k.Current()
|
||||
|
||||
k.mu.RLock()
|
||||
_, present := k.cache[startEpoch]
|
||||
k.mu.RUnlock()
|
||||
assert.False(t, present, "stale epoch %d should have been evicted from the cache", startEpoch)
|
||||
}
|
||||
|
||||
// TestKeyRing_KnownVectors locks down the HKDF derivation. Any change to the
|
||||
// salt / info strings / serialization that breaks distributed agreement on a
|
||||
// fleet upgrade will make these vectors fail. If you intentionally change the
|
||||
// scheme, regenerate these vectors AND rotate the keyring version constant.
|
||||
func TestKeyRing_KnownVectors(t *testing.T) {
|
||||
secret, err := hex.DecodeString("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f")
|
||||
require.NoError(t, err)
|
||||
|
||||
k, err := NewKeyRing(secret, time.Minute, 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Sign key for epoch 0 must derive deterministically.
|
||||
want := hex.EncodeToString(deriveEpochKey(secret, 0, keyringInfoSign))
|
||||
got := hex.EncodeToString(deriveEpochKey(k.masterSecret, 0, keyringInfoSign))
|
||||
assert.Equal(t, want, got, "epoch 0 sign key derivation must be deterministic")
|
||||
|
||||
// Master cookie key (no epoch component) must also be deterministic.
|
||||
wantCookie := hex.EncodeToString(deriveMasterCookieKey(secret))
|
||||
gotCookie := hex.EncodeToString(k.MasterCookieKey())
|
||||
assert.Equal(t, wantCookie, gotCookie, "master cookie key derivation must be deterministic")
|
||||
|
||||
// Cross-context separation: sign key for any epoch must NOT equal the
|
||||
// master cookie key. Domain separation is the whole point of using
|
||||
// distinct HKDF info strings.
|
||||
signKey := deriveEpochKey(secret, 42, keyringInfoSign)
|
||||
cookieKey := deriveMasterCookieKey(secret)
|
||||
assert.NotEqual(t, hex.EncodeToString(signKey), hex.EncodeToString(cookieKey),
|
||||
"sign and master-cookie contexts must produce different keys")
|
||||
|
||||
// Cross-epoch separation: same context, different epoch → different bytes.
|
||||
e0 := deriveEpochKey(secret, 0, keyringInfoSign)
|
||||
e1 := deriveEpochKey(secret, 1, keyringInfoSign)
|
||||
assert.NotEqual(t, hex.EncodeToString(e0), hex.EncodeToString(e1),
|
||||
"adjacent epochs must produce different sign keys")
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestChallengeCodeSeededStatically asserts that the default
|
||||
// NewChallengeRuntime path loads the build-time-obfuscated challenge code
|
||||
// once from initial_bundle.js.gz and serves it verbatim — there is no
|
||||
// runtime re-obfuscation or pool for the (public) library path anymore.
|
||||
func TestChallengeCodeSeededStatically(t *testing.T) {
|
||||
rt, err := NewChallengeRuntime(t.Context())
|
||||
require.NoError(t, err)
|
||||
|
||||
code := rt.getChallengeCode()
|
||||
require.NotEmpty(t, code, "challenge code must be seeded from the baked-in initial bundle")
|
||||
|
||||
// The hook sentinel must survive obfuscation so the dynamic key module
|
||||
// can find the registered hook (reservedStrings in obfuscate.js).
|
||||
require.Contains(t, code, hookSentinel,
|
||||
"obfuscated challenge code must preserve the hook sentinel verbatim")
|
||||
|
||||
// The accessor is a stable read of static state: repeated calls return
|
||||
// the identical string (no pool, no random selection).
|
||||
assert.Equal(t, code, rt.getChallengeCode(), "challenge code must be static across calls")
|
||||
}
|
||||
|
||||
// TestCryptoObfuscationDefaultPoolSize confirms that with no
|
||||
// WithCryptoObfuscationPoolSize option, the per-epoch dynamic module
|
||||
// cache holds exactly 1 variant per epoch — preserving the historical
|
||||
// single-variant-per-epoch behavior.
|
||||
func TestCryptoObfuscationDefaultPoolSize(t *testing.T) {
|
||||
rt, err := NewChallengeRuntime(t.Context())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, cryptoObfuscationPoolDefaultSize, rt.cryptoPoolSize,
|
||||
"default crypto pool size must be cryptoObfuscationPoolDefaultSize")
|
||||
|
||||
// The pre-warmer runs currentDynamicModule at construction; the
|
||||
// cache for the current epoch must hold exactly 1 variant.
|
||||
currentEpoch, _ := rt.keys.Current()
|
||||
|
||||
rt.dynamicModuleCacheMu.RLock()
|
||||
variants, ok := rt.dynamicModuleCache[currentEpoch]
|
||||
rt.dynamicModuleCacheMu.RUnlock()
|
||||
|
||||
require.True(t, ok, "current epoch must be cached after construction (pre-warmer ran)")
|
||||
require.Len(t, variants, 1, "default crypto pool keeps 1 variant per epoch")
|
||||
}
|
||||
|
||||
// TestCryptoObfuscationPoolSize asserts that when
|
||||
// WithCryptoObfuscationPoolSize is raised, the per-epoch cache holds
|
||||
// the configured number of distinct obfuscated variants of the SAME
|
||||
// epoch key, and currentDynamicModule picks from them.
|
||||
func TestCryptoObfuscationPoolSize(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("each crypto pool variant costs ~3s of obfuscator CPU; skipped in -short")
|
||||
}
|
||||
|
||||
const poolSize = 3
|
||||
rt, err := NewChallengeRuntime(t.Context(),
|
||||
WithCryptoObfuscationPoolSize(poolSize),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, poolSize, rt.cryptoPoolSize, "crypto pool size option must propagate")
|
||||
|
||||
currentEpoch, _ := rt.keys.Current()
|
||||
|
||||
rt.dynamicModuleCacheMu.RLock()
|
||||
variants, ok := rt.dynamicModuleCache[currentEpoch]
|
||||
rt.dynamicModuleCacheMu.RUnlock()
|
||||
|
||||
require.True(t, ok, "current epoch must be cached after construction")
|
||||
require.Len(t, variants, poolSize, "crypto pool must hold poolSize variants")
|
||||
|
||||
// Each variant must be a distinct obfuscation. javascript-obfuscator
|
||||
// with mangled identifier-names produces different byte output each
|
||||
// run, so duplicates are an obfuscator regression.
|
||||
for i := 1; i < poolSize; i++ {
|
||||
assert.NotEqual(t, variants[0], variants[i],
|
||||
"variants %d and 0 are byte-identical; obfuscator should produce distinct output per pass", i)
|
||||
}
|
||||
|
||||
// currentDynamicModule must return one of the cached variants.
|
||||
got, err := rt.currentDynamicModule(t.Context())
|
||||
require.NoError(t, err)
|
||||
found := slices.Contains(variants, got)
|
||||
assert.True(t, found, "currentDynamicModule returned a value not in the cached pool")
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// obfuscator.go embeds the wasm build of `javascript-obfuscator` and exposes
|
||||
// a single thread-safe entry point (ObfuscateJS) used by both the static and
|
||||
// dynamic bundle paths. The compiled wazero module is shared across calls;
|
||||
// each ObfuscateJS invocation instantiates a fresh module instance with its
|
||||
// own stdin/stdout buffers so concurrent obfuscations don't collide.
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
crand "crypto/rand"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/tetratelabs/wazero"
|
||||
)
|
||||
|
||||
//go:embed js/obfuscate/index.wasm.gz
|
||||
var obfuscatorWasmGz []byte
|
||||
|
||||
var (
|
||||
obfuscatorWasm []byte
|
||||
obfuscatorWasmOnce sync.Once
|
||||
)
|
||||
|
||||
// ObfuscateJS runs the input source through the embedded `javascript-obfuscator`
|
||||
// wasm module and returns the obfuscated output. Thread-safe: wazero allows
|
||||
// concurrent module instantiations from the same compiled module, which is
|
||||
// what makes the dynamic-module singleflight pattern work.
|
||||
func (c *ChallengeRuntime) ObfuscateJS(ctx context.Context, inputJS string) (string, error) {
|
||||
stdin := bytes.NewReader([]byte(inputJS))
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
config := wazero.NewModuleConfig().
|
||||
WithStdin(stdin).
|
||||
WithStdout(&stdout).
|
||||
WithStderr(&stderr).
|
||||
// Give wazero a real PRNG source and real wall/nano time so output is non-deterministic.
|
||||
WithRandSource(crand.Reader).
|
||||
WithSysWalltime().
|
||||
WithSysNanotime()
|
||||
|
||||
mod, err := c.r.InstantiateModule(ctx, c.obfuscatorMod, config)
|
||||
if err != nil {
|
||||
if stderr.Len() > 0 {
|
||||
return "", fmt.Errorf("wasm runtime error: %v | stderr: %s", err, stderr.String())
|
||||
}
|
||||
return "", fmt.Errorf("wasm instantiation error: %v", err)
|
||||
}
|
||||
mod.Close(ctx)
|
||||
|
||||
return stdout.String(), nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,289 @@
|
||||
syntax = "proto3";
|
||||
package fingerprint;
|
||||
|
||||
option go_package = ".;pb";
|
||||
|
||||
message FingerprintData {
|
||||
fingerprintSignals signals = 1;
|
||||
string fsid = 2;
|
||||
string nonce = 3;
|
||||
int64 time = 4;
|
||||
string url = 5;
|
||||
bool fastBotDetection = 6;
|
||||
fingerprintFastBotDetectionDetails fastBotDetectionDetails = 7;
|
||||
fingerprintBotAlias bot = 8;
|
||||
}
|
||||
|
||||
// ChallengeCookie is the sealed cookie payload. It wraps the validated
|
||||
// fingerprint together with metadata that influences re-challenge decisions
|
||||
// (e.g. the PoW difficulty the client proved). Keeping this separate from
|
||||
// FingerprintData lets us evolve the cookie envelope without polluting the
|
||||
// fingerprint model exposed to hook expressions.
|
||||
message ChallengeCookie {
|
||||
FingerprintData fingerprint = 1;
|
||||
// pow_difficulty is the PoW difficulty (in leading zero bits) the client
|
||||
// proved when this cookie was sealed.
|
||||
int32 pow_difficulty = 2;
|
||||
}
|
||||
|
||||
message fingerprintSignals {
|
||||
fingerprintAutomation automation = 1;
|
||||
fingerprintDevice device = 2;
|
||||
fingerprintBrowser browser = 3;
|
||||
fingerprintGraphics graphics = 4;
|
||||
fingerprintCodecs codecs = 5;
|
||||
fingerprintLocale locale = 6;
|
||||
fingerprintContexts contexts = 7;
|
||||
}
|
||||
|
||||
message fingerprintAutomation {
|
||||
bool webdriver = 1;
|
||||
bool webdriverWritable = 2;
|
||||
bool selenium = 3;
|
||||
bool cdp = 4;
|
||||
bool playwright = 5;
|
||||
string navigatorPropertyDescriptors = 6;
|
||||
}
|
||||
|
||||
message fingerprintDevice {
|
||||
int32 cpuCount = 1;
|
||||
int32 memory = 2;
|
||||
string platform = 3;
|
||||
fingerprintScreenResolution screenResolution = 4;
|
||||
fingerprintMultimediaDevices multimediaDevices = 5;
|
||||
fingerprintDeviceMediaQueries mediaQueries = 6;
|
||||
}
|
||||
|
||||
message fingerprintScreenResolution {
|
||||
int32 width = 1;
|
||||
int32 height = 2;
|
||||
int32 pixelDepth = 3;
|
||||
int32 colorDepth = 4;
|
||||
int32 availableWidth = 5;
|
||||
int32 availableHeight = 6;
|
||||
int32 innerWidth = 7;
|
||||
int32 innerHeight = 8;
|
||||
bool hasMultipleDisplays = 9;
|
||||
}
|
||||
|
||||
message fingerprintMultimediaDevices {
|
||||
int32 speakers = 1;
|
||||
int32 microphones = 2;
|
||||
int32 webcams = 3;
|
||||
}
|
||||
|
||||
message fingerprintDeviceMediaQueries {
|
||||
string prefersColorScheme = 1;
|
||||
bool prefersReducedMotion = 2;
|
||||
bool prefersReducedTransparency = 3;
|
||||
string colorGamut = 4;
|
||||
string pointer = 5;
|
||||
string anyPointer = 6;
|
||||
bool hover = 7;
|
||||
bool anyHover = 8;
|
||||
int32 colorDepth = 9;
|
||||
}
|
||||
|
||||
message fingerprintBrowser {
|
||||
string userAgent = 1;
|
||||
fingerprintBrowserFeatures features = 2;
|
||||
fingerprintBrowserPlugins plugins = 3;
|
||||
fingerprintBrowserExtensions extensions = 4;
|
||||
fingerprintBrowserHighEntropyValues highEntropyValues = 5;
|
||||
int32 etsl = 6;
|
||||
string maths = 7;
|
||||
fingerprintBrowserToSourceError toSourceError = 8;
|
||||
}
|
||||
|
||||
message fingerprintBrowserFeatures {
|
||||
string bitmask = 1;
|
||||
bool chrome = 2;
|
||||
bool brave = 3;
|
||||
bool applePaySupport = 4;
|
||||
bool opera = 5;
|
||||
bool serial = 6;
|
||||
bool attachShadow = 7;
|
||||
bool caches = 8;
|
||||
bool webAssembly = 9;
|
||||
bool buffer = 10;
|
||||
bool showModalDialog = 11;
|
||||
bool safari = 12;
|
||||
bool webkitPrefixedFunction = 13;
|
||||
bool mozPrefixedFunction = 14;
|
||||
bool usb = 15;
|
||||
bool browserCapture = 16;
|
||||
bool paymentRequestUpdateEvent = 17;
|
||||
bool pressureObserver = 18;
|
||||
bool audioSession = 19;
|
||||
bool selectAudioOutput = 20;
|
||||
bool barcodeDetector = 21;
|
||||
bool battery = 22;
|
||||
bool devicePosture = 23;
|
||||
bool documentPictureInPicture = 24;
|
||||
bool eyeDropper = 25;
|
||||
bool editContext = 26;
|
||||
bool fencedFrame = 27;
|
||||
bool sanitizer = 28;
|
||||
bool otpCredential = 29;
|
||||
}
|
||||
|
||||
message fingerprintBrowserPlugins {
|
||||
bool isValidPluginArray = 1;
|
||||
int32 pluginCount = 2;
|
||||
string pluginNamesHash = 3;
|
||||
bool pluginConsistency1 = 4;
|
||||
bool pluginOverflow = 5;
|
||||
}
|
||||
|
||||
message fingerprintBrowserExtensions {
|
||||
string bitmask = 1;
|
||||
repeated string extensions = 2;
|
||||
}
|
||||
|
||||
message fingerprintBrowserHighEntropyValues {
|
||||
string architecture = 1;
|
||||
string bitness = 2;
|
||||
repeated fingerprintBrandVersion brands = 3;
|
||||
bool mobile = 4;
|
||||
string model = 5;
|
||||
string platform = 6;
|
||||
string platformVersion = 7;
|
||||
string uaFullVersion = 8;
|
||||
}
|
||||
|
||||
message fingerprintBrandVersion {
|
||||
string brand = 1;
|
||||
string version = 2;
|
||||
}
|
||||
|
||||
message fingerprintBrowserToSourceError {
|
||||
string toSourceError = 1;
|
||||
bool hasToSource = 2;
|
||||
}
|
||||
|
||||
message fingerprintGraphics {
|
||||
fingerprintGraphicsWebGL webGL = 1;
|
||||
fingerprintGraphicsWebGPU webgpu = 2;
|
||||
fingerprintGraphicsCanvas canvas = 3;
|
||||
}
|
||||
|
||||
message fingerprintGraphicsWebGL {
|
||||
string vendor = 1;
|
||||
string renderer = 2;
|
||||
}
|
||||
|
||||
message fingerprintGraphicsWebGPU {
|
||||
string vendor = 1;
|
||||
string architecture = 2;
|
||||
string device = 3;
|
||||
string description = 4;
|
||||
}
|
||||
|
||||
message fingerprintGraphicsCanvas {
|
||||
bool hasModifiedCanvas = 1;
|
||||
string canvasFingerprint = 2;
|
||||
}
|
||||
|
||||
message fingerprintCodecs {
|
||||
string audioCanPlayTypeHash = 1;
|
||||
string videoCanPlayTypeHash = 2;
|
||||
string audioMediaSourceHash = 3;
|
||||
string videoMediaSourceHash = 4;
|
||||
string rtcAudioCapabilitiesHash = 5;
|
||||
string rtcVideoCapabilitiesHash = 6;
|
||||
bool hasMediaSource = 7;
|
||||
}
|
||||
|
||||
message fingerprintLocale {
|
||||
fingerprintLocaleInternationalization internationalization = 1;
|
||||
fingerprintLocaleLanguages languages = 2;
|
||||
}
|
||||
|
||||
message fingerprintLocaleInternationalization {
|
||||
string timezone = 1;
|
||||
string localeLanguage = 2;
|
||||
}
|
||||
|
||||
message fingerprintLocaleLanguages {
|
||||
repeated string languages = 1;
|
||||
string language = 2;
|
||||
}
|
||||
|
||||
message fingerprintContexts {
|
||||
fingerprintIframeContext iframe = 1;
|
||||
fingerprintWebWorkerContext webWorker = 2;
|
||||
}
|
||||
|
||||
message fingerprintIframeContext {
|
||||
bool webdriver = 1;
|
||||
string userAgent = 2;
|
||||
string platform = 3;
|
||||
int32 memory = 4;
|
||||
int32 cpuCount = 5;
|
||||
string language = 6;
|
||||
}
|
||||
|
||||
message fingerprintWebWorkerContext {
|
||||
string vendor = 1;
|
||||
string renderer = 2;
|
||||
string userAgent = 3;
|
||||
string language = 4;
|
||||
string platform = 5;
|
||||
int32 memory = 6;
|
||||
int32 cpuCount = 7;
|
||||
}
|
||||
|
||||
message fingerprintFastBotDetectionDetails {
|
||||
fingerprintDetectionResult headlessChromeScreenResolution = 1;
|
||||
fingerprintDetectionResult hasWebdriver = 2;
|
||||
fingerprintDetectionResult hasWebdriverWritable = 3;
|
||||
fingerprintDetectionResult hasSeleniumProperty = 4;
|
||||
fingerprintDetectionResult hasCDP = 5;
|
||||
fingerprintDetectionResult hasPlaywright = 6;
|
||||
fingerprintDetectionResult hasImpossibleDeviceMemory = 7;
|
||||
fingerprintDetectionResult hasHighCPUCount = 8;
|
||||
fingerprintDetectionResult hasMissingChromeObject = 9;
|
||||
fingerprintDetectionResult hasWebdriverIframe = 10;
|
||||
fingerprintDetectionResult hasWebdriverWorker = 11;
|
||||
fingerprintDetectionResult hasMismatchWebGLInWorker = 12;
|
||||
fingerprintDetectionResult hasMismatchPlatformIframe = 13;
|
||||
fingerprintDetectionResult hasMismatchPlatformWorker = 14;
|
||||
fingerprintDetectionResult hasSwiftshaderRenderer = 15;
|
||||
fingerprintDetectionResult hasUTCTimezone = 16;
|
||||
fingerprintDetectionResult hasMismatchLanguages = 17;
|
||||
fingerprintDetectionResult hasInconsistentEtsl = 18;
|
||||
fingerprintDetectionResult hasBotUserAgent = 19;
|
||||
fingerprintDetectionResult hasGPUMismatch = 20;
|
||||
fingerprintDetectionResult hasPlatformMismatch = 21;
|
||||
}
|
||||
|
||||
message fingerprintDetectionResult {
|
||||
bool detected = 1;
|
||||
}
|
||||
|
||||
message fingerprintBotAlias {
|
||||
bool headlessChromeScreenResolution = 1;
|
||||
bool webdriver = 2;
|
||||
bool webdriverWritable = 3;
|
||||
bool selenium = 4;
|
||||
bool cdp = 5;
|
||||
bool playwright = 6;
|
||||
bool impossibleDeviceMemory = 7;
|
||||
bool highCPUCount = 8;
|
||||
bool missingChromeObject = 9;
|
||||
bool webdriverIframe = 10;
|
||||
bool webdriverWorker = 11;
|
||||
bool mismatchWebGLInWorker = 12;
|
||||
bool mismatchPlatformIframe = 13;
|
||||
bool mismatchPlatformWorker = 14;
|
||||
bool swiftshaderRenderer = 15;
|
||||
bool utcTimezone = 16;
|
||||
bool anyDetected = 17;
|
||||
int32 detectedCount = 18;
|
||||
bool mismatchLanguages = 19;
|
||||
bool inconsistentEtsl = 20;
|
||||
bool botUserAgent = 21;
|
||||
bool gpuMismatch = 22;
|
||||
bool platformMismatch = 23;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package pb
|
||||
|
||||
// Dependencies:
|
||||
//
|
||||
// apt install protobuf-compiler
|
||||
//
|
||||
// keep this in sync with go.mod
|
||||
// go install google.golang.org/protobuf/cmd/[email protected]
|
||||
//
|
||||
// Not the same versions as google.golang.org/grpc
|
||||
// go list -m -versions google.golang.org/grpc/cmd/protoc-gen-go-grpc
|
||||
// go install google.golang.org/grpc/cmd/[email protected]
|
||||
|
||||
//go:generate protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative fingerprint.proto
|
||||
@@ -0,0 +1,77 @@
|
||||
"use strict";
|
||||
|
||||
var K = [
|
||||
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
|
||||
0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
||||
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
|
||||
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
||||
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
|
||||
0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
||||
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
|
||||
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
||||
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
|
||||
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
||||
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
|
||||
0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
||||
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
|
||||
0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
||||
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
|
||||
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
|
||||
];
|
||||
|
||||
function sha256(data) {
|
||||
var h0 = 0x6a09e667, h1 = 0xbb67ae85, h2 = 0x3c6ef372, h3 = 0xa54ff53a,
|
||||
h4 = 0x510e527f, h5 = 0x9b05688c, h6 = 0x1f83d9ab, h7 = 0x5be0cd19;
|
||||
var len = data.length, bitLen = len * 8;
|
||||
var padLen = 64 - ((len + 9) % 64);
|
||||
var totalLen = len + 1 + (padLen === 64 ? 0 : padLen) + 8;
|
||||
var msg = new Uint8Array(totalLen);
|
||||
msg.set(data); msg[len] = 0x80;
|
||||
var dv = new DataView(msg.buffer);
|
||||
dv.setUint32(totalLen - 4, bitLen, false);
|
||||
var w = new Int32Array(64);
|
||||
for (var offset = 0; offset < totalLen; offset += 64) {
|
||||
for (var i = 0; i < 16; i++) w[i] = dv.getInt32(offset + i * 4, false);
|
||||
for (var i = 16; i < 64; i++) {
|
||||
var s0 = (((w[i-15] >>> 7) | (w[i-15] << 25)) ^ ((w[i-15] >>> 18) | (w[i-15] << 14)) ^ (w[i-15] >>> 3)) | 0;
|
||||
var s1 = (((w[i-2] >>> 17) | (w[i-2] << 15)) ^ ((w[i-2] >>> 19) | (w[i-2] << 13)) ^ (w[i-2] >>> 10)) | 0;
|
||||
w[i] = (w[i-16] + s0 + w[i-7] + s1) | 0;
|
||||
}
|
||||
var a = h0, b = h1, c = h2, d = h3, e = h4, f = h5, g = h6, h = h7;
|
||||
for (var i = 0; i < 64; i++) {
|
||||
var S1 = (((e >>> 6) | (e << 26)) ^ ((e >>> 11) | (e << 21)) ^ ((e >>> 25) | (e << 7))) | 0;
|
||||
var ch = ((e & f) ^ (~e & g)) | 0;
|
||||
var t1 = (h + S1 + ch + K[i] + w[i]) | 0;
|
||||
var S0 = (((a >>> 2) | (a << 30)) ^ ((a >>> 13) | (a << 19)) ^ ((a >>> 22) | (a << 10))) | 0;
|
||||
var maj = ((a & b) ^ (a & c) ^ (b & c)) | 0;
|
||||
var t2 = (S0 + maj) | 0;
|
||||
h = g; g = f; f = e; e = (d + t1) | 0; d = c; c = b; b = a; a = (t1 + t2) | 0;
|
||||
}
|
||||
h0 = (h0 + a) | 0; h1 = (h1 + b) | 0; h2 = (h2 + c) | 0; h3 = (h3 + d) | 0;
|
||||
h4 = (h4 + e) | 0; h5 = (h5 + f) | 0; h6 = (h6 + g) | 0; h7 = (h7 + h) | 0;
|
||||
}
|
||||
var out = new Uint8Array(32);
|
||||
var ov = new DataView(out.buffer);
|
||||
ov.setUint32(0, h0, false); ov.setUint32(4, h1, false);
|
||||
ov.setUint32(8, h2, false); ov.setUint32(12, h3, false);
|
||||
ov.setUint32(16, h4, false); ov.setUint32(20, h5, false);
|
||||
ov.setUint32(24, h6, false); ov.setUint32(28, h7, false);
|
||||
return out;
|
||||
}
|
||||
|
||||
self.onmessage = function(e) {
|
||||
var prefix = e.data.p, difficulty = e.data.d, nonce = 0;
|
||||
var enc = new TextEncoder();
|
||||
while (true) {
|
||||
var candidate = prefix + nonce.toString(36);
|
||||
var hash = sha256(enc.encode(candidate));
|
||||
var fullBytes = difficulty >> 3, remainBits = difficulty & 7, ok = true;
|
||||
for (var i = 0; i < fullBytes; i++) { if (hash[i] !== 0) { ok = false; break; } }
|
||||
if (ok && remainBits > 0) {
|
||||
var mask = (0xff << (8 - remainBits)) & 0xff;
|
||||
if ((hash[fullBytes] & mask) !== 0) ok = false;
|
||||
}
|
||||
if (ok) { self.postMessage(nonce.toString(36)); return; }
|
||||
nonce++;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,75 @@
|
||||
// secret.go parses and validates the master-secret used by the keyring, and
|
||||
// generates a random fallback when none is configured. The secret is the
|
||||
// trust anchor for every per-epoch HMAC key and the cookie-sealing AES key,
|
||||
// so any change here is wire-protocol-affecting.
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
crand "crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// minSecretBytes is the smallest accepted master-secret length. 32 bytes is
|
||||
// the natural minimum for an HMAC-SHA256 key with full security margin; we
|
||||
// reject anything shorter as a configuration error rather than silently
|
||||
// padding it.
|
||||
const minSecretBytes = 32
|
||||
|
||||
// generateRandomSecret returns a fresh 32-byte secret suitable for use when
|
||||
// no master_secret is configured. Single-instance deployments are fine with
|
||||
// this; distributed deployments MUST configure a shared secret because each
|
||||
// instance generates an independent random one here.
|
||||
func generateRandomSecret() ([]byte, error) {
|
||||
buf := make([]byte, 32)
|
||||
if _, err := crand.Read(buf); err != nil {
|
||||
return nil, fmt.Errorf("generate random master secret: %w", err)
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// ParseConfiguredSecret accepts a configured master secret as either a hex
|
||||
// string (preferred — encodes raw bytes unambiguously) or a raw passphrase
|
||||
// (fallback for human-edited configs). The result is at least minSecretBytes.
|
||||
func ParseConfiguredSecret(value string) ([]byte, error) {
|
||||
if value == "" {
|
||||
return nil, errors.New("empty master secret")
|
||||
}
|
||||
|
||||
// Hex form: even length, hex digits only.
|
||||
if isHex(value) {
|
||||
raw, err := hex.DecodeString(value)
|
||||
if err == nil {
|
||||
if len(raw) < minSecretBytes {
|
||||
return nil, fmt.Errorf("hex master secret decodes to %d bytes; minimum is %d", len(raw), minSecretBytes)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
// Fall through to passphrase handling on hex parse failure — defensive.
|
||||
}
|
||||
|
||||
if len(value) < minSecretBytes {
|
||||
return nil, fmt.Errorf("passphrase master secret is %d bytes; minimum is %d", len(value), minSecretBytes)
|
||||
}
|
||||
|
||||
return []byte(value), nil
|
||||
}
|
||||
|
||||
func isHex(s string) bool {
|
||||
if s == "" || len(s)%2 != 0 {
|
||||
return false
|
||||
}
|
||||
for i := range len(s) {
|
||||
c := s[i]
|
||||
switch {
|
||||
case c >= '0' && c <= '9':
|
||||
case c >= 'a' && c <= 'f':
|
||||
case c >= 'A' && c <= 'F':
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestParseConfiguredSecret_Hex(t *testing.T) {
|
||||
// 32 raw bytes = 64 hex chars
|
||||
hex := strings.Repeat("ab", 32)
|
||||
got, err := ParseConfiguredSecret(hex)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, got, 32)
|
||||
}
|
||||
|
||||
func TestParseConfiguredSecret_HexTooShort(t *testing.T) {
|
||||
// 31 raw bytes = 62 hex chars → must be rejected
|
||||
hex := strings.Repeat("ab", 31)
|
||||
_, err := ParseConfiguredSecret(hex)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "minimum is 32")
|
||||
}
|
||||
|
||||
func TestParseConfiguredSecret_Passphrase(t *testing.T) {
|
||||
// "long enough passphrase, definitely > 32 bytes" — 47 bytes
|
||||
pw := "long enough passphrase, definitely > 32 bytes"
|
||||
got, err := ParseConfiguredSecret(pw)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []byte(pw), got)
|
||||
}
|
||||
|
||||
func TestParseConfiguredSecret_PassphraseTooShort(t *testing.T) {
|
||||
_, err := ParseConfiguredSecret("short")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestParseConfiguredSecret_Empty(t *testing.T) {
|
||||
_, err := ParseConfiguredSecret("")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestGenerateRandomSecret(t *testing.T) {
|
||||
a, err := generateRandomSecret()
|
||||
require.NoError(t, err)
|
||||
b, err := generateRandomSecret()
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, a, 32)
|
||||
assert.Len(t, b, 32)
|
||||
assert.NotEqual(t, a, b, "successive calls must yield different secrets")
|
||||
}
|
||||
|
||||
func TestNewChallengeRuntime_RejectsShortSecret(t *testing.T) {
|
||||
// Bypassing ParseConfiguredSecret to feed the runtime a too-short secret
|
||||
// directly via the option — the runtime itself must defend against this.
|
||||
short := []byte("too-short")
|
||||
_, err := NewChallengeRuntime(t.Context(), WithMasterSecret(short))
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "minimum")
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// spent_set.go is the single-use store that eliminates challenge replay: each
|
||||
// validated submission burns its per-challenge nonce `r`, so a replay fails.
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
)
|
||||
|
||||
// spentSetDefaultMaxEntries is a deep DoS backstop. Growth is sig+PoW-gated and
|
||||
// TTL-bounded (ticketAgeBackstop), so steady-state stays far below this. If the
|
||||
// cap is ever hit, LRU evicts the oldest (maybe still-live) `r`, letting that
|
||||
// one submission replay once — acceptable at this size.
|
||||
const spentSetDefaultMaxEntries = 1_000_000
|
||||
|
||||
// spentSet records consumed per-challenge nonces. Safe for concurrent use.
|
||||
type spentSet struct {
|
||||
// mu makes the Has/Set pair in checkAndInsert atomic (gcache locks each op
|
||||
// independently, which alone wouldn't stop two concurrent replays winning).
|
||||
mu sync.Mutex
|
||||
cache gcache.Cache
|
||||
|
||||
// maxEntries is the configured LRU cap, retained for introspection.
|
||||
maxEntries int
|
||||
}
|
||||
|
||||
func newSpentSet(maxEntries int) *spentSet {
|
||||
return &spentSet{cache: gcache.New(maxEntries).LRU().Build(), maxEntries: maxEntries}
|
||||
}
|
||||
|
||||
// checkAndInsert atomically records `r` as spent, returning true if it was
|
||||
// fresh and false if already present (replay). TTL matches the freshness
|
||||
// window, so an expired (absent) entry is also one the caller rejects anyway.
|
||||
func (s *spentSet) checkAndInsert(r string, ttl time.Duration) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.cache.Has(r) {
|
||||
return false
|
||||
}
|
||||
|
||||
// SetWithExpire can't fail here: we configure no serialization/eviction hook.
|
||||
_ = s.cache.SetWithExpire(r, struct{}{}, ttl)
|
||||
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSpentSet_CheckAndInsert(t *testing.T) {
|
||||
s := newSpentSet(spentSetDefaultMaxEntries)
|
||||
|
||||
// First insert of a fresh r succeeds.
|
||||
assert.True(t, s.checkAndInsert("r1", time.Minute))
|
||||
// Replay of the same r is rejected.
|
||||
assert.False(t, s.checkAndInsert("r1", time.Minute))
|
||||
// A different r is independent.
|
||||
assert.True(t, s.checkAndInsert("r2", time.Minute))
|
||||
}
|
||||
|
||||
// TestSpentSet_ConcurrentSameKey asserts the check-and-insert pair is atomic:
|
||||
// when N goroutines race to burn the same r, exactly one wins. Run with -race.
|
||||
func TestSpentSet_ConcurrentSameKey(t *testing.T) {
|
||||
s := newSpentSet(spentSetDefaultMaxEntries)
|
||||
|
||||
const goroutines = 64
|
||||
var wins int64
|
||||
var wg sync.WaitGroup
|
||||
start := make(chan struct{})
|
||||
|
||||
for range goroutines {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
if s.checkAndInsert("same-r", time.Minute) {
|
||||
atomic.AddInt64(&wins, 1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
close(start)
|
||||
wg.Wait()
|
||||
|
||||
assert.Equal(t, int64(1), wins, "exactly one goroutine must win the single-use race")
|
||||
}
|
||||
|
||||
// TestSpentSet_Expiry asserts an entry is re-accepted after its TTL lapses.
|
||||
// (In production the freshness check rejects such an aged r anyway; this only
|
||||
// guards the store's own expiry behavior.)
|
||||
func TestSpentSet_Expiry(t *testing.T) {
|
||||
s := newSpentSet(spentSetDefaultMaxEntries)
|
||||
|
||||
assert.True(t, s.checkAndInsert("r-exp", 20*time.Millisecond))
|
||||
assert.False(t, s.checkAndInsert("r-exp", 20*time.Millisecond))
|
||||
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
|
||||
assert.True(t, s.checkAndInsert("r-exp", 20*time.Millisecond),
|
||||
"entry must be re-accepted after its TTL expires")
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestSplitBundle_HookSentinelInBakedBundle is the smallest possible
|
||||
// regression guard for the split-bundle protocol: the static obfuscated
|
||||
// bundle that we ship in initial_bundle.js.gz MUST contain a literal
|
||||
// occurrence of the hook sentinel. Without it, the dynamic key module
|
||||
// would never find globalThis[hookName] and the challenge would silently
|
||||
// fail in every browser.
|
||||
//
|
||||
// The sentinel survives the high-obfuscation preset only because
|
||||
// obfuscate.js explicitly registers it via reservedStrings. If someone
|
||||
// removes that registration this test fires before any user notices.
|
||||
func TestSplitBundle_HookSentinelInBakedBundle(t *testing.T) {
|
||||
rt := &ChallengeRuntime{}
|
||||
require.NoError(t, rt.seedCacheFromInitialBundle())
|
||||
|
||||
bundle := rt.getChallengeCode()
|
||||
require.NotEmpty(t, bundle)
|
||||
|
||||
count := strings.Count(bundle, hookSentinel)
|
||||
assert.GreaterOrEqual(t, count, 1,
|
||||
"hook sentinel %q must survive obfuscation in the challenge code (was reservedStrings dropped from obfuscate.js?)",
|
||||
hookSentinel)
|
||||
}
|
||||
|
||||
// TestSplitBundle_DynamicModuleEmbedsCurrentEpochKey verifies the dynamic
|
||||
// module that GetChallengePage will serve actually contains the per-epoch
|
||||
// key. The output is heavily obfuscated, but the key bytes (hex-encoded)
|
||||
// MUST be present — if they aren't, the client can't HMAC the ticket.
|
||||
//
|
||||
// We do not assert the literal hex string survives intact (the obfuscator
|
||||
// transforms most string literals); instead we run the obfuscated module
|
||||
// through a JS engine and check it tries to call our hook with the right
|
||||
// key. That requires running JS — out of scope here, so we settle for: the
|
||||
// dynamic module is non-empty, references the hook sentinel, and doesn't
|
||||
// accidentally leak the hex key as a plain literal.
|
||||
func TestSplitBundle_DynamicModuleObfuscatesKey(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
rt := newChallengeRuntimeForSplitTest(t, keys)
|
||||
|
||||
got, err := rt.currentDynamicModule(t.Context())
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, got)
|
||||
|
||||
epoch, signKey := keys.Current()
|
||||
keyHex := hex.EncodeToString(signKey)
|
||||
|
||||
// The hook sentinel must appear (reservedStrings keeps it intact).
|
||||
assert.Contains(t, got, hookSentinel,
|
||||
"dynamic module is missing the hook sentinel %q", hookSentinel)
|
||||
|
||||
// The key MUST NOT appear in plain hex form. The obfuscator's
|
||||
// string-array transform should have encoded it. If the literal hex
|
||||
// shows up the obfuscation strength has dropped, defeating the whole
|
||||
// point of the split-bundle design.
|
||||
assert.NotContains(t, got, keyHex,
|
||||
"per-epoch key leaked into the dynamic module in plain hex (obfuscator regression?)")
|
||||
|
||||
// Sanity: epoch number can leak as a literal — it's not a secret.
|
||||
t.Logf("dynamic module size for epoch %d: %d bytes", epoch, len(got))
|
||||
}
|
||||
|
||||
// TestSplitBundle_DynamicModuleCachedPerEpoch confirms repeated calls in
|
||||
// the same epoch share the obfuscation cost (single-flight via cache).
|
||||
//
|
||||
// We pin the keyring's clock so the two calls are guaranteed to see the
|
||||
// same epoch — without this, the ~12s obfuscation cost can straddle a
|
||||
// rotation boundary on testKeyRing's 1-minute interval and the second
|
||||
// call ends up generating a different module under a different key.
|
||||
func TestSplitBundle_DynamicModuleCachedPerEpoch(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
pinned := time.Now()
|
||||
keys.now = func() time.Time { return pinned }
|
||||
|
||||
rt := newChallengeRuntimeForSplitTest(t, keys)
|
||||
|
||||
first, err := rt.currentDynamicModule(t.Context())
|
||||
require.NoError(t, err)
|
||||
|
||||
second, err := rt.currentDynamicModule(t.Context())
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, first, second, "second call must return the cached module byte-for-byte")
|
||||
}
|
||||
|
||||
// TestSplitBundle_DynamicModuleRebuildsOnEpochAdvance asserts that when the
|
||||
// keyring rolls to a new epoch, currentDynamicModule produces a
|
||||
// fresh module embedding the new key.
|
||||
func TestSplitBundle_DynamicModuleRebuildsOnEpochAdvance(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
rt := newChallengeRuntimeForSplitTest(t, keys)
|
||||
|
||||
first, err := rt.currentDynamicModule(t.Context())
|
||||
require.NoError(t, err)
|
||||
firstEpoch, _ := keys.Current()
|
||||
|
||||
// Advance the keyring's clock past the rotation interval.
|
||||
keys.now = func() time.Time { return time.Now().Add(2 * time.Minute) }
|
||||
|
||||
second, err := rt.currentDynamicModule(t.Context())
|
||||
require.NoError(t, err)
|
||||
secondEpoch, _ := keys.Current()
|
||||
|
||||
assert.NotEqual(t, firstEpoch, secondEpoch, "epoch did not advance")
|
||||
assert.NotEqual(t, first, second, "dynamic module did not change after epoch advance")
|
||||
}
|
||||
|
||||
// TestSplitBundle_DynamicModuleConcurrentSingleflight asserts that concurrent
|
||||
// callers for the same epoch share a single obfuscation pass via singleflight.
|
||||
// N concurrent goroutines call currentDynamicModule for the same epoch; only
|
||||
// ONE must actually run the obfuscator (the others share its result via
|
||||
// singleflight). If we ever hold the cache mutex across the obfuscation
|
||||
// pass, this test will time out OR show every caller doing its own
|
||||
// obfuscation.
|
||||
func TestSplitBundle_DynamicModuleConcurrentSingleflight(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
pinned := time.Now()
|
||||
keys.now = func() time.Time { return pinned }
|
||||
rt := newChallengeRuntimeForSplitTest(t, keys)
|
||||
|
||||
const N = 8
|
||||
results := make([]string, N)
|
||||
errs := make([]error, N)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(N)
|
||||
|
||||
start := time.Now()
|
||||
for i := range N {
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
out, err := rt.currentDynamicModule(t.Context())
|
||||
results[idx] = out
|
||||
errs[idx] = err
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
elapsed := time.Since(start)
|
||||
|
||||
for i, err := range errs {
|
||||
require.NoError(t, err, "goroutine %d failed", i)
|
||||
}
|
||||
|
||||
// All callers must have observed the same obfuscation output. If any
|
||||
// caller diverged, singleflight wasn't doing its job.
|
||||
for i := 1; i < N; i++ {
|
||||
require.Equal(t, results[0], results[i],
|
||||
"goroutines %d and 0 saw different module bytes; singleflight is broken", i)
|
||||
}
|
||||
|
||||
// Tighter bound: with N=8 goroutines and a single ~5s obfuscation,
|
||||
// the wall-clock should be roughly one obfuscation, not N. We use
|
||||
// 2x as the threshold to absorb scheduler noise; a regression where
|
||||
// every caller runs its own obfuscation would push elapsed past
|
||||
// N * 5s = 40s, way above the threshold.
|
||||
t.Logf("8 concurrent currentDynamicModule calls completed in %s", elapsed)
|
||||
require.Less(t, elapsed, 15*time.Second,
|
||||
"concurrent calls took %s; singleflight likely not coalescing (would expect 1 obfuscation, got serial)",
|
||||
elapsed)
|
||||
}
|
||||
|
||||
// TestSplitBundle_HTMLDoesNotContainSecret is the core security invariant
|
||||
// of the split-bundle design: the HTML challenge page MUST NOT contain the
|
||||
// per-epoch sign key in plain (hex) form. If it does, an attacker can
|
||||
// scrape it and forge tickets without running the obfuscated bundle —
|
||||
// exactly the failure mode the split-bundle protocol was designed to fix.
|
||||
func TestSplitBundle_HTMLDoesNotContainSecret(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
rt := newChallengeRuntimeForSplitTest(t, keys)
|
||||
|
||||
html, err := rt.GetChallengePage(t.Context(), "test-agent", 8)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, html)
|
||||
|
||||
_, signKey := keys.Current()
|
||||
keyHex := hex.EncodeToString(signKey)
|
||||
|
||||
assert.NotContains(t, html, keyHex,
|
||||
"the per-epoch sign key leaked into plain HTML \u2014 split-bundle invariant violated")
|
||||
}
|
||||
|
||||
// TestSplitBundle_HTMLLoadsFPScannerViaScriptTag asserts the rendered page
|
||||
// pulls the public fpscanner in via a <script src> tag (so it is cacheable and
|
||||
// served unobfuscated) rather than inlining it. fpscanner-only source strings
|
||||
// must NOT appear inline in the page.
|
||||
func TestSplitBundle_HTMLLoadsFPScannerViaScriptTag(t *testing.T) {
|
||||
keys := testKeyRing()
|
||||
rt := newChallengeRuntimeForSplitTest(t, keys)
|
||||
|
||||
html, err := rt.GetChallengePage(t.Context(), "test-agent", 8)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Contains(t, html, `<script src="`+ChallengeFPScannerPath+`">`,
|
||||
"challenge page must load fpscanner via a <script src> tag")
|
||||
require.NotContains(t, strings.ToLower(html), "swiftshader",
|
||||
"fpscanner source must be served separately, not inlined in the challenge page")
|
||||
}
|
||||
|
||||
// hookSentinel must match the constant used by both challenge.js (as
|
||||
// CSEC_HOOK_NAME) and obfuscate.js (in reservedStrings). Hardcoded here on
|
||||
// purpose so the test breaks if the JS-side constant is renamed without a
|
||||
// matching update.
|
||||
const hookSentinel = "__CSEC_CHALLENGE_HOOK_v1__"
|
||||
|
||||
// newChallengeRuntimeForSplitTest builds a runtime with WASM + the baked-in
|
||||
// initial bundle but a deterministic keyring, so split-bundle assertions
|
||||
// can be made against a known key.
|
||||
func newChallengeRuntimeForSplitTest(t *testing.T, keys *KeyRing) *ChallengeRuntime {
|
||||
t.Helper()
|
||||
rt, err := NewChallengeRuntime(t.Context(),
|
||||
WithMasterSecret(testSecret),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
rt.keys = keys
|
||||
// Reset the dynamic module cache because the constructor pre-warmed it
|
||||
// with rt.keys (which we just swapped). The next call regenerates.
|
||||
rt.dynamicModuleCacheMu.Lock()
|
||||
rt.dynamicModuleCache = make(map[int64][]string)
|
||||
rt.dynamicModuleCacheMu.Unlock()
|
||||
return rt
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
// Package challenge static_bundle.go handles the **challenge code** — the
|
||||
// crypto/glue JavaScript injected inline on the challenge page. It is obfuscated
|
||||
// once at build time (initial_bundle.js.gz) and loaded verbatim at startup; no
|
||||
// runtime re-obfuscation. The public fpscanner is served separately and
|
||||
// unobfuscated at ChallengeFPScannerPath. For the sensitive per-epoch HMAC sign
|
||||
// key, see dynamic_module.go.
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
challengejs "github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/js"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// initialBundleGz is the build-time obfuscated challenge code (via
|
||||
// `go generate ./pkg/appsec/challenge/js/...`), embedded so the runtime can
|
||||
// serve it immediately without paying the obfuscation cost at startup.
|
||||
//
|
||||
//go:embed initial_bundle.js.gz
|
||||
var initialBundleGz []byte
|
||||
|
||||
var (
|
||||
initialBundle string
|
||||
initialBundleOnce sync.Once
|
||||
initialBundleErr error
|
||||
)
|
||||
|
||||
// FPScannerJS is the public, unobfuscated fpscanner bundle served at
|
||||
// ChallengeFPScannerPath. Re-exported so the dispatcher can serve it via the
|
||||
// challenge package alongside PowWorkerJS.
|
||||
var FPScannerJS = challengejs.FPScannerJS
|
||||
|
||||
// seedCacheFromInitialBundle decompresses the build-time obfuscated challenge
|
||||
// code (initial_bundle.js.gz) and stores it as the static code served on every
|
||||
// challenge page. Cheap (~ms) — eliminates the obfuscation that startup would
|
||||
// otherwise pay.
|
||||
func (c *ChallengeRuntime) seedCacheFromInitialBundle() error {
|
||||
initialBundleOnce.Do(func() {
|
||||
decompressStart := time.Now()
|
||||
|
||||
if len(initialBundleGz) == 0 {
|
||||
initialBundleErr = errors.New("baked-in initial_bundle.js.gz is empty (was `go generate` run?)")
|
||||
return
|
||||
}
|
||||
|
||||
gz, err := gzip.NewReader(bytes.NewReader(initialBundleGz))
|
||||
if err != nil {
|
||||
initialBundleErr = fmt.Errorf("gzip reader for initial bundle: %w", err)
|
||||
return
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
decoded, err := io.ReadAll(gz)
|
||||
if err != nil {
|
||||
initialBundleErr = fmt.Errorf("decompress initial bundle: %w", err)
|
||||
return
|
||||
}
|
||||
initialBundle = string(decoded)
|
||||
|
||||
c.log().WithFields(log.Fields{
|
||||
"compressed_bytes": len(initialBundleGz),
|
||||
"decompressed_bytes": len(initialBundle),
|
||||
"duration_ms": time.Since(decompressStart).Milliseconds(),
|
||||
}).Debug("decompressed baked-in obfuscated challenge code")
|
||||
})
|
||||
|
||||
if initialBundleErr != nil {
|
||||
return initialBundleErr
|
||||
}
|
||||
if initialBundle == "" {
|
||||
return errors.New("initial bundle is empty after decompression")
|
||||
}
|
||||
|
||||
c.challengeCode = initialBundle
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildChallengeBundle substitutes the internal-path placeholders into the
|
||||
// (minified, not-yet-obfuscated) challenge code. Used only by the synchronous
|
||||
// fallback below — the normal path serves the pre-obfuscated initial bundle.
|
||||
func (*ChallengeRuntime) buildChallengeBundle() string {
|
||||
return strings.NewReplacer(
|
||||
"__CROWDSEC_SUBMIT_PATH__", ChallengeSubmitPath,
|
||||
"__CROWDSEC_POW_WORKER_PATH__", ChallengePowWorkerPath,
|
||||
).Replace(challengejs.ChallengeCode)
|
||||
}
|
||||
|
||||
// generateAndCacheChallengeJS is the synchronous fallback used when the
|
||||
// baked-in initial bundle is missing or corrupt at startup. It obfuscates the
|
||||
// challenge code in the foreground — undesirable but preferable to starting up
|
||||
// with no code to serve when `go generate` was not run. Steady-state operation
|
||||
// never hits this path.
|
||||
func (c *ChallengeRuntime) generateAndCacheChallengeJS(ctx context.Context) error {
|
||||
obfuscated, err := c.ObfuscateJS(ctx, c.buildChallengeBundle())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.challengeCode = obfuscated
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getChallengeCode returns the static, build-time-obfuscated challenge code
|
||||
// injected inline on the challenge page.
|
||||
func (c *ChallengeRuntime) getChallengeCode() string {
|
||||
return c.challengeCode
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
// ticket.go holds the per-challenge crypto handed to the browser and verified
|
||||
// on submit: a random per-challenge nonce `r`, the PoW salt + its MAC, and the
|
||||
// helpers to derive the per-challenge secret `s = HMAC(K_epoch, r)`. The browser
|
||||
// solves the PoW (leading-zero-bits) and signs its submission with `s`;
|
||||
// ValidateChallengeResponse (in challenge.go) re-derives `s`, checks the
|
||||
// signature and PoW, then burns `r` (single-use). PoW difficulty levels are
|
||||
// tuned for pure-JS SHA-256 through the obfuscated runtime; see the
|
||||
// PowDifficulty* constants below.
|
||||
|
||||
package challenge
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
crand "crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PoW difficulty levels in leading zero bits. Pure JS SHA-256 through the
|
||||
// obfuscator runs ~500-5000 ops/sec, so keep these conservative.
|
||||
const (
|
||||
PowDifficultyDisabled = 0 // no PoW required, nonce "0" always valid
|
||||
PowDifficultyLow = 10 // ~1024 avg iterations ≈ 0.2-2s
|
||||
PowDifficultyMedium = 12 // ~4096 avg iterations ≈ 1-8s
|
||||
PowDifficultyHigh = 15 // ~32768 avg iterations ≈ 7-60s
|
||||
PowDifficultyImpossible = 256 // full SHA-256 width: clients cannot solve, server always rejects
|
||||
|
||||
defaultPowDifficulty = PowDifficultyMedium
|
||||
)
|
||||
|
||||
// ticketAgeBackstop is a loose ceiling on accepted submission age in
|
||||
// verifyChallenge. The actual freshness gate is the keyring live
|
||||
// window (rotationInterval × maxLiveEpochs); this is a separate ceiling
|
||||
// that protects against operators configuring an unusually wide live
|
||||
// window. Loose enough not to interfere with real submissions on slow
|
||||
// clients (high-difficulty PoW) but tight enough to bound replay
|
||||
// surface in pathological configurations.
|
||||
const ticketAgeBackstop = 20 * time.Minute
|
||||
|
||||
// generateChallengeNonce returns a fresh 16-byte random per-challenge nonce
|
||||
// (`r`) as hex. `r` keys single-use bookkeeping (spent_set.go) and seeds the
|
||||
// per-challenge secret `s = HMAC(K_epoch, r)`. Error (not panic) on entropy
|
||||
// failure so only the current request fails.
|
||||
func generateChallengeNonce() (string, error) {
|
||||
buf := make([]byte, 16)
|
||||
if _, err := crand.Read(buf); err != nil {
|
||||
return "", fmt.Errorf("generate challenge nonce: %w", err)
|
||||
}
|
||||
|
||||
return hex.EncodeToString(buf), nil
|
||||
}
|
||||
|
||||
// deriveChallengeSecret computes the per-challenge signing secret
|
||||
// `s = HMAC(K_epoch, r)` (hex). `s` is never transmitted; client and server
|
||||
// derive it independently from the same per-epoch key.
|
||||
func deriveChallengeSecret(signKey []byte, r string) string {
|
||||
return hmacSHA256Hex(signKey, []byte(r))
|
||||
}
|
||||
|
||||
// epochForTimestamp converts a nanosecond UnixNano string (the format used in
|
||||
// challenge.go's ts) into the keyring's epoch identifier. Uses the same
|
||||
// rotation interval as the keyring so two instances always agree.
|
||||
func (c *ChallengeRuntime) epochForTimestamp(ts string) int64 {
|
||||
tsVal, err := strconv.ParseInt(ts, 10, 64)
|
||||
if err != nil || tsVal <= 0 {
|
||||
// Caller will reject the request via the liveness check anyway; return
|
||||
// an out-of-window sentinel epoch.
|
||||
return -1
|
||||
}
|
||||
return tsVal / int64(time.Second) / int64(c.keys.rotationInterval/time.Second)
|
||||
}
|
||||
|
||||
// generatePowPrefix returns a freshly-generated 16-byte random PoW salt
|
||||
// rendered as a hex string. Errors from crypto/rand.Read indicate a broken
|
||||
// kernel entropy pool, which is recoverable at the request layer (we can
|
||||
// reject the current challenge and let the client retry) — returning the
|
||||
// error rather than panicking keeps a single failing request from taking
|
||||
// down the whole WAF.
|
||||
func generatePowPrefix() (string, error) {
|
||||
buf := make([]byte, 16)
|
||||
if _, err := crand.Read(buf); err != nil {
|
||||
return "", fmt.Errorf("generate PoW prefix: %w", err)
|
||||
}
|
||||
|
||||
return hex.EncodeToString(buf), nil
|
||||
}
|
||||
|
||||
// computePowMAC authenticates a PoW salt as server-generated and bound to a
|
||||
// specific challenge (`r`) + timestamp + difficulty, signed with the per-epoch
|
||||
// key.
|
||||
func (c *ChallengeRuntime) computePowMAC(salt, r, ts string, difficulty int) string {
|
||||
epoch := c.epochForTimestamp(ts)
|
||||
signKey, ok := c.keys.SignKey(epoch)
|
||||
if !ok {
|
||||
_, signKey = c.keys.Current()
|
||||
}
|
||||
|
||||
h := hmac.New(sha256.New, signKey)
|
||||
h.Write([]byte(salt))
|
||||
h.Write([]byte(r))
|
||||
h.Write([]byte(ts))
|
||||
h.Write([]byte(strconv.Itoa(difficulty)))
|
||||
|
||||
return fmt.Sprintf("%x", h.Sum(nil))
|
||||
}
|
||||
|
||||
func hasLeadingZeroBits(hash []byte, bits int) bool {
|
||||
fullBytes := bits / 8
|
||||
remainBits := bits % 8
|
||||
|
||||
for i := range fullBytes {
|
||||
if hash[i] != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if remainBits > 0 {
|
||||
mask := byte(0xFF << (8 - remainBits))
|
||||
if hash[fullBytes]&mask != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// deriveFingerprintObfKey returns the keystream key for the fingerprint
|
||||
// payload: `HMAC(s, "fpenc"||r)`.
|
||||
func deriveFingerprintObfKey(s, r string) string {
|
||||
return hmacSHA256Hex([]byte(s), []byte("fpenc"+r))
|
||||
}
|
||||
|
||||
// deobfuscateFingerprint reverses the client-side repeating-key XOR + base64
|
||||
// applied to the fingerprint JSON, using the key from deriveFingerprintObfKey.
|
||||
func deobfuscateFingerprint(obfKey string, payload string) (string, error) {
|
||||
payloadBytes, err := base64.StdEncoding.DecodeString(payload)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to decode obfuscated fingerprint: %w", err)
|
||||
}
|
||||
|
||||
out := make([]byte, len(payloadBytes))
|
||||
|
||||
for i := range payloadBytes {
|
||||
out[i] = payloadBytes[i] ^ obfKey[i%len(obfKey)]
|
||||
}
|
||||
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// verifyChallenge gates timestamp freshness and authenticates the PoW-salt
|
||||
// binding, returning the per-epoch sign key (for deriving `s`) on success.
|
||||
// Stateless — any instance sharing the master secret can verify. Knowledge of
|
||||
// the per-epoch key is proven separately by the caller's `sig` check.
|
||||
func (c *ChallengeRuntime) verifyChallenge(clientR, clientTS, clientPowSalt, clientPowMAC string, clientDifficulty int) ([]byte, bool) {
|
||||
tsVal, err := strconv.ParseInt(clientTS, 10, 64)
|
||||
if err != nil || tsVal <= 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
age := time.Since(time.Unix(0, tsVal))
|
||||
if age < 0 || age > ticketAgeBackstop {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
epoch := c.epochForTimestamp(clientTS)
|
||||
signKey, ok := c.keys.SignKey(epoch)
|
||||
if !ok {
|
||||
// Epoch fell out of the live window — reject without leaking a usable
|
||||
// "signature mismatch" vs "stale epoch" distinction via timing.
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// MAC is recomputed over the client-submitted difficulty: a tampered `d`
|
||||
// won't match, so this is what authenticates the difficulty.
|
||||
expectedMAC := c.computePowMAC(clientPowSalt, clientR, clientTS, clientDifficulty)
|
||||
|
||||
if !hmac.Equal([]byte(clientPowMAC), []byte(expectedMAC)) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return signKey, true
|
||||
}
|
||||
|
||||
func hmacSHA256Hex(key, msg []byte) string {
|
||||
h := hmac.New(sha256.New, key)
|
||||
h.Write(msg)
|
||||
return fmt.Sprintf("%x", h.Sum(nil))
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package challenge
|
||||
|
||||
// Generated from IANA tzdb zone1970.tab (public domain). Maps an IANA
|
||||
// timezone name to the set of ISO-3166 alpha-2 country codes associated
|
||||
// with it. Regenerate with:
|
||||
//
|
||||
// awk -F'\t' '!/^#/ && NF>=3 { ... }' /usr/share/zoneinfo/zone1970.tab
|
||||
//
|
||||
// see the plan / repo docs for the full script.
|
||||
//
|
||||
// Historical aliases (Asia/Calcutta, Europe/Kiev, etc.) are in tzAliases
|
||||
// below so older browsers that still send those names resolve correctly.
|
||||
|
||||
var tzCountry = map[string][]string{
|
||||
"Europe/Andorra": {"AD"},
|
||||
"Asia/Dubai": {"AE", "OM", "RE", "SC", "TF"},
|
||||
"Asia/Kabul": {"AF"},
|
||||
"Europe/Tirane": {"AL"},
|
||||
"Asia/Yerevan": {"AM"},
|
||||
"Antarctica/Casey": {"AQ"},
|
||||
"Antarctica/Davis": {"AQ"},
|
||||
"Antarctica/Mawson": {"AQ"},
|
||||
"Antarctica/Palmer": {"AQ"},
|
||||
"Antarctica/Rothera": {"AQ"},
|
||||
"Antarctica/Troll": {"AQ"},
|
||||
"Antarctica/Vostok": {"AQ"},
|
||||
"America/Argentina/Buenos_Aires": {"AR"},
|
||||
"America/Argentina/Cordoba": {"AR"},
|
||||
"America/Argentina/Salta": {"AR"},
|
||||
"America/Argentina/Jujuy": {"AR"},
|
||||
"America/Argentina/Tucuman": {"AR"},
|
||||
"America/Argentina/Catamarca": {"AR"},
|
||||
"America/Argentina/La_Rioja": {"AR"},
|
||||
"America/Argentina/San_Juan": {"AR"},
|
||||
"America/Argentina/Mendoza": {"AR"},
|
||||
"America/Argentina/San_Luis": {"AR"},
|
||||
"America/Argentina/Rio_Gallegos": {"AR"},
|
||||
"America/Argentina/Ushuaia": {"AR"},
|
||||
"Pacific/Pago_Pago": {"AS", "UM"},
|
||||
"Europe/Vienna": {"AT"},
|
||||
"Australia/Lord_Howe": {"AU"},
|
||||
"Antarctica/Macquarie": {"AU"},
|
||||
"Australia/Hobart": {"AU"},
|
||||
"Australia/Melbourne": {"AU"},
|
||||
"Australia/Sydney": {"AU"},
|
||||
"Australia/Broken_Hill": {"AU"},
|
||||
"Australia/Brisbane": {"AU"},
|
||||
"Australia/Lindeman": {"AU"},
|
||||
"Australia/Adelaide": {"AU"},
|
||||
"Australia/Darwin": {"AU"},
|
||||
"Australia/Perth": {"AU"},
|
||||
"Australia/Eucla": {"AU"},
|
||||
"Asia/Baku": {"AZ"},
|
||||
"America/Barbados": {"BB"},
|
||||
"Asia/Dhaka": {"BD"},
|
||||
"Europe/Brussels": {"BE", "LU", "NL"},
|
||||
"Europe/Sofia": {"BG"},
|
||||
"Atlantic/Bermuda": {"BM"},
|
||||
"America/La_Paz": {"BO"},
|
||||
"America/Noronha": {"BR"},
|
||||
"America/Belem": {"BR"},
|
||||
"America/Fortaleza": {"BR"},
|
||||
"America/Recife": {"BR"},
|
||||
"America/Araguaina": {"BR"},
|
||||
"America/Maceio": {"BR"},
|
||||
"America/Bahia": {"BR"},
|
||||
"America/Sao_Paulo": {"BR"},
|
||||
"America/Campo_Grande": {"BR"},
|
||||
"America/Cuiaba": {"BR"},
|
||||
"America/Santarem": {"BR"},
|
||||
"America/Porto_Velho": {"BR"},
|
||||
"America/Boa_Vista": {"BR"},
|
||||
"America/Manaus": {"BR"},
|
||||
"America/Eirunepe": {"BR"},
|
||||
"America/Rio_Branco": {"BR"},
|
||||
"Asia/Thimphu": {"BT"},
|
||||
"Europe/Minsk": {"BY"},
|
||||
"America/Belize": {"BZ"},
|
||||
"America/St_Johns": {"CA"},
|
||||
"America/Halifax": {"CA"},
|
||||
"America/Glace_Bay": {"CA"},
|
||||
"America/Moncton": {"CA"},
|
||||
"America/Goose_Bay": {"CA"},
|
||||
"America/Toronto": {"CA", "BS"},
|
||||
"America/Iqaluit": {"CA"},
|
||||
"America/Winnipeg": {"CA"},
|
||||
"America/Resolute": {"CA"},
|
||||
"America/Rankin_Inlet": {"CA"},
|
||||
"America/Regina": {"CA"},
|
||||
"America/Swift_Current": {"CA"},
|
||||
"America/Edmonton": {"CA"},
|
||||
"America/Cambridge_Bay": {"CA"},
|
||||
"America/Inuvik": {"CA"},
|
||||
"America/Dawson_Creek": {"CA"},
|
||||
"America/Fort_Nelson": {"CA"},
|
||||
"America/Whitehorse": {"CA"},
|
||||
"America/Dawson": {"CA"},
|
||||
"America/Vancouver": {"CA"},
|
||||
"Europe/Zurich": {"CH", "DE", "LI"},
|
||||
"Africa/Abidjan": {"CI", "BF", "GH", "GM", "GN", "IS", "ML", "MR", "SH", "SL", "SN", "TG"},
|
||||
"Pacific/Rarotonga": {"CK"},
|
||||
"America/Santiago": {"CL"},
|
||||
"America/Coyhaique": {"CL"},
|
||||
"America/Punta_Arenas": {"CL"},
|
||||
"Pacific/Easter": {"CL"},
|
||||
"Asia/Shanghai": {"CN"},
|
||||
"Asia/Urumqi": {"CN"},
|
||||
"America/Bogota": {"CO"},
|
||||
"America/Costa_Rica": {"CR"},
|
||||
"America/Havana": {"CU"},
|
||||
"Atlantic/Cape_Verde": {"CV"},
|
||||
"Asia/Nicosia": {"CY"},
|
||||
"Asia/Famagusta": {"CY"},
|
||||
"Europe/Prague": {"CZ", "SK"},
|
||||
"Europe/Berlin": {"DE", "DK", "NO", "SE", "SJ"},
|
||||
"America/Santo_Domingo": {"DO"},
|
||||
"Africa/Algiers": {"DZ"},
|
||||
"America/Guayaquil": {"EC"},
|
||||
"Pacific/Galapagos": {"EC"},
|
||||
"Europe/Tallinn": {"EE"},
|
||||
"Africa/Cairo": {"EG"},
|
||||
"Africa/El_Aaiun": {"EH"},
|
||||
"Europe/Madrid": {"ES"},
|
||||
"Africa/Ceuta": {"ES"},
|
||||
"Atlantic/Canary": {"ES"},
|
||||
"Europe/Helsinki": {"FI", "AX"},
|
||||
"Pacific/Fiji": {"FJ"},
|
||||
"Atlantic/Stanley": {"FK"},
|
||||
"Pacific/Kosrae": {"FM"},
|
||||
"Atlantic/Faroe": {"FO"},
|
||||
"Europe/Paris": {"FR", "MC"},
|
||||
"Europe/London": {"GB", "GG", "IM", "JE"},
|
||||
"Asia/Tbilisi": {"GE"},
|
||||
"America/Cayenne": {"GF"},
|
||||
"Europe/Gibraltar": {"GI"},
|
||||
"America/Nuuk": {"GL"},
|
||||
"America/Danmarkshavn": {"GL"},
|
||||
"America/Scoresbysund": {"GL"},
|
||||
"America/Thule": {"GL"},
|
||||
"Europe/Athens": {"GR"},
|
||||
"Atlantic/South_Georgia": {"GS"},
|
||||
"America/Guatemala": {"GT"},
|
||||
"Pacific/Guam": {"GU", "MP"},
|
||||
"Africa/Bissau": {"GW"},
|
||||
"America/Guyana": {"GY"},
|
||||
"Asia/Hong_Kong": {"HK"},
|
||||
"America/Tegucigalpa": {"HN"},
|
||||
"America/Port-au-Prince": {"HT"},
|
||||
"Europe/Budapest": {"HU"},
|
||||
"Asia/Jakarta": {"ID"},
|
||||
"Asia/Pontianak": {"ID"},
|
||||
"Asia/Makassar": {"ID"},
|
||||
"Asia/Jayapura": {"ID"},
|
||||
"Europe/Dublin": {"IE"},
|
||||
"Asia/Jerusalem": {"IL"},
|
||||
"Asia/Kolkata": {"IN"},
|
||||
"Indian/Chagos": {"IO"},
|
||||
"Asia/Baghdad": {"IQ"},
|
||||
"Asia/Tehran": {"IR"},
|
||||
"Europe/Rome": {"IT", "SM", "VA"},
|
||||
"America/Jamaica": {"JM"},
|
||||
"Asia/Amman": {"JO"},
|
||||
"Asia/Tokyo": {"JP", "AU"},
|
||||
"Africa/Nairobi": {"KE", "DJ", "ER", "ET", "KM", "MG", "SO", "TZ", "UG", "YT"},
|
||||
"Asia/Bishkek": {"KG"},
|
||||
"Pacific/Tarawa": {"KI", "MH", "TV", "UM", "WF"},
|
||||
"Pacific/Kanton": {"KI"},
|
||||
"Pacific/Kiritimati": {"KI"},
|
||||
"Asia/Pyongyang": {"KP"},
|
||||
"Asia/Seoul": {"KR"},
|
||||
"Asia/Almaty": {"KZ"},
|
||||
"Asia/Qyzylorda": {"KZ"},
|
||||
"Asia/Qostanay": {"KZ"},
|
||||
"Asia/Aqtobe": {"KZ"},
|
||||
"Asia/Aqtau": {"KZ"},
|
||||
"Asia/Atyrau": {"KZ"},
|
||||
"Asia/Oral": {"KZ"},
|
||||
"Asia/Beirut": {"LB"},
|
||||
"Asia/Colombo": {"LK"},
|
||||
"Africa/Monrovia": {"LR"},
|
||||
"Europe/Vilnius": {"LT"},
|
||||
"Europe/Riga": {"LV"},
|
||||
"Africa/Tripoli": {"LY"},
|
||||
"Africa/Casablanca": {"MA"},
|
||||
"Europe/Chisinau": {"MD"},
|
||||
"Pacific/Kwajalein": {"MH"},
|
||||
"Asia/Yangon": {"MM", "CC"},
|
||||
"Asia/Ulaanbaatar": {"MN"},
|
||||
"Asia/Hovd": {"MN"},
|
||||
"Asia/Macau": {"MO"},
|
||||
"America/Martinique": {"MQ"},
|
||||
"Europe/Malta": {"MT"},
|
||||
"Indian/Mauritius": {"MU"},
|
||||
"Indian/Maldives": {"MV", "TF"},
|
||||
"America/Mexico_City": {"MX"},
|
||||
"America/Cancun": {"MX"},
|
||||
"America/Merida": {"MX"},
|
||||
"America/Monterrey": {"MX"},
|
||||
"America/Matamoros": {"MX"},
|
||||
"America/Chihuahua": {"MX"},
|
||||
"America/Ciudad_Juarez": {"MX"},
|
||||
"America/Ojinaga": {"MX"},
|
||||
"America/Mazatlan": {"MX"},
|
||||
"America/Bahia_Banderas": {"MX"},
|
||||
"America/Hermosillo": {"MX"},
|
||||
"America/Tijuana": {"MX"},
|
||||
"Asia/Kuching": {"MY", "BN"},
|
||||
"Africa/Maputo": {"MZ", "BI", "BW", "CD", "MW", "RW", "ZM", "ZW"},
|
||||
"Africa/Windhoek": {"NA"},
|
||||
"Pacific/Noumea": {"NC"},
|
||||
"Pacific/Norfolk": {"NF"},
|
||||
"Africa/Lagos": {"NG", "AO", "BJ", "CD", "CF", "CG", "CM", "GA", "GQ", "NE"},
|
||||
"America/Managua": {"NI"},
|
||||
"Asia/Kathmandu": {"NP"},
|
||||
"Pacific/Nauru": {"NR"},
|
||||
"Pacific/Niue": {"NU"},
|
||||
"Pacific/Auckland": {"NZ", "AQ"},
|
||||
"Pacific/Chatham": {"NZ"},
|
||||
"America/Panama": {"PA", "CA", "KY"},
|
||||
"America/Lima": {"PE"},
|
||||
"Pacific/Tahiti": {"PF"},
|
||||
"Pacific/Marquesas": {"PF"},
|
||||
"Pacific/Gambier": {"PF"},
|
||||
"Pacific/Port_Moresby": {"PG", "AQ", "FM"},
|
||||
"Pacific/Bougainville": {"PG"},
|
||||
"Asia/Manila": {"PH"},
|
||||
"Asia/Karachi": {"PK"},
|
||||
"Europe/Warsaw": {"PL"},
|
||||
"America/Miquelon": {"PM"},
|
||||
"Pacific/Pitcairn": {"PN"},
|
||||
"America/Puerto_Rico": {"PR", "AG", "CA", "AI", "AW", "BL", "BQ", "CW", "DM", "GD", "GP", "KN", "LC", "MF", "MS", "SX", "TT", "VC", "VG", "VI"},
|
||||
"Asia/Gaza": {"PS"},
|
||||
"Asia/Hebron": {"PS"},
|
||||
"Europe/Lisbon": {"PT"},
|
||||
"Atlantic/Madeira": {"PT"},
|
||||
"Atlantic/Azores": {"PT"},
|
||||
"Pacific/Palau": {"PW"},
|
||||
"America/Asuncion": {"PY"},
|
||||
"Asia/Qatar": {"QA", "BH"},
|
||||
"Europe/Bucharest": {"RO"},
|
||||
"Europe/Belgrade": {"RS", "BA", "HR", "ME", "MK", "SI"},
|
||||
"Europe/Kaliningrad": {"RU"},
|
||||
"Europe/Moscow": {"RU"},
|
||||
"Europe/Simferopol": {"RU", "UA"},
|
||||
"Europe/Kirov": {"RU"},
|
||||
"Europe/Volgograd": {"RU"},
|
||||
"Europe/Astrakhan": {"RU"},
|
||||
"Europe/Saratov": {"RU"},
|
||||
"Europe/Ulyanovsk": {"RU"},
|
||||
"Europe/Samara": {"RU"},
|
||||
"Asia/Yekaterinburg": {"RU"},
|
||||
"Asia/Omsk": {"RU"},
|
||||
"Asia/Novosibirsk": {"RU"},
|
||||
"Asia/Barnaul": {"RU"},
|
||||
"Asia/Tomsk": {"RU"},
|
||||
"Asia/Novokuznetsk": {"RU"},
|
||||
"Asia/Krasnoyarsk": {"RU"},
|
||||
"Asia/Irkutsk": {"RU"},
|
||||
"Asia/Chita": {"RU"},
|
||||
"Asia/Yakutsk": {"RU"},
|
||||
"Asia/Khandyga": {"RU"},
|
||||
"Asia/Vladivostok": {"RU"},
|
||||
"Asia/Ust-Nera": {"RU"},
|
||||
"Asia/Magadan": {"RU"},
|
||||
"Asia/Sakhalin": {"RU"},
|
||||
"Asia/Srednekolymsk": {"RU"},
|
||||
"Asia/Kamchatka": {"RU"},
|
||||
"Asia/Anadyr": {"RU"},
|
||||
"Asia/Riyadh": {"SA", "AQ", "KW", "YE"},
|
||||
"Pacific/Guadalcanal": {"SB", "FM"},
|
||||
"Africa/Khartoum": {"SD"},
|
||||
"Asia/Singapore": {"SG", "AQ", "MY"},
|
||||
"America/Paramaribo": {"SR"},
|
||||
"Africa/Juba": {"SS"},
|
||||
"Africa/Sao_Tome": {"ST"},
|
||||
"America/El_Salvador": {"SV"},
|
||||
"Asia/Damascus": {"SY"},
|
||||
"America/Grand_Turk": {"TC"},
|
||||
"Africa/Ndjamena": {"TD"},
|
||||
"Asia/Bangkok": {"TH", "CX", "KH", "LA", "VN"},
|
||||
"Asia/Dushanbe": {"TJ"},
|
||||
"Pacific/Fakaofo": {"TK"},
|
||||
"Asia/Dili": {"TL"},
|
||||
"Asia/Ashgabat": {"TM"},
|
||||
"Africa/Tunis": {"TN"},
|
||||
"Pacific/Tongatapu": {"TO"},
|
||||
"Europe/Istanbul": {"TR"},
|
||||
"Asia/Taipei": {"TW"},
|
||||
"Europe/Kyiv": {"UA"},
|
||||
"America/New_York": {"US"},
|
||||
"America/Detroit": {"US"},
|
||||
"America/Kentucky/Louisville": {"US"},
|
||||
"America/Kentucky/Monticello": {"US"},
|
||||
"America/Indiana/Indianapolis": {"US"},
|
||||
"America/Indiana/Vincennes": {"US"},
|
||||
"America/Indiana/Winamac": {"US"},
|
||||
"America/Indiana/Marengo": {"US"},
|
||||
"America/Indiana/Petersburg": {"US"},
|
||||
"America/Indiana/Vevay": {"US"},
|
||||
"America/Chicago": {"US"},
|
||||
"America/Indiana/Tell_City": {"US"},
|
||||
"America/Indiana/Knox": {"US"},
|
||||
"America/Menominee": {"US"},
|
||||
"America/North_Dakota/Center": {"US"},
|
||||
"America/North_Dakota/New_Salem": {"US"},
|
||||
"America/North_Dakota/Beulah": {"US"},
|
||||
"America/Denver": {"US"},
|
||||
"America/Boise": {"US"},
|
||||
"America/Phoenix": {"US", "CA"},
|
||||
"America/Los_Angeles": {"US"},
|
||||
"America/Anchorage": {"US"},
|
||||
"America/Juneau": {"US"},
|
||||
"America/Sitka": {"US"},
|
||||
"America/Metlakatla": {"US"},
|
||||
"America/Yakutat": {"US"},
|
||||
"America/Nome": {"US"},
|
||||
"America/Adak": {"US"},
|
||||
"Pacific/Honolulu": {"US"},
|
||||
"America/Montevideo": {"UY"},
|
||||
"Asia/Samarkand": {"UZ"},
|
||||
"Asia/Tashkent": {"UZ"},
|
||||
"America/Caracas": {"VE"},
|
||||
"Asia/Ho_Chi_Minh": {"VN"},
|
||||
"Pacific/Efate": {"VU"},
|
||||
"Pacific/Apia": {"WS"},
|
||||
"Africa/Johannesburg": {"ZA", "LS", "SZ"},
|
||||
}
|
||||
|
||||
// tzAliases maps deprecated / legacy timezone names that older browsers or
|
||||
// operating systems still emit to their canonical equivalents in tzCountry.
|
||||
// Keeping the aliases small and explicit rather than generated; pull more in
|
||||
// as we observe them in production traffic.
|
||||
var tzAliases = map[string]string{
|
||||
"Asia/Calcutta": "Asia/Kolkata",
|
||||
"Asia/Saigon": "Asia/Ho_Chi_Minh",
|
||||
"Asia/Katmandu": "Asia/Kathmandu",
|
||||
"Asia/Istanbul": "Europe/Istanbul",
|
||||
"Europe/Kiev": "Europe/Kyiv",
|
||||
"America/Buenos_Aires": "America/Argentina/Buenos_Aires",
|
||||
"America/Godthab": "America/Nuuk",
|
||||
}
|
||||
|
||||
// lookupTimezoneCountries returns the ISO country codes associated with an
|
||||
// IANA timezone, resolving aliases first. Returns nil for unknown zones —
|
||||
// callers treat nil as "no data" and decline to emit a mismatch signal.
|
||||
func lookupTimezoneCountries(tz string) []string {
|
||||
if tz == "" {
|
||||
return nil
|
||||
}
|
||||
if canonical, ok := tzAliases[tz]; ok {
|
||||
tz = canonical
|
||||
}
|
||||
return tzCountry[tz]
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
// Package cookie holds the AppsecCookie value type and its builder helpers.
|
||||
// The AppsecCookie shape is what WAF rules manipulate via the expr
|
||||
// environment (`AppsecCookie(name).Secure().HttpOnly()...`) and what the
|
||||
// challenge runtime hands back to the bouncer; encoding to/from
|
||||
// http.SetCookie strings is also done here.
|
||||
|
||||
package cookie
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultCookiePath = "/"
|
||||
defaultCookieSameSite = "Lax"
|
||||
)
|
||||
|
||||
const (
|
||||
SameSiteLax = "Lax"
|
||||
SameSiteStrict = "Strict"
|
||||
SameSiteNone = "None"
|
||||
)
|
||||
|
||||
// AppsecCookie is the value-type passed around through expr rules to build
|
||||
// up a Set-Cookie header. Rule authors construct it via cookie.AppsecCookie(name)
|
||||
// and chain the builder methods below; the final String() call renders it
|
||||
// to a Set-Cookie-compatible string.
|
||||
type AppsecCookie struct {
|
||||
Name string
|
||||
Expiration int64
|
||||
Val string
|
||||
PathVal string
|
||||
DomainName string
|
||||
SecureFlag bool
|
||||
HttpOnlyFlag bool
|
||||
SameSiteMode string
|
||||
}
|
||||
|
||||
// NewAppsecCookie returns a cookie pre-populated with sensible defaults.
|
||||
// Defaults: Path "/", SameSite=Lax, session lifetime, not secure/httponly unless set.
|
||||
func NewAppsecCookie(name string) *AppsecCookie {
|
||||
return (&AppsecCookie{Name: name}).withDefaults()
|
||||
}
|
||||
|
||||
// Cookie is a short alias for NewAppsecCookie.
|
||||
func Cookie(name string) *AppsecCookie {
|
||||
return NewAppsecCookie(name)
|
||||
}
|
||||
|
||||
func (c *AppsecCookie) withDefaults() *AppsecCookie {
|
||||
if c.PathVal == "" {
|
||||
c.PathVal = defaultCookiePath
|
||||
}
|
||||
|
||||
if c.SameSiteMode == "" {
|
||||
c.SameSiteMode = defaultCookieSameSite
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// Value sets the cookie's value.
|
||||
func (c *AppsecCookie) Value(value string) *AppsecCookie {
|
||||
c.Val = value
|
||||
return c
|
||||
}
|
||||
|
||||
// Path sets the cookie Path attribute; empty input restores the default ("/").
|
||||
func (c *AppsecCookie) Path(path string) *AppsecCookie {
|
||||
if path == "" {
|
||||
path = defaultCookiePath
|
||||
}
|
||||
|
||||
c.PathVal = path
|
||||
return c
|
||||
}
|
||||
|
||||
// Domain sets the cookie Domain attribute. Leave unset to scope the cookie
|
||||
// to the request host.
|
||||
func (c *AppsecCookie) Domain(domain string) *AppsecCookie {
|
||||
c.DomainName = domain
|
||||
return c
|
||||
}
|
||||
|
||||
// Secure marks the cookie Secure. Required when SameSite=None.
|
||||
func (c *AppsecCookie) Secure() *AppsecCookie {
|
||||
c.SecureFlag = true
|
||||
return c
|
||||
}
|
||||
|
||||
// HttpOnly marks the cookie HttpOnly so it cannot be read from JS.
|
||||
func (c *AppsecCookie) HttpOnly() *AppsecCookie {
|
||||
c.HttpOnlyFlag = true
|
||||
return c
|
||||
}
|
||||
|
||||
// SameSite sets the SameSite mode (Lax / Strict / None). Unknown values are
|
||||
// passed through verbatim; validation happens on String() rendering.
|
||||
func (c *AppsecCookie) SameSite(mode string) *AppsecCookie {
|
||||
c.SameSiteMode = mode
|
||||
return c
|
||||
}
|
||||
|
||||
// ExpiresAt sets an absolute expiration time (unix seconds).
|
||||
func (c *AppsecCookie) ExpiresAt(t time.Time) *AppsecCookie {
|
||||
c.Expiration = t.Unix()
|
||||
return c
|
||||
}
|
||||
|
||||
// ExpiresIn sets the expiration relative to now.
|
||||
func (c *AppsecCookie) ExpiresIn(d time.Duration) *AppsecCookie {
|
||||
c.Expiration = time.Now().Add(d).Unix()
|
||||
return c
|
||||
}
|
||||
|
||||
// String formats the cookie into a Set-Cookie compatible string.
|
||||
func (c *AppsecCookie) String() string {
|
||||
c.withDefaults()
|
||||
|
||||
cookie := http.Cookie{
|
||||
Name: c.Name,
|
||||
Value: c.Val,
|
||||
Path: c.PathVal,
|
||||
Domain: c.DomainName,
|
||||
Secure: c.SecureFlag,
|
||||
HttpOnly: c.HttpOnlyFlag,
|
||||
SameSite: toSameSite(c.SameSiteMode),
|
||||
}
|
||||
|
||||
if c.Expiration > 0 {
|
||||
cookie.Expires = time.Unix(c.Expiration, 0).UTC()
|
||||
cookie.MaxAge = int(time.Until(cookie.Expires).Seconds())
|
||||
}
|
||||
|
||||
// SameSite=None requires Secure=true per modern browsers; enforce to avoid invalid cookies.
|
||||
if cookie.SameSite == http.SameSiteNoneMode && !cookie.Secure {
|
||||
cookie.Secure = true
|
||||
}
|
||||
|
||||
return cookie.String()
|
||||
}
|
||||
|
||||
func toSameSite(mode string) http.SameSite {
|
||||
switch strings.ToLower(mode) {
|
||||
case "lax":
|
||||
return http.SameSiteLaxMode
|
||||
case "strict":
|
||||
return http.SameSiteStrictMode
|
||||
case "none":
|
||||
return http.SameSiteNoneMode
|
||||
default:
|
||||
return http.SameSiteDefaultMode
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package appsec
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
|
||||
)
|
||||
|
||||
const (
|
||||
// ModuleName is the acquisition module name carried on events produced by
|
||||
// the appsec datasource.
|
||||
ModuleName = "appsec"
|
||||
|
||||
// SourceWAF is the `source` field set on events generated by a WAF rule match.
|
||||
SourceWAF = "crowdsec-appsec"
|
||||
// SourceChallenge is the `source` field set on events generated by the
|
||||
// challenge lifecycle. It is intentionally distinct from SourceWAF so that
|
||||
// challenge scenarios can filter on it without colliding with WAF scenarios.
|
||||
SourceChallenge = "crowdsec-appsec-challenge"
|
||||
)
|
||||
|
||||
// ChallengeReason identifies a challenge lifecycle moment. Its string value is
|
||||
// set as the `challenge_event` field on events produced by
|
||||
// ChallengeEventFromRequest.
|
||||
type ChallengeReason string
|
||||
|
||||
const (
|
||||
ChallengeReasonRequested ChallengeReason = "requested" // challenge page served to the client
|
||||
ChallengeReasonSubmitted ChallengeReason = "submitted" // a challenge validation attempt was received
|
||||
ChallengeReasonFailed ChallengeReason = "failed" // submission failed crypto/PoW validation
|
||||
ChallengeReasonRejected ChallengeReason = "rejected" // submission decoded fine but an on_challenge_submit hook called RejectSubmission
|
||||
ChallengeReasonSolved ChallengeReason = "solved" // validation passed, cookie issued
|
||||
)
|
||||
|
||||
// baseEventFromRequest builds the common pipeline.Event skeleton shared by WAF
|
||||
// and challenge events: the request metadata in Parsed and the acquisition Line.
|
||||
func baseEventFromRequest(r *ParsedRequest, labels map[string]string, txUuid, source string) pipeline.Event {
|
||||
evt := pipeline.MakeEvent(false, pipeline.LOG, true)
|
||||
evt.Stage = "s00-raw"
|
||||
evt.Parsed = map[string]string{
|
||||
"source_ip": r.ClientIP,
|
||||
"target_host": r.Host,
|
||||
"target_uri": r.URI,
|
||||
"method": r.Method,
|
||||
"req_uuid": txUuid,
|
||||
"source": source,
|
||||
"remediation_cmpt_ip": r.RemoteAddrNormalized,
|
||||
}
|
||||
evt.Line = pipeline.Line{
|
||||
Time: time.Now(),
|
||||
Labels: labels,
|
||||
Process: true,
|
||||
Module: ModuleName,
|
||||
Src: ModuleName,
|
||||
Raw: "dummy-appsec-data", // we discard empty Line.Raw items :)
|
||||
}
|
||||
evt.Appsec = pipeline.AppsecEvent{}
|
||||
|
||||
return evt
|
||||
}
|
||||
|
||||
func EventFromRequest(r *ParsedRequest, labels map[string]string, txUuid string) (pipeline.Event, error) {
|
||||
return baseEventFromRequest(r, labels, txUuid, SourceWAF), nil
|
||||
}
|
||||
|
||||
// ChallengeEventInfo describes a single challenge lifecycle moment to be turned
|
||||
// into a pipeline.Event by ChallengeEventFromRequest.
|
||||
type ChallengeEventInfo struct {
|
||||
Reason ChallengeReason // requested | submitted | failed | rejected | solved
|
||||
FailReason string // set on failed (raw error message) or rejected (operator-supplied reason)
|
||||
FailErr error // set on failed; carries the sentinel-wrapped error so metrics can classify via errors.Is.
|
||||
Difficulty int // target PoW difficulty for this moment
|
||||
Fingerprint *challenge.FingerprintData // nil when none available (e.g. requested w/o cookie)
|
||||
}
|
||||
|
||||
// ChallengeEventFromRequest builds a LOG event for a challenge lifecycle moment.
|
||||
// It carries SourceChallenge so it never collides with WAF scenarios, plus the
|
||||
// reason, difficulty and (when available) fingerprint scalars for filtering.
|
||||
func ChallengeEventFromRequest(r *ParsedRequest, labels map[string]string, txUuid string, info ChallengeEventInfo) pipeline.Event {
|
||||
evt := baseEventFromRequest(r, labels, txUuid, SourceChallenge)
|
||||
|
||||
if r.HTTPRequest != nil {
|
||||
evt.Parsed["user_agent"] = r.HTTPRequest.UserAgent()
|
||||
}
|
||||
|
||||
evt.Parsed["challenge_event"] = string(info.Reason)
|
||||
evt.Parsed["challenge_difficulty"] = strconv.Itoa(info.Difficulty)
|
||||
if info.FailReason != "" {
|
||||
evt.Parsed["challenge_fail_reason"] = info.FailReason
|
||||
}
|
||||
|
||||
if fp := info.Fingerprint; fp != nil {
|
||||
// Expose the full fingerprint (all nested signals, bot-detection details,
|
||||
// etc.) so scenarios and alert-context can traverse it as
|
||||
// evt.Unmarshaled.fingerprint.Signals... Stored as a pointer so expr can
|
||||
// also call the *FingerprintData helper methods (Platform(), BotSignals(),
|
||||
// IsBot(), ...). It stays out of LAPI/CAPI unless explicitly mapped into Meta.
|
||||
evt.Unmarshaled["fingerprint"] = fp
|
||||
// Flat shortcuts for cheap scenario filtering.
|
||||
evt.Parsed["fsid"] = fp.FSID
|
||||
evt.Parsed["fingerprint_bot"] = strconv.FormatBool(fp.FastBotDetection.Bool())
|
||||
evt.Parsed["fingerprint_allowlisted"] = strconv.FormatBool(fp.Allowlisted)
|
||||
if fp.AllowlistReason != "" {
|
||||
evt.Parsed["fingerprint_allowlist_reason"] = fp.AllowlistReason
|
||||
}
|
||||
evt.Parsed["platform"] = fp.Signals.Device.Platform
|
||||
}
|
||||
|
||||
return evt
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package appsec
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
|
||||
)
|
||||
|
||||
func testChallengeRequest() *ParsedRequest {
|
||||
return &ParsedRequest{
|
||||
ClientIP: "1.2.3.4",
|
||||
Host: "example.com",
|
||||
URI: "/protected",
|
||||
Method: "GET",
|
||||
UUID: "test-uuid",
|
||||
RemoteAddrNormalized: "10.0.0.1",
|
||||
IsInBand: true,
|
||||
HTTPRequest: &http.Request{Header: http.Header{"User-Agent": []string{"test-agent"}}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestChallengeEventFromRequest(t *testing.T) {
|
||||
labels := map[string]string{"type": "appsec"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
info ChallengeEventInfo
|
||||
assert func(t *testing.T, evt pipeline.Event)
|
||||
}{
|
||||
{
|
||||
name: "requested carries the distinct source and reason",
|
||||
info: ChallengeEventInfo{Reason: ChallengeReasonRequested, Difficulty: 4},
|
||||
assert: func(t *testing.T, evt pipeline.Event) {
|
||||
require.Equal(t, pipeline.LOG, evt.Type)
|
||||
require.Equal(t, SourceChallenge, evt.Parsed["source"])
|
||||
require.Equal(t, string(ChallengeReasonRequested), evt.Parsed["challenge_event"])
|
||||
require.Equal(t, "4", evt.Parsed["challenge_difficulty"])
|
||||
// client/request metadata
|
||||
require.Equal(t, "1.2.3.4", evt.Parsed["source_ip"])
|
||||
require.Equal(t, "example.com", evt.Parsed["target_host"])
|
||||
require.Equal(t, "test-uuid", evt.Parsed["req_uuid"])
|
||||
require.Equal(t, "10.0.0.1", evt.Parsed["remediation_cmpt_ip"])
|
||||
require.Equal(t, "test-agent", evt.Parsed["user_agent"])
|
||||
require.Equal(t, ModuleName, evt.Line.Module)
|
||||
// no fingerprint → no fingerprint fields
|
||||
require.NotContains(t, evt.Parsed, "fsid")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "submitted has no fingerprint and no fail reason",
|
||||
info: ChallengeEventInfo{Reason: ChallengeReasonSubmitted},
|
||||
assert: func(t *testing.T, evt pipeline.Event) {
|
||||
require.Equal(t, string(ChallengeReasonSubmitted), evt.Parsed["challenge_event"])
|
||||
require.NotContains(t, evt.Parsed, "challenge_fail_reason")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "failed surfaces the fail reason",
|
||||
info: ChallengeEventInfo{Reason: ChallengeReasonFailed, FailReason: "invalid proof-of-work"},
|
||||
assert: func(t *testing.T, evt pipeline.Event) {
|
||||
require.Equal(t, string(ChallengeReasonFailed), evt.Parsed["challenge_event"])
|
||||
require.Equal(t, "invalid proof-of-work", evt.Parsed["challenge_fail_reason"])
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "solved surfaces fingerprint scalars",
|
||||
info: ChallengeEventInfo{
|
||||
Reason: ChallengeReasonSolved,
|
||||
Difficulty: 5,
|
||||
Fingerprint: &challenge.FingerprintData{
|
||||
FSID: "FS1_abc",
|
||||
FastBotDetection: challenge.FlexBool(true),
|
||||
Allowlisted: true,
|
||||
AllowlistReason: "googlebot",
|
||||
},
|
||||
},
|
||||
assert: func(t *testing.T, evt pipeline.Event) {
|
||||
require.Equal(t, string(ChallengeReasonSolved), evt.Parsed["challenge_event"])
|
||||
require.Equal(t, "FS1_abc", evt.Parsed["fsid"])
|
||||
require.Equal(t, "true", evt.Parsed["fingerprint_bot"])
|
||||
require.Equal(t, "true", evt.Parsed["fingerprint_allowlisted"])
|
||||
require.Equal(t, "googlebot", evt.Parsed["fingerprint_allowlist_reason"])
|
||||
// the full fingerprint struct must be available for traversal
|
||||
fp, ok := evt.Unmarshaled["fingerprint"].(*challenge.FingerprintData)
|
||||
require.True(t, ok, "fingerprint should be attached to Unmarshaled")
|
||||
require.Equal(t, "FS1_abc", fp.FSID)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
evt := ChallengeEventFromRequest(testChallengeRequest(), labels, "test-uuid", tc.info)
|
||||
tc.assert(t, evt)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// emitChallengeEvent must be a no-op (and not panic) when no output channel is wired.
|
||||
func TestEmitChallengeEventNoChannel(t *testing.T) {
|
||||
w := &AppsecRuntimeConfig{}
|
||||
require.NotPanics(t, func() {
|
||||
w.emitChallengeEvent(testChallengeRequest(), ChallengeEventInfo{Reason: ChallengeReasonRequested})
|
||||
})
|
||||
}
|
||||
|
||||
// emitChallengeEvent sends the built event on the wired channel.
|
||||
func TestEmitChallengeEventSendsOnChannel(t *testing.T) {
|
||||
out := make(chan pipeline.Event, 1)
|
||||
w := &AppsecRuntimeConfig{OutChan: out, Labels: map[string]string{"type": "appsec"}}
|
||||
|
||||
w.emitChallengeEvent(testChallengeRequest(), ChallengeEventInfo{Reason: ChallengeReasonFailed, FailReason: "boom"})
|
||||
|
||||
evt := <-out
|
||||
require.Equal(t, SourceChallenge, evt.Parsed["source"])
|
||||
require.Equal(t, string(ChallengeReasonFailed), evt.Parsed["challenge_event"])
|
||||
require.Equal(t, "boom", evt.Parsed["challenge_fail_reason"])
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package appsec
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
)
|
||||
|
||||
// Per-output-path mutexes as there can be concurrency. Outer mutex only guards the map
|
||||
// itself
|
||||
var (
|
||||
fpDumpMapMu sync.Mutex
|
||||
fpDumpLocks = map[string]*sync.Mutex{}
|
||||
)
|
||||
|
||||
func fpDumpLockFor(path string) *sync.Mutex {
|
||||
fpDumpMapMu.Lock()
|
||||
defer fpDumpMapMu.Unlock()
|
||||
m, ok := fpDumpLocks[path]
|
||||
if !ok {
|
||||
m = &sync.Mutex{}
|
||||
fpDumpLocks[path] = m
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
var fpLabelSanitizer = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
// sanitizeFpLabel returns a filesystem-safe slug for the supplied label.
|
||||
func sanitizeFpLabel(label string) string {
|
||||
s := fpLabelSanitizer.ReplaceAllString(strings.ToLower(label), "_")
|
||||
s = strings.Trim(s, "_")
|
||||
if s == "" {
|
||||
return "unlabeled"
|
||||
}
|
||||
if len(s) > 40 {
|
||||
s = strings.Trim(s[:40], "_")
|
||||
if s == "" {
|
||||
return "unlabeled"
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// fpDumpEntry is the on-disk record shape: one per line in the
|
||||
// labeled JSONL file.
|
||||
type fpDumpEntry struct {
|
||||
Label string `json:"label"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
ClientIP string `json:"client_ip,omitempty"`
|
||||
RemoteAddr string `json:"remote_addr,omitempty"`
|
||||
RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"`
|
||||
UserAgent string `json:"user_agent,omitempty"`
|
||||
Host string `json:"host,omitempty"`
|
||||
URI string `json:"uri,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Fingerprint *challenge.FingerprintData `json:"fingerprint"`
|
||||
}
|
||||
|
||||
// DumpFingerprint allows to dump the fingerprint + some context (ip, host, timestamp etc.)
|
||||
// to as JSONL file for later analysis.
|
||||
func DumpFingerprint(dir, label string, fp *challenge.FingerprintData, req *ParsedRequest) string {
|
||||
if fp == nil {
|
||||
log.Warnf("DumpFingerprint(%q) called with nil fingerprint, skipping", label)
|
||||
return ""
|
||||
}
|
||||
if dir == "" {
|
||||
log.Warnf("DumpFingerprint(%q): no dump directory configured, skipping", label)
|
||||
return ""
|
||||
}
|
||||
|
||||
path := filepath.Join(dir, fmt.Sprintf("crowdsec_fp_dump_%s.jsonl", sanitizeFpLabel(label)))
|
||||
|
||||
entry := fpDumpEntry{
|
||||
Label: label,
|
||||
Timestamp: time.Now().UTC(),
|
||||
Fingerprint: fp,
|
||||
}
|
||||
if req != nil {
|
||||
entry.ClientIP = req.ClientIP
|
||||
entry.RemoteAddr = req.RemoteAddr
|
||||
entry.RemoteAddrNormalized = req.RemoteAddrNormalized
|
||||
entry.UserAgent = req.Headers.Get("User-Agent")
|
||||
entry.Host = req.Host
|
||||
entry.URI = req.URI
|
||||
entry.Method = req.Method
|
||||
}
|
||||
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
log.Errorf("DumpFingerprint(%q): marshal: %s", label, err)
|
||||
return ""
|
||||
}
|
||||
|
||||
mu := fpDumpLockFor(path)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
fd, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
log.Errorf("DumpFingerprint(%q): open %s: %s", label, path, err)
|
||||
return ""
|
||||
}
|
||||
defer fd.Close()
|
||||
if _, err := fd.Write(append(line, '\n')); err != nil {
|
||||
log.Errorf("DumpFingerprint(%q): write %s: %s", label, path, err)
|
||||
return ""
|
||||
}
|
||||
log.Infof("fingerprint dumped (label=%q) to %s", label, path)
|
||||
return path
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package appsec
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
)
|
||||
|
||||
// fakeFingerprint returns a minimal but distinguishable *challenge.FingerprintData
|
||||
// suitable for asserting that DumpFingerprint preserved the value end-to-end.
|
||||
func fakeFingerprint(fsid string) *challenge.FingerprintData {
|
||||
return &challenge.FingerprintData{
|
||||
FSID: fsid,
|
||||
Nonce: "test-nonce",
|
||||
Time: 1717000000,
|
||||
URL: "https://example.test/path",
|
||||
}
|
||||
}
|
||||
|
||||
func fakeRequest() *ParsedRequest {
|
||||
return &ParsedRequest{
|
||||
// Distinct visitor vs. bouncer addresses so the test can prove
|
||||
// DumpFingerprint records the *real* source IP (ClientIP) and
|
||||
// not just the bouncer's TCP peer (RemoteAddr).
|
||||
ClientIP: "203.0.113.7",
|
||||
RemoteAddr: "127.0.0.1:54321",
|
||||
RemoteAddrNormalized: "127.0.0.1",
|
||||
Host: "example.test",
|
||||
URI: "/login",
|
||||
Method: "POST",
|
||||
Headers: http.Header{"User-Agent": []string{"Mozilla/5.0 (test)"}},
|
||||
}
|
||||
}
|
||||
|
||||
// readJSONL reads every line of the given path as a JSON object and returns them.
|
||||
func readJSONL(t *testing.T, path string) []map[string]any {
|
||||
t.Helper()
|
||||
fd, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %s", path, err)
|
||||
}
|
||||
defer fd.Close()
|
||||
var out []map[string]any
|
||||
sc := bufio.NewScanner(fd)
|
||||
// Fingerprints with full signals can be large; bump the buffer to avoid
|
||||
// false negatives on real-shape payloads, even though our test data is tiny.
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
|
||||
for sc.Scan() {
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(sc.Bytes(), &m); err != nil {
|
||||
t.Fatalf("invalid JSON line %q: %s", sc.Text(), err)
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
t.Fatalf("scan: %s", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSanitizeFpLabel(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"bot", "bot"},
|
||||
{"human – Chrome on macOS!", "human_chrome_on_macos"},
|
||||
{" ", "unlabeled"},
|
||||
{"", "unlabeled"},
|
||||
{"!!! @@@ ", "unlabeled"},
|
||||
{"Bot-1", "bot_1"},
|
||||
{"Bot_1", "bot_1"},
|
||||
// 50-char alpha label gets truncated to 40, trim of trailing underscore is a no-op.
|
||||
{strings.Repeat("a", 50), strings.Repeat("a", 40)},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := sanitizeFpLabel(tc.in)
|
||||
if got != tc.want {
|
||||
t.Errorf("sanitizeFpLabel(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpFingerprint_AppendsAndPreservesFields(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fp := fakeFingerprint("fsid-abc")
|
||||
req := fakeRequest()
|
||||
|
||||
path1 := DumpFingerprint(dir, "test-human", fp, req)
|
||||
if path1 == "" {
|
||||
t.Fatal("first DumpFingerprint returned empty path")
|
||||
}
|
||||
wantPath := filepath.Join(dir, "crowdsec_fp_dump_test_human.jsonl")
|
||||
if path1 != wantPath {
|
||||
t.Errorf("path = %q, want %q", path1, wantPath)
|
||||
}
|
||||
|
||||
// Second call with the same label must append, not truncate.
|
||||
path2 := DumpFingerprint(dir, "test-human", fakeFingerprint("fsid-def"), req)
|
||||
if path2 != path1 {
|
||||
t.Errorf("second call wrote to %q, want same file %q", path2, path1)
|
||||
}
|
||||
|
||||
lines := readJSONL(t, path1)
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("got %d JSONL lines, want 2 (append failed?)", len(lines))
|
||||
}
|
||||
|
||||
// Both lines must carry the operator-supplied label verbatim.
|
||||
for i, line := range lines {
|
||||
if line["label"] != "test-human" {
|
||||
t.Errorf("line %d label = %v, want %q", i, line["label"], "test-human")
|
||||
}
|
||||
// Timestamp must parse as RFC3339.
|
||||
ts, ok := line["timestamp"].(string)
|
||||
if !ok {
|
||||
t.Errorf("line %d timestamp not a string: %T", i, line["timestamp"])
|
||||
} else if _, err := time.Parse(time.RFC3339Nano, ts); err != nil {
|
||||
t.Errorf("line %d timestamp %q not RFC3339: %s", i, ts, err)
|
||||
}
|
||||
// Minimal request context preserved.
|
||||
// client_ip is the real visitor address — the field a SOC
|
||||
// operator actually wants when triaging a sample. Verify it
|
||||
// is distinct from remote_addr (which is the bouncer).
|
||||
if line["client_ip"] != "203.0.113.7" {
|
||||
t.Errorf("line %d client_ip = %v, want %q", i, line["client_ip"], "203.0.113.7")
|
||||
}
|
||||
if line["remote_addr"] != "127.0.0.1:54321" {
|
||||
t.Errorf("line %d remote_addr = %v", i, line["remote_addr"])
|
||||
}
|
||||
if line["normalized_remote_addr"] != "127.0.0.1" {
|
||||
t.Errorf("line %d normalized_remote_addr = %v", i, line["normalized_remote_addr"])
|
||||
}
|
||||
if line["user_agent"] != "Mozilla/5.0 (test)" {
|
||||
t.Errorf("line %d user_agent = %v", i, line["user_agent"])
|
||||
}
|
||||
if line["host"] != "example.test" {
|
||||
t.Errorf("line %d host = %v", i, line["host"])
|
||||
}
|
||||
if line["uri"] != "/login" {
|
||||
t.Errorf("line %d uri = %v", i, line["uri"])
|
||||
}
|
||||
if line["method"] != "POST" {
|
||||
t.Errorf("line %d method = %v", i, line["method"])
|
||||
}
|
||||
}
|
||||
|
||||
// Fingerprint payload must be preserved per-line (fsid differs across the two calls).
|
||||
wantFSIDs := []string{"fsid-abc", "fsid-def"}
|
||||
for i, line := range lines {
|
||||
fp, ok := line["fingerprint"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("line %d: fingerprint not an object: %T", i, line["fingerprint"])
|
||||
}
|
||||
if fp["fsid"] != wantFSIDs[i] {
|
||||
t.Errorf("line %d fingerprint.fsid = %v, want %q", i, fp["fsid"], wantFSIDs[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpFingerprint_DifferentLabelsGoToDifferentFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pHuman := DumpFingerprint(dir, "human", fakeFingerprint("a"), fakeRequest())
|
||||
pBot := DumpFingerprint(dir, "bot", fakeFingerprint("b"), fakeRequest())
|
||||
if pHuman == "" || pBot == "" {
|
||||
t.Fatalf("paths empty: %q %q", pHuman, pBot)
|
||||
}
|
||||
if pHuman == pBot {
|
||||
t.Fatalf("expected distinct files for distinct labels, got %q", pHuman)
|
||||
}
|
||||
|
||||
if got := len(readJSONL(t, pHuman)); got != 1 {
|
||||
t.Errorf("human file has %d lines, want 1", got)
|
||||
}
|
||||
if got := len(readJSONL(t, pBot)); got != 1 {
|
||||
t.Errorf("bot file has %d lines, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpFingerprint_NilFingerprintIsNoop(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
got := DumpFingerprint(dir, "anything", nil, fakeRequest())
|
||||
if got != "" {
|
||||
t.Errorf("nil fingerprint returned path %q, want empty", got)
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if strings.HasPrefix(e.Name(), "crowdsec_fp_dump_") {
|
||||
t.Errorf("nil fingerprint wrote a file: %s", e.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpFingerprint_EmptyDirIsNoop(t *testing.T) {
|
||||
got := DumpFingerprint("", "anything", fakeFingerprint("xx"), fakeRequest())
|
||||
if got != "" {
|
||||
t.Errorf("empty dir returned path %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpFingerprint_NilRequestStillDumps(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := DumpFingerprint(dir, "nofreq", fakeFingerprint("xx"), nil)
|
||||
if path == "" {
|
||||
t.Fatal("expected dump to succeed with nil request")
|
||||
}
|
||||
lines := readJSONL(t, path)
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("got %d lines, want 1", len(lines))
|
||||
}
|
||||
// Request fields are omitempty, so they should be absent from the JSON.
|
||||
for _, k := range []string{"client_ip", "remote_addr", "normalized_remote_addr", "user_agent", "host", "uri", "method"} {
|
||||
if _, ok := lines[0][k]; ok {
|
||||
t.Errorf("nil request still wrote %s=%v", k, lines[0][k])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpFingerprint_EmptyAndJunkLabelsCollapse(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p1 := DumpFingerprint(dir, "", fakeFingerprint("a"), nil)
|
||||
p2 := DumpFingerprint(dir, "!!! @@@ ", fakeFingerprint("b"), nil)
|
||||
if p1 == "" || p2 == "" {
|
||||
t.Fatalf("paths empty: %q %q", p1, p2)
|
||||
}
|
||||
wantPath := filepath.Join(dir, "crowdsec_fp_dump_unlabeled.jsonl")
|
||||
if p1 != wantPath || p2 != wantPath {
|
||||
t.Errorf("expected both junk labels to land in %q, got %q and %q", wantPath, p1, p2)
|
||||
}
|
||||
if got := len(readJSONL(t, p1)); got != 2 {
|
||||
t.Errorf("unlabeled file has %d lines, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpFingerprint_ConcurrentAppendIsLineSafe(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
const goroutines = 32
|
||||
const perGoroutine = 20
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
for range goroutines {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for range perGoroutine {
|
||||
if path := DumpFingerprint(dir, "race", fakeFingerprint("fsid"), fakeRequest()); path == "" {
|
||||
t.Error("DumpFingerprint returned empty path under contention")
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
path := filepath.Join(dir, "crowdsec_fp_dump_race.jsonl")
|
||||
lines := readJSONL(t, path) // this would fatal on any torn line
|
||||
if len(lines) != goroutines*perGoroutine {
|
||||
t.Errorf("got %d lines, want %d", len(lines), goroutines*perGoroutine)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package appsec
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/metrics"
|
||||
)
|
||||
|
||||
// Well-known MaxMind test vectors that exist in the test mmdb under
|
||||
// pkg/parser/testdata. Used to exercise the geoip-backed country
|
||||
// resolution inside EvaluateMismatches without mocks.
|
||||
const (
|
||||
testIPUS = "216.160.83.56" // Milton, US
|
||||
testIPGB = "81.2.69.142" // London, GB
|
||||
)
|
||||
|
||||
// setupGeoIP loads the same test mmdb the parser tests use so that
|
||||
// IPToCountryString has a working reader to query. Safe to call more than
|
||||
// once in a single test run.
|
||||
func setupGeoIP(t *testing.T) {
|
||||
t.Helper()
|
||||
require.NoError(t, exprhelpers.GeoIPInit("../parser/testdata/"), "geoip init failed")
|
||||
}
|
||||
|
||||
// fpEuropeParisCDP returns a fingerprint with Europe/Paris timezone and a
|
||||
// single fired library signal (CDP), so EvaluateMismatches can be driven
|
||||
// predictably.
|
||||
func fpEuropeParisCDP(t *testing.T) *challenge.FingerprintData {
|
||||
t.Helper()
|
||||
raw := `{
|
||||
"signals": {
|
||||
"device": {"platform": "MacIntel"},
|
||||
"browser": {
|
||||
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120",
|
||||
"highEntropyValues": {"platform": "macOS"}
|
||||
},
|
||||
"locale": {
|
||||
"internationalization": {"timezone": "Europe/Paris"},
|
||||
"languages": {"language": "en"}
|
||||
}
|
||||
},
|
||||
"fsid": "FS_RUNTIME", "nonce": "n", "time": 1, "url": "http://x/",
|
||||
"fastBotDetection": true,
|
||||
"fastBotDetectionDetails": {
|
||||
"hasCDP": {"detected": true, "severity": "high"}
|
||||
}
|
||||
}`
|
||||
fp := &challenge.FingerprintData{}
|
||||
require.NoError(t, json.Unmarshal([]byte(raw), fp))
|
||||
return fp
|
||||
}
|
||||
|
||||
func makeRuntime() *AppsecRuntimeConfig {
|
||||
return &AppsecRuntimeConfig{Logger: log.NewEntry(log.StandardLogger())}
|
||||
}
|
||||
|
||||
func TestEvaluateMismatches_GeoIPTimezoneMismatch_Fires(t *testing.T) {
|
||||
setupGeoIP(t)
|
||||
metrics.AppsecFingerprintMismatch.Reset()
|
||||
|
||||
w := makeRuntime()
|
||||
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
|
||||
state := &AppsecRequestState{Fingerprint: fpEuropeParisCDP(t)}
|
||||
req := &ParsedRequest{
|
||||
HTTPRequest: httpReq,
|
||||
ClientIP: testIPUS, // geoip resolves to "US"
|
||||
AppsecEngine: "test-engine",
|
||||
}
|
||||
|
||||
report := w.EvaluateMismatches(state, req)
|
||||
|
||||
// Europe/Paris fingerprint timezone but request geoips to US → the
|
||||
// soft timezone_country signal must fire. CDP already fires from the
|
||||
// library side.
|
||||
assert.True(t, report.Has(challenge.ReasonTimezoneCountry),
|
||||
"expected timezone_country to fire for Europe/Paris + US geoip")
|
||||
assert.True(t, report.Has(challenge.ReasonCDP))
|
||||
assert.Equal(t, 1, report.High(), "only CDP is high severity here")
|
||||
assert.Equal(t, 1, report.Low(), "timezone_country is low severity")
|
||||
|
||||
// Prometheus counters must reflect one bump per fired signal.
|
||||
assert.InDelta(t, 1.0, testutil.ToFloat64(
|
||||
metrics.AppsecFingerprintMismatch.With(prometheus.Labels{
|
||||
"reason": challenge.ReasonTimezoneCountry,
|
||||
"severity": challenge.SeverityLow,
|
||||
"appsec_engine": "test-engine",
|
||||
}),
|
||||
), 0)
|
||||
assert.InDelta(t, 1.0, testutil.ToFloat64(
|
||||
metrics.AppsecFingerprintMismatch.With(prometheus.Labels{
|
||||
"reason": challenge.ReasonCDP,
|
||||
"severity": challenge.SeverityHigh,
|
||||
"appsec_engine": "test-engine",
|
||||
}),
|
||||
), 0)
|
||||
}
|
||||
|
||||
func TestEvaluateMismatches_GeoIPConsistent_NoTimezoneSignal(t *testing.T) {
|
||||
setupGeoIP(t)
|
||||
metrics.AppsecFingerprintMismatch.Reset()
|
||||
|
||||
w := makeRuntime()
|
||||
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Browser reports Europe/London tz; client IP geolocates to GB.
|
||||
// No timezone_country mismatch should fire.
|
||||
raw := `{
|
||||
"signals": {
|
||||
"device": {"platform": "Win32"},
|
||||
"browser": {
|
||||
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120",
|
||||
"highEntropyValues": {"platform": "Windows"}
|
||||
},
|
||||
"locale": {
|
||||
"internationalization": {"timezone": "Europe/London"},
|
||||
"languages": {"language": "en"}
|
||||
}
|
||||
},
|
||||
"fsid": "FS_OK_GB", "nonce": "n", "time": 1, "url": "http://x/",
|
||||
"fastBotDetection": false,
|
||||
"fastBotDetectionDetails": {}
|
||||
}`
|
||||
fp := &challenge.FingerprintData{}
|
||||
require.NoError(t, json.Unmarshal([]byte(raw), fp))
|
||||
|
||||
state := &AppsecRequestState{Fingerprint: fp}
|
||||
req := &ParsedRequest{
|
||||
HTTPRequest: httpReq,
|
||||
ClientIP: testIPGB, // geoip resolves to "GB"
|
||||
AppsecEngine: "test-engine",
|
||||
}
|
||||
|
||||
report := w.EvaluateMismatches(state, req)
|
||||
|
||||
assert.False(t, report.Has(challenge.ReasonTimezoneCountry),
|
||||
"Europe/London + GB geoip must not fire timezone_country")
|
||||
assert.True(t, report.Empty(), "no signals expected at all")
|
||||
}
|
||||
|
||||
func TestEvaluateMismatches_CachesAcrossCalls(t *testing.T) {
|
||||
setupGeoIP(t)
|
||||
metrics.AppsecFingerprintMismatch.Reset()
|
||||
|
||||
w := makeRuntime()
|
||||
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
|
||||
state := &AppsecRequestState{Fingerprint: fpEuropeParisCDP(t)}
|
||||
req := &ParsedRequest{
|
||||
HTTPRequest: httpReq,
|
||||
ClientIP: testIPUS,
|
||||
AppsecEngine: "test-engine",
|
||||
}
|
||||
|
||||
first := w.EvaluateMismatches(state, req)
|
||||
second := w.EvaluateMismatches(state, req)
|
||||
third := w.EvaluateMismatches(state, req)
|
||||
|
||||
// All three calls must return the exact same pointer — state cache.
|
||||
assert.Same(t, first, second)
|
||||
assert.Same(t, second, third)
|
||||
assert.Same(t, first, state.LastMismatchReport)
|
||||
|
||||
// Observability fires once total (log + metric) despite three calls.
|
||||
// Verifies the emitMismatchObservability gate at state.LastMismatchReport.
|
||||
assert.InDelta(t, 1.0, testutil.ToFloat64(
|
||||
metrics.AppsecFingerprintMismatch.With(prometheus.Labels{
|
||||
"reason": challenge.ReasonTimezoneCountry,
|
||||
"severity": challenge.SeverityLow,
|
||||
"appsec_engine": "test-engine",
|
||||
}),
|
||||
), 0)
|
||||
}
|
||||
|
||||
func TestEvaluateMismatches_UnresolvableClientIP_NoTimezoneSignal(t *testing.T) {
|
||||
setupGeoIP(t)
|
||||
metrics.AppsecFingerprintMismatch.Reset()
|
||||
|
||||
w := makeRuntime()
|
||||
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
|
||||
state := &AppsecRequestState{Fingerprint: fpEuropeParisCDP(t)}
|
||||
req := &ParsedRequest{
|
||||
HTTPRequest: httpReq,
|
||||
ClientIP: "203.0.113.1", // not in the test mmdb
|
||||
AppsecEngine: "test-engine",
|
||||
}
|
||||
|
||||
report := w.EvaluateMismatches(state, req)
|
||||
|
||||
// With country == "" the timezone_country helper declines to decide,
|
||||
// so only the library CDP signal fires.
|
||||
assert.False(t, report.Has(challenge.ReasonTimezoneCountry))
|
||||
assert.True(t, report.Has(challenge.ReasonCDP))
|
||||
}
|
||||
|
||||
func TestEvaluateMismatches_EmptyReport_NoObservability(t *testing.T) {
|
||||
setupGeoIP(t)
|
||||
metrics.AppsecFingerprintMismatch.Reset()
|
||||
|
||||
w := makeRuntime()
|
||||
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Consistent fingerprint — no library signals, no custom helper fires.
|
||||
raw := `{
|
||||
"signals": {
|
||||
"device": {"platform": "Win32"},
|
||||
"browser": {
|
||||
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120",
|
||||
"highEntropyValues": {"platform": "Windows"}
|
||||
},
|
||||
"locale": {
|
||||
"internationalization": {"timezone": "Europe/London"},
|
||||
"languages": {"language": "en"}
|
||||
}
|
||||
},
|
||||
"fsid": "X", "nonce": "n", "time": 1, "url": "http://x/",
|
||||
"fastBotDetection": false,
|
||||
"fastBotDetectionDetails": {}
|
||||
}`
|
||||
fp := &challenge.FingerprintData{}
|
||||
require.NoError(t, json.Unmarshal([]byte(raw), fp))
|
||||
|
||||
state := &AppsecRequestState{Fingerprint: fp}
|
||||
req := &ParsedRequest{
|
||||
HTTPRequest: httpReq,
|
||||
ClientIP: testIPGB,
|
||||
AppsecEngine: "test-engine",
|
||||
}
|
||||
|
||||
report := w.EvaluateMismatches(state, req)
|
||||
assert.True(t, report.Empty())
|
||||
|
||||
// No metric bump for any reason when the report is empty.
|
||||
for _, reason := range challenge.KnownReasons() {
|
||||
sev := challenge.SeverityFor(reason)
|
||||
got := testutil.ToFloat64(metrics.AppsecFingerprintMismatch.With(prometheus.Labels{
|
||||
"reason": reason,
|
||||
"severity": sev,
|
||||
"appsec_engine": "test-engine",
|
||||
}))
|
||||
assert.InDelta(t, 0.0, got, 0, "no counter bump expected for %q", reason)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package appsec
|
||||
|
||||
import "github.com/expr-lang/expr/ast"
|
||||
|
||||
// This is not an actual patcher: we just walk the AST to check if we need to create a WASM VM for the challenge mode.
|
||||
type appsecExprPatcher struct {
|
||||
NeedWASMVM bool
|
||||
}
|
||||
|
||||
// challengeRuntimeCallees is the set of expr helper names whose presence in
|
||||
// any compiled rule body implies that the challenge runtime (WASM VM,
|
||||
// obfuscator, keyring) must be initialized.
|
||||
var challengeRuntimeCallees = map[string]struct{}{
|
||||
"SendChallenge": {},
|
||||
"GrantChallengeCookie": {},
|
||||
"RejectSubmission": {},
|
||||
"LogAccepted": {},
|
||||
}
|
||||
|
||||
func (p *appsecExprPatcher) Visit(node *ast.Node) { //nolint:gocritic // signature fixed by expr-lang ast.Visitor interface
|
||||
if n, ok := (*node).(*ast.CallNode); ok {
|
||||
if _, needs := challengeRuntimeCallees[n.Callee.String()]; needs {
|
||||
p.NeedWASMVM = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package appsec
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAppsecConfigBuildDetectsRequireValidChallenge(t *testing.T) {
|
||||
logger := log.New()
|
||||
logger.SetOutput(io.Discard)
|
||||
|
||||
// SendChallenge is only exposed in post_eval / on_challenge envs; the patcher
|
||||
// detects it from any stage and flags NeedWASMVM.
|
||||
cfg := AppsecConfig{
|
||||
Logger: log.NewEntry(logger),
|
||||
PostEval: []Hook{
|
||||
{
|
||||
Apply: []string{"SendChallenge()"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runtimeCfg, err := cfg.Build(t.Context(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, runtimeCfg.NeedWASMVM)
|
||||
}
|
||||
|
||||
func TestAppsecConfigBuildDoesNotDetectRequireValidChallengeWhenUnused(t *testing.T) {
|
||||
logger := log.New()
|
||||
logger.SetOutput(io.Discard)
|
||||
|
||||
cfg := AppsecConfig{
|
||||
Logger: log.NewEntry(logger),
|
||||
PreEval: []Hook{
|
||||
{
|
||||
Apply: []string{"SetRemediation(\"ban\")"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runtimeCfg, err := cfg.Build(t.Context(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, runtimeCfg.NeedWASMVM)
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package appsec
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/cookie"
|
||||
)
|
||||
|
||||
// TestAppsecTempResponse_Clone asserts the clone's reference-typed fields have
|
||||
// independent backing: mutating the original (as the out-of-band phase does)
|
||||
// must not be visible through the clone the HTTP handler holds.
|
||||
func TestAppsecTempResponse_Clone(t *testing.T) {
|
||||
orig := AppsecTempResponse{
|
||||
Action: ChallengeRemediation,
|
||||
UserHeaders: map[string][]string{"X-Test": {"a"}},
|
||||
UserHTTPCookies: []cookie.AppsecCookie{{Name: "c1"}},
|
||||
}
|
||||
|
||||
clone := orig.Clone()
|
||||
|
||||
// Out-of-band-style mutations on the original.
|
||||
orig.UserHeaders["X-New"] = []string{"z"} // new key
|
||||
orig.UserHeaders["X-Test"][0] = "MUT" // in-place inner-slice write
|
||||
orig.UserHTTPCookies = append(orig.UserHTTPCookies, cookie.AppsecCookie{Name: "c2"})
|
||||
|
||||
assert.NotContains(t, clone.UserHeaders, "X-New", "new keys must not leak into the clone")
|
||||
assert.Equal(t, "a", clone.UserHeaders["X-Test"][0], "inner slices must be copied, not aliased")
|
||||
require.Len(t, clone.UserHTTPCookies, 1, "cookie slice must be copied")
|
||||
assert.Equal(t, "c1", clone.UserHTTPCookies[0].Name)
|
||||
assert.Equal(t, ChallengeRemediation, clone.Action)
|
||||
}
|
||||
+244
-14
@@ -2,10 +2,83 @@ package appsec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/appsec/cookie"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
|
||||
)
|
||||
|
||||
// isLegitimateBotHelper wraps exprhelpers.IsLegitimateBot with the
|
||||
// per-request escape hatch: once ExemptFromChallenge was called, the verdict
|
||||
// is true without datafile or DNS checks. The standard rules gate
|
||||
// SendChallenge() on IsLegitimateBot(), so forcing the verdict true is what
|
||||
// whitelists the request from the challenge.
|
||||
func isLegitimateBotHelper(state *AppsecRequestState) func(string, string, string) bool {
|
||||
return func(ip string, ua string, path string) bool {
|
||||
return state.ChallengeExempt || exprhelpers.IsLegitimateBot(ip, ua, path)
|
||||
}
|
||||
}
|
||||
|
||||
// parseLogVerbosity maps an optional expr-side verbosity argument
|
||||
// ("minimal", "info", "verbose") to a FingerprintLogVerbosity. Empty /
|
||||
// missing returns FingerprintLogInfo (the default tier). Unknown values
|
||||
// emit a warning through the supplied logger and also fall back to
|
||||
// FingerprintLogInfo so the surrounding action (reject / log) still
|
||||
// takes effect — a typo in the verbosity arg shouldn't silently swallow
|
||||
// the operator's reject.
|
||||
func parseLogVerbosity(logger *log.Entry, verbosity []string) challenge.FingerprintLogVerbosity {
|
||||
if len(verbosity) == 0 {
|
||||
return challenge.FingerprintLogInfo
|
||||
}
|
||||
|
||||
switch strings.ToLower(strings.TrimSpace(verbosity[0])) {
|
||||
case "", "info":
|
||||
return challenge.FingerprintLogInfo
|
||||
case "minimal":
|
||||
return challenge.FingerprintLogMinimal
|
||||
case "verbose":
|
||||
return challenge.FingerprintLogVerbose
|
||||
default:
|
||||
if logger != nil {
|
||||
logger.Warnf("unknown fingerprint log verbosity %q; falling back to info", verbosity[0])
|
||||
}
|
||||
|
||||
return challenge.FingerprintLogInfo
|
||||
}
|
||||
}
|
||||
|
||||
// parseChallengeCookieTTLArg interprets the optional TTL argument to the
|
||||
// GrantChallengeCookie expr helper. Zero variadic args means "use the
|
||||
// runtime default" and yields a nil override. A single non-empty string is
|
||||
// parsed with time.ParseDuration (e.g. "1h", "30m"). More than one TTL
|
||||
// argument or an unparseable value is reported as an error so hook authors
|
||||
// get a precise diagnostic at evaluation time rather than a silent fallback.
|
||||
func parseChallengeCookieTTLArg(ttl []string) (*time.Duration, error) {
|
||||
if len(ttl) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if len(ttl) > 1 {
|
||||
return nil, fmt.Errorf("GrantChallengeCookie accepts at most one TTL argument, got %d", len(ttl))
|
||||
}
|
||||
if ttl[0] == "" {
|
||||
return nil, nil
|
||||
}
|
||||
d, err := time.ParseDuration(ttl[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid GrantChallengeCookie TTL %q: %w", ttl[0], err)
|
||||
}
|
||||
if d <= 0 {
|
||||
return nil, fmt.Errorf("GrantChallengeCookie TTL must be positive, got %s", d)
|
||||
}
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
func GetOnLoadEnv(w *AppsecRuntimeConfig) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"RemoveInBandRuleByID": w.DisableInBandRuleByID,
|
||||
@@ -17,6 +90,7 @@ func GetOnLoadEnv(w *AppsecRuntimeConfig) map[string]interface{} {
|
||||
"SetRemediationByTag": w.SetActionByTag,
|
||||
"SetRemediationByID": w.SetActionByID,
|
||||
"SetRemediationByName": w.SetActionByName,
|
||||
"SetChallengeDifficulty": w.SetChallengeDifficulty,
|
||||
"LoadAPISchemaWithName": w.LoadAPISchemaWithName,
|
||||
"LoadAPISchemaWithOptions": w.LoadAPISchemaWithOptions,
|
||||
"RegisterAPISchemaBodyDecoder": w.RegisterAPISchemaBodyDecoder,
|
||||
@@ -38,6 +112,8 @@ func GetPreEvalEnv(ctx context.Context, w *AppsecRuntimeConfig, state *AppsecReq
|
||||
"RemoveOutBandRuleByTag": func(tag string) error { return w.RemoveOutbandRuleByTag(state, tag) },
|
||||
"RemoveOutBandRuleByName": func(name string) error { return w.RemoveOutbandRuleByName(state, name) },
|
||||
"DropRequest": func(reason string) error { return w.DropRequest(state, request, reason) },
|
||||
"SetChallengeBody": func(body string) error { return w.SetChallengeBody(state, body) },
|
||||
"SetChallengeCookie": func(cookie cookie.AppsecCookie) error { return w.SetChallengeCookie(state, cookie) },
|
||||
"SetRemediationByTag": w.SetActionByTag,
|
||||
"SetRemediationByID": w.SetActionByID,
|
||||
"SetRemediationByName": w.SetActionByName,
|
||||
@@ -49,36 +125,190 @@ func GetPreEvalEnv(ctx context.Context, w *AppsecRuntimeConfig, state *AppsecReq
|
||||
state.PendingHTTPCode = &code
|
||||
return nil
|
||||
},
|
||||
"AppsecCookie": cookie.NewAppsecCookie,
|
||||
// SendChallenge is intentionally not exposed in pre_eval: a challenge must
|
||||
// only be issued after in-band WAF evaluation. Referencing it here fails to
|
||||
// compile at config load.
|
||||
"SetChallengeDifficulty": func(level string) error {
|
||||
return w.SetChallengeDifficultyPerRequest(state, level)
|
||||
},
|
||||
"GrantChallengeCookie": func(reason string, ttl ...string) error {
|
||||
ttlOverride, err := parseChallengeCookieTTLArg(ttl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return w.GrantChallengeCookie(state, request, reason, ttlOverride)
|
||||
},
|
||||
"fingerprint": state.Fingerprint,
|
||||
"ValidateRequestWithSchema": func(ref string) bool {
|
||||
return w.ValidateRequestWithSchema(ctx, state, request, ref)
|
||||
},
|
||||
"DisableBodyInspection": func() error { return w.DisableBodyInspection(state) },
|
||||
"IsLegitimateBot": isLegitimateBotHelper(state),
|
||||
"ExemptFromChallenge": func() error { state.ChallengeExempt = true; return nil },
|
||||
}
|
||||
}
|
||||
|
||||
func GetPostEvalEnv(w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest) map[string]interface{} {
|
||||
func GetPostEvalEnv(ctx context.Context, w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"IsInBand": request.IsInBand,
|
||||
"IsOutBand": request.IsOutBand,
|
||||
"DumpRequest": request.DumpRequest,
|
||||
"req": request.HTTPRequest,
|
||||
"hook_vars": state.HookVars,
|
||||
"SendChallenge": func() error {
|
||||
return w.SendChallenge(ctx, state, request)
|
||||
},
|
||||
"SetChallengeDifficulty": func(level string) error {
|
||||
return w.SetChallengeDifficultyPerRequest(state, level)
|
||||
},
|
||||
"GrantChallengeCookie": func(reason string, ttl ...string) error {
|
||||
ttlOverride, err := parseChallengeCookieTTLArg(ttl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return w.GrantChallengeCookie(state, request, reason, ttlOverride)
|
||||
},
|
||||
"DumpFingerprint": func(label string) string {
|
||||
return DumpFingerprint(w.FingerprintDumpDir, label, state.Fingerprint, request)
|
||||
},
|
||||
"fingerprint": state.Fingerprint,
|
||||
"hook_vars": state.HookVars,
|
||||
"IsLegitimateBot": isLegitimateBotHelper(state),
|
||||
"ExemptFromChallenge": func() error { state.ChallengeExempt = true; return nil },
|
||||
}
|
||||
}
|
||||
|
||||
func GetOnChallengeEnv(ctx context.Context, w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"req": request.HTTPRequest,
|
||||
"IsInBand": request.IsInBand,
|
||||
"fingerprint": state.Fingerprint,
|
||||
"SendChallenge": func() error {
|
||||
return w.SendChallenge(ctx, state, request)
|
||||
},
|
||||
"SetRemediation": func(action string) error {
|
||||
state.PendingAction = &action
|
||||
return nil
|
||||
},
|
||||
"SetReturnCode": func(code int) error {
|
||||
state.PendingHTTPCode = &code
|
||||
return nil
|
||||
},
|
||||
"DropRequest": func(reason string) error { return w.DropRequest(state, request, reason) },
|
||||
"SetChallengeDifficulty": func(level string) error {
|
||||
return w.SetChallengeDifficultyPerRequest(state, level)
|
||||
},
|
||||
|
||||
// EvaluateMismatches: aggregate fingerprint mismatch report. First
|
||||
// call per request computes the report, caches it on state, and
|
||||
// emits one Debug log line + per-signal Prometheus counter bumps;
|
||||
// subsequent calls return the cached pointer so rules can chain
|
||||
// `.High() >= 1 && .Has("cdp")` without redoing the work.
|
||||
//
|
||||
// Atomic helpers (fingerprint.UAMobileMismatch,
|
||||
// fingerprint.AcceptLanguageMismatch(req),
|
||||
// fingerprint.TimezoneCountryMismatch(country)) are methods on
|
||||
// `fingerprint` and can still be called directly from rules.
|
||||
"EvaluateMismatches": func() *challenge.MismatchReport {
|
||||
return w.EvaluateMismatches(state, request)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// GetOnChallengeSubmitEnv is the env exposed to on_challenge_submit hooks.
|
||||
// Deliberately narrow: the hook fires once during the challenge submission
|
||||
// JSON response, so anything that would change the response shape
|
||||
// (SendChallenge, SetRemediation, SetReturnCode, SetChallengeDifficulty,
|
||||
// DropRequest) is intentionally omitted to avoid breaking the client-side
|
||||
// JS handler. Operators wanting to escalate or block at the next request
|
||||
// should do so via pre_eval.
|
||||
func GetOnChallengeSubmitEnv(w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"req": request.HTTPRequest,
|
||||
"IsInBand": request.IsInBand,
|
||||
"fingerprint": state.Fingerprint,
|
||||
"RejectSubmission": func(reason string, verbosity ...string) error {
|
||||
if err := w.RejectSubmission(state, reason); err != nil {
|
||||
return err
|
||||
}
|
||||
// Emit the reject log immediately with the operator-chosen
|
||||
// verbosity. state.Fingerprint is populated at submit time
|
||||
// (set just before processHooks runs); LogRejected is
|
||||
// nil-safe so an unexpected nil here is a no-op rather than
|
||||
// a panic.
|
||||
state.Fingerprint.LogRejected(
|
||||
w.Logger,
|
||||
log.InfoLevel,
|
||||
request.ClientIP,
|
||||
request.RemoteAddrNormalized,
|
||||
state.SubmissionRejection.Reason,
|
||||
"on_challenge_submit rejected",
|
||||
parseLogVerbosity(w.Logger, verbosity),
|
||||
)
|
||||
// Terminal: halt later on_challenge_submit rules so a `filter: "true"`
|
||||
// `LogAccepted` can't fire for an already-rejected submission.
|
||||
state.HooksHalted = true
|
||||
return nil
|
||||
},
|
||||
// LogAccepted is intentionally exposed only here: per-request
|
||||
// cookie-validation acceptance is logged at Debug from the
|
||||
// internal ProcessOnChallengeRules path, so the only operator-
|
||||
// authored accept point is on a real challenge submission.
|
||||
"LogAccepted": func(msg string, verbosity ...string) error {
|
||||
state.Fingerprint.LogAccepted(
|
||||
w.Logger,
|
||||
log.InfoLevel,
|
||||
request.ClientIP,
|
||||
request.RemoteAddrNormalized,
|
||||
msg,
|
||||
parseLogVerbosity(w.Logger, verbosity),
|
||||
)
|
||||
return nil
|
||||
},
|
||||
// In on_challenge_submit the response is the challenge-submit JSON
|
||||
// envelope the client is already awaiting; a 307 redirect would
|
||||
// break its state machine. Route to the inline variant that
|
||||
// attaches the cookie to the existing envelope.
|
||||
"GrantChallengeCookie": func(reason string, ttl ...string) error {
|
||||
ttlOverride, err := parseChallengeCookieTTLArg(ttl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.GrantAllowlistCookieInline(state, request, reason, ttlOverride); err != nil {
|
||||
return err
|
||||
}
|
||||
// Terminal: halt later rules — the grant is final, and a following
|
||||
// rule could only overwrite the synthetic allowlist fingerprint.
|
||||
state.HooksHalted = true
|
||||
return nil
|
||||
},
|
||||
"EvaluateMismatches": func() *challenge.MismatchReport {
|
||||
return w.EvaluateMismatches(state, request)
|
||||
},
|
||||
"DumpFingerprint": func(label string) string {
|
||||
return DumpFingerprint(w.FingerprintDumpDir, label, state.Fingerprint, request)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func GetOnMatchEnv(w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest, evt pipeline.Event) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"evt": evt,
|
||||
"req": request.HTTPRequest,
|
||||
"hook_vars": state.HookVars,
|
||||
"IsInBand": request.IsInBand,
|
||||
"IsOutBand": request.IsOutBand,
|
||||
"SetRemediation": func(action string) error { return w.SetAction(state, action) },
|
||||
"SetReturnCode": func(code int) error { return w.SetHTTPCode(state, code) },
|
||||
"CancelEvent": func() error { return w.CancelEvent(state) },
|
||||
"SendEvent": func() error { return w.SendEvent(state) },
|
||||
"CancelAlert": func() error { return w.CancelAlert(state) },
|
||||
"SendAlert": func() error { return w.SendAlert(state) },
|
||||
"DumpRequest": request.DumpRequest,
|
||||
"evt": evt,
|
||||
"req": request.HTTPRequest,
|
||||
"hook_vars": state.HookVars,
|
||||
"IsInBand": request.IsInBand,
|
||||
"IsOutBand": request.IsOutBand,
|
||||
"SetRemediation": func(action string) error { return w.SetAction(state, action) },
|
||||
"SetReturnCode": func(code int) error { return w.SetHTTPCode(state, code) },
|
||||
"CancelEvent": func() error { return w.CancelEvent(state) },
|
||||
"SendEvent": func() error { return w.SendEvent(state) },
|
||||
"CancelAlert": func() error { return w.CancelAlert(state) },
|
||||
"SendAlert": func() error { return w.SendAlert(state) },
|
||||
"DumpRequest": request.DumpRequest,
|
||||
"SetChallengeBody": func(body string) error { return w.SetChallengeBody(state, body) },
|
||||
"SetChallengeCookie": func(cookie cookie.AppsecCookie) error { return w.SetChallengeCookie(state, cookie) },
|
||||
"AppsecCookie": cookie.NewAppsecCookie,
|
||||
"IsLegitimateBot": isLegitimateBotHelper(state),
|
||||
"ExemptFromChallenge": func() error { state.ChallengeExempt = true; return nil },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package appsec
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestLegitimateBotHooksCompile guards the env maps: a hook referencing
|
||||
// IsLegitimateBot/ExemptFromChallenge must compile in every phase exposing them.
|
||||
func TestLegitimateBotHooksCompile(t *testing.T) {
|
||||
for _, stage := range []hookStage{hookPreEval, hookPostEval, hookOnMatch} {
|
||||
h := &Hook{
|
||||
Filter: `IsLegitimateBot(req.RemoteAddr, req.UserAgent(), req.URL.Path)`,
|
||||
Apply: []string{`ExemptFromChallenge()`},
|
||||
}
|
||||
require.NoError(t, h.Build(t.Context(), stage, nil), "stage %v", stage)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExemptFromChallengeEscapeHatch verifies the per-request escape hatch:
|
||||
// once ExemptFromChallenge was called, IsLegitimateBot returns true without
|
||||
// consulting datafiles (none are loaded here) or DNS.
|
||||
func TestExemptFromChallengeEscapeHatch(t *testing.T) {
|
||||
state := &AppsecRequestState{HookVars: map[string]string{}}
|
||||
env := GetPreEvalEnv(t.Context(), &AppsecRuntimeConfig{}, state, &ParsedRequest{})
|
||||
|
||||
isLegit := env["IsLegitimateBot"].(func(string, string, string) bool)
|
||||
exempt := env["ExemptFromChallenge"].(func() error)
|
||||
|
||||
assert.False(t, isLegit("1.2.3.4", "googlebot", "/"))
|
||||
|
||||
require.NoError(t, exempt())
|
||||
assert.True(t, isLegit("1.2.3.4", "googlebot", "/"))
|
||||
assert.True(t, isLegit("garbage-ip", "", ""), "escape hatch bypasses all checks")
|
||||
|
||||
// the flag is per-request state: a fresh state starts clean
|
||||
freshEnv := GetPreEvalEnv(t.Context(), &AppsecRuntimeConfig{}, &AppsecRequestState{HookVars: map[string]string{}}, &ParsedRequest{})
|
||||
assert.False(t, freshEnv["IsLegitimateBot"].(func(string, string, string) bool)("1.2.3.4", "googlebot", "/"))
|
||||
}
|
||||
|
||||
// TestParseChallengeCookieTTLArg covers the GrantChallengeCookie optional TTL
|
||||
// argument parsing: no args / empty string yield a nil override (use runtime
|
||||
// default), a parseable duration yields a positive pointer, and malformed /
|
||||
// non-positive / multi-arg inputs surface as errors so hook authors see a
|
||||
// precise diagnostic instead of silent fallback.
|
||||
func TestParseChallengeCookieTTLArg(t *testing.T) {
|
||||
t.Run("no args → nil override", func(t *testing.T) {
|
||||
got, err := parseChallengeCookieTTLArg(nil)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, got)
|
||||
})
|
||||
|
||||
t.Run("empty string → nil override", func(t *testing.T) {
|
||||
got, err := parseChallengeCookieTTLArg([]string{""})
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, got)
|
||||
})
|
||||
|
||||
t.Run("valid duration", func(t *testing.T) {
|
||||
got, err := parseChallengeCookieTTLArg([]string{"1h30m"})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, 90*time.Minute, *got)
|
||||
})
|
||||
|
||||
t.Run("malformed duration", func(t *testing.T) {
|
||||
_, err := parseChallengeCookieTTLArg([]string{"forever"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "invalid GrantChallengeCookie TTL")
|
||||
})
|
||||
|
||||
t.Run("non-positive duration", func(t *testing.T) {
|
||||
_, err := parseChallengeCookieTTLArg([]string{"-5m"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "must be positive")
|
||||
})
|
||||
|
||||
t.Run("multiple TTL args rejected", func(t *testing.T) {
|
||||
_, err := parseChallengeCookieTTLArg([]string{"1h", "2h"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "at most one TTL argument")
|
||||
})
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gopkg.in/yaml.v3"
|
||||
@@ -26,11 +27,19 @@ type CrowdsecServiceCfg struct {
|
||||
BucketStateFile string `yaml:"state_input_file,omitempty"` // if we need to unserialize buckets at start
|
||||
BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
|
||||
BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode
|
||||
DNSCache *DNSCacheCfg `yaml:"dns_cache,omitempty"`
|
||||
|
||||
SimulationFilePath string `yaml:"-"`
|
||||
ContextToSend map[string][]string `yaml:"-"`
|
||||
}
|
||||
|
||||
// Cache config for DNS lookups (legit bots, rdns PO)
|
||||
type DNSCacheCfg struct {
|
||||
TTL *time.Duration `yaml:"ttl,omitempty"`
|
||||
NegativeTTL *time.Duration `yaml:"negative_ttl,omitempty"`
|
||||
Size *int `yaml:"size,omitempty"`
|
||||
}
|
||||
|
||||
var ErrNoAcquisitionDefined = errors.New("no acquisition_path or acquisition_dir specified")
|
||||
|
||||
func (c *CrowdsecServiceCfg) CollectAcquisitionFiles() ([]string, error) {
|
||||
|
||||
@@ -3,8 +3,11 @@ package csconfig
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/go-cs-lib/cstest"
|
||||
)
|
||||
@@ -195,3 +198,29 @@ func TestLoadCrowdsec(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSCacheCfg(t *testing.T) {
|
||||
yamlConfig := `
|
||||
acquisition_path: ./testdata/acquis.yaml
|
||||
dns_cache:
|
||||
ttl: 2h
|
||||
negative_ttl: 30s
|
||||
size: 4096
|
||||
`
|
||||
|
||||
cfg := CrowdsecServiceCfg{}
|
||||
require.NoError(t, yaml.Unmarshal([]byte(yamlConfig), &cfg))
|
||||
|
||||
require.NotNil(t, cfg.DNSCache)
|
||||
require.NotNil(t, cfg.DNSCache.TTL)
|
||||
assert.Equal(t, 2*time.Hour, *cfg.DNSCache.TTL)
|
||||
require.NotNil(t, cfg.DNSCache.NegativeTTL)
|
||||
assert.Equal(t, 30*time.Second, *cfg.DNSCache.NegativeTTL)
|
||||
require.NotNil(t, cfg.DNSCache.Size)
|
||||
assert.Equal(t, 4096, *cfg.DNSCache.Size)
|
||||
|
||||
// the section is optional
|
||||
bare := CrowdsecServiceCfg{}
|
||||
require.NoError(t, yaml.Unmarshal([]byte("acquisition_path: ./testdata/acquis.yaml"), &bare))
|
||||
assert.Nil(t, bare.DNSCache)
|
||||
}
|
||||
|
||||
@@ -26,7 +26,12 @@ type Hub struct {
|
||||
}
|
||||
|
||||
// GetDataDir returns the data directory, where data sets are installed.
|
||||
// Empty for a hub created without local configuration (e.g. zero-value Hub in tests).
|
||||
func (h *Hub) GetDataDir() string {
|
||||
if h.local == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return h.local.InstallDataDir
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
// Package dnscache provides process-wide cached DNS lookups: PTR records,
|
||||
// forward (A/AAAA) records, and forward-confirmed reverse DNS (FCrDNS)
|
||||
// composed from the two. Results — including failures — are cached in a
|
||||
// shared LRU, and concurrent lookups for the same key coalesce via
|
||||
// singleflight, so a given DNS fact costs at most one query per TTL.
|
||||
//
|
||||
// Consumers: the parser's reverse_dns enricher (PTR only) and the
|
||||
// IsLegitimateBot appsec helper (FCrDNS).
|
||||
package dnscache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
// Resolver is satisfied by *net.Resolver; swapped for a fake in tests.
|
||||
type Resolver interface {
|
||||
LookupAddr(ctx context.Context, addr string) ([]string, error)
|
||||
LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
|
||||
}
|
||||
|
||||
var resolver Resolver = net.DefaultResolver
|
||||
|
||||
func SetResolver(r Resolver) {
|
||||
resolver = r
|
||||
}
|
||||
|
||||
const (
|
||||
lookupTimeout = 3 * time.Second // per lookup (PTR or forward)
|
||||
maxPTRNames = 3 // PTR fan-out cap; legit hosts return 1
|
||||
|
||||
defaultPositiveTTL = 1 * time.Hour
|
||||
defaultNegativeTTL = 5 * time.Minute
|
||||
defaultCacheSize = 16384
|
||||
)
|
||||
|
||||
var (
|
||||
positiveTTL = defaultPositiveTTL
|
||||
negativeTTL = defaultNegativeTTL
|
||||
cacheSize = defaultCacheSize
|
||||
|
||||
cacheOnce sync.Once
|
||||
cache gcache.Cache
|
||||
sf singleflight.Group
|
||||
)
|
||||
|
||||
func Configure(posTTL time.Duration, negTTL time.Duration, size int) {
|
||||
if posTTL > 0 {
|
||||
positiveTTL = posTTL
|
||||
}
|
||||
|
||||
if negTTL > 0 {
|
||||
negativeTTL = negTTL
|
||||
}
|
||||
|
||||
if size > 0 {
|
||||
cacheSize = size
|
||||
}
|
||||
}
|
||||
|
||||
func Purge() {
|
||||
if cache != nil {
|
||||
cache.Purge()
|
||||
}
|
||||
}
|
||||
|
||||
func cachedLookup[T any](key string, lookup func() T, empty func(T) bool) T {
|
||||
cacheOnce.Do(func() {
|
||||
cache = gcache.New(cacheSize).LRU().Build()
|
||||
})
|
||||
|
||||
if val, err := cache.Get(key); err == nil {
|
||||
return val.(T)
|
||||
}
|
||||
|
||||
val, _, _ := sf.Do(key, func() (any, error) {
|
||||
// Re-check the cache inside the singleflight callback in case
|
||||
// another goroutine populated it between the fast-path read and
|
||||
// our entry into Do.
|
||||
if val, err := cache.Get(key); err == nil {
|
||||
return val.(T), nil
|
||||
}
|
||||
|
||||
result := lookup()
|
||||
|
||||
ttl := positiveTTL
|
||||
if empty(result) {
|
||||
ttl = negativeTTL
|
||||
}
|
||||
|
||||
if err := cache.SetWithExpire(key, result, ttl); err != nil {
|
||||
log.Debugf("failed to cache DNS result for %s: %s", key, err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
})
|
||||
|
||||
return val.(T)
|
||||
}
|
||||
|
||||
func PTRRecords(addr netip.Addr) []string {
|
||||
return cachedLookup("ptr:"+addr.String(), func() []string {
|
||||
return lookupPTR(addr)
|
||||
}, func(names []string) bool { return len(names) == 0 })
|
||||
}
|
||||
|
||||
func ForwardIPs(host string) []netip.Addr {
|
||||
host = strings.TrimSuffix(strings.ToLower(host), ".")
|
||||
|
||||
return cachedLookup("fwd:"+host, func() []netip.Addr {
|
||||
return lookupForward(host)
|
||||
}, func(ips []netip.Addr) bool { return len(ips) == 0 })
|
||||
}
|
||||
|
||||
func ForwardConfirmedNames(addr netip.Addr) []string {
|
||||
verified := []string{}
|
||||
|
||||
names := PTRRecords(addr)
|
||||
|
||||
for _, name := range names[:min(len(names), maxPTRNames)] {
|
||||
name = strings.TrimSuffix(strings.ToLower(name), ".")
|
||||
|
||||
if slices.Contains(ForwardIPs(name), addr.Unmap()) {
|
||||
verified = append(verified, name)
|
||||
}
|
||||
}
|
||||
|
||||
return verified
|
||||
}
|
||||
|
||||
func lookupPTR(addr netip.Addr) []string {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), lookupTimeout)
|
||||
defer cancel()
|
||||
|
||||
names, err := resolver.LookupAddr(ctx, addr.String())
|
||||
if err != nil {
|
||||
log.Debugf("dnscache: PTR lookup failed for %s: %s", addr, err)
|
||||
return []string{}
|
||||
}
|
||||
|
||||
return names
|
||||
}
|
||||
|
||||
func lookupForward(host string) []netip.Addr {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), lookupTimeout)
|
||||
defer cancel()
|
||||
|
||||
ips, err := resolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
log.Debugf("dnscache: forward lookup failed for %s: %s", host, err)
|
||||
return []netip.Addr{}
|
||||
}
|
||||
|
||||
addrs := []netip.Addr{}
|
||||
|
||||
for _, ip := range ips {
|
||||
if addr, ok := netip.AddrFromSlice(ip.IP); ok {
|
||||
addrs = append(addrs, addr.Unmap())
|
||||
}
|
||||
}
|
||||
|
||||
return addrs
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package dnscache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeResolver implements Resolver from in-memory maps so no test ever hits
|
||||
// real DNS. ptrCalls/fwdCalls let tests assert how many lookups ran.
|
||||
type fakeResolver struct {
|
||||
ptr map[string][]string // ip → PTR names
|
||||
fwd map[string][]net.IPAddr // hostname → addresses
|
||||
delay time.Duration
|
||||
ptrCalls atomic.Int32
|
||||
fwdCalls atomic.Int32
|
||||
}
|
||||
|
||||
func (f *fakeResolver) LookupAddr(_ context.Context, addr string) ([]string, error) {
|
||||
f.ptrCalls.Add(1)
|
||||
time.Sleep(f.delay)
|
||||
|
||||
names, ok := f.ptr[addr]
|
||||
if !ok {
|
||||
return nil, &net.DNSError{Err: "no such host", Name: addr, IsNotFound: true}
|
||||
}
|
||||
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (f *fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) {
|
||||
f.fwdCalls.Add(1)
|
||||
|
||||
ips, ok := f.fwd[host]
|
||||
if !ok {
|
||||
return nil, &net.DNSError{Err: "no such host", Name: host, IsNotFound: true}
|
||||
}
|
||||
|
||||
return ips, nil
|
||||
}
|
||||
|
||||
// setupDNSTest installs the fake resolver and guarantees an empty cache,
|
||||
// restoring the default resolver when the test ends.
|
||||
func setupDNSTest(t *testing.T, r *fakeResolver) {
|
||||
t.Helper()
|
||||
|
||||
SetResolver(r)
|
||||
Purge()
|
||||
t.Cleanup(func() {
|
||||
SetResolver(net.DefaultResolver)
|
||||
Purge()
|
||||
})
|
||||
}
|
||||
|
||||
func TestPTRRecordsRawNames(t *testing.T) {
|
||||
resolver := &fakeResolver{
|
||||
ptr: map[string][]string{"1.1.1.1": {"One.One.one.one."}},
|
||||
}
|
||||
setupDNSTest(t, resolver)
|
||||
|
||||
// names come back raw: trailing dot and case preserved (the reverse_dns
|
||||
// parser enricher exposes them as-is)
|
||||
assert.Equal(t, []string{"One.One.one.one."}, PTRRecords(netip.MustParseAddr("1.1.1.1")))
|
||||
|
||||
// failures return an empty slice, and are cached too
|
||||
assert.Empty(t, PTRRecords(netip.MustParseAddr("203.0.113.1")))
|
||||
assert.Empty(t, PTRRecords(netip.MustParseAddr("203.0.113.1")))
|
||||
assert.Equal(t, int32(2), resolver.ptrCalls.Load(), "failed lookup must be cached as negative")
|
||||
}
|
||||
|
||||
func TestForwardIPsKeyNormalization(t *testing.T) {
|
||||
resolver := &fakeResolver{
|
||||
fwd: map[string][]net.IPAddr{"host.example.com": {{IP: net.ParseIP("192.0.2.1")}}},
|
||||
}
|
||||
setupDNSTest(t, resolver)
|
||||
|
||||
want := []netip.Addr{netip.MustParseAddr("192.0.2.1")}
|
||||
|
||||
// trailing dot and case variants hit the same cache entry
|
||||
assert.Equal(t, want, ForwardIPs("Host.Example.COM."))
|
||||
assert.Equal(t, want, ForwardIPs("host.example.com"))
|
||||
assert.Equal(t, int32(1), resolver.fwdCalls.Load(), "normalized variants must share one cache entry")
|
||||
}
|
||||
|
||||
func TestForwardConfirmedNames(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
resolver *fakeResolver
|
||||
ip string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "happy path, trailing dot and mixed case PTR",
|
||||
resolver: &fakeResolver{
|
||||
ptr: map[string][]string{"66.249.66.1": {"Crawl-66-249-66-1.GoogleBot.com."}},
|
||||
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
|
||||
},
|
||||
ip: "66.249.66.1",
|
||||
expected: []string{"crawl-66-249-66-1.googlebot.com"},
|
||||
},
|
||||
{
|
||||
name: "spoofed PTR, forward lookup does not confirm",
|
||||
resolver: &fakeResolver{
|
||||
ptr: map[string][]string{"203.0.113.5": {"crawl.googlebot.com."}},
|
||||
fwd: map[string][]net.IPAddr{"crawl.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
|
||||
},
|
||||
ip: "203.0.113.5",
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "no PTR record",
|
||||
resolver: &fakeResolver{},
|
||||
ip: "203.0.113.7",
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "PTR fan-out capped, extra names ignored",
|
||||
resolver: &fakeResolver{
|
||||
ptr: map[string][]string{"203.0.113.8": {"a.example.com.", "b.example.com.", "c.example.com.", "d.example.com."}},
|
||||
fwd: map[string][]net.IPAddr{"d.example.com": {{IP: net.ParseIP("203.0.113.8")}}},
|
||||
},
|
||||
ip: "203.0.113.8",
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "IPv4-mapped forward answer still confirms",
|
||||
resolver: &fakeResolver{
|
||||
ptr: map[string][]string{"192.0.2.9": {"mapped.example.com."}},
|
||||
fwd: map[string][]net.IPAddr{"mapped.example.com": {{IP: net.ParseIP("192.0.2.9").To16()}}},
|
||||
},
|
||||
ip: "192.0.2.9",
|
||||
expected: []string{"mapped.example.com"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
setupDNSTest(t, tc.resolver)
|
||||
assert.Equal(t, tc.expected, ForwardConfirmedNames(netip.MustParseAddr(tc.ip)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestForwardConfirmedNamesSharesPTRCache(t *testing.T) {
|
||||
resolver := &fakeResolver{
|
||||
ptr: map[string][]string{"66.249.66.1": {"crawl-66-249-66-1.googlebot.com."}},
|
||||
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
|
||||
}
|
||||
setupDNSTest(t, resolver)
|
||||
|
||||
addr := netip.MustParseAddr("66.249.66.1")
|
||||
|
||||
// a plain PTR consumer warms the cache for the FCrDNS consumer
|
||||
require.NotEmpty(t, PTRRecords(addr))
|
||||
assert.Equal(t, []string{"crawl-66-249-66-1.googlebot.com"}, ForwardConfirmedNames(addr))
|
||||
assert.Equal(t, int32(1), resolver.ptrCalls.Load(), "FCrDNS must reuse the cached PTR entry")
|
||||
|
||||
// and the composed result is fully cached on repeat
|
||||
assert.Equal(t, []string{"crawl-66-249-66-1.googlebot.com"}, ForwardConfirmedNames(addr))
|
||||
assert.Equal(t, int32(1), resolver.ptrCalls.Load())
|
||||
assert.Equal(t, int32(1), resolver.fwdCalls.Load())
|
||||
}
|
||||
|
||||
func TestSingleflight(t *testing.T) {
|
||||
resolver := &fakeResolver{
|
||||
ptr: map[string][]string{"66.249.66.1": {"crawl-66-249-66-1.googlebot.com."}},
|
||||
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
|
||||
delay: 50 * time.Millisecond,
|
||||
}
|
||||
setupDNSTest(t, resolver)
|
||||
|
||||
addr := netip.MustParseAddr("66.249.66.1")
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for range 10 {
|
||||
wg.Go(func() {
|
||||
assert.Equal(t, []string{"crawl-66-249-66-1.googlebot.com"}, ForwardConfirmedNames(addr))
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
assert.Equal(t, int32(1), resolver.ptrCalls.Load(), "concurrent cold lookups must coalesce")
|
||||
}
|
||||
|
||||
func TestConfigureZeroValuesKeepDefaults(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
positiveTTL = defaultPositiveTTL
|
||||
negativeTTL = defaultNegativeTTL
|
||||
cacheSize = defaultCacheSize
|
||||
})
|
||||
|
||||
Configure(0, 0, 0)
|
||||
assert.Equal(t, defaultPositiveTTL, positiveTTL)
|
||||
assert.Equal(t, defaultNegativeTTL, negativeTTL)
|
||||
assert.Equal(t, defaultCacheSize, cacheSize)
|
||||
|
||||
Configure(2*time.Hour, 0, 0)
|
||||
assert.Equal(t, 2*time.Hour, positiveTTL)
|
||||
assert.Equal(t, defaultNegativeTTL, negativeTTL)
|
||||
assert.Equal(t, defaultCacheSize, cacheSize)
|
||||
|
||||
Configure(0, 10*time.Minute, 4096)
|
||||
assert.Equal(t, 2*time.Hour, positiveTTL)
|
||||
assert.Equal(t, 10*time.Minute, negativeTTL)
|
||||
assert.Equal(t, 4096, cacheSize)
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package exprhelpers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/dnscache"
|
||||
)
|
||||
|
||||
var dataFileBots map[string][]*botEntry
|
||||
|
||||
// botEntry is one bot definition (one JSON line in a "bots" data file).
|
||||
// Matching semantics: (UA && PATH) && (IP || RANGE || RDNS) — the optional
|
||||
// user_agent and paths regexes are preconditions, then the source must be
|
||||
// verified by at least one identity check (exact IP, CIDR range, or
|
||||
// forward-confirmed reverse DNS). UA-only entries are rejected at load time:
|
||||
// a User-Agent is trivially spoofable.
|
||||
//
|
||||
// rdns patterns are regexes matched against the FCrDNS-verified names
|
||||
// (lowercase, no trailing dot). Anchor them — `(^|\.)googlebot\.com$` —
|
||||
// or "evilgooglebot.com.attacker.net" matches `googlebot\.com` too.
|
||||
type botEntry struct {
|
||||
Name string `json:"name"`
|
||||
UserAgent string `json:"user_agent,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
IPs []string `json:"ips,omitempty"`
|
||||
Ranges []string `json:"ranges,omitempty"`
|
||||
RDNS []string `json:"rdns,omitempty"`
|
||||
|
||||
// compiled at load time
|
||||
uaRegex *regexp.Regexp
|
||||
pathRegexes []*regexp.Regexp
|
||||
ipSet map[netip.Addr]struct{}
|
||||
prefixes []netip.Prefix
|
||||
rdnsRegexes []*regexp.Regexp
|
||||
}
|
||||
|
||||
func compileBotRegex(pattern string) (*regexp.Regexp, error) {
|
||||
return regexp.Compile("(?i)" + pattern) // Force case insensitive match
|
||||
}
|
||||
|
||||
func botFileInit(filename string, line string) error {
|
||||
entry := &botEntry{}
|
||||
|
||||
dec := json.NewDecoder(strings.NewReader(line))
|
||||
dec.DisallowUnknownFields()
|
||||
|
||||
if err := dec.Decode(entry); err != nil {
|
||||
return fmt.Errorf("failed to parse JSON line in %s: %w", filename, err)
|
||||
}
|
||||
|
||||
if entry.Name == "" {
|
||||
return fmt.Errorf("missing mandatory 'name' field in %s: %s", filename, line)
|
||||
}
|
||||
|
||||
if len(entry.IPs)+len(entry.Ranges)+len(entry.RDNS) == 0 {
|
||||
return fmt.Errorf("bot entry '%s' in %s has no identity verification (need at least one of ips/ranges/rdns)", entry.Name, filename)
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
if entry.UserAgent != "" {
|
||||
if entry.uaRegex, err = compileBotRegex(entry.UserAgent); err != nil {
|
||||
return fmt.Errorf("invalid user_agent regex for bot entry '%s' in %s: %w", entry.Name, filename, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, p := range entry.Paths {
|
||||
re, err := compileBotRegex(p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid path regex '%s' for bot entry '%s' in %s: %w", p, entry.Name, filename, err)
|
||||
}
|
||||
|
||||
entry.pathRegexes = append(entry.pathRegexes, re)
|
||||
}
|
||||
|
||||
entry.ipSet = make(map[netip.Addr]struct{}, len(entry.IPs))
|
||||
|
||||
for _, ip := range entry.IPs {
|
||||
addr, err := netip.ParseAddr(ip)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid IP '%s' for bot entry '%s' in %s: %w", ip, entry.Name, filename, err)
|
||||
}
|
||||
|
||||
entry.ipSet[addr.Unmap()] = struct{}{}
|
||||
}
|
||||
|
||||
for _, r := range entry.Ranges {
|
||||
prefix, err := netip.ParsePrefix(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid CIDR range '%s' for bot entry '%s' in %s: %w", r, entry.Name, filename, err)
|
||||
}
|
||||
|
||||
entry.prefixes = append(entry.prefixes, prefix.Masked())
|
||||
}
|
||||
|
||||
for _, p := range entry.RDNS {
|
||||
// an empty pattern matches every PTR-confirmed host: almost
|
||||
// certainly a mistake, reject it
|
||||
if p == "" {
|
||||
return fmt.Errorf("empty rdns pattern for bot entry '%s' in %s", entry.Name, filename)
|
||||
}
|
||||
|
||||
re, err := compileBotRegex(p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid rdns regex '%s' for bot entry '%s' in %s: %w", p, entry.Name, filename, err)
|
||||
}
|
||||
|
||||
entry.rdnsRegexes = append(entry.rdnsRegexes, re)
|
||||
}
|
||||
|
||||
dataFileBots[filename] = append(dataFileBots[filename], entry)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseBotAddr normalizes a source address as found in HTTP contexts:
|
||||
// bare IP, "ip:port", "[v6]:port". The zone is stripped and IPv4-mapped
|
||||
// IPv6 is unmapped so comparisons against load-time-parsed IPs/ranges are
|
||||
// consistent.
|
||||
func parseBotAddr(s string) (netip.Addr, bool) {
|
||||
addr, err := netip.ParseAddr(s)
|
||||
if err != nil {
|
||||
host, _, splitErr := net.SplitHostPort(s)
|
||||
if splitErr != nil {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
|
||||
if addr, err = netip.ParseAddr(host); err != nil {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
}
|
||||
|
||||
return addr.WithZone("").Unmap(), true
|
||||
}
|
||||
|
||||
// IsLegitimateBot reports whether the request (source address, User-Agent,
|
||||
// path) matches a bot definition from the loaded "bots" data files:
|
||||
// (UA && PATH) && (IP || RANGE || RDNS). Fail-closed: an unparseable
|
||||
// address or a DNS failure means "not a legitimate bot", never an error.
|
||||
//
|
||||
// The expensive FCrDNS resolution runs at most once per call — after the
|
||||
// cheap checks, against all candidate entries at once — and is cached per IP.
|
||||
func IsLegitimateBot(ip string, ua string, path string) bool {
|
||||
if len(dataFileBots) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
addr, ok := parseBotAddr(ip)
|
||||
if !ok {
|
||||
log.Debugf("IsLegitimateBot: invalid source address '%s'", ip)
|
||||
return false
|
||||
}
|
||||
|
||||
var rdnsCandidates []*botEntry
|
||||
|
||||
for _, entries := range dataFileBots {
|
||||
for _, entry := range entries {
|
||||
if entry.uaRegex != nil && !entry.uaRegex.MatchString(ua) {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(entry.pathRegexes) > 0 && !matchAnyRegex(entry.pathRegexes, path) {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, found := entry.ipSet[addr]; found {
|
||||
log.Debugf("IsLegitimateBot: %s verified as '%s' via exact IP", ip, entry.Name)
|
||||
return true
|
||||
}
|
||||
|
||||
for _, prefix := range entry.prefixes {
|
||||
if prefix.Contains(addr) {
|
||||
log.Debugf("IsLegitimateBot: %s verified as '%s' via range %s", ip, entry.Name, prefix)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if len(entry.rdnsRegexes) > 0 {
|
||||
rdnsCandidates = append(rdnsCandidates, entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(rdnsCandidates) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, name := range dnscache.ForwardConfirmedNames(addr) {
|
||||
for _, entry := range rdnsCandidates {
|
||||
if matchAnyRegex(entry.rdnsRegexes, name) {
|
||||
log.Debugf("IsLegitimateBot: %s verified as '%s' via FCrDNS (%s)", ip, entry.Name, name)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func matchAnyRegex(regexes []*regexp.Regexp, s string) bool {
|
||||
for _, re := range regexes {
|
||||
if re.MatchString(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
package exprhelpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/dnscache"
|
||||
)
|
||||
|
||||
// fakeResolver implements dnscache.Resolver from in-memory maps so no test
|
||||
// ever hits real DNS. ptrCalls lets tests assert how many lookups ran.
|
||||
// (The cache/singleflight internals are tested in pkg/dnscache; here the
|
||||
// fake only backs the end-to-end IsLegitimateBot checks.)
|
||||
type fakeResolver struct {
|
||||
ptr map[string][]string // ip → PTR names
|
||||
fwd map[string][]net.IPAddr // hostname → addresses
|
||||
ptrCalls atomic.Int32
|
||||
}
|
||||
|
||||
func (f *fakeResolver) LookupAddr(_ context.Context, addr string) ([]string, error) {
|
||||
f.ptrCalls.Add(1)
|
||||
|
||||
names, ok := f.ptr[addr]
|
||||
if !ok {
|
||||
return nil, &net.DNSError{Err: "no such host", Name: addr, IsNotFound: true}
|
||||
}
|
||||
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (f *fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) {
|
||||
ips, ok := f.fwd[host]
|
||||
if !ok {
|
||||
return nil, &net.DNSError{Err: "no such host", Name: host, IsNotFound: true}
|
||||
}
|
||||
|
||||
return ips, nil
|
||||
}
|
||||
|
||||
// setupBotTest resets the datafile globals and, when a fake resolver is
|
||||
// given, installs it in pkg/dnscache with an empty cache for isolation.
|
||||
func setupBotTest(t *testing.T, resolver *fakeResolver) {
|
||||
t.Helper()
|
||||
|
||||
err := Init(nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
if resolver != nil {
|
||||
dnscache.SetResolver(resolver)
|
||||
dnscache.Purge()
|
||||
t.Cleanup(func() {
|
||||
dnscache.SetResolver(net.DefaultResolver)
|
||||
dnscache.Purge()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBotFileInit(t *testing.T) {
|
||||
setupBotTest(t, nil)
|
||||
|
||||
err := FileInit("testdata", "test_data_bots.json", "bots")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, ok := dataFileBots["test_data_bots.json"]
|
||||
require.True(t, ok, "test_data_bots.json should be loaded")
|
||||
require.Len(t, entries, 4, "should have 4 entries (skipping comment and empty line)")
|
||||
|
||||
googlebot := entries[0]
|
||||
assert.Equal(t, "googlebot", googlebot.Name)
|
||||
assert.NotNil(t, googlebot.uaRegex)
|
||||
assert.Empty(t, googlebot.pathRegexes)
|
||||
assert.Len(t, googlebot.rdnsRegexes, 2)
|
||||
|
||||
uptimerobot := entries[1]
|
||||
assert.Len(t, uptimerobot.pathRegexes, 2)
|
||||
assert.Len(t, uptimerobot.prefixes, 2)
|
||||
assert.Len(t, uptimerobot.ipSet, 1)
|
||||
|
||||
// double load is a no-op, not an error
|
||||
err = FileInit("testdata", "test_data_bots.json", "bots")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, dataFileBots["test_data_bots.json"], 4)
|
||||
}
|
||||
|
||||
func TestBotFileInitErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
expectedErr string
|
||||
}{
|
||||
{"invalid JSON", "test_data_bots_invalid.json", "failed to parse JSON line"},
|
||||
{"missing name", "test_data_bots_no_name.json", "missing mandatory 'name' field"},
|
||||
{"no identity check", "test_data_bots_no_identity.json", "no identity verification"},
|
||||
{"bad user_agent regex", "test_data_bots_bad_regex.json", "invalid user_agent regex"},
|
||||
{"bad rdns regex", "test_data_bots_bad_rdns.json", "invalid rdns regex"},
|
||||
{"bad CIDR", "test_data_bots_bad_cidr.json", "invalid CIDR range"},
|
||||
{"bad IP", "test_data_bots_bad_ip.json", "invalid IP"},
|
||||
{"unknown field", "test_data_bots_unknown_field.json", "unknown field"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
setupBotTest(t, nil)
|
||||
|
||||
err := FileInit("testdata", tc.filename, "bots")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tc.expectedErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBotFileUnknownType(t *testing.T) {
|
||||
setupBotTest(t, nil)
|
||||
|
||||
// singular "bot" is a typo, must be rejected by existsInFileMaps
|
||||
err := FileInit("testdata", "test_data_bots.json", "bot")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unknown data type")
|
||||
}
|
||||
|
||||
func TestIsLegitimateBotNoDNS(t *testing.T) {
|
||||
resolver := &fakeResolver{}
|
||||
setupBotTest(t, resolver)
|
||||
|
||||
err := FileInit("testdata", "test_data_bots.json", "bots")
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ip string
|
||||
ua string
|
||||
path string
|
||||
expected bool
|
||||
}{
|
||||
{"exact IP", "10.1.2.3", "", "/", true},
|
||||
{"exact IPv6", "2001:db8::42", "", "/", true},
|
||||
{"exact IP with port", "10.1.2.3:54321", "", "/", true},
|
||||
{"IPv4-mapped IPv6", "::ffff:10.1.2.3", "", "/", true},
|
||||
{"range hit", "192.0.2.77", "anything", "/any/path", true},
|
||||
{"range miss", "192.0.3.77", "anything", "/any/path", false},
|
||||
{"UA+path+range", "69.162.124.230", "UptimeRobot/2.0", "/health/db", true},
|
||||
{"UA+path+IP", "216.144.250.150", "uptimerobot", "/status", true},
|
||||
{"UA+path+v6 range with port", "[2607:ff68:107::25]:443", "UptimeRobot/2.0", "/health", true},
|
||||
{"UA case-insensitive", "69.162.124.230", "UPTIMEROBOT", "/health", true},
|
||||
{"path case-insensitive", "69.162.124.230", "uptimerobot", "/HEALTH", true},
|
||||
{"path mismatch", "69.162.124.230", "uptimerobot", "/index.html", false},
|
||||
{"UA mismatch", "69.162.124.230", "definitely-not-it", "/health", false},
|
||||
{"empty UA against UA filter", "69.162.124.230", "", "/health", false},
|
||||
{"garbage IP", "not-an-ip", "uptimerobot", "/health", false},
|
||||
{"empty IP", "", "uptimerobot", "/health", false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.expected, IsLegitimateBot(tc.ip, tc.ua, tc.path))
|
||||
})
|
||||
}
|
||||
|
||||
// none of the above may have triggered a DNS lookup: either they matched
|
||||
// on IP/range, or they failed a precondition / address parsing.
|
||||
// "range miss" and "path/UA mismatch" don't reach the googlebot entry's
|
||||
// rdns because its UA precondition doesn't match those UAs.
|
||||
assert.Equal(t, int32(0), resolver.ptrCalls.Load(), "no DNS lookup expected")
|
||||
}
|
||||
|
||||
func TestIsLegitimateBotNoFiles(t *testing.T) {
|
||||
resolver := &fakeResolver{}
|
||||
setupBotTest(t, resolver)
|
||||
|
||||
assert.False(t, IsLegitimateBot("10.1.2.3", "googlebot", "/"))
|
||||
assert.Equal(t, int32(0), resolver.ptrCalls.Load())
|
||||
}
|
||||
|
||||
func TestIsLegitimateBotFCrDNS(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
resolver *fakeResolver
|
||||
ip string
|
||||
ua string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "happy path, trailing dot and mixed case PTR",
|
||||
resolver: &fakeResolver{
|
||||
ptr: map[string][]string{"66.249.66.1": {"Crawl-66-249-66-1.GoogleBot.com."}},
|
||||
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
|
||||
},
|
||||
ip: "66.249.66.1",
|
||||
ua: "Mozilla/5.0 (compatible; Googlebot/2.1)",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "spoofed PTR, forward lookup does not confirm",
|
||||
resolver: &fakeResolver{
|
||||
ptr: map[string][]string{"203.0.113.5": {"crawl.googlebot.com."}},
|
||||
fwd: map[string][]net.IPAddr{"crawl.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
|
||||
},
|
||||
ip: "203.0.113.5",
|
||||
ua: "googlebot",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "anchored pattern rejects lookalike domain",
|
||||
resolver: &fakeResolver{
|
||||
ptr: map[string][]string{"203.0.113.6": {"evilgooglebot.com."}},
|
||||
fwd: map[string][]net.IPAddr{"evilgooglebot.com": {{IP: net.ParseIP("203.0.113.6")}}},
|
||||
},
|
||||
ip: "203.0.113.6",
|
||||
ua: "googlebot",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "no PTR record",
|
||||
resolver: &fakeResolver{
|
||||
ptr: map[string][]string{},
|
||||
},
|
||||
ip: "203.0.113.7",
|
||||
ua: "googlebot",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "forward confirms via second domain suffix",
|
||||
resolver: &fakeResolver{
|
||||
ptr: map[string][]string{"66.249.66.2": {"rate-limited-proxy.google.com."}},
|
||||
fwd: map[string][]net.IPAddr{"rate-limited-proxy.google.com": {{IP: net.ParseIP("66.249.66.2")}}},
|
||||
},
|
||||
ip: "66.249.66.2",
|
||||
ua: "googlebot",
|
||||
expected: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
setupBotTest(t, tc.resolver)
|
||||
|
||||
err := FileInit("testdata", "test_data_bots.json", "bots")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, tc.expected, IsLegitimateBot(tc.ip, tc.ua, "/"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsLegitimateBotDNSCache(t *testing.T) {
|
||||
resolver := &fakeResolver{
|
||||
ptr: map[string][]string{"66.249.66.1": {"crawl-66-249-66-1.googlebot.com."}},
|
||||
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
|
||||
}
|
||||
setupBotTest(t, resolver)
|
||||
|
||||
err := FileInit("testdata", "test_data_bots.json", "bots")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, IsLegitimateBot("66.249.66.1", "googlebot", "/"))
|
||||
assert.True(t, IsLegitimateBot("66.249.66.1", "googlebot", "/other"))
|
||||
assert.Equal(t, int32(1), resolver.ptrCalls.Load(), "second call must be served from cache")
|
||||
|
||||
// negative results are cached too
|
||||
assert.False(t, IsLegitimateBot("203.0.113.9", "googlebot", "/"))
|
||||
assert.False(t, IsLegitimateBot("203.0.113.9", "googlebot", "/"))
|
||||
assert.Equal(t, int32(2), resolver.ptrCalls.Load(), "failed lookup must be cached as negative")
|
||||
}
|
||||
|
||||
// bot data files are loaded per-file via FileInit(dir, name, "bots"), driven
|
||||
// by the data reference on each appsec-rule (see LoadCollection).
|
||||
func TestFileInitBots(t *testing.T) {
|
||||
setupBotTest(t, nil)
|
||||
|
||||
botsDir := t.TempDir()
|
||||
|
||||
write := func(name, content string) {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(botsDir, name), []byte(content), 0o644))
|
||||
}
|
||||
|
||||
write("google.json", `{"name":"googlebot","user_agent":"googlebot","rdns":["googlebot.com"]}`)
|
||||
write("partners.json", `{"name":"partner","ranges":["192.0.2.0/24"]}`)
|
||||
|
||||
require.NoError(t, FileInit(botsDir, "google.json", "bots"))
|
||||
require.NoError(t, FileInit(botsDir, "partners.json", "bots"))
|
||||
|
||||
assert.Len(t, dataFileBots, 2)
|
||||
assert.Contains(t, dataFileBots, "google.json")
|
||||
assert.Contains(t, dataFileBots, "partners.json")
|
||||
|
||||
// any-file semantics: an entry from the second file matches
|
||||
assert.True(t, IsLegitimateBot("192.0.2.10", "", "/"))
|
||||
}
|
||||
|
||||
func TestFileInitBotsInvalidFile(t *testing.T) {
|
||||
setupBotTest(t, nil)
|
||||
|
||||
botsDir := t.TempDir()
|
||||
// first line is valid, second has no identity verification: the
|
||||
// malformed line makes the whole load fail.
|
||||
require.NoError(t, os.WriteFile(filepath.Join(botsDir, "bad.json"),
|
||||
[]byte(`{"name":"ok","ips":["10.0.0.1"]}`+"\n"+`{"name":"ua-only","user_agent":"x"}`), 0o644))
|
||||
|
||||
require.Error(t, FileInit(botsDir, "bad.json", "bots"))
|
||||
}
|
||||
|
||||
func TestParseBotAddr(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
expected string
|
||||
ok bool
|
||||
}{
|
||||
{"1.2.3.4", "1.2.3.4", true},
|
||||
{"1.2.3.4:8080", "1.2.3.4", true},
|
||||
{"2001:db8::1", "2001:db8::1", true},
|
||||
{"[2001:db8::1]:443", "2001:db8::1", true},
|
||||
{"::ffff:1.2.3.4", "1.2.3.4", true},
|
||||
{"fe80::1%eth0", "fe80::1", true},
|
||||
{"garbage", "", false},
|
||||
{"", "", false},
|
||||
{"1.2.3.4:port:extra", "", false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.in, func(t *testing.T) {
|
||||
addr, ok := parseBotAddr(tc.in)
|
||||
assert.Equal(t, tc.ok, ok)
|
||||
|
||||
if tc.ok {
|
||||
assert.Equal(t, tc.expected, addr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -531,6 +531,13 @@ var exprFuncs = []exprCustomFunc{
|
||||
new(func(string) *net.IPNet),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "IPToCountry",
|
||||
function: IPToCountry,
|
||||
signature: []any{
|
||||
new(func(string) string),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "JA4H",
|
||||
function: JA4H,
|
||||
|
||||
@@ -21,6 +21,36 @@ func GeoIPEnrich(params ...any) (any, error) {
|
||||
return city, nil
|
||||
}
|
||||
|
||||
// IPToCountryString resolves an IP address string to its ISO-3166 alpha-2
|
||||
// country code, returning an empty string on any nil/parse/lookup failure.
|
||||
// Intended for internal Go callers; expr rules use IPToCountry.
|
||||
func IPToCountryString(ip string) string {
|
||||
if geoIPCityReader == nil || ip == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
parsedIP := net.ParseIP(ip)
|
||||
if parsedIP == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
city, err := geoIPCityReader.City(parsedIP)
|
||||
if err != nil || city == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return city.Country.IsoCode
|
||||
}
|
||||
|
||||
// IPToCountry wraps IPToCountryString for expr. Lets rules write
|
||||
// `IPToCountry(req.RemoteAddr)` instead of
|
||||
// `GeoIPEnrich(req.RemoteAddr)?.Country.IsoCode` when they only need the
|
||||
// country and want nil-safety.
|
||||
func IPToCountry(params ...any) (any, error) {
|
||||
ip, _ := params[0].(string)
|
||||
return IPToCountryString(ip), nil
|
||||
}
|
||||
|
||||
func GeoIPASNEnrich(params ...any) (any, error) {
|
||||
if geoIPASNReader == nil {
|
||||
return nil, nil
|
||||
|
||||
@@ -122,6 +122,7 @@ func Init(databaseClient *database.Client) error {
|
||||
dataFileRegex = make(map[string][]*regexp.Regexp)
|
||||
dataFileRe2 = make(map[string][]*re2.Regexp)
|
||||
dataFileMap = make(map[string]*fileMapEntry)
|
||||
dataFileBots = make(map[string][]*botEntry)
|
||||
dbClient = databaseClient
|
||||
|
||||
XMLCacheInit()
|
||||
@@ -131,12 +132,15 @@ func Init(databaseClient *database.Client) error {
|
||||
|
||||
// ResetDataFiles clears all datafile-related global variables.
|
||||
// This should be called during HUP reload to ensure clean state.
|
||||
// The DNS cache (pkg/dnscache) is deliberately kept: DNS facts don't change
|
||||
// with the configuration, and a reload shouldn't trigger a re-lookup storm.
|
||||
func ResetDataFiles() {
|
||||
dataFile = make(map[string][]string)
|
||||
dataFileRegex = make(map[string][]*regexp.Regexp)
|
||||
dataFileRe2 = make(map[string][]*re2.Regexp)
|
||||
dataFileRegexCache = make(map[string]gcache.Cache)
|
||||
dataFileMap = make(map[string]*fileMapEntry)
|
||||
dataFileBots = make(map[string][]*botEntry)
|
||||
}
|
||||
|
||||
func RegexpCacheInit(filename string, cacheCfg enrichment.DataProvider) error {
|
||||
@@ -240,6 +244,10 @@ func FileInit(directory string, filename string, fileType string) error {
|
||||
if err := fileMapInit(filename, scanner.Text()); err != nil {
|
||||
return err
|
||||
}
|
||||
case "bots":
|
||||
if err := botFileInit(filename, scanner.Text()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -322,6 +330,8 @@ func existsInFileMaps(filename string, ftype string) (bool, error) {
|
||||
_, ok = dataFile[filename]
|
||||
case "map":
|
||||
_, ok = dataFileMap[filename]
|
||||
case "bots":
|
||||
_, ok = dataFileBots[filename]
|
||||
default:
|
||||
err = fmt.Errorf("unknown data type '%s' for : '%s'", ftype, filename)
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user