Waf challenge mode (#4268)

This commit is contained in:
blotus
2026-07-01 17:30:50 +02:00
committed by GitHub
parent 1cc082a463
commit 1f5e9c8dea
114 changed files with 18566 additions and 492 deletions
-3
View File
@@ -19,9 +19,6 @@ on:
paths-ignore:
- "README.md"
pull_request:
branches:
- master
- releases/**
paths-ignore:
- "README.md"
-3
View File
@@ -8,9 +8,6 @@ on:
paths-ignore:
- "README.md"
pull_request:
branches:
- master
- releases/**
paths-ignore:
- "README.md"
+87 -85
View File
@@ -11,13 +11,10 @@ on:
- master
- releases/**
paths-ignore:
- 'README.md'
- "README.md"
pull_request:
branches:
- master
- releases/**
paths-ignore:
- 'README.md'
- "README.md"
# these env variables are for localstack, so we can emulate aws services
env:
@@ -36,13 +33,13 @@ jobs:
localstack:
image: localstack/localstack:3.0
ports:
- 4566:4566 # Localstack exposes all services on the same port
- 4566:4566 # Localstack exposes all services on the same port
env:
DEBUG: ""
LAMBDA_EXECUTOR: ""
KINESIS_ERROR_PROBABILITY: ""
KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
# e.g sqs url will get localhost if we don't set this env to map our service
options: >-
--name=localstack
@@ -125,95 +122,100 @@ jobs:
--health-retries 5
--health-start-period 30s
steps:
- name: Check out CrowdSec repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
submodules: true
- name: Check out CrowdSec repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
submodules: false
- name: "Set up Go"
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with:
go-version-file: go.mod
- name: "Set up Go"
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with:
go-version-file: go.mod
- name: Run "go generate" and check for changes
run: |
set -e
# ensure the version of 'protoc' matches the one that generated the files
PROTOBUF_VERSION="35.0"
JAVY_VERSION="8.1.1"
# don't pollute the repo
pushd $HOME
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d $HOME/.protoc
curl -OL https://github.com/bytecodealliance/javy/releases/download/v${JAVY_VERSION}/javy-x86_64-linux-v${JAVY_VERSION}.gz
mkdir -p $HOME/.javy
gzip -d javy-x86_64-linux-v${JAVY_VERSION}.gz
mv javy-x86_64-linux-v${JAVY_VERSION} $HOME/.javy/javy
chmod +x $HOME/.javy/javy
popd
export PATH="$HOME/.protoc/bin:$HOME/.javy/:$PATH"
go install google.golang.org/protobuf/cmd/[email protected]
go install google.golang.org/grpc/cmd/[email protected]
go generate ./...
protoc --version
if [[ $(git status --porcelain) ]]; then
echo "Error: Uncommitted changes found after running 'go generate'. Please commit all generated code."
git diff
exit 1
else
echo "No changes detected after running 'go generate'."
fi
- name: Run "go generate" and check for changes
run: |
set -e
# ensure the version of 'protoc' matches the one that generated the files
PROTOBUF_VERSION="21.12"
# don't pollute the repo
pushd $HOME
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d $HOME/.protoc
popd
export PATH="$HOME/.protoc/bin:$PATH"
go install google.golang.org/protobuf/cmd/[email protected]
go install google.golang.org/grpc/cmd/[email protected]
go generate ./...
protoc --version
if [[ $(git status --porcelain) ]]; then
echo "Error: Uncommitted changes found after running 'go generate'. Please commit all generated code."
git diff
exit 1
else
echo "No changes detected after running 'go generate'."
fi
- name: Create localstack streams
run: |
- name: Create localstack streams
run: |
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
- name: Generate codecov configuration
run: |
- name: Generate codecov configuration
run: |
.github/generate-codecov-yml.sh > .github/codecov.yml
- name: Ensure we can do a dynamic build
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
make build
- name: Ensure we can do a dynamic build
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
make build
- name: Ensure we can do a static build
run: |
make clean build BUILD_STATIC=1
- name: Ensure we can do a static build
run: |
make clean build BUILD_STATIC=1
- name: Unit tests
run: |
go install gotest.tools/[email protected]
# make the repo read-only, with the exception of coverage output
touch coverage.out
chmod -R a-w .
chmod u+w coverage.out
make testcover
# ignore changes to codecov.yml
if [[ $(git status --porcelain -- . ":(exclude).github/codecov.yml") ]]; then
echo "Error: Unit tests should not create or alter files inside the repository. Please use the appropriate testing helpers or otherwise temporary locations."
git diff --name-only
exit 1
fi
chmod -R u+w .
- name: Unit tests
run: |
go install gotest.tools/[email protected]
# make the repo read-only, with the exception of coverage output
touch coverage.out
chmod -R a-w .
chmod u+w coverage.out
make testcover
# ignore changes to codecov.yml
if [[ $(git status --porcelain -- . ":(exclude).github/codecov.yml") ]]; then
echo "Error: Unit tests should not create or alter files inside the repository. Please use the appropriate testing helpers or otherwise temporary locations."
git diff --name-only
exit 1
fi
chmod -R u+w .
# check if some component stubs are missing
- name: "Build profile: minimal"
run: |
make build BUILD_PROFILE=minimal
# check if some component stubs are missing
- name: "Build profile: minimal"
run: |
make build BUILD_PROFILE=minimal
# check that the freebsd build is not broken
- name: "Build target: freebsd"
run: |
make build BUILD_STATIC=1 BUILD_RE2_WASM=1 BUILD_SQLITE=modernc GOOS=freebsd
# check that the freebsd build is not broken
- name: "Build target: freebsd"
run: |
make build BUILD_STATIC=1 BUILD_RE2_WASM=1 BUILD_SQLITE=modernc GOOS=freebsd
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
with:
files: coverage.out
flags: unit-linux
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
with:
files: coverage.out
flags: unit-linux
token: ${{ secrets.CODECOV_TOKEN }}
- name: golangci-lint
uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1
with:
version: v2.10
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
- name: golangci-lint
uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1
with:
version: v2.10
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
+4
View File
@@ -66,3 +66,7 @@ msi
__pycache__
*.py[cod]
*.egg-info
# local agent/tooling dirs (not part of the repo)
.agents/
.claude/
+3
View File
@@ -10,3 +10,6 @@
[submodule "tests/lib/bats-support"]
path = test/lib/bats-support
url = https://github.com/bats-core/bats-support.git
[submodule "pkg/appsec/challenge/js/fpscanner/fpscanner"]
path = pkg/appsec/challenge/js/fpscanner/fpscanner
url = https://github.com/antoinevastel/fpscanner
+12
View File
@@ -337,6 +337,10 @@ linters:
# A wildcard `*` can use as a struct name (ex: `*.MethodName`).
exclusions:
- "HCLogAdapter.SetLevel"
# Unmarshal methods must have a pointer receiver by nature; getters on
# the same type legitimately use value receivers (same rationale as the
# built-in Marshal* exclusions, which don't cover Unmarshal*).
- "*.UnmarshalJSON"
revive:
severity: error
@@ -548,6 +552,14 @@ linters:
path: pkg/acquisition/modules/victorialogs/internal/vlclient/vl_client.go
text: 'confusing-naming: Method ''QueryRange'' differs only by capitalization to method ''queryRange'' in the same source file'
# pkg/dnscache lookups are deliberately detached from the request
# context: their results are shared across requests (singleflight +
# cache), so one caller's cancellation must not abort the lookup for
# the others; a fixed timeout bounds them instead.
- linters:
- contextcheck
text: '(lookupPTR|lookupForward)` should pass the context parameter'
# tolerate complex functions in tests for now
- linters:
- maintidx
+17
View File
@@ -320,6 +320,23 @@ cscli: ## Build cscli
crowdsec: ## Build crowdsec
@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
# Regenerate the WAF challenge JS artifacts: the bundled fpscanner source
# (fpscanner/bundle.js), the obfuscator runtime (obfuscate/index.wasm.gz),
# and the build-time pre-obfuscated initial bundle (initial_bundle.js.gz).
# These are committed to the tree so a normal `make build` does NOT rebuild
# them — only run this target after intentionally changing challenge.js,
# fpscanner sources, the obfuscator wrapper, or the dynamic-module template.
#
# Requires `javy` (https://github.com/bytecodealliance/javy/releases) on
# PATH. The full obfuscation pass is slow (~1 minute) because it runs
# javascript-obfuscator's "high-obfuscation" preset over the full bundle;
# this is the cost we are paying once at generate time so the runtime
# doesn't pay it on every startup.
.PHONY: generate-challenge-js
generate-challenge-js: ## Regenerate WAF challenge JS bundles (requires javy on PATH)
@command -v javy >/dev/null 2>&1 || (echo "Error: javy is not installed. Download it from https://github.com/bytecodealliance/javy/releases and put it on PATH." && exit 1)
$(GO) generate ./pkg/appsec/challenge/js/...
testenv:
ifeq ($(TEST_LOCAL_ONLY),)
@echo 'NOTE: You need to run "make localstack" in a separate shell, "make localstack-stop" to terminate it; or define the envvar TEST_LOCAL_ONLY to some value.'
+3 -1
View File
@@ -75,7 +75,9 @@ func expandAlias(args []string) []string {
case "lapi":
ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine")
case "appsec":
ret = append(ret, "appsec-engine", "appsec-rule")
ret = append(ret, "appsec-engine", "appsec-rule", "appsec-challenge")
case "bot-detection":
ret = append(ret, "appsec-challenge", "appsec-challenge-infra")
default:
ret = append(ret, section)
}
@@ -0,0 +1,189 @@
package climetrics
import (
"fmt"
"io"
"sort"
"github.com/jedib0t/go-pretty/v6/table"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/core/cstable"
)
// statAppsecChallenge holds two views of the bot-detection counters:
//
// - Funnel: per-engine aggregates rendered as one row per appsec_engine,
// with columns for each lifecycle/outcome bucket. Mirrors the shape
// statAppsecEngine uses.
// - Reasons: per-engine per-kind per-reason breakdown so operators can see
// which RejectSubmission / GrantChallengeCookie strings (and which
// bounded protocol/cookie reasons) are firing without dropping into
// Prometheus. Rendered as a second table below the funnel.
type statAppsecChallenge struct {
Funnel map[string]map[string]int `json:"funnel"`
Reasons map[string]map[string]map[string]int `json:"reasons"` // engine -> kind -> reason -> count
}
func newStatAppsecChallenge() *statAppsecChallenge {
return &statAppsecChallenge{
Funnel: map[string]map[string]int{},
Reasons: map[string]map[string]map[string]int{},
}
}
func (*statAppsecChallenge) Description() (string, string) {
return "Bot Detection Metrics",
`Measures the challenge lifecycle of the AppSec component.`
}
func (s *statAppsecChallenge) Process(appsecEngine, metric string, val int) {
if _, ok := s.Funnel[appsecEngine]; !ok {
s.Funnel[appsecEngine] = make(map[string]int)
}
s.Funnel[appsecEngine][metric] += val
}
// ProcessReason records a per-kind/reason datapoint. Empty reasons are
// dropped.
func (s *statAppsecChallenge) ProcessReason(appsecEngine, kind, reason string, val int) {
if reason == "" {
return
}
if _, ok := s.Reasons[appsecEngine]; !ok {
s.Reasons[appsecEngine] = map[string]map[string]int{}
}
if _, ok := s.Reasons[appsecEngine][kind]; !ok {
s.Reasons[appsecEngine][kind] = map[string]int{}
}
s.Reasons[appsecEngine][kind][reason] += val
}
func (s *statAppsecChallenge) Table(out io.Writer, wantColor string, noUnit bool, showEmpty bool) {
t := cstable.New(out, wantColor).Writer
t.AppendHeader(table.Row{"Bot Detection", "Requested", "Submitted", "Solved", "Granted", "Protocol Failures", "Submissions Rejected", "Cookies Invalid"})
keys := []string{"requested", "submitted", "solved", "granted", "rejected_protocol", "rejected_submission", "rejected_cookie"}
numRows, err := metricsToTable(t, s.Funnel, keys, noUnit)
if err != nil {
log.Warningf("while collecting appsec challenge stats: %s", err)
return
}
if numRows == 0 && !showEmpty {
return
}
title, _ := s.Description()
t.SetTitle(title)
fmt.Fprintln(out, t.Render())
// Detail tables: per-engine breakdown of the user-supplied
// (GrantChallengeCookie / RejectSubmission) and bounded-vocabulary
// (protocol / cookie) reasons. One table per outcome (accepted /
// rejected) so the kind dimension only needs to show up where it
// actually varies.
s.renderAccepted(out, wantColor, noUnit)
s.renderRejected(out, wantColor, noUnit)
}
// rejectedKindDisplay maps a rejected sub-kind to the label shown in the
// rejected breakdown table.
var rejectedKindDisplay = map[string]string{
"protocol": "protocol",
"submission": "submission",
"cookie": "cookie",
}
// rejectedKindOrder fixes the row order so the rendered table is stable
// across runs and groups related kinds together.
var rejectedKindOrder = []string{"protocol", "submission", "cookie"}
// renderAccepted renders the per-engine breakdown of GrantChallengeCookie
// reasons. Only kind="granted" carries a reason (kind="solved" has none by
// design), so the Kind column is dropped here and the table footer carries
// a grand total across all engines and reasons.
func (s *statAppsecChallenge) renderAccepted(out io.Writer, wantColor string, noUnit bool) {
engines := make([]string, 0, len(s.Reasons))
for e, byKind := range s.Reasons {
if _, ok := byKind["granted"]; ok {
engines = append(engines, e)
}
}
if len(engines) == 0 {
return
}
sort.Strings(engines)
t := cstable.New(out, wantColor).Writer
t.AppendHeader(table.Row{"Appsec Engine", "Reason", "Count"})
total := int64(0)
for _, engine := range engines {
counts := s.Reasons[engine]["granted"]
reasons := make([]string, 0, len(counts))
for r := range counts {
reasons = append(reasons, r)
}
sort.Strings(reasons)
for _, r := range reasons {
t.AppendRow(table.Row{engine, r, formatNumber(int64(counts[r]), !noUnit)})
total += int64(counts[r])
}
}
t.AppendFooter(table.Row{"Total", "", formatNumber(total, !noUnit)})
t.SetTitle("Bot Detection — Accepted")
fmt.Fprintln(out, t.Render())
}
// renderRejected renders the per-engine per-kind per-reason breakdown of
// rejected outcomes. Kind matters here (protocol vs submission vs cookie),
// and the table footer carries a grand total across all rows.
func (s *statAppsecChallenge) renderRejected(out io.Writer, wantColor string, noUnit bool) {
engines := make([]string, 0, len(s.Reasons))
for e, byKind := range s.Reasons {
for _, kind := range rejectedKindOrder {
if _, ok := byKind[kind]; ok {
engines = append(engines, e)
break
}
}
}
if len(engines) == 0 {
return
}
sort.Strings(engines)
t := cstable.New(out, wantColor).Writer
t.AppendHeader(table.Row{"Appsec Engine", "Kind", "Reason", "Count"})
total := int64(0)
for _, engine := range engines {
byKind := s.Reasons[engine]
for _, kind := range rejectedKindOrder {
counts, ok := byKind[kind]
if !ok {
continue
}
reasons := make([]string, 0, len(counts))
for r := range counts {
reasons = append(reasons, r)
}
sort.Strings(reasons)
for _, r := range reasons {
t.AppendRow(table.Row{engine, rejectedKindDisplay[kind], r, formatNumber(int64(counts[r]), !noUnit)})
total += int64(counts[r])
}
}
}
t.AppendFooter(table.Row{"Total", "", "", formatNumber(total, !noUnit)})
t.SetTitle("Bot Detection — Rejected")
fmt.Fprintln(out, t.Render())
}
@@ -0,0 +1,62 @@
package climetrics
import (
"fmt"
"io"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/core/cstable"
)
// statAppsecChallengeInfra holds the process-global housekeeping counters of
// the bot-detection challenge runtime (key rotation, obfuscation, cache
// eviction). Unlike the lifecycle funnel these carry no appsec_engine label
// — the challenge runtime is a single per-process instance shared by every
// engine — so this is a flat metric -> count map rather than per-engine.
type statAppsecChallengeInfra map[string]int
func (statAppsecChallengeInfra) Description() (string, string) {
return "Bot Detection Infrastructure Metrics",
`Tracks the internal upkeep of the AppSec challenge runtime: signing-key rotation, JS re-obfuscation and cache eviction.`
}
func (s statAppsecChallengeInfra) Process(metric string, val int) {
s[metric] += val
}
// infraRow pairs a metric key with the label shown in the table, in a fixed
// order so the rendered output is stable across runs.
var infraRows = []struct {
key string
label string
}{
{"kepoch_generated", "Signing key regenerated"},
{"kepoch_evicted", "Signing key evicted"},
{"reobfuscation_dynamic", "Re-obfuscation (dynamic module)"},
{"reobfuscation_library", "Re-obfuscation (library bundle)"},
{"dynamic_module_evicted", "Dynamic module evicted"},
}
func (s statAppsecChallengeInfra) Table(out io.Writer, wantColor string, noUnit bool, showEmpty bool) {
t := cstable.New(out, wantColor).Writer
t.AppendHeader(table.Row{"Metric", "Count"})
numRows := 0
for _, r := range infraRows {
val, ok := s[r.key]
if !ok {
continue
}
t.AppendRow(table.Row{r.label, formatNumber(int64(val), !noUnit)})
numRows++
}
if numRows == 0 && !showEmpty {
return
}
title, _ := s.Description()
t.SetTitle(title)
fmt.Fprintln(out, t.Render())
}
@@ -27,9 +27,9 @@ func (s statAppsecEngine) Process(appsecEngine, metric string, val int) {
func (s statAppsecEngine) Table(out io.Writer, wantColor string, noUnit bool, showEmpty bool) {
t := cstable.New(out, wantColor).Writer
t.AppendHeader(table.Row{"Appsec Engine", "Processed", "Blocked"})
t.AppendHeader(table.Row{"Appsec Engine", "Processed", "Blocked", "Ch. Requested", "Ch. Accepted", "Ch. Rejected"})
keys := []string{"processed", "blocked"}
keys := []string{"processed", "blocked", "challenge_requested", "challenge_accepted", "challenge_rejected"}
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting appsec stats: %s", err)
+166 -69
View File
@@ -24,20 +24,22 @@ type metricStore map[string]metricSection
func NewMetricStore() metricStore {
return metricStore{
"acquisition": statAcquis{},
"alerts": statAlert{},
"bouncers": &statBouncer{},
"appsec-engine": statAppsecEngine{},
"appsec-rule": statAppsecRule{},
"decisions": statDecision{},
"lapi": statLapi{},
"lapi-bouncer": statLapiBouncer{},
"lapi-decisions": statLapiDecision{},
"lapi-machine": statLapiMachine{},
"parsers": statParser{},
"scenarios": statBucket{},
"stash": statStash{},
"whitelists": statWhitelist{},
"acquisition": statAcquis{},
"alerts": statAlert{},
"bouncers": &statBouncer{},
"appsec-engine": statAppsecEngine{},
"appsec-rule": statAppsecRule{},
"appsec-challenge": newStatAppsecChallenge(),
"appsec-challenge-infra": statAppsecChallengeInfra{},
"decisions": statDecision{},
"lapi": statLapi{},
"lapi-bouncer": statLapiBouncer{},
"lapi-decisions": statLapiDecision{},
"lapi-machine": statLapiMachine{},
"parsers": statParser{},
"scenarios": statBucket{},
"stash": statStash{},
"whitelists": statWhitelist{},
}
}
@@ -57,6 +59,59 @@ func (ms metricStore) Fetch(ctx context.Context, url string, db *database.Client
return nil
}
// metricLabels holds the label values extracted from a single Prometheus point.
type metricLabels struct {
name string
source string
machine string
bouncer string
route string
method string
reason string
origin string
action string
appsecEngine string
appsecRule string
// kind is carried by the appsec-challenge accepted/rejected counters
// to distinguish sub-outcomes; empty for every other metric.
kind string
// bundle is carried by the challenge re-obfuscation counter
// (dynamic vs library); empty for every other metric.
bundle string
mtype string
}
func extractLabels(p MetricPoint) metricLabels {
name, ok := p.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", p.Labels)
}
source, ok := p.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v for %s", p.Labels, p.Name)
} else if srctype, ok := p.Labels["type"]; ok {
source = srctype + ":" + source
}
return metricLabels{
name: name,
source: source,
machine: p.Labels["machine"],
bouncer: p.Labels["bouncer"],
route: p.Labels["route"],
method: p.Labels["method"],
reason: p.Labels["reason"],
origin: p.Labels["origin"],
action: p.Labels["action"],
appsecEngine: p.Labels["appsec_engine"],
appsecRule: p.Labels["rule_name"],
kind: p.Labels["kind"],
bundle: p.Labels["bundle"],
mtype: p.Labels["type"],
}
}
func (ms metricStore) processPrometheusMetrics(result []MetricPoint) {
mAcquis := ms["acquisition"].(statAcquis)
mAlert := ms["alerts"].(statAlert)
@@ -77,115 +132,157 @@ func (ms metricStore) processPrometheusMetrics(result []MetricPoint) {
continue
}
name, ok := p.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", p.Labels)
}
l := extractLabels(p)
source, ok := p.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v for %s", p.Labels, p.Name)
} else {
if srctype, ok := p.Labels["type"]; ok {
source = srctype + ":" + source
}
}
ival := int(p.Value)
machine := p.Labels["machine"]
bouncer := p.Labels["bouncer"]
route := p.Labels["route"]
method := p.Labels["method"]
reason := p.Labels["reason"]
origin := p.Labels["origin"]
action := p.Labels["action"]
appsecEngine := p.Labels["appsec_engine"]
appsecRule := p.Labels["rule_name"]
mtype := p.Labels["type"]
ival := int(p.Value)
switch p.Name {
switch p.Name {
//
// buckets
//
case metrics.BucketsInstantiationMetricName:
mBucket.Process(name, "instantiation", ival)
mBucket.Process(l.name, "instantiation", ival)
case metrics.BucketsCurrentCountMetricName:
mBucket.Process(name, "curr_count", ival)
mBucket.Process(l.name, "curr_count", ival)
case metrics.BucketsOverflowMetricName:
mBucket.Process(name, "overflow", ival)
mBucket.Process(l.name, "overflow", ival)
case metrics.BucketPouredMetricName:
mBucket.Process(name, "pour", ival)
mAcquis.Process(source, "pour", ival)
mBucket.Process(l.name, "pour", ival)
mAcquis.Process(l.source, "pour", ival)
case metrics.BucketsUnderflowMetricName:
mBucket.Process(name, "underflow", ival)
mBucket.Process(l.name, "underflow", ival)
//
// parsers
//
case metrics.GlobalParserHitsMetricName:
mAcquis.Process(source, "reads", ival)
mAcquis.Process(l.source, "reads", ival)
case metrics.GlobalParserHitsOkMetricName:
mAcquis.Process(source, "parsed", ival)
mAcquis.Process(l.source, "parsed", ival)
case metrics.GlobalParserHitsKoMetricName:
mAcquis.Process(source, "unparsed", ival)
mAcquis.Process(l.source, "unparsed", ival)
case metrics.NodesHitsMetricName:
mParser.Process(name, "hits", ival)
mParser.Process(l.name, "hits", ival)
case metrics.NodesHitsOkMetricName:
mParser.Process(name, "parsed", ival)
mParser.Process(l.name, "parsed", ival)
case metrics.NodesHitsKoMetricName:
mParser.Process(name, "unparsed", ival)
mParser.Process(l.name, "unparsed", ival)
//
// whitelists
//
case metrics.NodesWlHitsMetricName:
mWhitelist.Process(name, reason, "hits", ival)
mWhitelist.Process(l.name, l.reason, "hits", ival)
case metrics.NodesWlHitsOkMetricName:
mWhitelist.Process(name, reason, "whitelisted", ival)
mWhitelist.Process(l.name, l.reason, "whitelisted", ival)
// track as well whitelisted lines at acquis level
mAcquis.Process(source, "whitelisted", ival)
mAcquis.Process(l.source, "whitelisted", ival)
//
// lapi
//
case metrics.LapiRouteHitsMetricName:
mLapi.Process(route, method, ival)
mLapi.Process(l.route, l.method, ival)
case metrics.LapiMachineHitsMetricName:
mLapiMachine.Process(machine, route, method, ival)
mLapiMachine.Process(l.machine, l.route, l.method, ival)
case metrics.LapiBouncerHitsMetricName:
mLapiBouncer.Process(bouncer, route, method, ival)
mLapiBouncer.Process(l.bouncer, l.route, l.method, ival)
case metrics.LapiNilDecisionsMetricName, metrics.LapiNonNilDecisionsMetricName:
mLapiDecision.Process(bouncer, p.Name, ival)
mLapiDecision.Process(l.bouncer, p.Name, ival)
//
// decisions
//
case metrics.GlobalActiveDecisionsMetricName:
mDecision.Process(reason, origin, action, ival)
mDecision.Process(l.reason, l.origin, l.action, ival)
case metrics.GlobalAlertsMetricName:
mAlert.Process(reason, ival)
mAlert.Process(l.reason, ival)
//
// stash
//
case metrics.CacheMetricName:
mStash.Process(name, mtype, ival)
mStash.Process(l.name, l.mtype, ival)
//
// appsec
//
case "cs_appsec_reqs_total":
mAppsecEngine.Process(appsecEngine, "processed", ival)
mAppsecEngine.Process(l.appsecEngine, "processed", ival)
case "cs_appsec_block_total":
mAppsecEngine.Process(appsecEngine, "blocked", ival)
mAppsecEngine.Process(l.appsecEngine, "blocked", ival)
case "cs_appsec_rule_hits":
mAppsecRule.Process(appsecEngine, appsecRule, "triggered", ival)
mAppsecRule.Process(l.appsecEngine, l.appsecRule, "triggered", ival)
default:
// bot detection / challenge metrics live in their own handler to
// keep this switch's complexity in check.
if ms.processAppsecChallengeMetric(p.Name, l, ival) {
continue
}
log.Debugf("unknown: %+v", p.Name)
continue
}
}
}
// processAppsecChallengeMetric handles the bot-detection / challenge lifecycle
// and infrastructure metrics. It lives apart from processPrometheusMetrics
// because its nested kind/bundle switches would otherwise push that function
// over the cyclomatic-complexity limit. Returns false for unrecognized metrics.
func (ms metricStore) processAppsecChallengeMetric(name string, l metricLabels, ival int) bool {
mAppsecEngine := ms["appsec-engine"].(statAppsecEngine)
mAppsecChallenge := ms["appsec-challenge"].(*statAppsecChallenge)
mAppsecChallengeInfra := ms["appsec-challenge-infra"].(statAppsecChallengeInfra)
switch name {
//
// bot detection / challenge lifecycle
//
case metrics.AppsecChallengeRequestedMetricName:
mAppsecEngine.Process(l.appsecEngine, "challenge_requested", ival)
mAppsecChallenge.Process(l.appsecEngine, "requested", ival)
case metrics.AppsecChallengeSubmittedMetricName:
mAppsecChallenge.Process(l.appsecEngine, "submitted", ival)
case metrics.AppsecChallengeAcceptedMetricName:
mAppsecEngine.Process(l.appsecEngine, "challenge_accepted", ival)
switch l.kind {
case "solved":
mAppsecChallenge.Process(l.appsecEngine, "solved", ival)
case "granted":
mAppsecChallenge.Process(l.appsecEngine, "granted", ival)
mAppsecChallenge.ProcessReason(l.appsecEngine, "granted", l.reason, ival)
}
case metrics.AppsecChallengeRejectedMetricName:
mAppsecEngine.Process(l.appsecEngine, "challenge_rejected", ival)
switch l.kind {
case "protocol":
mAppsecChallenge.Process(l.appsecEngine, "rejected_protocol", ival)
mAppsecChallenge.ProcessReason(l.appsecEngine, "protocol", l.reason, ival)
case "submission":
mAppsecChallenge.Process(l.appsecEngine, "rejected_submission", ival)
mAppsecChallenge.ProcessReason(l.appsecEngine, "submission", l.reason, ival)
case "cookie":
mAppsecChallenge.Process(l.appsecEngine, "rejected_cookie", ival)
mAppsecChallenge.ProcessReason(l.appsecEngine, "cookie", l.reason, ival)
}
//
// bot detection / challenge infrastructure (process-global, no engine label)
//
case metrics.AppsecChallengeKepochGeneratedMetricName:
mAppsecChallengeInfra.Process("kepoch_generated", ival)
case metrics.AppsecChallengeKepochEvictedMetricName:
mAppsecChallengeInfra.Process("kepoch_evicted", ival)
case metrics.AppsecChallengeReobfuscationMetricName:
switch l.bundle {
case "dynamic":
mAppsecChallengeInfra.Process("reobfuscation_dynamic", ival)
case "library":
mAppsecChallengeInfra.Process("reobfuscation_library", ival)
}
case metrics.AppsecChallengeDynamicModuleEvictedMetricName:
mAppsecChallengeInfra.Process("dynamic_module_evicted", ival)
default:
return false
}
return true
}
func (ms metricStore) Format(out io.Writer, wantColor string, sections []string, outputFormat string, noUnit bool) error {
// copy only the sections we want
want := map[string]metricSection{}
+28
View File
@@ -17,6 +17,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/dnscache"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/metrics"
@@ -31,6 +32,8 @@ func initCrowdsec(ctx context.Context, cConfig *csconfig.Config, hub *cwhub.Hub,
return nil, nil, fmt.Errorf("while loading context: %w", err)
}
configureDNSCache(cConfig.Crowdsec.DNSCache)
err = exprhelpers.GeoIPInit(hub.GetDataDir())
if err != nil {
// GeoIP databases are not mandatory, do not make crowdsec fail if they are not present
@@ -70,6 +73,31 @@ func initCrowdsec(ctx context.Context, cConfig *csconfig.Config, hub *cwhub.Hub,
return csParsers, datasources, nil
}
func configureDNSCache(cfg *csconfig.DNSCacheCfg) {
if cfg == nil {
return
}
var (
ttl, negTTL time.Duration
size int
)
if cfg.TTL != nil {
ttl = *cfg.TTL
}
if cfg.NegativeTTL != nil {
negTTL = *cfg.NegativeTTL
}
if cfg.Size != nil {
size = *cfg.Size
}
dnscache.Configure(ttl, negTTL, size)
}
func startParserRoutines(ctx context.Context, g *errgroup.Group, cConfig *csconfig.Config, parsers *parser.Parsers, stageCollector *parser.StageParseCollector) {
for idx := range cConfig.Crowdsec.ParserRoutinesCount {
log.WithField("idx", idx).Info("Starting parser routine")
+3 -1
View File
@@ -31,6 +31,7 @@ require (
github.com/crowdsecurity/grokky v0.2.2
github.com/crowdsecurity/machineid v1.0.3
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/evanw/esbuild v0.27.2
github.com/expr-lang/expr v1.17.8
github.com/fatih/color v1.19.0
github.com/fsnotify/fsnotify v1.10.1
@@ -80,6 +81,7 @@ require (
github.com/slack-go/slack v0.23.0
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.11.1
github.com/tetratelabs/wazero v1.11.0
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
github.com/wasilibs/go-re2 v1.11.0
github.com/xhit/go-simple-mail/v2 v2.16.0
@@ -191,6 +193,7 @@ require (
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.18.5 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
github.com/magefile/mage v1.17.1 // indirect
@@ -226,7 +229,6 @@ require (
github.com/shopspring/decimal v1.4.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/tetratelabs/wazero v1.11.0 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.2.0 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
+3
View File
@@ -157,6 +157,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/evanw/esbuild v0.27.2 h1:3xBEws9y/JosfewXMM2qIyHAi+xRo8hVx475hVkJfNg=
github.com/evanw/esbuild v0.27.2/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48=
github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM=
github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -734,6 +736,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -4,6 +4,8 @@ import (
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
@@ -13,6 +15,8 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
)
@@ -1348,6 +1352,275 @@ func TestAppsecPhaseScopedHooks(t *testing.T) {
runTests(t, tests)
}
func TestAppsecOnChallengeHooks(t *testing.T) {
powWorkerURL, err := url.Parse(challenge.ChallengePowWorkerPath)
require.NoError(t, err)
tests := []appsecRuleTest{
{
name: "post_eval issues a challenge when no cookie is present",
expected_load_ok: true,
post_eval: []appsec.Hook{
{Apply: []string{"SendChallenge()"}},
},
input_request: appsec.ParsedRequest{
RemoteAddr: "1.2.3.4",
Method: "GET",
URI: "/protected",
HTTPRequest: &http.Request{Host: "example.com"},
},
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.Equal(t, appsec.ChallengeRemediation, responses[0].Action)
require.NotEmpty(t, responses[0].UserHTTPBodyContent)
require.Contains(t, responses[0].UserHeaders["Content-Type"], "text/html")
// Issuing the challenge page must emit a single "requested" event
// on a source distinct from the WAF one.
require.Len(t, events, 1)
require.Equal(t, pipeline.LOG, events[0].Type)
require.Equal(t, appsec.SourceChallenge, events[0].Parsed["source"])
require.Equal(t, string(appsec.ChallengeReasonRequested), events[0].Parsed["challenge_event"])
},
},
{
name: "on_challenge: no cookie → user hooks skipped (filter would nil-deref)",
expected_load_ok: true,
// This filter would nil-deref if it ran without a fingerprint. The
// dispatcher must skip it when no cookie is present.
on_challenge: []appsec.Hook{
{Filter: "fingerprint.Bot.MismatchWebGLInWorker", Apply: []string{"DropRequest('bot')"}},
},
// Must reference SendChallenge() somewhere to force ChallengeRuntime init.
post_eval: []appsec.Hook{
{Filter: "false", Apply: []string{"SendChallenge()"}},
},
input_request: appsec.ParsedRequest{
RemoteAddr: "1.2.3.4",
Method: "GET",
URI: "/protected",
HTTPRequest: &http.Request{Host: "example.com"},
},
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.False(t, responses[0].InBandInterrupt, "on_challenge must not fire without a fingerprint")
},
},
{
name: "on_challenge: PoW worker path served, WAF evaluation skipped",
expected_load_ok: true,
inband_rules: []appsec_rule.CustomRule{
{
Name: "rule1",
Zones: []string{"ARGS"},
Variables: []string{"foo"},
Match: appsec_rule.Match{Type: "regex", Value: "^toto"},
Transform: []string{"lowercase"},
},
},
// Force the challenge runtime to be initialized by referencing SendChallenge()
// in a hook that won't match this request.
on_challenge: []appsec.Hook{
{Filter: "false", Apply: []string{"SendChallenge()"}},
},
input_request: appsec.ParsedRequest{
RemoteAddr: "1.2.3.4",
Method: "GET",
URI: challenge.ChallengePowWorkerPath,
Args: url.Values{"foo": []string{"toto"}}, // would normally trigger rule1
HTTPRequest: &http.Request{Host: "example.com", URL: powWorkerURL},
},
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Empty(t, events, "WAF should not have evaluated the infrastructure path")
require.Len(t, responses, 1)
require.Equal(t, appsec.ChallengeRemediation, responses[0].Action)
require.Equal(t, challenge.PowWorkerJS, responses[0].UserHTTPBodyContent)
require.Contains(t, responses[0].UserHeaders["Content-Type"], "application/javascript")
},
},
{
name: "on_challenge: invalid submission returns failed body, no hooks run",
expected_load_ok: true,
// Unconditional DropRequest in on_challenge must NOT fire on an
// invalid submission — the dispatcher returns the failed JSON
// body and skips user hooks.
on_challenge: []appsec.Hook{
{Apply: []string{"DropRequest('should not fire')"}},
},
// Force ChallengeRuntime init.
post_eval: []appsec.Hook{
{Filter: "false", Apply: []string{"SendChallenge()"}},
},
input_request: appsec.ParsedRequest{
RemoteAddr: "1.2.3.4",
Method: "POST",
URI: challenge.ChallengeSubmitPath,
HTTPRequest: func() *http.Request {
u, _ := url.Parse(challenge.ChallengeSubmitPath)
return &http.Request{Host: "example.com", URL: u, Method: http.MethodPost}
}(),
},
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.Equal(t, appsec.ChallengeRemediation, responses[0].Action)
require.JSONEq(t, `{"status":"failed"}`, responses[0].UserHTTPBodyContent)
require.False(t, responses[0].InBandInterrupt, "on_challenge hooks must not run on invalid submission")
// A submission attempt emits "submitted", then "failed" (with a reason).
require.Len(t, events, 2)
require.Equal(t, appsec.SourceChallenge, events[0].Parsed["source"])
require.Equal(t, string(appsec.ChallengeReasonSubmitted), events[0].Parsed["challenge_event"])
require.Equal(t, string(appsec.ChallengeReasonFailed), events[1].Parsed["challenge_event"])
require.NotEmpty(t, events[1].Parsed["challenge_fail_reason"])
},
},
{
// Mock LAPI (see testAppSecEngine in appsec_test.go) exposes 5.4.3.2
// as an allowlisted IP. SendChallenge() must be a no-op for it: no
// challenge HTML, no Set-Cookie, response stays at the default pass
// action so the request goes through cleanly.
name: "allowlisted IP: post_eval SendChallenge() is suppressed",
expected_load_ok: true,
post_eval: []appsec.Hook{
{Apply: []string{"SendChallenge()"}},
},
input_request: appsec.ParsedRequest{
ClientIP: "5.4.3.2",
RemoteAddr: "5.4.3.2",
Method: "GET",
URI: "/protected",
HTTPRequest: &http.Request{Host: "example.com"},
},
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action,
"allowlisted IP must not be challenged")
require.Equal(t, appsec.AllowRemediation, appsecResponse.Action)
require.Empty(t, responses[0].UserHTTPBodyContent, "no challenge HTML must be served")
require.Empty(t, responses[0].UserHTTPCookies, "no challenge cookie must be issued")
require.False(t, responses[0].InBandInterrupt)
},
},
{
// Same allowlisted IP, but inside a /24 CIDR entry (5.4.4.0/24) —
// confirms range matches are honored the same way as exact IPs.
name: "allowlisted CIDR: post_eval SendChallenge() is suppressed",
expected_load_ok: true,
post_eval: []appsec.Hook{
{Apply: []string{"SendChallenge()"}},
},
input_request: appsec.ParsedRequest{
ClientIP: "5.4.4.42",
RemoteAddr: "5.4.4.42",
Method: "GET",
URI: "/protected",
HTTPRequest: &http.Request{Host: "example.com"},
},
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action)
require.Equal(t, appsec.AllowRemediation, appsecResponse.Action)
require.Empty(t, responses[0].UserHTTPBodyContent)
require.Empty(t, responses[0].UserHTTPCookies)
},
},
{
// Allowlisted IPs hitting infrastructure paths must not get the PoW
// worker JS served either — ProcessOnChallengeRules short-circuits
// before the path-based branch, so the request flows through normal
// WAF processing.
name: "allowlisted IP: PoW worker path is not served",
expected_load_ok: true,
// Reference SendChallenge() so ChallengeRuntime initializes.
on_challenge: []appsec.Hook{
{Filter: "false", Apply: []string{"SendChallenge()"}},
},
input_request: appsec.ParsedRequest{
ClientIP: "5.4.3.2",
RemoteAddr: "5.4.3.2",
Method: "GET",
URI: challenge.ChallengePowWorkerPath,
HTTPRequest: &http.Request{Host: "example.com", URL: powWorkerURL},
},
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action,
"allowlisted IP must not receive the PoW worker JS")
require.NotEqual(t, challenge.PowWorkerJS, responses[0].UserHTTPBodyContent)
require.Equal(t, appsec.AllowRemediation, appsecResponse.Action)
},
},
{
// SendChallenge is restricted to on_challenge + in-band post_eval
name: "pre_eval SendChallenge() fails to load",
expected_load_ok: false,
pre_eval: []appsec.Hook{
{Apply: []string{"SendChallenge()"}},
},
input_request: appsec.ParsedRequest{
RemoteAddr: "1.2.3.4",
Method: "GET",
URI: "/protected",
HTTPRequest: &http.Request{Host: "example.com"},
},
},
{
// SendChallenge in an out-of-band post_eval hook is rejected at runtime
name: "outofband post_eval SendChallenge() is rejected",
expected_load_ok: true,
outofband_post_eval: []appsec.Hook{
{Apply: []string{"SendChallenge()"}},
},
input_request: appsec.ParsedRequest{
RemoteAddr: "1.2.3.4",
Method: "GET",
URI: "/protected",
HTTPRequest: &http.Request{Host: "example.com"},
},
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.NotEqual(t, appsec.ChallengeRemediation, responses[0].Action,
"out-of-band SendChallenge must not issue a challenge")
require.Empty(t, responses[0].UserHTTPBodyContent, "no challenge HTML must be served")
},
},
{
// A challenged request is served the challenge and never reaches the
// backend;
name: "challenge skips out-of-band evaluation",
expected_load_ok: true,
post_eval: []appsec.Hook{
{Apply: []string{"SendChallenge()"}},
},
outofband_rules: []appsec_rule.CustomRule{
{
Name: "oob-rule",
Zones: []string{"ARGS"},
Variables: []string{"foo"},
Match: appsec_rule.Match{Type: "regex", Value: "^toto"},
Transform: []string{"lowercase"},
},
},
input_request: appsec.ParsedRequest{
RemoteAddr: "1.2.3.4",
Method: "GET",
URI: "/protected",
Args: url.Values{"foo": []string{"toto"}}, // would match oob-rule
HTTPRequest: &http.Request{Host: "example.com"},
},
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.Equal(t, appsec.ChallengeRemediation, responses[0].Action)
// Only the challenge "requested" event must be present; the OOB rule
// was skipped, so no appsec match event is emitted.
require.Len(t, events, 1)
require.Equal(t, appsec.SourceChallenge, events[0].Parsed["source"])
require.Equal(t, string(appsec.ChallengeReasonRequested), events[0].Parsed["challenge_event"])
},
},
}
runTests(t, tests)
}
// minimal schema used by the hook_vars-in-event tests: /users only accepts
// a POST whose body has a required username field.
const testUsersSchema = `openapi: 3.0.0
@@ -1602,3 +1875,91 @@ func mustParseURL(raw string) *url.URL {
}
return u
}
// TestAppsecLegitimateBotHooks exercises the IsLegitimateBot/ExemptFromChallenge
// helpers through the full runner: a bot definition is loaded from a
// legit_bots directory (as AppsecConfig.Build does with the hub data dir),
// an in-band rule matches the request, and a pre_eval hook downgrades the
// remediation for verified bots only.
func TestAppsecLegitimateBotHooks(t *testing.T) {
require.NoError(t, exprhelpers.Init(nil))
datadir := t.TempDir()
botsDir := filepath.Join(datadir, "legit_bots")
require.NoError(t, os.Mkdir(botsDir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(botsDir, "bots.json"),
[]byte(`{"name":"testbot","user_agent":"testbot","ranges":["192.0.2.0/24"]}`), 0o644))
require.NoError(t, exprhelpers.FileInit(botsDir, "bots.json", "bots"))
banRule := appsec_rule.CustomRule{
Name: "rule1",
Zones: []string{"ARGS"},
Variables: []string{"foo"},
Match: appsec_rule.Match{Type: "regex", Value: "^toto"},
Transform: []string{"lowercase"},
}
bypassHook := appsec.Hook{
Filter: `IsLegitimateBot(req.RemoteAddr, req.UserAgent(), req.URL.Path)`,
Apply: []string{`SetRemediation("allow")`},
}
botRequest := func(remoteAddr string, ua string) appsec.ParsedRequest {
return appsec.ParsedRequest{
RemoteAddr: remoteAddr,
Method: "GET",
URI: "/crawl",
Args: url.Values{"foo": []string{"toto"}},
HTTPRequest: &http.Request{
Host: "example.com",
RemoteAddr: remoteAddr,
URL: mustParseURL("http://example.com/crawl"),
Header: http.Header{"User-Agent": []string{ua}},
},
}
}
tests := []appsecRuleTest{
{
name: "verified bot bypasses the ban",
expected_load_ok: true,
inband_rules: []appsec_rule.CustomRule{banRule},
pre_eval: []appsec.Hook{bypassHook},
// the IP is in the declared range; ip:port proves address normalization end to end
input_request: botRequest("192.0.2.10:34567", "Mozilla/5.0 (compatible; TestBot/1.0)"),
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.True(t, responses[0].InBandInterrupt, "the rule still matches, only the remediation changes")
require.Equal(t, appsec.AllowRemediation, responses[0].Action)
},
},
{
name: "spoofed UA from a foreign IP is still banned",
expected_load_ok: true,
inband_rules: []appsec_rule.CustomRule{banRule},
pre_eval: []appsec.Hook{bypassHook},
input_request: botRequest("203.0.113.9:34567", "Mozilla/5.0 (compatible; TestBot/1.0)"),
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.True(t, responses[0].InBandInterrupt)
require.Equal(t, appsec.BanRemediation, responses[0].Action)
},
},
{
name: "ExemptFromChallenge escape hatch short-circuits the checks",
expected_load_ok: true,
inband_rules: []appsec_rule.CustomRule{banRule},
pre_eval: []appsec.Hook{
{Apply: []string{"ExemptFromChallenge()"}},
bypassHook,
},
// neither the UA nor the IP matches any definition
input_request: botRequest("203.0.113.9:34567", "curl/8.0"),
output_asserts: func(events []pipeline.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) {
require.Len(t, responses, 1)
require.True(t, responses[0].InBandInterrupt)
require.Equal(t, appsec.AllowRemediation, responses[0].Action)
},
},
}
runTests(t, tests)
}
+76 -21
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"time"
@@ -129,6 +130,15 @@ func (r *AppsecRunner) Init(datadir string) error {
r.logger.Tracef("Loaded inband rules: %+v", r.AppsecInbandEngine.GetRuleGroup().GetRules())
r.logger.Tracef("Loaded outband rules: %+v", r.AppsecOutbandEngine.GetRuleGroup().GetRules())
// Materialize the fingerprint-dump dir under data_dir. Failure doesn't prevent startup.
r.AppsecRuntime.FingerprintDumpDir = filepath.Join(datadir, "fingerprint_dumps")
if err := os.MkdirAll(r.AppsecRuntime.FingerprintDumpDir, 0o700); err != nil {
r.logger.Warnf("could not create fingerprint dump dir %q: %s; DumpFingerprint() will be a no-op", r.AppsecRuntime.FingerprintDumpDir, err)
r.AppsecRuntime.FingerprintDumpDir = ""
} else if err := os.Chmod(r.AppsecRuntime.FingerprintDumpDir, 0o700); err != nil {
r.logger.Warnf("could not chmod fingerprint dump dir %q to 0700: %s", r.AppsecRuntime.FingerprintDumpDir, err)
}
return nil
}
@@ -142,10 +152,8 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR
}
defer func() {
state.Tx.ProcessLogging()
//We don't close the transaction here, as it will reset coraza internal state and break variable tracking
err := r.AppsecRuntime.ProcessPostEvalRules(state, request)
// We don't close the transaction here, as it would reset coraza internal state and break variable tracking.
err := r.AppsecRuntime.ProcessPostEvalRules(ctx, state, request)
if err != nil {
r.logger.Errorf("unable to process PostEval rules: %s", err)
}
@@ -158,6 +166,13 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR
//FIXME: should we abort here ?
}
// User has requested valid challenge, but we did not find a valid cookie
// Immediately return, everything has been set already
if state.RequireChallenge {
r.logger.Debug("serving challenge")
return nil
}
if state.DropInfo(request) != nil {
r.logger.Debug("drop helper triggered during pre_eval, skipping WAF evaluation")
return nil
@@ -177,6 +192,11 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR
r.logger.Debugf("request body exceeded maximum allowed size but body inspection is disabled, allowing request")
}
defer func() {
state.Tx.ProcessLogging()
// We don't close the transaction here, as it would reset coraza internal state and break variable tracking.
}()
state.Tx.ProcessConnection(request.ClientIP, 0, "", 0)
for k, v := range request.Args {
@@ -241,14 +261,35 @@ func (r *AppsecRunner) processRequest(ctx context.Context, state *appsec.AppsecR
func (r *AppsecRunner) ProcessInBandRules(ctx context.Context, state *appsec.AppsecRequestState, request *appsec.ParsedRequest) error {
tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID)
state.Tx = tx
// Even if we have no inband rules, we might have pre-eval or post-eval rules to process
// Even if we have no inband rules, we might have pre-eval, post-eval or on_challenge hooks to process
if len(r.AppsecRuntime.InBandRules) == 0 &&
len(r.AppsecRuntime.CommonHooks.PreEval) == 0 &&
len(r.AppsecRuntime.InBandHooks.PreEval) == 0 &&
len(r.AppsecRuntime.CommonHooks.PostEval) == 0 &&
len(r.AppsecRuntime.InBandHooks.PostEval) == 0 {
len(r.AppsecRuntime.InBandHooks.PostEval) == 0 &&
len(r.AppsecRuntime.CompiledOnChallenge) == 0 &&
r.AppsecRuntime.ChallengeRuntime == nil {
return nil
}
// on_challenge runs before any WAF work: it serves PoW infrastructure paths,
// validates submissions, and populates state.Fingerprint from the cookie.
if err := r.AppsecRuntime.ProcessOnChallengeRules(ctx, state, request); err != nil {
r.logger.Errorf("unable to process OnChallenge rules: %s", err)
}
// Infrastructure paths (PoW worker, challenge submit) already set up the
// full response — skip pre_eval, WAF evaluation and post_eval entirely.
if state.RequireChallenge {
r.logger.Debugf("challenge response set by on_challenge, skipping WAF evaluation")
return nil
}
if state.DropInfo(request) != nil {
r.logger.Debug("drop helper triggered during on_challenge, skipping WAF evaluation")
return nil
}
err := r.processRequest(ctx, state, request)
return err
}
@@ -268,13 +309,8 @@ func (r *AppsecRunner) ProcessOutOfBandRules(ctx context.Context, state *appsec.
}
func (r *AppsecRunner) handleInBandInterrupt(ctx context.Context, state *appsec.AppsecRequestState, request *appsec.ParsedRequest) {
if allowed, reason := r.appsecAllowlistsClient.IsAllowlisted(request.ClientIP); allowed {
r.logger.Infof("%s is allowlisted by %s, skipping", request.ClientIP, reason)
return
}
//create the associated event for crowdsec itself
evt, err := EventFromRequest(request, r.Labels, state.Tx.ID())
evt, err := appsec.EventFromRequest(request, r.Labels, state.Tx.ID())
if err != nil {
//let's not interrupt the pipeline for this
r.logger.Errorf("unable to create event from request : %s", err)
@@ -361,13 +397,7 @@ func copyHookVars(evt *pipeline.Event, state *appsec.AppsecRequestState) {
}
func (r *AppsecRunner) handleOutBandInterrupt(ctx context.Context, state *appsec.AppsecRequestState, request *appsec.ParsedRequest) {
if allowed, reason := r.appsecAllowlistsClient.IsAllowlisted(request.ClientIP); allowed {
r.logger.Infof("%s is allowlisted by %s, skipping", request.ClientIP, reason)
return
}
evt, err := EventFromRequest(request, r.Labels, state.Tx.ID())
evt, err := appsec.EventFromRequest(request, r.Labels, state.Tx.ID())
if err != nil {
//let's not interrupt the pipeline for this
r.logger.Errorf("unable to create event from request : %s", err)
@@ -435,6 +465,19 @@ func (r *AppsecRunner) handleRequest(ctx context.Context, request *appsec.Parsed
logger.Debug("Request received in runner")
r.AppsecRuntime.ClearResponse(&state)
// Allowlisted IPs bypass every form of appsec processing (in-band rules,
// hooks, challenge issuance, on_challenge cookie validation, out-of-band
// rules). We send the default pass response straight back to the bouncer.
if r.appsecAllowlistsClient != nil {
if allowed, reason := r.appsecAllowlistsClient.IsAllowlisted(request.ClientIP); allowed {
logger.Infof("%s is allowlisted by %s, skipping WAF processing", request.ClientIP, reason)
// No Clone() needed (unlike the in-band send below): we return
// immediately, so no out-of-band phase races this response.
request.ResponseChannel <- state.Response
return
}
}
request.IsInBand = true
request.IsOutBand = false
@@ -468,8 +511,20 @@ func (r *AppsecRunner) handleRequest(ctx context.Context, request *appsec.Parsed
r.logger.Errorf("unable to close inband transaction: %s", err)
}
// send back the result to the HTTP handler for the InBand part
request.ResponseChannel <- state.Response
// Clone as the out-of-band phase might mutate the response
request.ResponseChannel <- state.Response.Clone()
// A challenge was served, so the request never reaches the backend;
if state.RequireChallenge {
globalParsingElapsed := time.Since(startGlobalParsing)
metrics.AppsecGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(globalParsingElapsed.Seconds())
return
}
// Challenge remediation is intentionally a no-op for OOB matches: the inband
// response has already been sent to the visitor at this point, so there is
// nothing left to challenge. OOB matches still feed alerts/events.
// (captcha gets the same treatment for the same reason.)
//Now let's process the out of band rules
+31 -11
View File
@@ -13,11 +13,11 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/appsec/allowlists"
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
)
@@ -31,15 +31,17 @@ type appsecRuleTest struct {
// schemas registers OpenAPI schemas (ref → YAML content) on the
// RequestValidator after Build, so hooks can refer to them via
// ValidateRequestWithSchema(ref).
schemas map[string]string
on_load []appsec.Hook
pre_eval []appsec.Hook
post_eval []appsec.Hook
on_match []appsec.Hook
schemas map[string]string
on_load []appsec.Hook
pre_eval []appsec.Hook
post_eval []appsec.Hook
on_match []appsec.Hook
on_challenge []appsec.Hook
// Phase-scoped hooks (dispatched only during the matching phase)
inband_on_match []appsec.Hook
inband_pre_eval []appsec.Hook
inband_post_eval []appsec.Hook
inband_on_challenge []appsec.Hook
outofband_on_match []appsec.Hook
outofband_pre_eval []appsec.Hook
outofband_post_eval []appsec.Hook
@@ -119,6 +121,7 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
PreEval: test.pre_eval,
PostEval: test.post_eval,
OnMatch: test.on_match,
OnChallenge: test.on_challenge,
BouncerBlockedHTTPCode: test.BouncerBlockedHTTPCode,
UserBlockedHTTPCode: test.UserBlockedHTTPCode,
UserPassedHTTPCode: test.UserPassedHTTPCode,
@@ -127,11 +130,12 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
}
// Set phase-scoped hooks if any are provided
if len(test.inband_on_match) > 0 || len(test.inband_pre_eval) > 0 || len(test.inband_post_eval) > 0 {
if len(test.inband_on_match) > 0 || len(test.inband_pre_eval) > 0 || len(test.inband_post_eval) > 0 || len(test.inband_on_challenge) > 0 {
appsecCfg.InBand = &appsec.AppsecPhaseConfig{
OnMatch: test.inband_on_match,
PreEval: test.inband_pre_eval,
PostEval: test.inband_post_eval,
OnMatch: test.inband_on_match,
PreEval: test.inband_pre_eval,
PostEval: test.inband_post_eval,
OnChallenge: test.inband_on_challenge,
}
}
@@ -146,6 +150,9 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
hub := cwhub.Hub{}
AppsecRuntime, err := appsecCfg.Build(t.Context(), &hub)
if err != nil {
if !test.expected_load_ok {
return
}
t.Fatalf("unable to build appsec runtime : %s", err)
}
for ref, schemaYAML := range test.schemas {
@@ -155,10 +162,23 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
}
AppsecRuntime.InBandRules = []appsec.AppsecCollection{{Rules: inbandRules, NativeRules: nativeInbandRules}}
AppsecRuntime.OutOfBandRules = []appsec.AppsecCollection{{Rules: outofbandRules, NativeRules: nativeOutofbandRules}}
// Hooks using SendChallenge() or on_challenge hooks require the WASM
// challenge runtime; mirror what pkg/acquisition/modules/appsec/config.go
// does in production. We share a single runtime across tests — building
// one is expensive (~15-20s for WASM obfuscator warm-up), and none of the
// tests mutate runtime-level state like default difficulty.
if AppsecRuntime.NeedWASMVM {
AppsecRuntime.ChallengeRuntime = getSharedChallengeRuntime(t)
}
appsecRunnerUUID := uuid.New().String()
//we copy AppsecRutime for each runner
wrt := *AppsecRuntime
wrt.Logger = logger
// mirror production wiring (run.go / config.go) so challenge lifecycle
// events are emitted on the same channel the harness collects from.
wrt.OutChan = OutChan
wrt.Labels = map[string]string{"foo": "bar"}
mux, urlx, teardown := setupLapi()
defer teardown()
@@ -242,7 +262,7 @@ func testAppSecEngine(t *testing.T, test appsecRuleTest) {
idle.Reset(idleDuration)
}
responses :=[]appsec.AppsecTempResponse{}
responses := []appsec.AppsecTempResponse{}
events := []pipeline.Event{}
done := make(chan struct{})
+88 -10
View File
@@ -9,6 +9,7 @@ import (
"fmt"
"net/http"
"os"
"strings"
"time"
yaml "github.com/goccy/go-yaml"
@@ -20,6 +21,9 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/apiclient/useragent"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/appsec/allowlists"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/metrics"
)
@@ -121,6 +125,61 @@ func loadCertPool(caCertPath string, logger log.FieldLogger) (*x509.CertPool, er
return caCertPool, nil
}
// expandAppsecConfigEntry resolves a single appsec_config(s) entry into the list
// of appsec-config item names to load. A literal entry is returned untouched. An entry containing a glob meta-character
// is matched against the installed appsec-configs with the same matcher used
// to expand appsec-rule patterns; it errors when no installed config matches.
func expandAppsecConfigEntry(entry string, hub *cwhub.Hub) ([]string, error) {
if !strings.ContainsAny(entry, "*?") {
return []string{entry}, nil
}
var matches []string
for _, item := range hub.GetInstalledByType(cwhub.APPSEC_CONFIGS, true) {
matched, err := exprhelpers.Match(entry, item.Name)
if err != nil {
return nil, fmt.Errorf("while matching %q against %q: %w", entry, item.Name, err)
}
if matched.(bool) {
matches = append(matches, item.Name)
}
}
if len(matches) == 0 {
return nil, fmt.Errorf("no installed appsec-config matches pattern %q", entry)
}
return matches, nil
}
// resolveAppsecConfigEntries expands every appsec_config(s) entry and returns
// the de-duplicated list of config names to load, in first-seen order.
func resolveAppsecConfigEntries(entries []string, hub *cwhub.Hub) ([]string, error) {
seen := make(map[string]struct{})
var toLoad []string
for _, entry := range entries {
names, err := expandAppsecConfigEntry(entry, hub)
if err != nil {
return nil, fmt.Errorf("unable to resolve appsec_config %q: %w", entry, err)
}
for _, name := range names {
if _, ok := seen[name]; ok {
continue
}
seen[name] = struct{}{}
toLoad = append(toLoad, name)
}
}
return toLoad, nil
}
func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.Entry, _ metrics.AcquisitionMetricsLevel) error {
if w.hub == nil {
return errors.New("appsec datasource requires a hub. this is a bug, please report")
@@ -177,13 +236,19 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E
if err := appsecCfg.LoadByPath(w.config.AppsecConfigPath); err != nil {
return fmt.Errorf("unable to load appsec_config: %w", err)
}
} else if w.config.AppsecConfig != "" {
if err := appsecCfg.Load(w.config.AppsecConfig, w.hub); err != nil {
return fmt.Errorf("unable to load appsec_config: %w", err)
} else if w.config.AppsecConfig != "" || len(w.config.AppsecConfigs) > 0 {
entries := w.config.AppsecConfigs
if w.config.AppsecConfig != "" {
entries = []string{w.config.AppsecConfig}
}
} else if len(w.config.AppsecConfigs) > 0 {
for _, appsecConfig := range w.config.AppsecConfigs {
if err := appsecCfg.Load(appsecConfig, w.hub); err != nil {
names, err := resolveAppsecConfigEntries(entries, w.hub)
if err != nil {
return err
}
for _, name := range names {
if err := appsecCfg.Load(name, w.hub); err != nil {
return fmt.Errorf("unable to load appsec_config: %w", err)
}
}
@@ -199,7 +264,23 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E
return fmt.Errorf("unable to build appsec_config: %w", err)
}
if appsecRuntime.NeedWASMVM {
logger.Info("Initializing WASM runtime for challenge obfuscation")
challengeOpts, err := challenge.BuildOptions(appsecCfg.Challenge, appsecCfg.Logger)
if err != nil {
return fmt.Errorf("unable to build challenge options: %w", err)
}
challengeRuntime, err := challenge.NewChallengeRuntime(ctx, challengeOpts...)
if err != nil {
return fmt.Errorf("unable to create challenge runtime: %w", err)
}
appsecRuntime.ChallengeRuntime = challengeRuntime
}
w.AppsecRuntime = appsecRuntime
w.AppsecRuntime.Labels = w.config.Labels
w.AppsecRuntime.DataDir = w.hub.GetDataDir()
err = w.AppsecRuntime.ProcessOnLoadRules()
@@ -213,14 +294,11 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E
for nbRoutine := range w.config.Routines {
appsecRunnerUUID := uuid.New().String()
// we copy AppsecRuntime for each runner
wrt := *w.AppsecRuntime
wrt.Logger = w.logger.Dup().WithField("runner_uuid", appsecRunnerUUID)
runner := AppsecRunner{
inChan: w.InChan,
UUID: appsecRunnerUUID,
logger: w.logger.WithField("runner_uuid", appsecRunnerUUID),
AppsecRuntime: &wrt,
AppsecRuntime: w.AppsecRuntime,
Labels: w.config.Labels,
appsecAllowlistsClient: w.appsecAllowlistClient,
}
@@ -0,0 +1,157 @@
package appsecacquisition
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// hubWithConfigs builds an in-memory hub holding the given appsec-configs and
// marks the listed ones as installed (an installed item is one with a
// LocalPath, see cwhub.ItemState.IsInstalled).
func hubWithConfigs(t *testing.T, all, installed []string) *cwhub.Hub {
t.Helper()
tempDir := t.TempDir()
local := &csconfig.LocalHubCfg{
HubDir: filepath.Join(tempDir, "hub"),
HubIndexFile: filepath.Join(tempDir, "hub", ".index.json"),
InstallDir: filepath.Join(tempDir, "install"),
InstallDataDir: filepath.Join(tempDir, "data"),
}
require.NoError(t, os.MkdirAll(local.HubDir, 0o755))
require.NoError(t, os.MkdirAll(local.InstallDir, 0o755))
require.NoError(t, os.MkdirAll(local.InstallDataDir, 0o755))
index := `{"appsec-configs": {`
for i, name := range all {
if i > 0 {
index += ","
}
index += `"` + name + `": {"path": "appsec-configs/` + name + `.yaml", "version": "1.0", "versions": {"1.0": {"digest": "aa"}}}`
}
index += "}}"
require.NoError(t, os.WriteFile(local.HubIndexFile, []byte(index), 0o644))
hub, err := cwhub.NewHub(local, nil)
require.NoError(t, err)
require.NoError(t, hub.Load())
for _, name := range installed {
item := hub.GetItem(cwhub.APPSEC_CONFIGS, name)
require.NotNilf(t, item, "appsec-config %q missing from test hub", name)
item.State.LocalPath = filepath.Join(local.InstallDir, name+".yaml")
}
return hub
}
func TestExpandAppsecConfigEntry(t *testing.T) {
all := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "crowdsecurity/uninstalled", "custom/mine"}
installed := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "custom/mine"}
tests := []struct {
name string
entry string
expected []string
expectedErr string
}{
{
// Literals pass through untouched, even unknown ones, so the
// per-name error still surfaces later from AppsecConfig.Load.
name: "literal passthrough",
entry: "crowdsecurity/vpatch",
expected: []string{"crowdsecurity/vpatch"},
},
{
name: "unknown literal passthrough",
entry: "does/not-exist",
expected: []string{"does/not-exist"},
},
{
// Sorted (case-insensitive) order from the hub; uninstalled is skipped.
name: "wildcard expands installed only",
entry: "crowdsecurity/*",
expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch"},
},
{
name: "wildcard matches all",
entry: "*",
expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch", "custom/mine"},
},
{
name: "wildcard with no match errors",
entry: "nope/*",
expectedErr: `no installed appsec-config matches pattern "nope/*"`,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
hub := hubWithConfigs(t, all, installed)
got, err := expandAppsecConfigEntry(tc.entry, hub)
if tc.expectedErr != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tc.expectedErr)
return
}
require.NoError(t, err)
assert.Equal(t, tc.expected, got)
})
}
}
func TestResolveAppsecConfigEntries(t *testing.T) {
all := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "crowdsecurity/uninstalled", "custom/mine"}
installed := []string{"crowdsecurity/vpatch", "crowdsecurity/generic", "custom/mine"}
tests := []struct {
name string
entries []string
expected []string
}{
{
// "*" already pulls in vpatch; the explicit literal must not load it
// a second time. First-seen order from the wildcard expansion wins.
name: "wildcard plus overlapping literal dedups",
entries: []string{"*", "crowdsecurity/vpatch"},
expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch", "custom/mine"},
},
{
name: "same literal twice collapses to one",
entries: []string{"crowdsecurity/vpatch", "crowdsecurity/vpatch"},
expected: []string{"crowdsecurity/vpatch"},
},
{
name: "unknown literal passes through once",
entries: []string{"does/not-exist", "does/not-exist"},
expected: []string{"does/not-exist"},
},
{
// Overlapping patterns: generic matches both, must appear once.
name: "overlapping patterns dedup",
entries: []string{"crowdsecurity/*", "*/generic"},
expected: []string{"crowdsecurity/generic", "crowdsecurity/vpatch"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
hub := hubWithConfigs(t, all, installed)
got, err := resolveAppsecConfigEntries(tc.entries, hub)
require.NoError(t, err)
assert.Equal(t, tc.expected, got)
})
}
}
+2 -1
View File
@@ -3,6 +3,7 @@ package appsecacquisition
import (
"github.com/crowdsecurity/crowdsec/pkg/acquisition/registry"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/types"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
)
var (
@@ -14,7 +15,7 @@ var (
_ types.LAPIClientAware = (*Source)(nil)
)
const ModuleName = "appsec"
const ModuleName = appsec.ModuleName
//nolint:gochecknoinits
func init() {
+19 -1
View File
@@ -11,6 +11,15 @@ func (*Source) GetMetrics() []prometheus.Collector {
metrics.AppsecReqCounter,
metrics.AppsecBlockCounter,
metrics.AppsecRuleHits,
metrics.AppsecFingerprintMismatch,
metrics.AppsecChallengeRequested,
metrics.AppsecChallengeSubmitted,
metrics.AppsecChallengeAccepted,
metrics.AppsecChallengeRejected,
metrics.AppsecChallengeKepochGenerated,
metrics.AppsecChallengeKepochEvicted,
metrics.AppsecChallengeReobfuscation,
metrics.AppsecChallengeDynamicModuleEvicted,
metrics.AppsecValidationOKCounter,
metrics.AppsecValidationFailedCounter,
metrics.AppsecOutbandParsingHistogram,
@@ -24,9 +33,18 @@ func (*Source) GetAggregMetrics() []prometheus.Collector {
metrics.AppsecReqCounter,
metrics.AppsecBlockCounter,
metrics.AppsecRuleHits,
metrics.AppsecFingerprintMismatch,
metrics.AppsecChallengeRequested,
metrics.AppsecChallengeSubmitted,
metrics.AppsecChallengeAccepted,
metrics.AppsecChallengeRejected,
metrics.AppsecChallengeKepochGenerated,
metrics.AppsecChallengeKepochEvicted,
metrics.AppsecChallengeReobfuscation,
metrics.AppsecChallengeDynamicModuleEvicted,
metrics.AppsecOutbandParsingHistogram,
metrics.AppsecValidationOKCounter,
metrics.AppsecValidationFailedCounter,
metrics.AppsecOutbandParsingHistogram,
metrics.AppsecInbandParsingHistogram,
metrics.AppsecGlobalParsingHistogram,
}
+8
View File
@@ -105,6 +105,12 @@ func (w *Source) listenAndServe(ctx context.Context, t *tomb.Tomb) error {
w.logger.Errorf("Error shutting down Appsec server: %s", err.Error())
}
if w.AppsecRuntime != nil && w.AppsecRuntime.ChallengeRuntime != nil {
if err := w.AppsecRuntime.ChallengeRuntime.Close(ctx); err != nil {
w.logger.Errorf("Error closing challenge runtime: %s", err)
}
}
if w.config.ListenSocket != "" {
if err := os.Remove(w.config.ListenSocket); err != nil {
if !errors.Is(err, fs.ErrNotExist) {
@@ -133,6 +139,8 @@ func (w *Source) StreamingAcquisition(ctx context.Context, out chan pipeline.Eve
t.Go(func() error {
defer trace.ReportPanic()
w.AppsecRuntime.OutChan = out
for _, runner := range w.AppsecRunners {
runner.outChan = out
@@ -0,0 +1,30 @@
package appsecacquisition
import (
"sync"
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
)
// sharedChallengeRuntime lazily creates a single ChallengeRuntime for the
// whole package test run. NewChallengeRuntime runs the obfuscator WASM to
// generate a challenge JS bundle (~15-20s), so spinning one up per test is
// prohibitively slow and unnecessary for integration tests that don't mutate
// runtime-level state.
var (
sharedChallengeRuntimeOnce sync.Once
sharedChallengeRuntimeInst *challenge.ChallengeRuntime
sharedChallengeRuntimeErr error
)
func getSharedChallengeRuntime(t *testing.T) *challenge.ChallengeRuntime {
t.Helper()
sharedChallengeRuntimeOnce.Do(func() {
sharedChallengeRuntimeInst, sharedChallengeRuntimeErr = challenge.NewChallengeRuntime(t.Context())
})
require.NoError(t, sharedChallengeRuntimeErr)
return sharedChallengeRuntimeInst
}
-31
View File
@@ -288,37 +288,6 @@ func containsAll(excludedZones []string, matchedZones []string) bool {
return true
}
func EventFromRequest(r *appsec.ParsedRequest, labels map[string]string, txUuid string) (pipeline.Event, error) {
evt := pipeline.MakeEvent(false, pipeline.LOG, true)
// def needs fixing
evt.Stage = "s00-raw"
evt.Parsed = map[string]string{
"source_ip": r.ClientIP,
"target_host": r.Host,
"target_uri": r.URI,
"method": r.Method,
"req_uuid": txUuid,
"source": "crowdsec-appsec",
"remediation_cmpt_ip": r.RemoteAddrNormalized,
// TBD:
// http_status
// user_agent
}
evt.Line = pipeline.Line{
Time: time.Now(),
// should we add some info like listen addr/port/path ?
Labels: labels,
Process: true,
Module: ModuleName,
Src: ModuleName,
Raw: "dummy-appsec-data", // we discard empty Line.Raw items :)
}
evt.Appsec = pipeline.AppsecEvent{}
return evt, nil
}
type ruleData struct {
ID int
Name string
+777 -37
View File
File diff suppressed because it is too large Load Diff
+440
View File
@@ -0,0 +1,440 @@
package appsec
import (
"net/http"
"net/url"
"sync"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
)
// sharedChallengeRuntime lazily creates a single ChallengeRuntime for the
// whole package test run. NewChallengeRuntime runs the obfuscator WASM to
// generate a challenge JS bundle (~15-20s), so spinning one up per test is
// prohibitively slow and unnecessary for logic-only dispatcher tests.
var (
sharedChallengeRuntimeOnce sync.Once
sharedChallengeRuntimeInst *challenge.ChallengeRuntime
sharedChallengeRuntimeErr error
)
func getSharedChallengeRuntime(t *testing.T) *challenge.ChallengeRuntime {
t.Helper()
sharedChallengeRuntimeOnce.Do(func() {
// Use context.Background so the runtime survives across tests.
// Goroutines it spawns leak for the duration of the test binary; that's
// acceptable here since the binary exits after tests complete.
sharedChallengeRuntimeInst, sharedChallengeRuntimeErr = challenge.NewChallengeRuntime(t.Context())
})
require.NoError(t, sharedChallengeRuntimeErr)
return sharedChallengeRuntimeInst
}
// newChallengeTestRuntime builds an AppsecRuntimeConfig wired with a real
// ChallengeRuntime for dispatcher tests. Provided on_challenge hooks are
// compiled via buildHookList, so tests can assert that user expressions run.
func newChallengeTestRuntime(t *testing.T, hooks []Hook) *AppsecRuntimeConfig {
t.Helper()
cr := getSharedChallengeRuntime(t)
compiled, err := buildHookList(t.Context(), hooks, hookOnChallenge, &appsecExprPatcher{})
require.NoError(t, err)
logger := log.NewEntry(log.StandardLogger())
cfg := &AppsecConfig{
Name: "test-challenge",
BouncerBlockedHTTPCode: http.StatusForbidden,
BouncerPassedHTTPCode: http.StatusOK,
UserBlockedHTTPCode: http.StatusForbidden,
UserPassedHTTPCode: http.StatusOK,
DefaultPassAction: AllowRemediation,
DefaultRemediation: BanRemediation,
}
return &AppsecRuntimeConfig{
Name: "test-challenge",
Config: cfg,
Logger: logger,
ChallengeRuntime: cr,
CompiledOnChallenge: compiled,
}
}
func newInBandRequest(method, path string, body []byte) *ParsedRequest {
u, _ := url.Parse(path)
return &ParsedRequest{
HTTPRequest: &http.Request{
Method: method,
URL: u,
Header: http.Header{"User-Agent": []string{"go-test"}},
},
Body: body,
IsInBand: true,
}
}
func TestProcessOnChallengeRulesNilRuntime(t *testing.T) {
rt := &AppsecRuntimeConfig{
Logger: log.NewEntry(log.StandardLogger()),
Config: &AppsecConfig{},
}
state := &AppsecRequestState{}
req := newInBandRequest(http.MethodGet, "/", nil)
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
assert.False(t, state.RequireChallenge)
assert.Nil(t, state.Fingerprint)
}
func TestProcessOnChallengeRulesServesPowWorker(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
req := newInBandRequest(http.MethodGet, challenge.ChallengePowWorkerPath, nil)
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
assert.True(t, state.RequireChallenge)
assert.Equal(t, ChallengeRemediation, state.Response.Action)
assert.Equal(t, http.StatusOK, state.Response.UserHTTPResponseCode)
assert.Equal(t, challenge.PowWorkerJS, state.Response.UserHTTPBodyContent)
assert.Contains(t, state.Response.UserHeaders["Content-Type"], "application/javascript")
}
func TestProcessOnChallengeRulesSubmitInvalidBody(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
// Empty body → ValidateChallengeResponse fails; dispatcher returns failed JSON body.
req := newInBandRequest(http.MethodPost, challenge.ChallengeSubmitPath, nil)
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
assert.True(t, state.RequireChallenge)
assert.JSONEq(t, bodyChallengeFailed, state.Response.UserHTTPBodyContent)
assert.Contains(t, state.Response.UserHeaders["Content-Type"], "application/json")
// No cookie is issued on failure.
assert.Empty(t, state.Response.UserHTTPCookies)
}
func TestProcessOnChallengeRulesNoCookieSkipsUserHooks(t *testing.T) {
// A hook that would unconditionally flip PendingAction must NOT run when
// there is no fingerprint to inspect — this prevents nil-deref panics on
// filters like `fingerprint.Bot.X`.
rt := newChallengeTestRuntime(t, []Hook{
{Apply: []string{"SetRemediation('allow')"}},
})
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
req := newInBandRequest(http.MethodGet, "/", nil)
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
assert.Nil(t, state.Fingerprint)
assert.Nil(t, state.PendingAction, "user hooks must not run when fingerprint is nil")
}
func TestProcessOnChallengeRulesInvalidCookieSkipsUserHooks(t *testing.T) {
rt := newChallengeTestRuntime(t, []Hook{
{Apply: []string{"SetReturnCode(418)"}},
})
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
// Attach an invalid cookie; ValidCookie fails silently and fingerprint stays nil.
u, _ := url.Parse("/")
httpReq := &http.Request{
Method: http.MethodGet,
URL: u,
Header: http.Header{
"User-Agent": []string{"go-test"},
"Cookie": []string{challenge.ChallengeCookieName + "=garbage"},
},
}
req := &ParsedRequest{HTTPRequest: httpReq, IsInBand: true}
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
assert.Nil(t, state.Fingerprint)
assert.Nil(t, state.PendingHTTPCode, "user hooks must not run when fingerprint is nil")
}
func TestSendChallengeNoOpWhenStoredDifficultyMeetsTarget(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
// Simulate a client that already proved the runtime-default difficulty.
state.Fingerprint = &challenge.FingerprintData{}
state.CookiePowDifficulty = rt.ChallengeRuntime.Difficulty()
req := newInBandRequest(http.MethodGet, "/protected", nil)
require.NoError(t, rt.SendChallenge(t.Context(), state, req))
assert.False(t, state.RequireChallenge)
assert.Empty(t, state.Response.UserHTTPBodyContent)
assert.NotEqual(t, ChallengeRemediation, state.Response.Action)
}
func TestSendChallengeReIssuesWhenTargetExceedsStored(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
// Client proved medium; this request demands impossible.
state.Fingerprint = &challenge.FingerprintData{}
state.CookiePowDifficulty = challenge.PowDifficultyMedium
impossible := challenge.PowDifficultyImpossible
state.ChallengeDifficulty = &impossible
req := newInBandRequest(http.MethodGet, "/protected", nil)
require.NoError(t, rt.SendChallenge(t.Context(), state, req))
assert.True(t, state.RequireChallenge)
assert.Equal(t, ChallengeRemediation, state.Response.Action)
assert.Contains(t, state.Response.UserHeaders["Content-Type"], "text/html")
assert.NotEmpty(t, state.Response.UserHTTPBodyContent)
}
func TestSendChallengeNoOpWhenTargetLoweredBelowStored(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
// Client proved high; admin lowered the global target.
state.Fingerprint = &challenge.FingerprintData{}
state.CookiePowDifficulty = challenge.PowDifficultyHigh
low := challenge.PowDifficultyLow
state.ChallengeDifficulty = &low
req := newInBandRequest(http.MethodGet, "/protected", nil)
require.NoError(t, rt.SendChallenge(t.Context(), state, req))
assert.False(t, state.RequireChallenge)
}
func TestProcessOnChallengeRulesInvalidSubmissionSkipsUserHooks(t *testing.T) {
// A hook that would unconditionally flip PendingAction must NOT run when
// the submission is invalid — the failed JSON body is the only response.
rt := newChallengeTestRuntime(t, []Hook{
{Apply: []string{"SetRemediation('allow')"}},
})
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
// Empty body → ValidateChallengeResponse fails.
req := newInBandRequest(http.MethodPost, challenge.ChallengeSubmitPath, nil)
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, req))
assert.True(t, state.RequireChallenge)
assert.JSONEq(t, bodyChallengeFailed, state.Response.UserHTTPBodyContent)
assert.Nil(t, state.PendingAction, "user hooks must not run on invalid submission")
}
// TestRejectSubmissionSetsState confirms the helper sets the state field;
// downstream ProcessOnChallengeRules submit branch keys off this to refuse
// cookie issuance.
func TestRejectSubmissionSetsState(t *testing.T) {
rt := &AppsecRuntimeConfig{
Logger: log.NewEntry(log.StandardLogger()),
Config: &AppsecConfig{},
}
state := &AppsecRequestState{}
require.NoError(t, rt.RejectSubmission(state, "bot detected"))
require.NotNil(t, state.SubmissionRejection)
assert.Equal(t, "bot detected", state.SubmissionRejection.Reason)
// Empty reason gets a default placeholder.
state2 := &AppsecRequestState{}
require.NoError(t, rt.RejectSubmission(state2, ""))
assert.Equal(t, "submission rejected by on_challenge_submit", state2.SubmissionRejection.Reason)
}
// TestGrantChallengeCookieIssuesRedirect confirms the helper mints a
// cookie, stamps allowlist fingerprint + bypass state, and produces a
// 307 challenge response carrying the Location header and Set-Cookie
// back to the visitor. The 307 is necessary because GenerateResponse
// only serializes UserCookies on ChallengeRemediation responses.
func TestGrantChallengeCookieIssuesRedirect(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
req := newInBandRequest(http.MethodGet, "/protected?a=1", nil)
require.NoError(t, rt.GrantChallengeCookie(state, req, "Googlebot/2.1", nil))
require.NotNil(t, state.Fingerprint)
assert.True(t, state.Fingerprint.Allowlisted)
assert.Equal(t, "Googlebot/2.1", state.Fingerprint.AllowlistReason)
assert.True(t, state.ChallengeBypassed, "ChallengeBypassed must be set to suppress later SendChallenge")
assert.Equal(t, ChallengeRemediation, state.Response.Action, "must produce a challenge-action response so the cookie is serialized")
assert.Equal(t, http.StatusTemporaryRedirect, state.Response.UserHTTPResponseCode, "must be a 307 redirect")
assert.True(t, state.RequireChallenge, "RequireChallenge must be flagged so the wire response carries the cookie")
require.Len(t, state.Response.UserHTTPCookies, 1, "allowlist cookie must be appended")
require.NotNil(t, state.Response.UserHeaders)
require.Contains(t, state.Response.UserHeaders, "Location")
assert.Equal(t, []string{"/protected?a=1"}, state.Response.UserHeaders["Location"])
assert.Equal(t, challenge.GrantRedirectBody, state.Response.UserHTTPBodyContent, "body must be the redirect fallback page")
}
// TestGrantAllowlistCookieInlineNoRedirect confirms the submit-phase
// variant attaches the cookie without producing a redirect response.
// on_challenge_submit returns the challenge-submit JSON envelope the
// client is already awaiting; a 307 would break the client JS state
// machine.
func TestGrantAllowlistCookieInlineNoRedirect(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
req := newInBandRequest(http.MethodPost, challenge.ChallengeSubmitPath, nil)
require.NoError(t, rt.GrantAllowlistCookieInline(state, req, "submit-allowlist", nil))
require.NotNil(t, state.Fingerprint)
assert.True(t, state.Fingerprint.Allowlisted)
assert.Equal(t, "submit-allowlist", state.Fingerprint.AllowlistReason)
assert.True(t, state.ChallengeBypassed)
require.Len(t, state.Response.UserHTTPCookies, 1, "cookie must ride on the existing envelope")
assert.NotEqual(t, http.StatusTemporaryRedirect, state.Response.UserHTTPResponseCode, "inline variant must NOT emit a redirect status")
assert.NotContains(t, state.Response.UserHeaders, "Location", "inline variant must not set Location")
}
// TestSendChallengeNoOpAfterGrantChallengeCookie covers the bypass guard:
// after GrantChallengeCookie sets ChallengeBypassed, a subsequent
// SendChallenge call in the same request is a no-op (must not overwrite
// the 307 redirect response body with a challenge page).
func TestSendChallengeNoOpAfterGrantChallengeCookie(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
req := newInBandRequest(http.MethodGet, "/", nil)
require.NoError(t, rt.GrantChallengeCookie(state, req, "ua-pass", nil))
require.True(t, state.ChallengeBypassed)
cookiesBefore := len(state.Response.UserHTTPCookies)
bodyBefore := state.Response.UserHTTPBodyContent
statusBefore := state.Response.UserHTTPResponseCode
require.NoError(t, rt.SendChallenge(t.Context(), state, req))
assert.Len(t, state.Response.UserHTTPCookies, cookiesBefore, "cookie set must be unchanged")
assert.Equal(t, bodyBefore, state.Response.UserHTTPBodyContent, "redirect body must not be overwritten")
assert.Equal(t, statusBefore, state.Response.UserHTTPResponseCode, "redirect status must not be overwritten")
}
// TestProcessOnChallengeRulesAllowlistCookiePropagatesFlag asserts the
// cookie-valid branch of ProcessOnChallengeRules copies the Allowlisted +
// AllowlistReason fields from the decoded cookie into state.Fingerprint.
func TestProcessOnChallengeRulesAllowlistCookiePropagatesFlag(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
// Mint an allowlist cookie via the public helper, then re-attach it to a
// regular request and run the dispatcher.
mintReq := newInBandRequest(http.MethodGet, "/", nil)
ck, err := rt.ChallengeRuntime.SealAllowlistCookie(mintReq.HTTPRequest, "test-bypass", nil)
require.NoError(t, err)
u, _ := url.Parse("/protected")
getReq := &ParsedRequest{
HTTPRequest: &http.Request{
Method: http.MethodGet,
URL: u,
Header: http.Header{
"User-Agent": []string{"go-test"},
"Cookie": []string{challenge.ChallengeCookieName + "=" + ck.Val},
},
},
IsInBand: true,
}
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, getReq))
require.NotNil(t, state.Fingerprint, "valid allowlist cookie must populate fingerprint")
assert.True(t, state.Fingerprint.Allowlisted)
assert.Equal(t, "test-bypass", state.Fingerprint.AllowlistReason)
assert.True(t, state.ChallengeBypassed,
"allowlist cookie must set ChallengeBypassed so later SendChallenge is a no-op")
}
// TestAllowlistCookieRoundtripBypassesSendChallenge covers the actual bug
// reported against GrantChallengeCookie: a follow-up request carrying the
// minted allowlist cookie must not be re-challenged. Without the
// ChallengeBypassed propagation in the cookie-valid branch, SendChallenge
// would happily render a challenge page on top of the cookie.
func TestAllowlistCookieRoundtripBypassesSendChallenge(t *testing.T) {
rt := newChallengeTestRuntime(t, nil)
mintReq := newInBandRequest(http.MethodGet, "/", nil)
ck, err := rt.ChallengeRuntime.SealAllowlistCookie(mintReq.HTTPRequest, "roundtrip", nil)
require.NoError(t, err)
u, _ := url.Parse("/protected")
replayReq := &ParsedRequest{
HTTPRequest: &http.Request{
Method: http.MethodGet,
URL: u,
Header: http.Header{
"User-Agent": []string{"go-test"},
"Cookie": []string{challenge.ChallengeCookieName + "=" + ck.Val},
},
},
IsInBand: true,
}
state := &AppsecRequestState{}
state.ResetResponse(rt.Config)
require.NoError(t, rt.ProcessOnChallengeRules(t.Context(), state, replayReq))
require.NotNil(t, state.Fingerprint)
require.True(t, state.ChallengeBypassed, "replayed allowlist cookie must flip ChallengeBypassed")
// Anything downstream calling SendChallenge must now be a no-op.
bodyBefore := state.Response.UserHTTPBodyContent
statusBefore := state.Response.UserHTTPResponseCode
actionBefore := state.Response.Action
require.NoError(t, rt.SendChallenge(t.Context(), state, replayReq))
assert.Equal(t, actionBefore, state.Response.Action, "SendChallenge must not set ChallengeRemediation")
assert.NotEqual(t, ChallengeRemediation, state.Response.Action,
"a replayed allowlist cookie must not produce a challenge response")
assert.Equal(t, bodyBefore, state.Response.UserHTTPBodyContent, "challenge body must not be written")
assert.Equal(t, statusBefore, state.Response.UserHTTPResponseCode, "challenge status must not overwrite")
assert.False(t, state.RequireChallenge, "RequireChallenge must stay false on bypassed replay")
}
// TestGenerateResponseChallengeNilHeaders guards against a regression where a
// `default_remediation: challenge` match reaches GenerateResponse with no
// UserHeaders map set (only the internal challenge-serving paths set headers).
// Writing the default CSP into that nil map used to panic.
func TestGenerateResponseChallengeNilHeaders(t *testing.T) {
rt := &AppsecRuntimeConfig{
Logger: log.NewEntry(log.StandardLogger()),
Config: &AppsecConfig{
UserBlockedHTTPCode: http.StatusForbidden,
BouncerBlockedHTTPCode: http.StatusForbidden,
},
}
resp := AppsecTempResponse{Action: ChallengeRemediation, UserHeaders: nil}
require.NotPanics(t, func() {
_, body := rt.GenerateResponse(resp, rt.Logger)
require.NotNil(t, body.UserHeaders)
assert.Equal(t, []string{challenge.DefaultChallengeCSP}, body.UserHeaders["Content-Security-Policy"])
})
}
+143 -15
View File
@@ -4,6 +4,7 @@ import (
"os"
"path/filepath"
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
@@ -326,24 +327,130 @@ func TestBuildNilPhaseConfig(t *testing.T) {
assert.Empty(t, rt.OutOfBandHooks.OnMatch)
}
func TestLoadAPISchemaRejectsPathTraversal(t *testing.T) {
cases := []string{
"../escape.yaml",
"../../etc/passwd",
"/etc/passwd",
func TestBuildOnChallengeTopLevelAndInband(t *testing.T) {
cfg := AppsecConfig{
Logger: log.NewEntry(log.StandardLogger()),
DefaultRemediation: "ban",
OnChallenge: []Hook{
{Apply: []string{"SetRemediation('allow')"}},
},
InBand: &AppsecPhaseConfig{
OnChallenge: []Hook{
{Filter: "IsInBand == true", Apply: []string{"SetReturnCode(418)"}},
},
},
}
for _, filename := range cases {
t.Run(filename, func(t *testing.T) {
rt := &AppsecRuntimeConfig{
Logger: log.NewEntry(log.StandardLogger()),
DataDir: t.TempDir(),
}
hub := &cwhub.Hub{}
rt, err := cfg.Build(t.Context(), hub)
require.NoError(t, err)
err := rt.LoadAPISchemaWithName("ref", filename)
require.Error(t, err)
require.Contains(t, err.Error(), "must be relative to schemas")
})
// Both top-level and inband on_challenge hooks end up in CompiledOnChallenge.
require.Len(t, rt.CompiledOnChallenge, 2)
assert.NotNil(t, rt.CompiledOnChallenge[0].ApplyExpr)
assert.NotNil(t, rt.CompiledOnChallenge[1].FilterExpr)
// Defining on_challenge must force the challenge runtime on.
assert.True(t, rt.NeedWASMVM)
}
func TestBuildOnChallengeUnderOutofbandRejected(t *testing.T) {
cfg := AppsecConfig{
Logger: log.NewEntry(log.StandardLogger()),
DefaultRemediation: "ban",
OutOfBand: &AppsecPhaseConfig{
OnChallenge: []Hook{{Apply: []string{"SetRemediation('allow')"}}},
},
}
hub := &cwhub.Hub{}
_, err := cfg.Build(t.Context(), hub)
require.Error(t, err)
assert.Contains(t, err.Error(), "on_challenge hooks are only valid in-band")
}
func TestLoadByPathOnChallengeYAML(t *testing.T) {
cfg := newTestConfig()
f := writeTempYAML(t, `
name: test-on-challenge
on_challenge:
- filter: "fingerprint != nil"
apply:
- SetRemediation('allow')
inband:
on_challenge:
- apply:
- SendChallenge()
`)
require.NoError(t, cfg.LoadByPath(f))
assert.Len(t, cfg.OnChallenge, 1)
require.NotNil(t, cfg.InBand)
assert.Len(t, cfg.InBand.OnChallenge, 1)
}
// TestLoadByPathChallengeBlockMergesAcrossFiles confirms that a `challenge:`
// block in an appsec-config YAML is parsed onto AppsecConfig.Challenge, and
// that loading a second config with a disjoint subset of fields *merges*
// rather than replacing — fields set in earlier configs survive when not
// touched by later ones, and overlapping fields take the later value.
func TestLoadByPathChallengeBlockMergesAcrossFiles(t *testing.T) {
cfg := newTestConfig()
first := writeTempYAML(t, `
name: base
challenge:
master_secret: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
key_rotation_interval: 5m
max_live_epochs: 3
cookie_ttl: 12h
`)
require.NoError(t, cfg.LoadByPath(first))
require.NotNil(t, cfg.Challenge)
require.NotNil(t, cfg.Challenge.MasterSecret)
assert.Equal(t, 5*time.Minute, *cfg.Challenge.KeyRotationInterval)
assert.Equal(t, 3, *cfg.Challenge.MaxLiveEpochs)
assert.Equal(t, 12*time.Hour, *cfg.Challenge.CookieTTL)
// Second config overrides CookieTTL and adds crypto/spent-set fields,
// leaving master_secret / rotation / max_live_epochs from the first
// config untouched. This is the multi-config last-wins-per-field
// behavior collections rely on.
second := writeTempYAML(t, `
name: overlay
challenge:
cookie_ttl: 1h
crypto_obfuscation_pool_size: 2
spent_set_max_entries: 250000
`)
require.NoError(t, cfg.LoadByPath(second))
require.NotNil(t, cfg.Challenge)
assert.Equal(t, 1*time.Hour, *cfg.Challenge.CookieTTL, "later config overrides cookie_ttl")
require.NotNil(t, cfg.Challenge.MasterSecret, "master_secret from first config must survive")
assert.Equal(t, 5*time.Minute, *cfg.Challenge.KeyRotationInterval)
require.NotNil(t, cfg.Challenge.CryptoObfuscationPoolSize)
assert.Equal(t, 2, *cfg.Challenge.CryptoObfuscationPoolSize, "new field from second config appears")
require.NotNil(t, cfg.Challenge.SpentSetMaxEntries)
assert.Equal(t, 250000, *cfg.Challenge.SpentSetMaxEntries)
}
// TestLoadByPathRejectsRemovedLibraryObfuscationKeys confirms that an
// appsec-config carrying any of the removed `library_*` obfuscation keys
// fails to parse (strict YAML mode). fpscanner is now served unobfuscated,
// so these knobs are gone and stale configs must fail loudly rather than
// silently doing nothing.
func TestLoadByPathRejectsRemovedLibraryObfuscationKeys(t *testing.T) {
for _, key := range []string{
"library_runtime_obfuscation_enabled: true",
"library_obfuscation_pool_size: 2",
"library_obfuscation_refresh_interval: 30m",
} {
cfg := newTestConfig()
f := writeTempYAML(t, "name: stale\nchallenge:\n "+key+"\n")
err := cfg.LoadByPath(f)
require.Error(t, err, "stale key %q must be rejected", key)
assert.Contains(t, err.Error(), "library_",
"strict YAML parse error must name the unknown field")
}
}
@@ -366,3 +473,24 @@ func TestBuildOnLoadStaysOutOfPhaseHooks(t *testing.T) {
assert.Empty(t, rt.CommonHooks.PostEval)
assert.Empty(t, rt.CommonHooks.OnMatch)
}
func TestLoadAPISchemaRejectsPathTraversal(t *testing.T) {
cases := []string{
"../escape.yaml",
"../../etc/passwd",
"/etc/passwd",
}
for _, filename := range cases {
t.Run(filename, func(t *testing.T) {
rt := &AppsecRuntimeConfig{
Logger: log.NewEntry(log.StandardLogger()),
DataDir: t.TempDir(),
}
err := rt.LoadAPISchemaWithName("ref", filename)
require.Error(t, err)
require.Contains(t, err.Error(), "must be relative to schemas")
})
}
}
+129 -83
View File
@@ -10,12 +10,13 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/enrichment"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
)
type AppsecCollection struct {
Rules []string
NativeRules []string
Rules []string
NativeRules []string
}
const APPSEC_RULE = "appsec-rule"
@@ -33,7 +34,7 @@ type AppsecCollectionConfig struct {
Labels map[string]any `yaml:"labels"` // Labels is K:V list aiming at providing context the overflow
Data any `yaml:"data"` // Ignore it
Data []*enrichment.DataProvider `yaml:"data"`
hash string
version string
}
@@ -49,103 +50,148 @@ type RulesDetails struct {
// Is using the id is a good idea ? might be too specific to coraza and not easily reusable
var AppsecRulesDetails = make(map[int]RulesDetails)
// matchPattern reports whether appsecRule.Name matches pattern.
func matchPattern(pattern, name string, logger *log.Entry) bool {
tmpMatch, err := exprhelpers.Match(pattern, name)
if err != nil {
logger.Errorf("unable to match %s with %s : %s", name, pattern, err)
return false
}
matched, ok := tmpMatch.(bool)
if !ok {
logger.Errorf("unable to match %s with %s : result is not a boolean", name, pattern)
return false
}
return matched
}
// loadNativeRulesFromFile reads a single seclang rules file and returns its non-comment, non-empty lines.
func loadNativeRulesFromFile(fullPath string, logger *log.Entry) []string {
c, err := os.ReadFile(fullPath)
if err != nil {
logger.Errorf("unable to read file %s : %s", fullPath, err)
return nil
}
rules := make([]string, 0)
for line := range strings.SplitSeq(string(c), "\n") {
if strings.HasPrefix(line, "#") {
continue
}
if strings.TrimSpace(line) == "" {
continue
}
rules = append(rules, line)
}
return rules
}
// loadSecLangFilesRules resolves the seclang file globs and appends their rules to appsecCol.
func loadSecLangFilesRules(appsecRule AppsecCollectionConfig, hub *cwhub.Hub, logger *log.Entry, appsecCol *AppsecCollection) {
for _, rulesFile := range appsecRule.SecLangFilesRules {
logger.Debugf("Adding rules from %s", rulesFile)
globPattern := filepath.Join(hub.GetDataDir(), rulesFile)
matches, err := filepath.Glob(globPattern)
if err != nil {
logger.Errorf("unable to glob %s : %s", rulesFile, err)
continue
}
if len(matches) == 0 {
logger.Warnf("no file matched pattern %s", globPattern)
continue
}
for _, fullPath := range matches {
appsecCol.NativeRules = append(appsecCol.NativeRules, loadNativeRulesFromFile(fullPath, logger)...)
}
}
}
// registerRuleDetails records the metadata and debug flags for a converted custom rule.
func registerRuleDetails(appsecRule AppsecCollectionConfig, ruleName string, rulesId []uint32, logger *log.Entry) {
// We only take the first id, as it's the one of the "main" rule
if _, ok := AppsecRulesDetails[int(rulesId[0])]; !ok {
AppsecRulesDetails[int(rulesId[0])] = RulesDetails{
LogLevel: log.InfoLevel,
Hash: appsecRule.hash,
Version: appsecRule.version,
Name: appsecRule.Name,
}
} else {
logger.Warnf("conflicting id %d for rule %s !", rulesId[0], ruleName)
}
for _, id := range rulesId {
SetRuleDebug(int(id), appsecRule.Debug)
}
}
// loadCustomRules converts the YAML custom rules and appends the resulting seclang rules to appsecCol.
func loadCustomRules(appsecRule AppsecCollectionConfig, logger *log.Entry, appsecCol *AppsecCollection) error {
for _, rule := range appsecRule.Rules {
rule.Severity = appsecRule.Severity
strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name, appsecRule.Description)
if err != nil {
logger.Errorf("unable to convert rule %s : %s", appsecRule.Name, err)
return err
}
logger.Debugf("Adding rule %s", strRule)
appsecCol.Rules = append(appsecCol.Rules, strRule)
registerRuleDetails(appsecRule, rule.Name, rulesId, logger)
}
return nil
}
// initRuleData initializes the data files referenced by an appsec rule.
func initRuleData(appsecRule AppsecCollectionConfig, hub *cwhub.Hub, logger *log.Entry) {
for _, appsecRuleData := range appsecRule.Data {
if appsecRuleData.DestPath == "" {
logger.Errorf("missing dest_path for rule %s : %+v", appsecRule.Name, appsecRuleData)
continue
}
if err := exprhelpers.FileInit(hub.GetDataDir(), appsecRuleData.DestPath, appsecRuleData.Type); err != nil {
logger.Errorf("unable to initialize data file %s : %s", appsecRuleData.DestPath, err)
continue
}
}
}
func LoadCollection(pattern string, logger *log.Entry, hub *cwhub.Hub) ([]AppsecCollection, error) {
ret := make([]AppsecCollection, 0)
for _, appsecRule := range appsecRules {
tmpMatch, err := exprhelpers.Match(pattern, appsecRule.Name)
if err != nil {
logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err)
continue
}
matched, ok := tmpMatch.(bool)
if !ok {
logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err)
continue
}
if !matched {
if !matchPattern(pattern, appsecRule.Name, logger) {
continue
}
appsecCol := AppsecCollection{}
if appsecRule.SecLangFilesRules != nil {
for _, rulesFile := range appsecRule.SecLangFilesRules {
logger.Debugf("Adding rules from %s", rulesFile)
globPattern := filepath.Join(hub.GetDataDir(), rulesFile)
matches, err := filepath.Glob(globPattern)
if err != nil {
logger.Errorf("unable to glob %s : %s", rulesFile, err)
continue
}
if len(matches) == 0 {
logger.Warnf("no file matched pattern %s", globPattern)
continue
}
for _, fullPath := range matches {
c, err := os.ReadFile(fullPath)
if err != nil {
logger.Errorf("unable to read file %s : %s", rulesFile, err)
continue
}
for line := range strings.SplitSeq(string(c), "\n") {
if strings.HasPrefix(line, "#") {
continue
}
if strings.TrimSpace(line) == "" {
continue
}
appsecCol.NativeRules = append(appsecCol.NativeRules, line)
}
}
}
}
loadSecLangFilesRules(appsecRule, hub, logger, &appsecCol)
if appsecRule.SecLangRules != nil {
logger.Tracef("Adding inline rules %+v", appsecRule.SecLangRules)
appsecCol.NativeRules = append(appsecCol.NativeRules, appsecRule.SecLangRules...)
}
if appsecRule.Rules != nil {
for _, rule := range appsecRule.Rules {
rule.Severity = appsecRule.Severity
strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name, appsecRule.Description)
if err != nil {
logger.Errorf("unable to convert rule %s : %s", appsecRule.Name, err)
return nil, err
}
logger.Debugf("Adding rule %s", strRule)
appsecCol.Rules = append(appsecCol.Rules, strRule)
// We only take the first id, as it's the one of the "main" rule
if _, ok := AppsecRulesDetails[int(rulesId[0])]; !ok {
AppsecRulesDetails[int(rulesId[0])] = RulesDetails{
LogLevel: log.InfoLevel,
Hash: appsecRule.hash,
Version: appsecRule.version,
Name: appsecRule.Name,
}
} else {
logger.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name)
}
for _, id := range rulesId {
SetRuleDebug(int(id), appsecRule.Debug)
}
}
if err := loadCustomRules(appsecRule, logger, &appsecCol); err != nil {
return nil, err
}
initRuleData(appsecRule, hub, logger)
ret = append(ret, appsecCol)
}
+888
View File
@@ -0,0 +1,888 @@
# WAF Bot Detection / Challenge — Design, Review, Usage
This document covers the CrowdSec WAF challenge subsystem: how it is built, what
reviewers should look for when touching it, and how operators / collection
authors are expected to use it.
The implementation lives under [pkg/appsec/challenge](.) and is wired into the
acquisition pipeline by [pkg/acquisition/modules/appsec](../../acquisition/modules/appsec).
---
## 1. Design
### 1.1 What the feature does
The challenge subsystem lets the WAF interpose a browser-side proof-of-work +
device-fingerprint check between a visitor and the protected origin. A visitor
who solves the challenge receives a sealed cookie carrying their measured
fingerprint; subsequent requests from the same browser can be evaluated by
operator-defined hooks (`on_challenge`, `on_challenge_submit`) without
re-challenging.
The challenge runtime is built **lazily**: it only spins up if at least one
loaded hook calls `SendChallenge()`, `GrantChallengeCookie()`, or
`RejectSubmission()`. This is detected by [`appsecExprPatcher`](../patcher.go)
walking the compiled expression AST and setting
`AppsecRuntimeConfig.NeedWASMVM`.
### 1.2 Component map
```
┌──────────────────────────────────────────────────────────────────────┐
│ Acquisition module (pkg/acquisition/modules/appsec/config.go) │
│ - HTTP listener, ParsedRequest plumbing │
│ - Loads appsec-config(s), builds AppsecRuntimeConfig │
│ - If NeedWASMVM: challenge.BuildOptions → NewChallengeRuntime │
└──────────────────────────────────────────────────────────────────────┘
│
▼
┌──────────────────────────────────────────────────────────────────────┐
│ pkg/appsec (AppsecConfig / AppsecRuntimeConfig) │
│ - YAML schema, hook compilation, request-lifecycle dispatcher │
│ - ProcessOnChallengeRules: cookie validation, /submit handling │
│ - SendChallenge / GrantChallengeCookie / RejectSubmission helpers │
└──────────────────────────────────────────────────────────────────────┘
│
▼
┌──────────────────────────────────────────────────────────────────────┐
│ pkg/appsec/challenge (ChallengeRuntime) │
│ - KeyRing: HKDF-derived per-epoch keys + long-lived cookie key │
│ - Crypto: AES-GCM cookie seal/open, per-challenge + PoW MAC HMACs │
│ - Obfuscator: wazero-hosted WASM running javascript-obfuscator │
│ - Challenge code: single static build-time obfuscated variant │
│ - Dynamic module pool: per-epoch key script, obfuscated variants │
│ - Fingerprint mismatch report aggregator │
└──────────────────────────────────────────────────────────────────────┘
```
### 1.3 Configuration model
Challenge tuning lives **inside the appsec-config**. The relevant fields are on [`challenge.Config`](config.go), referenced
from `AppsecConfig.Challenge`:
```go
type Config struct {
MasterSecret *string
KeyRotationInterval *time.Duration
MaxLiveEpochs *int
CookieTTL *time.Duration
CryptoObfuscationPoolSize *int
}
```
`challenge.BuildOptions(c *Config) ([]Option, error)` translates the merged
config into the option list consumed by `NewChallengeRuntime`.
### 1.4 Cryptographic core
**Notation.** These names are used consistently throughout this doc, the wire
protocol, and the code. Note the two distinct "nonces": the **challenge nonce
`r`** and the **PoW nonce `n`** (the AES-GCM nonce inside the cookie envelope is
a third, unrelated value — written `gcm_nonce`).
| Symbol | Name | On the wire? | Definition |
|---|---|---|---|
| `master_secret` | master secret | no (config) | long-lived root secret, ≥ 32 B |
| `K_epoch` | per-epoch key | no (derived) | `HKDF(master_secret, epoch)`; rotates each interval |
| `r` | challenge nonce | yes (`r`) | random 16 B per challenge; identifies it for single-use |
| `ts` | timestamp | yes (`ts`) | `time.Now().UnixNano()`; selects the epoch |
| `s` | per-challenge secret | **no** | `HMAC(K_epoch, r)`; signs the submission |
| `p` | PoW salt | yes (`p`) | random 16 B the client hashes for the PoW |
| `n` | PoW nonce | yes (`n`) | PoW solution — makes `SHA256(p‖n)` have ≥ N leading zero bits |
| `m` | PoW MAC | yes (`m`) | `HMAC(K_epoch, p‖r‖ts)`; proves `p` is server-issued |
| `f` | fingerprint payload | yes (`f`) | obfuscated + base64 device fingerprint |
| `encKey` | fingerprint obfuscation key | **no** | `HMAC(s, "fpenc"‖r)` |
| `sig` | submission signature | yes (`sig`) | `HMAC(s, r‖ts‖n‖f)` |
The submit POST therefore carries exactly `f, r, ts, sig, n, p, m` (§1.6); `s`
and `encKey` are derived on both ends and never transmitted.
**At a glance** — every cryptographic operation (all keys are 256-bit; every
MAC/signature comparison uses constant-time `hmac.Equal`):
| Operation | Primitive | Key | Input → output |
|---|---|---|---|
| Derive operational keys | HKDF-SHA256 (salt `crowdsec-challenge-keyring-v1`) | `master_secret` | info `epoch-sign:<epoch>` → `K_epoch`; info `cookie-master` → master cookie key |
| Per-challenge secret `s` | HMAC-SHA256 | `K_epoch` | `r` → `s` |
| PoW MAC `m` | HMAC-SHA256 | `K_epoch` | `p‖r‖ts` → `m` |
| Proof-of-work | SHA-256 | — | `p‖n` accepted iff ≥ N leading zero bits |
| Fingerprint obf. key `encKey` | HMAC-SHA256 | `s` | `"fpenc"‖r` → `encKey` |
| Fingerprint payload `f` | repeating-key XOR + base64 | `encKey` | fingerprint JSON → `f` (light obfuscation, **not** confidentiality — §2.6) |
| Submission signature `sig` | HMAC-SHA256 | `s` | `r‖ts‖n‖f` → `sig` |
| Cookie seal / open | AES-256-GCM | HKDF-derived from master cookie key | proto envelope, AAD = `User-Agent` → sealed cookie |
The subsections below detail each; the validation order is in §1.6.
#### Master secret
- Configured as hex (preferred — even-length, hex digits only) or as a
passphrase. Both forms must be at least **32 bytes** decoded — the natural
minimum for HMAC-SHA256 with full security margin.
- If unconfigured, [`generateRandomSecret`](secret.go) produces a fresh 32-byte
secret and the runtime logs a warning. Suitable only for single-instance
deployments: a restart invalidates outstanding cookies, and distributed
WAFs can't agree on the same value.
#### KeyRing — per-epoch derivation
[`pkg/appsec/challenge/keyring.go`](keyring.go) derives two independent key
families from the master secret via HKDF-SHA256 (salt
`"crowdsec-challenge-keyring-v1"`):
| Family | HKDF info | Lifetime | Use |
|---|---|---|---|
| Per-epoch key (`K_epoch`) | `"epoch-sign"` | One rotation interval | Per-challenge secret `s` + PoW MAC `m` |
| Master cookie key | `"cookie-master"` | Lifetime of master secret | AES-GCM seal/open of challenge cookies |
An **epoch** is a time bucket of `rotationInterval` seconds, computed as
`now.Unix() / rotation_interval_seconds`. All instances sharing the same
master secret + rotation interval derive bit-identical epoch keys with no
coordination — the basis for distributed verification of the per-challenge
crypto (replay protection is separately stateful; see §2.1).
The keyring keeps a sliding window of `MaxLiveEpochs` past epochs plus the
current epoch plus a one-step clock-skew slack. Submissions signed by any key
inside the live window verify; older ones are rejected. Cached entries
outside the window are evicted whenever a new epoch is derived.
Defaults: `keyringDefaultRotation = 5 minutes`, `keyringDefaultMaxLive = 3`,
`keyringMinRotation = 30 seconds`.
#### Cookie envelope (v0)
[`pkg/appsec/challenge/crypto.go`](crypto.go) seals cookies under the
long-lived master cookie key with AES-256-GCM:
```
wire format: version_byte(1) || gcm_nonce(12) || ciphertext
plaintext : not_after_be8(8) || flags(1) || reason_len_be(2)
|| reason_bytes(0..256)
|| protobuf_envelope
AAD : user-agent bytes
```
Three things to note:
1. **Cookie expiry is enforced by the embedded `not_after` timestamp**, not
by keyring eviction. This intentional decouples cookie TTL (operator
policy, e.g. 12h–24h) from the keyring live window (the window in which an
extracted per-epoch key stays usable, e.g. 15m). Long-lived cookies don't
widen that signing window.
2. **User-Agent is the GCM authentication tag.** A cookie cannot be replayed
across UAs — `openCookie` fails authentication on mismatch.
3. **Allowlist cookies** set `cookieFlagAllowlisted = 0x01` and carry an
operator reason (`MaxAllowlistReasonLen = 256` bytes — bounded to keep the
final cookie comfortably under the browser 4 KB limit). Real-submission
cookies leave the flag clear and the reason empty.
#### Per-challenge secret, PoW MAC, and single-use
Each challenge render carries a fresh 16-byte challenge nonce `r`. From it the
client derives the **per-challenge secret** `s`, which is *never transmitted*:
- `s = HMAC(K_epoch, r)` — the per-challenge secret. Both client and server
derive it independently (the client from the `K_epoch` delivered in the
obfuscated dynamic module; the server from the keyring). It never appears on
the wire.
- `m = HMAC(K_epoch, p || r || ts)` — the PoW MAC; authenticates the PoW salt
`p` as server-issued and bound to this challenge, so a client can't pick a
favourable salt.
The submission is signed `sig = HMAC(s, r || ts || n || f)`. A valid `sig` proves
the submitter knows `K_epoch` — normally by running our obfuscated JS to read it
from the dynamic module, though a determined attacker can extract it (§2.6). See
[ticket.go](ticket.go) (`deriveChallengeSecret`, `computePowMAC`,
`verifyChallenge`).
`ts` is `time.Now().UnixNano()` formatted as a string; the verifier derives the
epoch from `ts` itself and looks up the same key. **Challenge *issuance* is
stateless** (any instance with the same master secret can render and verify the
crypto), but **submission is single-use**: the validated `r` is burned in an
in-memory spent-set ([spent_set.go](spent_set.go)), so a captured submission
cannot be replayed against that instance. This is the load-bearing replay
control; the per-epoch key obfuscation is only a non-load-bearing speed-bump
(§2.6). The spent-set is **per-instance**, so single-use assumes all reverse
proxies route to one appsec instance — see §2.1 / §3.4.
Freshness is doubly gated: keyring liveness (epoch in live window) and a loose
backstop of `ticketAgeBackstop = 20 minutes` — which also bounds the spent-set
TTL.
#### Single-use tickets
[`spent_set.go`](spent_set.go) records consumed challenge nonces (`r`) to
enforce single-use. It wraps **gcache** (already used by `pkg/cache`): a bounded
LRU (`spent_set_max_entries`, default `spentSetDefaultMaxEntries`) with a per-entry TTL equal to the freshness window,
so there is no janitor goroutine or hand-rolled cap. A single mutex makes the
`Has`/`SetWithExpire` pair in `checkAndInsert` atomic, so two concurrent
identical submissions can't both win. The burn happens **last** in
`ValidateChallengeResponse` (after freshness, PoW-MAC, PoW, and `sig` all pass),
so the set only ever grows for crypto+PoW-valid submissions — its growth is
sig+PoW-gated and TTL-bounded.
PoW levels (in leading SHA-256 zero bits) live as constants in
[`ticket.go`](ticket.go):
| Level | Bits | Approx work |
|---|---|---|
| `Disabled` | 0 | nonce `"0"` always valid |
| `Low` | 10 | ~1k hashes, 0.2–2 s |
| `Medium` (default) | 12 | ~4k hashes, 1–8 s |
| `High` | 15 | ~32k hashes, 7–60 s |
| `Impossible` | 256 | unsolvable — hard block |
### 1.5 JS obfuscation pipeline
The browser receives four logical pieces of JS at every challenge:
1. **Obfuscated challenge code** — the small crypto/glue bundle (SHA-256, HMAC,
PoW orchestration, fingerprint XOR, submission, hook registration). Injected
**inline** on the challenge page. A single static build-time obfuscated
variant (no runtime pool — see below).
2. **Dynamic key module** — a ~30-line script embedding the current epoch's key
`K_epoch` as a hex literal, so the client can derive the per-challenge secret
`s = HMAC(K_epoch, r)` and sign the submission. Injected inline.
3. **fpscanner bundle** — the public `fpscanner` device-fingerprint scanner,
served **unobfuscated** as its own file at
`/crowdsec-internal/challenge/fpscanner.js` and loaded via a plain
`<script src>` tag. It assigns `globalThis.CrowdsecFingerprintScanner`; the
challenge code reads that global at use time. Public, non-sensitive code —
cacheable.
4. **PoW worker** — separate JS served at `/crowdsec-internal/challenge/pow-worker.js`.
#### The obfuscator itself
A pre-built `javascript-obfuscator` distribution compiled to WASM lives at
[`js/obfuscate/index.wasm.gz`](js/obfuscate) (regeneration is wired through
`go:generate` in [`js/generate.go`](js/generate.go)). At runtime
[`obfuscator.go`](obfuscator.go) hosts it under [wazero](https://wazero.io):
- The WASM module is **decompressed and compiled once** at startup
(`CompileModule` ~4–5 s); per-call cost is just `InstantiateModule`.
- Each obfuscation pass produces byte-distinct output even for the same input
(mangled identifier names, random transforms).
- A pass on the **challenge code** is cheap (~5 s of CPU) — it is just the small
crypto/glue bundle; the bulk of the old library (fpscanner) is no longer
obfuscated at all. A pass on the **dynamic module** also takes ~5 s.
#### Challenge code (single static variant)
The challenge crypto/glue code is **always** obfuscated, but it is a **single
static build-time variant** — there is no runtime pool or refresher for it
anymore. The build pipeline splits the two concerns:
- [`js/cmd/bundle`](js/cmd/bundle) emits two outputs: `fpscanner/fpscanner.js`
(the minified, **unobfuscated** fpscanner IIFE, embedded as
`challengejs.FPScannerJS` / re-exported as `challenge.FPScannerJS` and served
raw at `/crowdsec-internal/challenge/fpscanner.js`) and `challenge_code.js`
(the obfuscation input).
- [`js/cmd/initialbundle`](js/cmd/initialbundle) substitutes the path
placeholders into `challenge_code.js` and obfuscates it into
`../initial_bundle.js.gz`, embedded into the binary.
On startup [`seedCacheFromInitialBundle()`](static_bundle.go) loads that single
baked-in variant into the `challengeCode string` field on `ChallengeRuntime`;
every visitor receives it inline via [`getChallengeCode()`](static_bundle.go).
There is no per-visitor byte variance on the challenge code, and the fpscanner
bundle (the public, bulky part) is served as-is.
#### Dynamic module pool
[`dynamic_module.go`](dynamic_module.go) renders a small template
([`dynamic_module.js.tmpl`](dynamic_module.js.tmpl)) with the current epoch's
key + epoch number, obfuscates it, and caches `crypto_obfuscation_pool_size`
variants per live epoch.
Two non-obvious mechanisms keep this fast:
- **`singleflight` deduplication**: if N concurrent requests arrive right
after a rotation, only one runs the obfuscator per variant slot; the others
block on its result. Worst-case latency at the rotation boundary is one
obfuscation pass, not N serialized passes.
- **Pre-warming**: a goroutine fires `1/4 × rotation_interval` before each
boundary (capped 1–30 s) and obfuscates the full pool for the upcoming
epoch. The first request after rotation finds the cache populated.
The obfuscator config (in [`js/obfuscate/obfuscate.js`](js/obfuscate))
reserves the hook symbol `__CSEC_CHALLENGE_HOOK_v1__` so the challenge code and
the dynamic module can find each other by name across independent obfuscation
passes. The key material itself is **not** reserved — it's deliberately
encoded by the obfuscator's string-array transforms.
### 1.6 Request lifecycle
The acquisition module's `appsecHandler` parses every incoming HTTP request
into a `ParsedRequest` and pushes it into the runner pool. Each runner calls
`AppsecRunner.ProcessInBandRules`, which **first** invokes
`AppsecRuntimeConfig.ProcessOnChallengeRules` and only proceeds to WAF
evaluation if no challenge response was assembled.
`ProcessOnChallengeRules` dispatches three cases:
| Path | Method | Handling |
|---|---|---|
| `/crowdsec-internal/challenge/pow-worker.js` | GET | Static `PowWorkerJS` body served |
| `/crowdsec-internal/challenge/fpscanner.js` | GET | Static (unobfuscated) `FPScannerJS` body served |
| `/crowdsec-internal/challenge/submit` | POST | `ValidateChallengeResponse` → single-use burn + fingerprint de-obfuscate → `on_challenge_submit` hooks → `bodyChallengeOK` + Set-Cookie *or* `bodyChallengeRejected` |
| anything else | any | Existing `__crowdsec_challenge` cookie validated → `state.Fingerprint` populated → `on_challenge` hooks |
For a submission, `ValidateChallengeResponse`:
1. Parses the form (`f, r, ts, sig, n, p, m, d` — obfuscated fingerprint,
challenge nonce, timestamp, submission signature, PoW nonce, PoW salt,
PoW MAC, and the issued PoW difficulty).
2. Calls `verifyChallenge` to gate freshness, derive the epoch from `ts`, look
up the per-epoch key, and verify `m == HMAC(K_epoch, p || r || ts || d)`.
Binding `d` into the MAC authenticates the difficulty: a client cannot
downgrade an escalated per-request challenge without invalidating `m`.
3. Rejects outright if the (now-authenticated) difficulty is `Impossible`.
4. Verifies the PoW: `hasLeadingZeroBits(SHA-256(p || n), d)` — using the
issued per-request difficulty, not the runtime-global default.
5. Derives `s = HMAC(K_epoch, r)` (never transmitted) and verifies
`sig == HMAC(s, r || ts || n || f)`.
6. Burns `r` in the spent-set — **single-use**; a replay is rejected here.
7. De-obfuscates `f` (repeating-key XOR keyed by `HMAC(s, "fpenc" || r)`) and
JSON-unmarshals into `FingerprintData`.
8. Seals a v0 cookie carrying that fingerprint **and the proven difficulty `d`**
under the long-lived master cookie key with a fresh `not_after`, so the next
request's re-challenge check compares against the work actually performed.
The challenge page itself (`GetChallengePage`) builds the response by:
1. Resolving difficulty (per-request override beats runtime default).
2. Reading the single static obfuscated challenge code and picking a random
dynamic module variant (the unobfuscated fpscanner bundle is referenced by
`<script src>`, not inlined).
3. Generating `ts`, the challenge nonce `r`, the PoW salt `p`, and the PoW MAC `m`.
4. Rendering [`challenge.html.tmpl`](challenge.html.tmpl) with all of the
above.
### 1.7 Hook surface
Six hook stages, two of which are challenge-specific:
| Hook | When | Env | Notable helpers |
|---|---|---|---|
| `on_load` | Engine startup, once | runtime config | `SetChallengeDifficulty` (sets runtime default) |
| `pre_eval` | Before WAF | request + state | `SendChallenge`, `GrantChallengeCookie(reason, ttl?)`, `SetChallengeDifficulty` (per-req) |
| `post_eval` | After WAF | request + state | same as `pre_eval` |
| `on_match` | Per-rule match | event + state | `SetRemediation`, `SetReturnCode` |
| `on_challenge` | Valid cookie/submission, fingerprint available | fingerprint + state | `SendChallenge`, `SetChallengeDifficulty`, `EvaluateMismatches`, `fingerprint.*` |
| `on_challenge_submit` | POST `/submit` after crypto validation | fingerprint + state | `RejectSubmission`, `GrantChallengeCookie(reason, ttl?)` (inline), `EvaluateMismatches` |
`on_challenge` is skipped if `state.Fingerprint` is nil (no valid cookie or
submission). `on_challenge_submit` is deliberately narrow: only helpers that
preserve the challenge-submit JSON envelope shape are exposed — a 307 redirect
from this phase would break the client JS state machine.
### 1.8 Fingerprint mismatch report
[`fingerprint_mismatch.go`](fingerprint_mismatch.go) aggregates two families
of signals into a `MismatchReport`:
**Library-native** (from `fpscanner`):
CDP, WebDriver flavours, Selenium, Playwright, headless Chrome, impossible
memory, high CPU count, WebGL/GPU/platform mismatches, bot user-agent —
mostly *high* severity. UTC timezone is *medium*; swiftshader and language
mismatches are *low*.
**Custom helpers**:
- `UAMobileMismatch()` — UA claims mobile but inner viewport ≥ 1024 px
(catches desktop UA switchers). Medium.
- `AcceptLanguageMismatch(req)` — request `Accept-Language` base differs from
`navigator.language` base. Medium.
- `TimezoneCountryMismatch(country)` — timezone not valid for GeoIP-derived
country. Low (travellers / VPNs legitimately trigger it).
`EvaluateMismatches()` is the hook-side entry point; it caches the result on
`state.LastMismatchReport` so chained calls like
`EvaluateMismatches().High() >= 1 && EvaluateMismatches().Has("cdp")` don't
recompute. First call also emits a debug log line and bumps per-signal
Prometheus counters.
### 1.9 Allowlist bypass
`GrantChallengeCookie(reason, ttl?)` is the operator escape hatch for trusted
clients (Googlebot, internal probes, paying partners…). It mints a sealed
cookie with the `cookieFlagAllowlisted` bit set and the operator reason
embedded, stamps a synthetic `Allowlisted` fingerprint on `state.Fingerprint`,
and flips `state.ChallengeBypassed` so any later `SendChallenge` in the same
request is a no-op.
Two variants, both exposed under the same name to the operator:
- In `pre_eval` / `post_eval`: issues HTTP 307 to the same URI with the
cookie attached. The visitor bounces back through the WAF with the cookie
present and the next iteration short-circuits on the allowlist branch.
Required because plain `AllowRemediation` responses don't carry cookies
through the bouncer protocol; only `ChallengeRemediation` does.
- In `on_challenge_submit`: appends the cookie to the existing
`{"status":"ok"}` JSON envelope.
The optional `ttl` argument is parsed via `time.ParseDuration` and overrides
the runtime-global `cookie_ttl` for that single cookie only. Empty / missing
falls back to the runtime default; malformed / non-positive surfaces as an
expression error so authors see a precise diagnostic.
### 1.10 Decision logging
Accept / reject decisions go through `FingerprintData.LogAccepted` /
`LogRejected` in [`fingerprint_helpers.go`](fingerprint_helpers.go).
Both are nil-safe; the caller picks the log level (Debug for
per-request sites, Info for rarer / operator-authored events).
The asymmetry to remember when reading these helpers: **accept logs
always emit the baseline flags** (`is_bot`, `allowlisted`) so a clean
acceptance is explicit, while **reject logs emit only positive
information** — `"headless": false` is noise on a forensic artefact.
`signals` is enumerated by walking `libDetections`, so it cannot
drift from `MismatchReport.Reasons()`.
For submit-phase events the verbosity is operator-controlled via the
expr helpers `RejectSubmission(reason, verbosity?)` and
`LogAccepted(msg, verbosity?)` (see §3.2); both helpers log at Info,
and the verbosity argument tunes only the field-set richness, not the
level. `ValidateChallengeResponse` in `challenge.go` is deliberately
silent on the success path — the caller in `appsec.go` (and the expr
wrappers in `waf_helpers.go`) own the `*log.Entry`, so they own the
accept / reject log.
For the field reference and example output, see §3.1.
---
## 2. Review guidelines
Things to actively look for when reviewing changes that touch this subsystem.
### 2.1 Invariants that must not break
- **Stateless *crypto*, stateful *single-use*.** All per-challenge / PoW-MAC /
cookie crypto must remain derivable from the master secret + epoch + request data
(no node-local randomness baked into verifiable material), so any instance can
verify any other's signatures. **However, replay protection is intentionally
stateful**: the challenge nonce `r` is burned in an in-memory spent-set
([spent_set.go](spent_set.go)). The design assumes **all web servers talk to a
single appsec instance** — single-use holds within that instance. A
multi-instance fleet without a shared spent-set would not enforce fleet-wide
single-use (a captured submission could replay against a *different* instance);
that is accepted by assumption. Cookie sealing/validation stays stateless under
the master cookie key, so only the brief challenge *handshake* is instance-
sticky — the 12h cookie validates on any instance.
- **`master_secret` policy.** A misconfigured master_secret must FAIL the
config load — it must never silently fall back to a random secret when the
operator clearly intended to set one. The "random secret" path is reserved
for the *no `master_secret` at all* case, and must always emit the existing
warning. [`BuildOptions`](config.go) bubbles `ParseConfiguredSecret` errors
for this reason.
- **Secret-logging discipline.** `master_secret` is never logged. `k_epoch` /
`s` may be logged **only at debug**, behind an `IsLevelEnabled(debug)` guard
so they aren't formatted at higher levels (§1.11). A stray `Info`/`Warn`
carrying `k_epoch` is a key leak — keep the guard.
- **Cookie / keyring decoupling.** Cookie TTL is enforced inside the sealed
envelope (`not_after`), not by keyring eviction. Don't tie cookie validity
to live-epoch membership — that would force cookie TTL ≤ keyring window,
which is the opposite of what the design wants.
- **User-Agent AAD.** Removing the UA AAD from `sealCookieV0` / `openCookie`
breaks cookie-binding-to-browser. If a change wants per-UA tolerance,
introduce a *separate* validation path; don't loosen the GCM tag.
- **`NeedWASMVM` detection coverage.** Any new expr helper that needs the
challenge runtime must be added to `challengeRuntimeCallees` in
[`patcher.go`](../patcher.go). Forgetting this lets a hook reference a
helper without spinning the runtime up at all — `nil` deref at request
time.
- **`state.ChallengeBypassed` honor.** Helpers that produce a challenge
response (`SendChallenge`, future variants) must early-return when
`ChallengeBypassed` is set. Two `Set-Cookie`s and a redirect with a body
is incoherent for the bouncer protocol.
- **`on_challenge_submit` narrowness.** The submit-phase env intentionally
omits `SendChallenge`, `SetRemediation`, `SetReturnCode`,
`SetChallengeDifficulty`, `DropRequest`. Adding any of these here will
silently break the client JS state machine. If you need this behaviour,
set state and let the *next* request's `pre_eval` act on it.
### 2.2 Concurrency hazards
- **Dynamic module pool**: writes go through `singleflight` and a mutex.
Don't bypass either. Walking `dynamicModuleCache` without holding
`dynamicModuleCacheMu` is a race; mutating it inside a
`singleflight.Do` callback is fine.
- **Challenge code**: held in a single immutable `challengeCode string` field,
seeded once at startup by `seedCacheFromInitialBundle()` and read via
`getChallengeCode()`. There is no pool, mutex, or refresher to coordinate —
it is set-once and read-only thereafter.
- **Pre-warmer lifetime**: the pre-warmer reads from the keyring and writes
into the dynamic module cache. It must observe context cancellation. If
you add a new goroutine, plumb the same context.
- **Obfuscator instantiation**: `Runtime.InstantiateModule` is safe to call
concurrently against the same `CompiledModule`; the actual JS execution
isn't a concern because each call gets its own module instance.
### 2.3 Performance and resource shape
- **Compile-once, instantiate-per-call** is the established WASM pattern.
If a change moves WASM compilation onto the request path, that adds ~4–5
seconds of CPU per challenge served.
- **Pool sizes have CPU cost**, not just memory cost. Each
`crypto_obfuscation_pool_size` increment is one extra ~5 s obfuscation
pass per epoch rotation. The challenge code is obfuscated once at build
time, so it adds no runtime obfuscation cost at all.
- **Pre-warm lead time** is `1/4 × rotation_interval`, capped at 30 s and
floored at 1 s. Shrinking the rotation interval below ~20 s makes the
pre-warmer too tight for the obfuscation pass; the test
`TestCryptoObfuscationPoolSize` in
[obfuscation_pools_test.go](obfuscation_pools_test.go) is currently
flaky around this boundary and is unrelated to most config changes.
### 2.4 Config and merge semantics
- **`challenge.Config` fields must stay pointer-typed.** Switching a field
to a value type breaks the multi-config merge — `MergeFrom` relies on
`nil` to mean "unset, keep dst's value". A `bool` value type would always
look set to `false`.
- **`LoadByPath` merge order is "last wins" per field.** Tests asserting
this live in `pkg/appsec/appsec_config_test.go::TestLoadByPathChallengeBlockMergesAcrossFiles`.
Don't switch to first-wins or error-on-conflict without an explicit
design decision — collection composability depends on it.
- **YAML strict parsing.** [`Source.UnmarshalConfig`](../../acquisition/modules/appsec/config.go)
uses `yaml.Strict()`. This is the desired loud-failure surface for stale
`challenge_*` keys in acquisition YAML left over from earlier branches.
Don't loosen it.
### 2.5 Test hooks
Targeted tests live alongside the code:
- [`config_test.go`](config_test.go) — `MergeFrom`, `BuildOptions`,
per-call cookie TTL override.
- [`challenge_test.go`](challenge_test.go) — leading-zero-bits PoW check,
end-to-end render/submit.
- [`keyring_test.go`](keyring_test.go) + [`keyring_integration_test.go`](keyring_integration_test.go)
— epoch math, key derivation determinism, liveness window.
- [`secret_test.go`](secret_test.go) — hex vs passphrase, min-size guard.
- [`initial_bundle_test.go`](initial_bundle_test.go), [`split_bundle_test.go`](split_bundle_test.go)
— baked-in bundle integrity. `initial_bundle_test.go` also asserts fpscanner
is **not** present in the obfuscated challenge bundle and that `FPScannerJS`
exposes the `globalThis.CrowdsecFingerprintScanner` global.
- [`obfuscation_pools_test.go`](obfuscation_pools_test.go) — asserts the
challenge code is seeded **statically** (single build-time variant, no
runtime pool) and exercises the crypto/dynamic-module pool behaviour (note:
`TestCryptoObfuscationPoolSize` is flaky; it depends on the JS obfuscator
producing byte-distinct output across passes).
- [`pkg/appsec/appsec_challenge_test.go`](../appsec_challenge_test.go) —
`GrantChallengeCookie` 307 path, inline submit variant,
`state.ChallengeBypassed` guard, allowlist flag propagation.
- [`pkg/appsec/waf_helpers_test.go`](../waf_helpers_test.go) — TTL
argument parsing.
Run subsets when iterating:
```
go test ./pkg/appsec/challenge/ -run 'TestConfig|TestBuildOptions|TestSealAllowlistCookieTTLOverride'
go test ./pkg/appsec/ -run 'TestGrantChallengeCookie|TestProcessOnChallengeRules'
```
The combined `./pkg/appsec/... ./pkg/acquisition/modules/appsec/...` run
hits ~10 min wall clock because of the obfuscation passes; expect to time
out unless you split by package.
### 2.6 Things that look wrong but aren't
- `GrantChallengeCookie` issues a **307** even though it semantically
*allows* the visitor. This is because the bouncer protocol only
serialises `UserCookies` on a `ChallengeRemediation` response; a plain
`AllowRemediation` would drop the `Set-Cookie`. The 307 is the
minimal-friction way to deliver the cookie *and* bounce the visitor
through the WAF a second time so the allowlist branch hits.
- `mintAllowlistCookie` deliberately **overwrites any prior `state.Fingerprint`**
set by the cookie-valid branch. Operator-explicit allowlist beats a
pre-existing real-submission fingerprint. Filter on `req.Headers` *before*
the allowlist rule fires if you want the real signal preserved.
- The dynamic module template embeds the per-epoch key as a hex literal in
plain JS source before obfuscation. That's intentional — the obfuscator's
string-array transforms encode it, and per-epoch rotation bounds the
exposure. Note that with per-challenge single-use tickets the obfuscation is
**no longer load-bearing**: extracting `K_epoch` lets an attacker self-mint
valid submissions *without ever requesting a challenge page* — issuance is
stateless, so the server records no issued `r`, and the attacker can pick a
fresh `r` each time. Each forged submission still costs one PoW solve and is
usable once before the spent-set burns it. The real bound is therefore **one
PoW solve per forged submission, for the lifetime of the live epoch window** —
no worse than the pre-PR ticket scheme. Raising the JS reverse-engineering bar
is tracked separately.
- The fingerprint payload `f` is "encrypted" with a repeating-key XOR
(`deriveFingerprintObfKey` / `deobfuscateFingerprint`). This is **deliberately
light obfuscation, not confidentiality**: a known-plaintext attack recovers
only the per-challenge, single-use key `HMAC(s, "fpenc" || r)`, which is
one-way-derived (`K_epoch → s → obfKey`) and cannot leak `s`/`K_epoch` or
forge `sig`. Transport confidentiality is TLS's job; integrity is `sig`'s.
Don't "upgrade" it to WebCrypto AES-GCM — routing `K_epoch`-derived key
material through `crypto.subtle.*` creates a trivially-hookable key-
interception boundary, which is worse.
- Challenge submission is single-use but issuance is not: an attacker spamming
GET challenge pages costs nothing extra (issuance is stateless). The
spent-set only grows on *valid* submissions (sig+PoW-gated) and entries
expire after the freshness window, so it can't be cheaply flooded.
---
## 3. Intended usage
This section is for collection authors and operators shipping bot
detection.
### 3.1 Minimal collection setup
Install a collection that ships an appsec-config exercising the challenge
hooks:
```sh
cscli collections install crowdsecurity/appsec-bot-detection
```
A complete appsec-config for the feature looks like:
```yaml
name: crowdsecurity/appsec-bot-detection
default_remediation: ban
inband_rules:
- crowdsecurity/base-config
- crowdsecurity/vpatch-*
challenge:
# Required for distributed (multi-WAF) deployments. Hex (preferred) or
# passphrase; min 32 bytes / characters. If unset the runtime generates an
# ephemeral random secret and logs a warning (single-instance only).
master_secret: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
# Optional tuning — defaults shown.
key_rotation_interval: 5m
max_live_epochs: 3
cookie_ttl: 12h
# max_cookie_size: 4096 # encoded cookie size ceiling (seal + open); bounds envelope allocation
crypto_obfuscation_pool_size: 1
# Hooks define WHEN to challenge and WHO gets allowlisted.
inband:
pre_eval:
# Allowlist trusted reverse-resolved Googlebot. Mint a 24h cookie
# explicitly with the optional TTL argument.
- filter: "req.Headers.Get('User-Agent') contains 'Googlebot'"
apply:
- GrantChallengeCookie("Googlebot", "24h")
# Challenge everything else with a measurable fingerprint signal we want
# to inspect on subsequent requests.
- filter: "true"
apply:
- SendChallenge()
on_challenge:
# After a valid submission OR cookie present: escalate to ban if the
# fingerprint exhibits strong bot indicators.
- filter: "EvaluateMismatches().High() >= 1 || EvaluateMismatches().Has('cdp')"
apply:
- SetRemediation("ban")
# Soft escalation: raise difficulty on weak signals.
- filter: "EvaluateMismatches().Medium() >= 2"
apply:
- SetChallengeDifficulty("high")
- SendChallenge()
on_challenge_submit:
# Catch automation that solves the PoW but fails atomic fingerprint
# consistency checks. Refuse to issue a cookie for these and dump
# the full fingerprint detail into the reject log.
- filter: "fingerprint.UAMobileMismatch() || fingerprint.AcceptLanguageMismatch(req)"
apply:
- RejectSubmission("ua_or_lang_mismatch", "verbose")
# Clean submission: log the acceptance with default-verbosity fields
# so operators have an auditable accept event per visitor.
- filter: "true"
apply:
- LogAccepted("challenge submission accepted")
```
#### What the decision logs look like
Every fingerprint accept / reject above produces a structured log
entry via `FingerprintData.LogAccepted` / `LogRejected`. Operators
correlate on `fsid` / `source` and can index these directly.
Examples at the default `"info"` verbosity (what the expr helpers
`LogAccepted` / `RejectSubmission` emit when called without a second
argument):
```
level=info msg="challenge submission accepted" source=203.0.113.7 bouncer=10.0.0.5 fsid=FS1_abc ua="Mozilla/5.0…" platform=macOS is_bot=false signals="[]" allowlisted=false is_mobile=false
level=info msg="on_challenge_submit rejected" source=203.0.113.7 bouncer=10.0.0.5 fsid=FS1_abc reason=ua_or_lang_mismatch ua="Mozilla/5.0…" platform=macOS signals="[cdp]" automation=true
level=info msg="granted allowlist challenge cookie via 307 redirect" source=66.249.66.1 bouncer=10.0.0.5 location=/ ua="Googlebot/2.1" allowlisted=true allowlist_reason=Googlebot signals="[]"
level=debug msg="valid challenge cookie" source=203.0.113.7 bouncer=10.0.0.5 fsid=FS1_abc ua="Mozilla/5.0…" platform=macOS is_bot=false signals="[]" allowlisted=false
```
The verbosity string (`"minimal"`, `"info"` default, `"verbose"`) is
passed as the optional last argument to both `LogAccepted` and
`RejectSubmission`. Bad values emit a warning and fall back to
`"info"`.
**Always present (`"minimal"` verbosity):**
| Field | Type | Notes |
|---|---|---|
| `source` | string | Real visitor IP (`request.ClientIP`, set by the bouncer via X-Forwarded-For or equivalent). Same field name as `fingerprint mismatch` logs for correlation |
| `bouncer` | string | Connection-level peer of the appsec listener (`request.RemoteAddrNormalized`) — the bouncer / gateway that forwarded the request. Useful in multi-WAF setups or when debugging an X-Forwarded-For chain |
| `fsid` | string | Per-fingerprint identifier; omitted when empty |
| `ua` | string | Browser-reported User-Agent; omitted when empty |
| `platform` | string | High-entropy client-hint platform, falling back to `navigator.platform`; omitted when empty |
| `is_bot` | bool (accept) / only-if-true (reject) | Library fast-bot verdict |
| `signals` | `[]string` | Names of library bot signals that fired (e.g. `["cdp", "headless_screen_resolution"]`); empty on clean fingerprints. Enumerated from `libDetections`, so always in sync with `MismatchReport.Reasons()` |
| `allowlisted` | bool (accept) / only-if-true (reject) | True for cookies minted by `GrantChallengeCookie` |
| `allowlist_reason` | string | Present only when `allowlisted == true` and the reason is non-empty |
| `reason` | string | **Reject-only**, always present. The operator-facing cause (e.g. the string passed to `RejectSubmission`) |
**`"info"` verbosity adds** `is_mobile` plus the category roll-ups
(`automation` / `headless` / `mismatch` / `impossible_device`); the
roll-ups only appear when they fired.
**`"verbose"` verbosity adds** `timezone`, `language`, `cpu_count`,
`memory`, `url`, `nonce`, `fp_time`. Zero / empty values are omitted.
Custom mismatches (`UAMobileMismatch`, `AcceptLanguageMismatch`,
`TimezoneCountryMismatch`) are **not** in `signals` — they need
request / geo context and are emitted separately on the `fingerprint
mismatch` line, with their own `reasons` field. Correlate on `fsid` /
`source`.
### 3.2 YAML field reference
#### `challenge:` block
| Key | Type | Default | Notes |
|---|---|---|---|
| `master_secret` | string | random + warning | Hex (preferred) or passphrase, ≥ 32 bytes |
| `key_rotation_interval` | duration | 5m | Min 30 s; must be the same across all instances |
| `max_live_epochs` | int | 3 | Past epochs accepted; sized to cover the freshness window |
| `cookie_ttl` | duration | 12h | Decoupled from rotation interval; can be long (24h+) |
| `max_cookie_size` | int | 4096 | Encoded cookie size ceiling, enforced on seal and open. Bounds the memory allocated from the (attacker-supplied) fingerprint envelope — an over-allocation DoS guard. Matches the per-cookie size browsers guarantee; raise only for non-browser clients that tolerate larger cookies |
| `crypto_obfuscation_pool_size` | int | 1 | Variants of dynamic key module per epoch; each costs ~5 s of CPU per rotation |
| `log_level` | string | inherits | `debug` / `info` / `warning` (etc.) for the challenge runtime only. ⚠️ **`debug` writes the per-epoch key `k_epoch` to the logs** (key rotation + per challenge) — forgeable signing material; a diagnostic mode only, **not for production** (see §1.10). Unset inherits the appsec logger's level. |
#### Hook helpers
| Helper | Available in | Effect |
|---|---|---|
| `SendChallenge()` | `pre_eval`, `post_eval`, `on_challenge` | Serve a challenge page (no-op if visitor already passed an equal-or-harder PoW, or `ChallengeBypassed`) |
| `GrantChallengeCookie(reason, ttl?)` | `pre_eval`, `post_eval` | Mint allowlist cookie + 307 redirect |
| `GrantChallengeCookie(reason, ttl?)` | `on_challenge_submit` | Mint allowlist cookie inline on submit response |
| `RejectSubmission(reason, verbosity?)` | `on_challenge_submit` | Refuse to issue cookie for an otherwise-valid submission; emits an Info reject log immediately. `verbosity ∈ {minimal, info (default), verbose}` controls how much fingerprint detail rides on the log; bad values warn and fall back to info |
| `LogAccepted(msg, verbosity?)` | `on_challenge_submit` | Emit an Info accept log with the current fingerprint. Same `verbosity` vocabulary as `RejectSubmission`. No-op when no fingerprint is present |
| `SetChallengeDifficulty(level)` | `on_load` | Set runtime default (`"disabled"`, `"low"`, `"medium"`, `"high"`, `"impossible"`) |
| `SetChallengeDifficulty(level)` | `pre_eval`, `post_eval`, `on_challenge` | Per-request override (does NOT change runtime default) |
| `EvaluateMismatches()` | `on_challenge`, `on_challenge_submit` | Returns `MismatchReport`; cached per request |
| `fingerprint.UAMobileMismatch()` | wherever `fingerprint` is non-nil | Atomic check, no aggregation |
| `fingerprint.AcceptLanguageMismatch(req)` | same | Atomic check |
| `fingerprint.TimezoneCountryMismatch(country)` | same | Atomic check |
#### `MismatchReport` API
```
.Count() total signals fired
.Empty() true if none fired
.High() / .Medium() / .Low() count by severity
.BySeverity(s) count for "high" / "medium" / "low"
.Has(reason) true if a specific signal fired
.Reasons() stable-ordered list of fired reason keys
.String() "reason1(sev),reason2(sev)" — handy for logs
```
### 3.3 Multi-config composition
Multiple appsec-configs can be merged by listing them in acquisition:
```yaml
# acquis.yaml
appsec_configs:
- crowdsecurity/appsec-default # base rules, no challenge block
- crowdsecurity/appsec-bot-detection # challenge block + hooks
- mycorp/internal-allowlist # adds an extra pre_eval hook
```
Within the `challenge:` block, **each later config's non-nil fields override
earlier ones, field by field.** A collection that only wants to ship a
specific `cookie_ttl` can do so without redeclaring the rest:
```yaml
# mycorp/short-cookie-overlay
name: mycorp/short-cookie-overlay
challenge:
cookie_ttl: 30m # overrides whatever the upstream config set
```
Other top-level YAML keys follow the existing appsec-config patterns: rules
and hooks are *appended* across configs; scalars like `default_remediation`,
HTTP codes, and challenge fields are *overridden*.
### 3.4 Single-instance vs distributed deployment
What is stateless vs stateful matters here:
- **Stateless (works fleet-wide, no coordination):** challenge crypto
verification (`sig` / PoW-MAC) and cookie seal/open. Any instance sharing the
`master_secret` + `key_rotation_interval` validates any other's challenges and
cookies.
- **Stateful (per-instance):** single-use replay protection. The spent-set that
burns each `r` is **in-memory and local to one appsec instance** — it is *not*
shared across instances (no Redis / shared store).
| Scenario | What to do |
|---|---|
| Single WAF, dev | Skip `master_secret` entirely. Ephemeral cookies on restart, warning logged. |
| Single WAF, prod | Set `master_secret`. Restarts don't invalidate outstanding cookies. Single-use replay protection is fully effective (one local spent-set). |
| Multi-WAF, prod | Set the **same** `master_secret` *and* `key_rotation_interval` on every instance — crypto/cookie verification then works without coordination. **But:** because the spent-set is per-instance, a captured submission could be **replayed against a *different* instance**. The design therefore assumes **all reverse proxies route challenge traffic to a single appsec/WAF instance** (or that the front-end pins the challenge handshake to one instance). Running independent instances behind a load balancer means single-use degrades to per-instance — replay across instances is possible until a shared spent-set is added. |
For multi-instance, also make sure the WAFs' clocks are reasonably aligned —
the keyring tolerates one rotation interval of clock skew, no more.
### 3.5 Tuning checklist
- **Default-friendly** (low CPU, single-instance): no `challenge:` block, or
just `master_secret` + `cookie_ttl`. Acceptable for most deployments.
- **High-traffic, wants per-visitor JS variance**: bump
`crypto_obfuscation_pool_size` to 2–3. Costs `N × 5 s` of CPU per
rotation.
- **fpscanner is public, served raw**: the fpscanner bundle at
`/crowdsec-internal/challenge/fpscanner.js` is unobfuscated, cacheable public
code — there is no runtime knob to obfuscate it, and scrapers keying on that
static bundle are not a threat the runtime tries to defeat. The only
per-visitor JS variance available is on the dynamic key module
(`crypto_obfuscation_pool_size`); the challenge crypto code is a single
static build-time obfuscated variant.
- **Long-lived cookies for known clients**: set a long `cookie_ttl`
(24h, 7 days) — safe because cookie expiry is enforced by `not_after`,
not by the much shorter keyring window.
- **Aggressive bot detection**: combine `EvaluateMismatches()` thresholds
with atomic checks (`fingerprint.UAMobileMismatch`, etc.) and consider
`SetChallengeDifficulty("high")` for suspect fingerprints. Reserve
`"impossible"` for known-bad: solving it server-side is rejected, so it
acts as a soft block via the challenge flow rather than a hard `ban`.
### 3.6 Operational notes
- The challenge runtime endpoints (`/crowdsec-internal/challenge/*`) are
served on the same listener as the appsec acquisition module. They are
not configurable independently.
- The default cookie name is `__crowdsec_challenge`. It is `HttpOnly`,
`Path=/`, `SameSite=Lax`, and `Secure` when the request was HTTPS.
- Operator-issued allowlist cookies carry the reason string in their
payload (≤ 256 bytes) — visible in logs and in the cookie value, **not**
in the Set-Cookie attributes. Don't put secrets in the reason.
- The "impossible" difficulty produces a challenge page the client
cannot solve. It functions as a soft block: visitors see a challenge
loop instead of a 403, which is often the desired UX for ambiguous
cases.
+720
View File
@@ -0,0 +1,720 @@
// Package challenge implements the AppSec WAF challenge mode: a PoW-gated
// landing page, a fingerprint collection bundle, and the surrounding key
// rotation + cookie machinery. This file holds the runtime orchestration
// (lifecycle, HTTP entry points, template rendering, hook plumbing) and
// delegates specialized concerns to sibling files in the same package:
//
// - keyring.go / crypto.go / ticket.go — per-epoch HKDF keys, AES-GCM
// cookie seal/unseal, ticket signing
// - static_bundle.go — public fpscanner/JS bundle
// - dynamic_module.go — sensitive per-epoch sign-key module
// - obfuscator.go — wazero wrapper around the JS obfuscator
// - fingerprint*.go — fingerprint wire shape + helpers + mismatch report
// - config.go / secret.go — YAML config + master-secret handling
package challenge
import (
"bytes"
"compress/gzip"
"context"
"crypto/hmac"
"crypto/sha256"
_ "embed"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"text/template"
"time"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/pb"
"github.com/crowdsecurity/crowdsec/pkg/appsec/cookie"
"github.com/crowdsecurity/crowdsec/pkg/logging"
log "github.com/sirupsen/logrus"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
"golang.org/x/sync/singleflight"
)
// Internal URL paths the challenge runtime intercepts. Bouncers MUST forward
// these to the WAF unmodified; they are served by the appsec dispatcher
// (pkg/appsec/appsec.go) rather than by the protected origin.
const (
ChallengeJSPath = "/crowdsec-internal/challenge/challenge.js"
ChallengeSubmitPath = "/crowdsec-internal/challenge/submit"
ChallengePowWorkerPath = "/crowdsec-internal/challenge/pow-worker.js"
ChallengeFPScannerPath = "/crowdsec-internal/challenge/fpscanner.js"
)
// Sentinel errors (reasons) returned by ValidateChallengeResponse.
var (
ErrChallengeFields = errors.New("missing required fields in challenge response")
ErrChallengeTicket = errors.New("invalid ticket in challenge response")
ErrChallengeDifficulty = errors.New("challenge difficulty is impossible")
ErrChallengePoW = errors.New("invalid proof-of-work in challenge response")
ErrChallengeHMAC = errors.New("invalid HMAC in challenge response")
ErrChallengePayload = errors.New("invalid challenge response payload")
)
// ChallengeCookieName is the name of the sealed cookie carrying the
// successfully-validated fingerprint between requests.
const ChallengeCookieName = "__crowdsec_challenge"
// cryptoObfuscationPoolDefaultSize is how many obfuscations of the per-epoch
// key module to keep per live epoch. Each variant embeds the same key
// differently (per-visitor byte variance); default 1 keeps prior behavior.
const cryptoObfuscationPoolDefaultSize = 1
// defaultCookieTTL is the default challenge-cookie validity. Decoupled from the
// keyring window (enforced by not_after in the envelope), so cookies can
// outlive the per-epoch signing window without widening forgery exposure.
const defaultCookieTTL = 12 * time.Hour
// DefaultChallengeCSP is the Content-Security-Policy header used on the
// challenge page when the operator hasn't configured a custom one. Allows
// inline script/style (the challenge runtime injects both) and blob workers
// (the PoW worker is loaded from a blob URL).
const DefaultChallengeCSP = "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; worker-src 'self' blob:;"
//go:embed challenge.html.tmpl
var htmlTemplate string
// grantRedirectBody is the body of the 307 from GrantChallengeCookie — a no-JS
// fallback for HTTP clients that don't auto-follow Location. Static, so no
// per-request parsing is needed.
//
//go:embed grant_redirect.html.tmpl
var GrantRedirectBody string
// PowWorkerJS is the JavaScript PoW worker shipped to the browser at
// ChallengePowWorkerPath. Served as-is — obfuscation here would only slow
// down the per-visitor PoW loop without adding security value.
//
//go:embed pow-worker.js
var PowWorkerJS string
// ChallengeRuntime is the per-process state for the challenge mode: wazero
// instance, signing/cookie keys, obfuscation pools, fingerprint key
// rotation. One instance is shared across all appsec runners (see
// pkg/acquisition/modules/appsec/config.go). Construct via NewChallengeRuntime.
type ChallengeRuntime struct {
r wazero.Runtime
obfuscatorMod wazero.CompiledModule
// challengeCode is the build-time-obfuscated challenge crypto/glue code,
// decompressed once at startup from initial_bundle.js.gz and injected inline
// on every challenge page. Static for the life of the process (no runtime
// re-obfuscation or pool); the sensitive per-epoch key module lives in
// dynamic_module.go.
challengeCode string
powDifficulty int
// keys derives per-epoch sign keys (per-challenge secret / PoW MAC HMACs)
// and the long-lived master cookie key from the master secret. Routing all
// HMAC/AEAD through it makes rotation a bookkeeping change; instances
// sharing the secret + rotation interval derive identical keys.
keys *KeyRing
// cryptoPoolSize is how many obfuscations of the per-epoch key module to
// keep per live epoch (per-visitor byte variance over the same key);
// default 1.
cryptoPoolSize int
// dynamicModuleCache memoizes the obfuscated per-epoch key module variants
// (epoch → cryptoPoolSize renderings); currentDynamicModule picks one at
// random. dynamicModuleSF coalesces concurrent obfuscations for the same
// epoch so a rotation thundering-herd pays at most one obfuscation latency
// per slot, not N. The mutex guards only the map, never the obfuscation.
dynamicModuleCache map[int64][]string
dynamicModuleCacheMu sync.RWMutex
dynamicModuleSF singleflight.Group
// cookieTTL is how long a challenge cookie stays valid. Enforced by the
// not_after stamp in the sealed envelope (crypto.go), not keyring eviction,
// so it can exceed the per-epoch signing window.
cookieTTL time.Duration
// maxCookieLen is the size ceiling enforced when sealing and opening cookies
// (crypto.go). Bounds the allocation derived from the attacker-influenced
// fingerprint envelope. Defaults to MaxCookieLen (the browser limit).
maxCookieLen int
// htmlTpl is the challenge HTML template, parsed once and reused so
// GetChallengePage doesn't re-parse per request.
htmlTpl *template.Template
// spent burns consumed per-challenge nonces (`r`) to enforce single-use and
// eliminate replay (in-memory, single instance — see spent_set.go).
spent *spentSet
// logger is the component logger; its level is set at construction
// (WithLogger / BuildOptions). All challenge logging routes through it so
// `log_level` controls verbosity independently of the global logger.
logger *log.Entry
}
// Option configures a ChallengeRuntime at construction time.
type Option func(*runtimeOptions)
// Field semantics mirror the YAML Config in config.go;
type runtimeOptions struct {
masterSecret []byte // must be >= minSecretBytes when non-nil
rotationInterval time.Duration
maxLiveEpochs int
cookieTTL time.Duration
maxCookieLen int
cryptoObfuscationPoolSize int
spentSetMaxEntries int
logger *log.Entry // nil → default "challenge" sublogger
}
func WithLogger(logger *log.Entry) Option {
return func(o *runtimeOptions) {
o.logger = logger
}
}
// log returns the component logger, defaulting to a standard "challenge"
// sublogger when unset (runtimes built directly in tests don't set it).
func (c *ChallengeRuntime) log() *log.Entry {
if c.logger != nil {
return c.logger
}
return logging.SubLogger(log.StandardLogger(), "challenge", 0)
}
// WithMasterSecret sets the long-lived shared secret (see Config.MasterSecret).
func WithMasterSecret(secret []byte) Option {
return func(o *runtimeOptions) {
o.masterSecret = secret
}
}
// WithRotationInterval sets the per-epoch key rotation period (see Config.KeyRotationInterval).
func WithRotationInterval(d time.Duration) Option {
return func(o *runtimeOptions) {
o.rotationInterval = d
}
}
// WithMaxLiveEpochs sets how many past epochs the keyring keeps accepting (see Config.MaxLiveEpochs).
func WithMaxLiveEpochs(n int) Option {
return func(o *runtimeOptions) {
o.maxLiveEpochs = n
}
}
// WithCookieTTL sets challenge-cookie validity (see Config.CookieTTL); zero/negative is ignored.
func WithCookieTTL(ttl time.Duration) Option {
return func(o *runtimeOptions) {
if ttl > 0 {
o.cookieTTL = ttl
}
}
}
// WithMaxCookieLen sets the cookie size ceiling (see Config.MaxCookieSize);
// zero/negative is ignored, leaving the MaxCookieLen default in effect.
func WithMaxCookieLen(n int) Option {
return func(o *runtimeOptions) {
if n > 0 {
o.maxCookieLen = n
}
}
}
// WithCryptoObfuscationPoolSize sets the per-epoch sign-key obfuscation pool size
// (see Config.CryptoObfuscationPoolSize); values below 1 are ignored.
func WithCryptoObfuscationPoolSize(n int) Option {
return func(o *runtimeOptions) {
if n >= 1 {
o.cryptoObfuscationPoolSize = n
}
}
}
// WithSpentSetMaxEntries caps the replay-protection LRU (see Config.SpentSetMaxEntries); values below 1 are ignored.
func WithSpentSetMaxEntries(n int) Option {
return func(o *runtimeOptions) {
if n >= 1 {
o.spentSetMaxEntries = n
}
}
}
// DifficultyFromLevel resolves a named level ("low", "medium", "high") to
// a PoW difficulty in leading zero bits. Case-insensitive.
func DifficultyFromLevel(level string) (int, error) {
switch strings.ToLower(level) {
case "disabled":
return PowDifficultyDisabled, nil
case "low":
return PowDifficultyLow, nil
case "medium":
return PowDifficultyMedium, nil
case "high":
return PowDifficultyHigh, nil
case "impossible":
return PowDifficultyImpossible, nil
default:
return 0, fmt.Errorf("unknown challenge difficulty %q (expected disabled, low, medium, high, or impossible)", level)
}
}
// Difficulty returns the current default PoW difficulty (in leading zero bits).
func (c *ChallengeRuntime) Difficulty() int {
return c.powDifficulty
}
// SetDifficulty sets the default PoW difficulty from a named level.
func (c *ChallengeRuntime) SetDifficulty(level string) error {
bits, err := DifficultyFromLevel(level)
if err != nil {
return err
}
c.powDifficulty = bits
return nil
}
// NewChallengeRuntime builds and starts a ChallengeRuntime: it initializes the
// wazero runtime, decompresses the baked-in library bundle, derives the master
// secret + keyring, pre-warms the dynamic-module cache, and (if configured)
// spawns the background obfuscation refresher. Safe for concurrent use across
// all appsec runners. Pass WithXxx options to override defaults.
// compileObfuscatorModule decompresses the baked-in obfuscator WASM (once,
// process-wide) and compiles it for the given runtime. Pre-compiling lets each
// ObfuscateJS call merely instantiate the module instead of re-parsing the WASM
// bytes, which would otherwise cost ~4-5s per call.
func compileObfuscatorModule(ctx context.Context, r wazero.Runtime) (wazero.CompiledModule, error) {
var obfuscatorWasmErr error
obfuscatorWasmOnce.Do(func() {
zr, err := gzip.NewReader(bytes.NewReader(obfuscatorWasmGz))
if err != nil {
obfuscatorWasmErr = fmt.Errorf("failed to create gzip reader for obfuscator wasm: %w", err)
return
}
defer zr.Close()
obfuscatorWasm, err = io.ReadAll(zr)
if err != nil {
obfuscatorWasmErr = fmt.Errorf("failed to decompress obfuscator wasm: %w", err)
return
}
})
if obfuscatorWasmErr != nil {
return nil, obfuscatorWasmErr
}
compiledMod, err := r.CompileModule(ctx, obfuscatorWasm)
if err != nil {
return nil, fmt.Errorf("failed to compile obfuscator wasm module: %w", err)
}
return compiledMod, nil
}
func NewChallengeRuntime(ctx context.Context, opts ...Option) (*ChallengeRuntime, error) {
resolvedOpts := runtimeOptions{}
for _, opt := range opts {
opt(&resolvedOpts)
}
logger := resolvedOpts.logger
if logger == nil {
logger = logging.SubLogger(log.StandardLogger(), "challenge", 0)
}
secret := resolvedOpts.masterSecret
if secret == nil {
var err error
secret, err = generateRandomSecret()
if err != nil {
return nil, err
}
logger.Warn("no master secret configured for the WAF challenge runtime; generated an ephemeral random secret. " +
"Distributed (multi-WAF) deployments MUST configure a shared master_secret in the appsec config; " +
"single-instance deployments will see outstanding challenge cookies invalidated on restart.")
} else if len(secret) < minSecretBytes {
return nil, fmt.Errorf("master secret is %d bytes; minimum is %d", len(secret), minSecretBytes)
}
rotationInterval := resolvedOpts.rotationInterval
if rotationInterval == 0 {
rotationInterval = keyringDefaultRotation
}
keys, err := NewKeyRing(secret, rotationInterval, resolvedOpts.maxLiveEpochs)
if err != nil {
return nil, fmt.Errorf("build challenge keyring: %w", err)
}
keys.logger = logger
cookieTTL := resolvedOpts.cookieTTL
if cookieTTL <= 0 {
cookieTTL = defaultCookieTTL
}
maxCookieLen := resolvedOpts.maxCookieLen
if maxCookieLen <= 0 {
maxCookieLen = MaxCookieLen
}
cryptoPoolSize := resolvedOpts.cryptoObfuscationPoolSize
if cryptoPoolSize <= 0 {
cryptoPoolSize = cryptoObfuscationPoolDefaultSize
}
spentSetMaxEntries := resolvedOpts.spentSetMaxEntries
if spentSetMaxEntries <= 0 {
spentSetMaxEntries = spentSetDefaultMaxEntries
}
r := wazero.NewRuntime(ctx)
// No need to keep the closer around, we can just close the runtime itself when stopping
if _, err := wasi_snapshot_preview1.Instantiate(ctx, r); err != nil {
return nil, fmt.Errorf("failed to instantiate WASI: %w", err)
}
compiledMod, err := compileObfuscatorModule(ctx, r)
if err != nil {
return nil, err
}
// We use text/template instead of html/template because the data we send
// is pretty much hardcoded and trusted; html/template would escape the JS
// we inject. Parsed once here so GetChallengePage doesn't re-parse on
// every request.
htmlTpl, err := template.New("challenge").Parse(htmlTemplate)
if err != nil {
return nil, fmt.Errorf("parse challenge html template: %w", err)
}
challengeRuntime := &ChallengeRuntime{
r: r,
obfuscatorMod: compiledMod,
powDifficulty: defaultPowDifficulty,
keys: keys,
cryptoPoolSize: cryptoPoolSize,
dynamicModuleCache: make(map[int64][]string),
cookieTTL: cookieTTL,
maxCookieLen: maxCookieLen,
htmlTpl: htmlTpl,
spent: newSpentSet(spentSetMaxEntries),
logger: logger,
}
// Load the build-time-obfuscated challenge code from the baked-in bundle so
// we can serve immediately.
if err := challengeRuntime.seedCacheFromInitialBundle(); err != nil {
// Initial bundle missing/corrupt (e.g. `go generate` not run): fall back
// to obfuscating the challenge code synchronously.
logger.Warnf("failed to load baked-in initial challenge bundle (%v); falling back to synchronous generation", err)
if err := challengeRuntime.generateAndCacheChallengeJS(ctx); err != nil {
return nil, fmt.Errorf("failed to generate initial challenge bundle: %w", err)
}
}
// Pre-warm the current epoch's dynamic module so the first GetChallengePage
// doesn't pay the ~5s obfuscation cost on the request path.
if _, err := challengeRuntime.currentDynamicModule(ctx); err != nil {
return nil, fmt.Errorf("warm dynamic key module: %w", err)
}
// The dynamic-module pre-warmer is always on — the per-epoch key must be
// re-obfuscated on every rotation (bounded cost, one pass per variant). The
// challenge code itself is static (build-time obfuscated), so there is no
// library refresher anymore.
go challengeRuntime.dynamicModulePreWarmer(ctx)
logger.WithFields(log.Fields{
"rotation_interval": rotationInterval,
"cookie_ttl": cookieTTL,
"max_cookie_len": maxCookieLen,
"pow_difficulty": defaultPowDifficulty,
"crypto_pool_size": cryptoPoolSize,
}).Info("WAF challenge runtime initialized")
return challengeRuntime, nil
}
func (c *ChallengeRuntime) Close(ctx context.Context) error {
if c == nil || c.r == nil {
return nil
}
return c.r.Close(ctx)
}
// GetChallengePage renders the challenge HTML page with the given PoW difficulty.
// If difficulty is 0, the default difficulty is used.
func (c *ChallengeRuntime) GetChallengePage(ctx context.Context, userAgent string, difficulty int) (string, error) {
_ = userAgent
if difficulty <= 0 {
difficulty = c.powDifficulty
}
challengeCode := c.getChallengeCode()
if challengeCode == "" {
if err := c.generateAndCacheChallengeJS(ctx); err != nil {
return "", fmt.Errorf("failed to generate challenge JS: %w", err)
}
challengeCode = c.getChallengeCode()
if challengeCode == "" {
return "", errors.New("challenge JS cache is empty")
}
}
// Issuance is stateless (only submission is stateful — see the single-use
// burn in ValidateChallengeResponse). `r` seeds the per-challenge secret
// `s = HMAC(K_epoch, r)` the client derives from the obfuscated dynamic
// module, so `s` never appears in plain HTML; the PoW MAC binds the salt to
// `r`+ts so a client can't pick a favorable salt.
ts := fmt.Sprintf("%d", time.Now().UnixNano())
r, err := generateChallengeNonce()
if err != nil {
return "", err
}
powSalt, err := generatePowPrefix()
if err != nil {
return "", fmt.Errorf("generate PoW salt: %w", err)
}
powMAC := c.computePowMAC(powSalt, r, ts, difficulty)
if c.log().Logger.IsLevelEnabled(log.DebugLevel) {
issEpoch, issKey := c.keys.Current()
c.log().WithFields(log.Fields{
"r": r,
"epoch": issEpoch,
"k_epoch": fmt.Sprintf("%x", issKey),
"difficulty": difficulty,
}).Debug("issued challenge")
}
// The challenge code only carries the hook registration; the dynamic module
// carries the per-epoch K, so K never appears in plain HTML.
dynamicModule, err := c.currentDynamicModule(ctx)
if err != nil {
return "", fmt.Errorf("build dynamic key module: %w", err)
}
var renderedPage strings.Builder
if err := c.htmlTpl.Execute(&renderedPage, map[string]interface{}{
"JSChallenge": challengeCode,
"DynamicModule": dynamicModule,
"FPScannerPath": ChallengeFPScannerPath,
"PowDifficulty": difficulty,
"PowPrefix": powSalt,
"PowMAC": powMAC,
"Timestamp": ts,
"R": r,
}); err != nil {
return "", fmt.Errorf("render challenge page: %w", err)
}
return renderedPage.String(), nil
}
// ValidateChallengeResponse parses a submit POST and runs the full chain:
// freshness + PoW-salt authenticity + difficulty, PoW solution,
// the submission signature `sig` (keyed by the never-transmitted s = HMAC(K_epoch, r)),
// a single-use burn of `r` (replay protection),
// and fingerprint de-obfuscation. On success it returns the sealed
// cookie, decoded FingerprintData, and the proven PoW difficulty; failures
// return a generic error so the caller doesn't leak which stage failed.
func (c *ChallengeRuntime) ValidateChallengeResponse(request *http.Request, body []byte) (*cookie.AppsecCookie, FingerprintData, int, error) {
vars, err := url.ParseQuery(string(body))
if err != nil {
return nil, FingerprintData{}, 0, fmt.Errorf("%w: %w", ErrChallengePayload, err)
}
encryptedFingerprint := vars.Get("f")
clientR := vars.Get("r")
clientTS := vars.Get("ts")
clientSig := vars.Get("sig")
clientNonce := vars.Get("n")
clientPowSalt := vars.Get("p")
clientPowMAC := vars.Get("m")
clientDifficultyStr := vars.Get("d")
clientPath := vars.Get("u")
if encryptedFingerprint == "" || clientR == "" || clientTS == "" || clientSig == "" || clientNonce == "" || clientPowSalt == "" || clientPowMAC == "" || clientDifficultyStr == "" || clientPath == "" {
return nil, FingerprintData{}, 0, errors.New("missing required fields in challenge response")
}
// Override the request path so the WAF sees the original URL the client
request.URL.Path = clientPath
// The difficulty the client claims it solved. It is untrusted until the PoW
// MAC (which binds it) is verified in verifyChallenge below. Bound to the
// valid PoW range so a malformed value can't reach the PoW/seal logic.
clientDifficulty, err := strconv.Atoi(clientDifficultyStr)
if err != nil || clientDifficulty < PowDifficultyDisabled || clientDifficulty > PowDifficultyImpossible {
return nil, FingerprintData{}, 0, errors.New("invalid ticket in challenge response")
}
// Server-issued `r` is a 16-byte nonce in hex (generateChallengeNonce):
// exactly 32 hex chars. Reject other shapes early so a K_epoch holder can't
// bloat the spent-set with oversized keys, and to keep the key space canonical.
if _, err := hex.DecodeString(clientR); err != nil || len(clientR) != 32 {
return nil, FingerprintData{}, 0, errors.New("invalid ticket in challenge response")
}
// Verify freshness + PoW-salt/difficulty authenticity and recover the
// per-epoch sign key (stateless). Key knowledge is proven by `sig` below.
signKey, ok := c.verifyChallenge(clientR, clientTS, clientPowSalt, clientPowMAC, clientDifficulty)
if !ok {
return nil, FingerprintData{}, 0, errors.New("invalid ticket in challenge response")
}
// Impossible difficulty is a deliberate hard-block: never accept anything.
// clientDifficulty is now MAC-authenticated, so this enforces the difficulty
// actually issued for this request, not the runtime-global default.
if clientDifficulty >= PowDifficultyImpossible {
return nil, FingerprintData{}, 0, ErrChallengeDifficulty
}
// Verify proof-of-work: SHA256(powSalt + nonce) must have required leading zero bits
powHash := sha256.Sum256([]byte(clientPowSalt + clientNonce))
if !hasLeadingZeroBits(powHash[:], clientDifficulty) {
return nil, FingerprintData{}, 0, ErrChallengePoW
}
// Verify the submission signature sig = HMAC(s, r||ts||n||f), where the
// secret s = HMAC(K_epoch, r) is never transmitted — a valid sig proves the
// client derived s from the per-epoch key in the obfuscated dynamic module.
s := deriveChallengeSecret(signKey, clientR)
expectedSig := hmacSHA256Hex([]byte(s), []byte(clientR+clientTS+clientNonce+encryptedFingerprint))
if !hmac.Equal([]byte(clientSig), []byte(expectedSig)) {
return nil, FingerprintData{}, 0, errors.New("invalid HMAC in challenge response")
}
// Single-use: burn `r` (rejects replays). Done last so the spent-set only
// grows on fully-valid submissions.
if !c.spent.checkAndInsert(clientR, ticketAgeBackstop) {
return nil, FingerprintData{}, 0, errors.New("challenge response already used")
}
obfKey := deriveFingerprintObfKey(s, clientR)
fingerprint, err := deobfuscateFingerprint(obfKey, encryptedFingerprint)
if err != nil {
return nil, FingerprintData{}, 0, fmt.Errorf("failed to deobfuscate fingerprint: %w", err)
}
var fpData FingerprintData
if err := json.Unmarshal([]byte(fingerprint), &fpData); err != nil {
return nil, FingerprintData{}, 0, fmt.Errorf("%w: failed to unmarshal fingerprint data: %w", ErrChallengePayload, err)
}
// Debug diagnostic: a validated submission. Guarded so `k_epoch` (forgeable
// signing material — DESIGN.md §2.1) is only formatted at debug.
if c.log().Logger.IsLevelEnabled(log.DebugLevel) {
c.log().WithFields(log.Fields{
"r": clientR,
"epoch": c.epochForTimestamp(clientTS),
"k_epoch": fmt.Sprintf("%x", signKey),
"fsid": fpData.FSID,
"is_bot": fpData.FastBotDetection,
}).Debug("validated submission")
}
// Seal the difficulty the client actually proved (MAC-authenticated above),
// so the next request's re-challenge check compares against real work and an
// escalated per-request difficulty survives the cookie round-trip.
envelope := &pb.ChallengeCookie{
Fingerprint: fpData.ToProto(),
PowDifficulty: int32(clientDifficulty),
}
// Seal under the long-lived master cookie key. The embedded not_after makes
// the server validity window exactly c.cookieTTL (independent of key
// rotation); the browser Max-Age below matches so both expire together.
notAfter := time.Now().Add(c.cookieTTL).Unix()
cookieValue, err := sealCookieV0(envelope, c.keys.MasterCookieKey(), notAfter, 0, "", []byte(request.UserAgent()), c.maxCookieLen)
if err != nil {
return nil, FingerprintData{}, 0, fmt.Errorf("failed to seal challenge cookie: %w", err)
}
ck := cookie.NewAppsecCookie(ChallengeCookieName).HttpOnly().Path("/").SameSite(cookie.SameSiteLax).ExpiresIn(c.cookieTTL).Value(cookieValue)
if request.URL.Scheme == "https" {
ck = ck.Secure()
}
return ck, fpData, clientDifficulty, nil
}
// SealAllowlistCookie mints an allowlist-bypass cookie (no fingerprint, with
// the operator reason) so GrantChallengeCookie can let trusted bots skip the
// challenge UI while still hitting on_challenge rules via fingerprint.Allowlisted.
// not_after honors c.cookieTTL unless ttlOverride (>0) is given; reason is
// bounded by MaxAllowlistReasonLen (crypto.go).
func (c *ChallengeRuntime) SealAllowlistCookie(request *http.Request, reason string, ttlOverride *time.Duration) (*cookie.AppsecCookie, error) {
if c == nil {
return nil, errors.New("challenge runtime not initialized")
}
ttl := c.cookieTTL
if ttlOverride != nil && *ttlOverride > 0 {
ttl = *ttlOverride
}
notAfter := time.Now().Add(ttl).Unix()
cookieValue, err := sealCookieV0(&pb.ChallengeCookie{}, c.keys.MasterCookieKey(), notAfter, cookieFlagAllowlisted, reason, []byte(request.UserAgent()), c.maxCookieLen)
if err != nil {
return nil, fmt.Errorf("failed to seal allowlist cookie: %w", err)
}
ck := cookie.NewAppsecCookie(ChallengeCookieName).HttpOnly().Path("/").SameSite(cookie.SameSiteLax).ExpiresIn(ttl).Value(cookieValue)
if request.URL.Scheme == "https" {
ck = ck.Secure()
}
return ck, nil
}
// CookieData bundles the decoded fingerprint with cookie-envelope metadata for
// re-challenge decisions. Allowlisted/AllowlistReason mark cookies minted by
// SealAllowlistCookie; they are zero for real-submission cookies.
type CookieData struct {
Fingerprint FingerprintData
PowDifficulty int
Allowlisted bool
AllowlistReason string
}
// ValidCookie unseals and validates a challenge cookie: envelope (version,
// AES-GCM tag), not_after expiry, and UA-pinning (a stolen cookie is useless to
// a different client). On any failure (tampered/expired/UA-mismatch/unknown
// version) it returns an error and the caller should treat the request as
// cookieless.
func (c *ChallengeRuntime) ValidCookie(ck *http.Cookie, userAgent string) (*CookieData, error) {
if ck == nil {
return nil, errors.New("nil cookie")
}
envelope, err := openCookie(ck.Value, c.keys.MasterCookieKey(), []byte(userAgent), c.maxCookieLen)
if err != nil {
return nil, fmt.Errorf("invalid challenge cookie: %w", err)
}
return &CookieData{
Fingerprint: fingerprintDataFromProto(envelope.Envelope.GetFingerprint()),
PowDifficulty: int(envelope.Envelope.GetPowDifficulty()),
Allowlisted: envelope.Allowlisted,
AllowlistReason: envelope.AllowlistReason,
}, nil
}
File diff suppressed because one or more lines are too long
+346
View File
@@ -0,0 +1,346 @@
// --- Pure JS SHA-256 (RFC 6234) ---
const K = [
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147,
0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
];
function sha256(data) {
let h0 = 0x6a09e667,
h1 = 0xbb67ae85,
h2 = 0x3c6ef372,
h3 = 0xa54ff53a,
h4 = 0x510e527f,
h5 = 0x9b05688c,
h6 = 0x1f83d9ab,
h7 = 0x5be0cd19;
const len = data.length;
const bitLen = len * 8;
const padLen = 64 - ((len + 9) % 64);
const totalLen = len + 1 + (padLen === 64 ? 0 : padLen) + 8;
const msg = new Uint8Array(totalLen);
msg.set(data);
msg[len] = 0x80;
const dv = new DataView(msg.buffer);
dv.setUint32(totalLen - 4, bitLen, false);
const w = new Int32Array(64);
for (let offset = 0; offset < totalLen; offset += 64) {
for (let i = 0; i < 16; i++) {
w[i] = dv.getInt32(offset + i * 4, false);
}
for (let i = 16; i < 64; i++) {
const s0 =
(((w[i - 15] >>> 7) | (w[i - 15] << 25)) ^
((w[i - 15] >>> 18) | (w[i - 15] << 14)) ^
(w[i - 15] >>> 3)) |
0;
const s1 =
(((w[i - 2] >>> 17) | (w[i - 2] << 15)) ^
((w[i - 2] >>> 19) | (w[i - 2] << 13)) ^
(w[i - 2] >>> 10)) |
0;
w[i] = (w[i - 16] + s0 + w[i - 7] + s1) | 0;
}
let a = h0,
b = h1,
c = h2,
d = h3,
e = h4,
f = h5,
g = h6,
h = h7;
for (let i = 0; i < 64; i++) {
const S1 =
(((e >>> 6) | (e << 26)) ^
((e >>> 11) | (e << 21)) ^
((e >>> 25) | (e << 7))) |
0;
const ch = ((e & f) ^ (~e & g)) | 0;
const temp1 = (h + S1 + ch + K[i] + w[i]) | 0;
const S0 =
(((a >>> 2) | (a << 30)) ^
((a >>> 13) | (a << 19)) ^
((a >>> 22) | (a << 10))) |
0;
const maj = ((a & b) ^ (a & c) ^ (b & c)) | 0;
const temp2 = (S0 + maj) | 0;
h = g;
g = f;
f = e;
e = (d + temp1) | 0;
d = c;
c = b;
b = a;
a = (temp1 + temp2) | 0;
}
h0 = (h0 + a) | 0;
h1 = (h1 + b) | 0;
h2 = (h2 + c) | 0;
h3 = (h3 + d) | 0;
h4 = (h4 + e) | 0;
h5 = (h5 + f) | 0;
h6 = (h6 + g) | 0;
h7 = (h7 + h) | 0;
}
const out = new Uint8Array(32);
const ov = new DataView(out.buffer);
ov.setUint32(0, h0, false);
ov.setUint32(4, h1, false);
ov.setUint32(8, h2, false);
ov.setUint32(12, h3, false);
ov.setUint32(16, h4, false);
ov.setUint32(20, h5, false);
ov.setUint32(24, h6, false);
ov.setUint32(28, h7, false);
return out;
}
// --- Helpers ---
function toHex(bytes) {
let hex = "";
for (let i = 0; i < bytes.length; i++) {
hex += (bytes[i] >> 4).toString(16) + (bytes[i] & 0xf).toString(16);
}
return hex;
}
function encode(str) {
return new TextEncoder().encode(str);
}
function sha256Hex(str) {
return toHex(sha256(encode(str)));
}
// --- HMAC-SHA256 ---
function hmacSHA256(key, message) {
if (key.length > 64) {
key = sha256(key);
}
const paddedKey = new Uint8Array(64);
paddedKey.set(key);
const ipad = new Uint8Array(64);
const opad = new Uint8Array(64);
for (let i = 0; i < 64; i++) {
ipad[i] = paddedKey[i] ^ 0x36;
opad[i] = paddedKey[i] ^ 0x5c;
}
const inner = new Uint8Array(64 + message.length);
inner.set(ipad);
inner.set(message, 64);
const innerHash = sha256(inner);
const outer = new Uint8Array(64 + 32);
outer.set(opad);
outer.set(innerHash, 64);
return sha256(outer);
}
function hmacSHA256Hex(keyStr, msgStr) {
return toHex(hmacSHA256(encode(keyStr), encode(msgStr)));
}
// hexToBytes decodes a hex string into a Uint8Array. Used to materialize the
// per-epoch HMAC key delivered as hex by the dynamic module.
function hexToBytes(hex) {
const out = new Uint8Array(hex.length / 2);
for (let i = 0; i < out.length; i++) {
out[i] = parseInt(hex.substr(i * 2, 2), 16);
}
return out;
}
function hmacSHA256HexKey(keyHex, msgStr) {
return toHex(hmacSHA256(hexToBytes(keyHex), encode(msgStr)));
}
// --- Proof-of-Work (offloaded to Web Worker) ---
const powWorkerPath = "__CROWDSEC_POW_WORKER_PATH__";
function solvePoWAsync(prefix, difficulty) {
if (difficulty <= 0) {
return Promise.resolve("0");
}
try {
const worker = new Worker(powWorkerPath);
return new Promise((resolve) => {
worker.onmessage = (e) => {
worker.terminate();
resolve(e.data);
};
worker.onerror = () => {
worker.terminate();
resolve(solvePoWMainThread(prefix, difficulty));
};
worker.postMessage({ p: prefix, d: difficulty });
});
} catch (_) {
return Promise.resolve(solvePoWMainThread(prefix, difficulty));
}
}
function solvePoWMainThread(prefix, difficulty) {
let nonce = 0;
while (true) {
const candidate = prefix + nonce.toString(36);
const hash = sha256(encode(candidate));
const fullBytes = difficulty >> 3;
const remainBits = difficulty & 7;
let ok = true;
for (let i = 0; i < fullBytes; i++) {
if (hash[i] !== 0) {
ok = false;
break;
}
}
if (ok && remainBits > 0) {
const mask = (0xff << (8 - remainBits)) & 0xff;
if ((hash[fullBytes] & mask) !== 0) ok = false;
}
if (ok) return nonce.toString(36);
nonce++;
}
}
// --- Fingerprint encryption (XOR) ---
function encryptFingerprint(key, fingerprint) {
const keyBytes = encode(key);
const textBytes = encode(fingerprint);
const encrypted = new Uint8Array(textBytes.length);
for (let i = 0; i < textBytes.length; i++) {
encrypted[i] = textBytes[i] ^ keyBytes[i % keyBytes.length];
}
const binaryString = String.fromCharCode(...encrypted);
return btoa(binaryString);
}
// --- Non-secret per-request values (plain template injection) ---
//
// Set on `globalThis` by a plain <script> the server renders before this
// module. None are secret:
// _ts — server time (forgeable but freshness-windowed)
// _powP — random PoW salt (server-issued so clients can't pick easy salts)
// _powM — HMAC(_powP, _r, _ts) under the per-epoch key
// _powD — PoW difficulty
// _r — per-challenge nonce; seeds s = HMAC(K, r)
// The secret is the per-epoch K, delivered only by the dynamic key module; the
// derived s is never transmitted.
const ts = typeof _ts !== "undefined" ? _ts : "";
const powPrefix = typeof _powP !== "undefined" ? _powP : "";
const powMAC = typeof _powM !== "undefined" ? _powM : "";
const powDifficulty = typeof _powD !== "undefined" ? _powD : 12;
const r = typeof _r !== "undefined" ? _r : "";
const submitPath = "__CROWDSEC_SUBMIT_PATH__";
// --- Challenge status reporting ---
function reportChallengeStatus(status) {
if (typeof window.crowdsecSetChallengeStatus === "function") {
window.crowdsecSetChallengeStatus(status);
return;
}
window.dispatchEvent(
new CustomEvent("crowdsec-challenge-status", {
detail: { status },
}),
);
}
// --- Main flow ---
// The static bundle registers a hook on globalThis; the dynamic key module
// (loaded after) calls it with { key, epoch }. Key-dependent crypto lives here;
// the static bundle holds no hardcoded K. The hook name survives obfuscation
// via `reservedStrings` so both bundles meet on the same global.
const CSEC_HOOK_NAME = "__CSEC_CHALLENGE_HOOK_v1__";
async function runChallenge(epochKey) {
// Fail closed if the fpscanner bundle didn't load.
const Scanner = globalThis.CrowdsecFingerprintScanner;
if (typeof Scanner !== "function") {
reportChallengeStatus("fail");
return;
}
const [nonce, fpResult] = await Promise.all([
solvePoWAsync(powPrefix, powDifficulty),
new Scanner().collectFingerprint({ encrypt: false }),
]);
// Per-challenge secret s = HMAC(K_epoch, r). Never transmitted; the server
// derives the same s from its per-epoch key and the cleartext r. epochKey is
// hex-encoded K; HMAC over its raw bytes.
const s = hmacSHA256HexKey(epochKey, r);
// Fingerprint payload obfuscation key, derived from the secret s (NOT a
// confidentiality guarantee — light obfuscation only).
const encKey = hmacSHA256Hex(s, "fpenc" + r);
const f = encryptFingerprint(encKey, JSON.stringify(fpResult));
// Submission signature binds (r, ts, nonce, f) under the secret s.
const sig = hmacSHA256Hex(s, r + ts + nonce + f);
return fetch(submitPath, {
method: "POST",
credentials: "same-origin",
body: new URLSearchParams({
f: f,
r: r,
ts: ts,
sig: sig,
n: nonce,
p: powPrefix,
m: powMAC,
d: powDifficulty,
u: window.location.pathname,
}),
})
.then((response) => response.json())
.then((data) => {
const status = typeof data?.status === "string" ? data.status : "fail";
reportChallengeStatus(status);
})
.catch(() => {
reportChallengeStatus("fail");
});
}
// Register the hook. The dynamic module (loaded after this) reads the same
// CSEC_HOOK_NAME string and invokes runChallenge with the per-epoch key.
globalThis[CSEC_HOOK_NAME] = function (params) {
// Defensive: the dynamic module must pass a non-empty string key.
if (!params || typeof params.key !== "string" || params.key.length === 0) {
reportChallengeStatus("fail");
return;
}
runChallenge(params.key);
};
+704
View File
@@ -0,0 +1,704 @@
package challenge
import (
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHasLeadingZeroBits(t *testing.T) {
tests := []struct {
name string
hash []byte
bits int
expect bool
}{
{
name: "0 bits required",
hash: []byte{0xff, 0xff},
bits: 0,
expect: true,
},
{
name: "8 bits - first byte zero",
hash: []byte{0x00, 0xff},
bits: 8,
expect: true,
},
{
name: "8 bits - first byte not zero",
hash: []byte{0x01, 0xff},
bits: 8,
expect: false,
},
{
name: "16 bits - two zero bytes",
hash: []byte{0x00, 0x00, 0xff},
bits: 16,
expect: true,
},
{
name: "10 bits - partial byte check passes",
hash: []byte{0x00, 0x20}, // 0b00000000 0b00100000 → 10 leading zeros
bits: 10,
expect: true,
},
{
name: "10 bits - partial byte check fails",
hash: []byte{0x00, 0x40}, // 0b00000000 0b01000000 → 9 leading zeros
bits: 10,
expect: false,
},
{
name: "1 bit - first bit zero",
hash: []byte{0x7f},
bits: 1,
expect: true,
},
{
name: "1 bit - first bit set",
hash: []byte{0x80},
bits: 1,
expect: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expect, hasLeadingZeroBits(tt.hash, tt.bits))
})
}
}
func TestGeneratePowPrefix(t *testing.T) {
p1 := mustGeneratePowPrefix(t)
p2 := mustGeneratePowPrefix(t)
assert.Len(t, p1, 32) // 16 bytes = 32 hex chars
assert.Len(t, p2, 32)
assert.NotEqual(t, p1, p2) // random, should differ
}
func TestDifficultyFromLevel(t *testing.T) {
bits, err := DifficultyFromLevel("low")
require.NoError(t, err)
assert.Equal(t, PowDifficultyLow, bits)
bits, err = DifficultyFromLevel("MEDIUM")
require.NoError(t, err)
assert.Equal(t, PowDifficultyMedium, bits)
bits, err = DifficultyFromLevel("High")
require.NoError(t, err)
assert.Equal(t, PowDifficultyHigh, bits)
bits, err = DifficultyFromLevel("disabled")
require.NoError(t, err)
assert.Equal(t, PowDifficultyDisabled, bits)
bits, err = DifficultyFromLevel("IMPOSSIBLE")
require.NoError(t, err)
assert.Equal(t, PowDifficultyImpossible, bits)
_, err = DifficultyFromLevel("extreme")
assert.Error(t, err)
}
func TestSetDifficulty(t *testing.T) {
c := newTestRuntimeWithDifficulty(defaultPowDifficulty)
require.NoError(t, c.SetDifficulty("low"))
assert.Equal(t, PowDifficultyLow, c.powDifficulty)
require.NoError(t, c.SetDifficulty("MEDIUM"))
assert.Equal(t, PowDifficultyMedium, c.powDifficulty)
require.NoError(t, c.SetDifficulty("High"))
assert.Equal(t, PowDifficultyHigh, c.powDifficulty)
require.NoError(t, c.SetDifficulty("impossible"))
assert.Equal(t, PowDifficultyImpossible, c.powDifficulty)
require.Error(t, c.SetDifficulty("extreme"))
assert.Equal(t, PowDifficultyImpossible, c.powDifficulty) // unchanged on error
}
func TestDeriveChallengeSecret(t *testing.T) {
secret := []byte("0123456789abcdef0123456789abcdef")
keys, err := NewKeyRing(secret, time.Minute, 3)
require.NoError(t, err)
c := &ChallengeRuntime{keys: keys}
ts := fmt.Sprintf("%d", time.Now().UnixNano())
epoch := c.epochForTimestamp(ts)
signKey, ok := c.keys.SignKey(epoch)
require.True(t, ok)
r1, err := generateChallengeNonce()
require.NoError(t, err)
r2, err := generateChallengeNonce()
require.NoError(t, err)
s1 := deriveChallengeSecret(signKey, r1)
s1again := deriveChallengeSecret(signKey, r1)
s2 := deriveChallengeSecret(signKey, r2)
assert.Len(t, s1, 64) // HMAC-SHA256 hex
assert.Equal(t, s1, s1again) // deterministic in (signKey, r)
assert.NotEqual(t, s1, s2) // distinct per-challenge r → distinct s
// Matches HMAC-SHA256 under the per-epoch sign key.
assert.Equal(t, hmacSHA256Hex(signKey, []byte(r1)), s1)
// The fingerprint obfuscation key is derived from s and r, distinct from s.
obf := deriveFingerprintObfKey(s1, r1)
assert.Len(t, obf, 64)
assert.NotEqual(t, s1, obf)
assert.Equal(t, obf, deriveFingerprintObfKey(s1, r1)) // deterministic
}
func TestVerifyChallenge(t *testing.T) {
keys, err := NewKeyRing([]byte("0123456789abcdef0123456789abcdef"), time.Minute, 3)
require.NoError(t, err)
c := &ChallengeRuntime{keys: keys}
r, ts := freshChallenge(t)
salt := mustGeneratePowPrefix(t)
const diff = PowDifficultyMedium
mac := c.computePowMAC(salt, r, ts, diff)
// Valid challenge → returns the epoch sign key.
signKey, ok := c.verifyChallenge(r, ts, salt, mac, diff)
assert.True(t, ok)
assert.NotEmpty(t, signKey)
// Wrong r (MAC bound to a different r).
_, ok = c.verifyChallenge("ffffffffffffffffffffffffffffffff", ts, salt, mac, diff)
assert.False(t, ok)
// Malformed timestamp.
_, ok = c.verifyChallenge(r, "9999999999999999999", salt, mac, diff)
assert.False(t, ok)
// Forged MAC.
_, ok = c.verifyChallenge(r, ts, salt, "forged-mac", diff)
assert.False(t, ok)
// MAC from a different salt.
otherSalt := mustGeneratePowPrefix(t)
_, ok = c.verifyChallenge(r, ts, otherSalt, mac, diff)
assert.False(t, ok)
// Tampered difficulty: the MAC binds difficulty, so claiming a lower one
// than was issued must fail (the per-request escalation bypass).
_, ok = c.verifyChallenge(r, ts, salt, mac, PowDifficultyLow)
assert.False(t, ok)
// Expired timestamp (too old).
oldTS := fmt.Sprintf("%d", time.Now().Add(-3*ticketAgeBackstop).UnixNano())
oldMAC := c.computePowMAC(salt, r, oldTS, diff)
_, ok = c.verifyChallenge(r, oldTS, salt, oldMAC, diff)
assert.False(t, ok)
// Cross-secret rejection: a challenge signed by a different master secret
// must NOT validate.
otherKeys, err := NewKeyRing([]byte("ffffffffffffffffffffffffffffffff"), time.Minute, 3)
require.NoError(t, err)
other := &ChallengeRuntime{keys: otherKeys}
otherMAC := other.computePowMAC(salt, r, ts, diff)
_, ok = c.verifyChallenge(r, ts, salt, otherMAC, diff)
assert.False(t, ok)
}
// TestDistributedAgreement asserts that two ChallengeRuntime instances
// configured with the same master_secret derive bit-identical per-challenge
// secrets and PoW MACs — the property that makes load-balanced deployments
// work (challenge issuance is stateless; only the spent-set is local).
func TestDistributedAgreement(t *testing.T) {
secret := []byte("shared-secret-shared-secret-shar") // 32 bytes
keysA, err := NewKeyRing(secret, time.Minute, 3)
require.NoError(t, err)
keysB, err := NewKeyRing(secret, time.Minute, 3)
require.NoError(t, err)
a := &ChallengeRuntime{keys: keysA}
b := &ChallengeRuntime{keys: keysB}
r, ts := freshChallenge(t)
salt := "deadbeefcafebabe"
// Both instances agree on the PoW MAC...
assert.Equal(t, a.computePowMAC(salt, r, ts, PowDifficultyMedium), b.computePowMAC(salt, r, ts, PowDifficultyMedium))
// ...and on the per-challenge secret derived from the epoch for ts.
epoch := a.epochForTimestamp(ts)
keyA, ok := a.keys.SignKey(epoch)
require.True(t, ok)
keyB, ok := b.keys.SignKey(epoch)
require.True(t, ok)
assert.Equal(t, deriveChallengeSecret(keyA, r), deriveChallengeSecret(keyB, r))
// And a challenge issued by A validates against B.
signKey, ok := b.verifyChallenge(r, ts, salt, a.computePowMAC(salt, r, ts, PowDifficultyMedium), PowDifficultyMedium)
assert.True(t, ok)
assert.NotEmpty(t, signKey)
}
// solvePoWGo is a Go implementation of the PoW solver matching the JS client.
func solvePoWGo(prefix string, difficulty int) string {
for nonce := 0; ; nonce++ {
candidate := prefix + formatBase36(nonce)
hash := sha256.Sum256([]byte(candidate))
if hasLeadingZeroBits(hash[:], difficulty) {
return formatBase36(nonce)
}
}
}
func formatBase36(n int) string {
if n == 0 {
return "0"
}
const digits = "0123456789abcdefghijklmnopqrstuvwxyz"
var buf [64]byte
i := len(buf)
for n > 0 {
i--
buf[i] = digits[n%36]
n /= 36
}
return string(buf[i:])
}
func TestPoWVerification(t *testing.T) {
prefix := mustGeneratePowPrefix(t)
difficulty := 8
nonce := solvePoWGo(prefix, difficulty)
powHash := sha256.Sum256([]byte(prefix + nonce))
assert.True(t, hasLeadingZeroBits(powHash[:], difficulty))
}
// testSecret is a fixed master secret used by all test helpers below so that
// tickets / MACs / cookies issued in one helper validate in another.
var testSecret = []byte("test-secret-test-secret-test-sec") // 32 bytes
// testKeyRing returns a fresh KeyRing built around testSecret. Returned by
// value would defeat the cache, so we hand back a pointer and rely on the
// caller to discard it after the test.
func testKeyRing() *KeyRing {
k, err := NewKeyRing(testSecret, time.Minute, 3)
if err != nil {
panic(err)
}
return k
}
// newTestRuntime returns a minimal ChallengeRuntime suitable for the helpers
// in this file (no WASM, no obfuscation cache — those tests construct full
// runtimes via NewChallengeRuntime). Includes a spent-set so
// ValidateChallengeResponse's single-use burn works.
func newTestRuntime() *ChallengeRuntime {
return &ChallengeRuntime{keys: testKeyRing(), spent: newSpentSet(spentSetDefaultMaxEntries)}
}
// newTestRuntimeWithDifficulty is a small convenience for the many ValidateChallengeResponse tests.
func newTestRuntimeWithDifficulty(d int) *ChallengeRuntime {
return &ChallengeRuntime{keys: testKeyRing(), powDifficulty: d, spent: newSpentSet(spentSetDefaultMaxEntries)}
}
// freshChallenge generates a per-request nonce+timestamp pair (matching
// GetChallengePage's `r` and `ts`).
func freshChallenge(tb testing.TB) (r, ts string) {
tb.Helper()
ts = fmt.Sprintf("%d", time.Now().UnixNano())
var err error
r, err = generateChallengeNonce()
require.NoError(tb, err)
return r, ts
}
// mustGeneratePowPrefix is the test-only wrapper around generatePowPrefix
// that aborts the test if crypto/rand fails (which would mean the test
// machine is broken in a way that invalidates the run anyway).
func mustGeneratePowPrefix(tb testing.TB) string {
tb.Helper()
p, err := generatePowPrefix()
require.NoError(tb, err)
return p
}
// buildValidBody constructs a valid challenge POST body with an empty
// fingerprint. Existing tests rely on this signature; for tests that need
// to assert fingerprint round-trip through the seal/unseal chain, use
// buildValidBodyWithFingerprint.
func buildValidBody(tb testing.TB, difficulty int, r, ts string) string {
return buildValidBodyWithFingerprint(tb, difficulty, r, ts, FingerprintData{})
}
// buildValidBodyWithFingerprint mirrors buildValidBody but lets the caller
// supply a populated FingerprintData so tests can assert that the submitted
// payload survives the full obfuscate → sign → deobfuscate → unmarshal chain.
// It mirrors the client (challenge.js): derive s = HMAC(K_epoch, r), obfuscate
// f under HMAC(s, "fpenc"||r), and sign sig = HMAC(s, r||ts||n||f).
func buildValidBodyWithFingerprint(tb testing.TB, difficulty int, r, ts string, fp FingerprintData) string {
c := newTestRuntime()
salt := mustGeneratePowPrefix(tb)
powMAC := c.computePowMAC(salt, r, ts, difficulty)
nonce := solvePoWGo(salt, difficulty)
// Derive s exactly as the server will, from the per-epoch sign key for ts.
// Mirror computePowMAC's fallback for out-of-window timestamps (used by the
// expired-timestamp test) so the body is still constructible; the server
// rejects on freshness before the signature is ever checked.
epoch := c.epochForTimestamp(ts)
signKey, ok := c.keys.SignKey(epoch)
if !ok {
_, signKey = c.keys.Current()
}
s := deriveChallengeSecret(signKey, r)
fpJSON, err := json.Marshal(fp)
require.NoError(tb, err)
obfKey := deriveFingerprintObfKey(s, r)
keyBytes := []byte(obfKey)
obfuscated := make([]byte, len(fpJSON))
for i := range fpJSON {
obfuscated[i] = fpJSON[i] ^ keyBytes[i%len(keyBytes)]
}
f := base64.StdEncoding.EncodeToString(obfuscated)
sig := hmacSHA256Hex([]byte(s), []byte(r+ts+nonce+f))
return url.Values{
"f": {f},
"r": {r},
"ts": {ts},
"sig": {sig},
"n": {nonce},
"p": {salt},
"m": {powMAC},
"d": {strconv.Itoa(difficulty)},
"u": {"/"},
}.Encode()
}
func TestValidateChallengeResponse(t *testing.T) {
c := newTestRuntimeWithDifficulty(8)
r, ts := freshChallenge(t)
body := buildValidBody(t, c.powDifficulty, r, ts)
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
req.Header.Set("User-Agent", "test-agent")
ck, fpResult, _, err := c.ValidateChallengeResponse(req, []byte(body))
require.NoError(t, err)
assert.NotNil(t, ck)
assert.NotNil(t, fpResult)
}
// TestValidateChallengeResponse_Replay asserts single-use: a valid submission
// is accepted once, and re-submitting the identical body is rejected because
// its per-challenge nonce `r` has been burned.
func TestValidateChallengeResponse_Replay(t *testing.T) {
c := newTestRuntimeWithDifficulty(8)
r, ts := freshChallenge(t)
body := buildValidBody(t, c.powDifficulty, r, ts)
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
req.Header.Set("User-Agent", "test-agent")
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
require.NoError(t, err)
// Replay the exact same body — must be rejected as already used.
req2, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
req2.Header.Set("User-Agent", "test-agent")
_, _, _, err = c.ValidateChallengeResponse(req2, []byte(body))
assert.ErrorContains(t, err, "already used")
}
// TestValidateChallengeResponse_FingerprintRoundTrip is the explicit
// happy-path guard for the full client→server seal/unseal chain. Other
// ValidateChallengeResponse tests only assert that a well-formed submission
// is accepted; this one verifies that the FingerprintData the browser
// submits is the same FingerprintData the server gets back, then survives
// the cookie Seal → Open round-trip too. Regression guard for any future
// change to encryption, HMAC keying, or proto conversion.
func TestValidateChallengeResponse_FingerprintRoundTrip(t *testing.T) {
c := &ChallengeRuntime{keys: testKeyRing(), powDifficulty: 8, cookieTTL: time.Hour, spent: newSpentSet(spentSetDefaultMaxEntries)}
submitted := FingerprintData{
FSID: "fsid-abcdef",
Nonce: "client-nonce-xyz",
Time: 1700000000,
URL: "https://example.com/protected",
FastBotDetection: FlexBool(false),
}
r, ts := freshChallenge(t)
body := buildValidBodyWithFingerprint(t, c.powDifficulty, r, ts, submitted)
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
require.NoError(t, err)
req.Header.Set("User-Agent", "test-agent")
ck, decoded, _, err := c.ValidateChallengeResponse(req, []byte(body))
require.NoError(t, err)
require.NotNil(t, ck)
// The fingerprint returned by ValidateChallengeResponse must match what
// the browser submitted (XOR'd, base64'd, then decrypted server-side).
assert.Equal(t, submitted.FSID, decoded.FSID)
assert.Equal(t, submitted.Nonce, decoded.Nonce)
assert.Equal(t, submitted.Time, decoded.Time)
assert.Equal(t, submitted.URL, decoded.URL)
// The sealed cookie must round-trip through ValidCookie and yield the
// same fingerprint contents — guards the proto conversion + AES-GCM
// seal/unseal chain.
parsed, err := http.ParseSetCookie(ck.String())
require.NoError(t, err)
cd, err := c.ValidCookie(parsed, "test-agent")
require.NoError(t, err)
require.NotNil(t, cd)
assert.Equal(t, submitted.FSID, cd.Fingerprint.FSID)
assert.Equal(t, submitted.URL, cd.Fingerprint.URL)
assert.Equal(t, c.powDifficulty, cd.PowDifficulty)
}
func TestValidateChallengeResponse_MultipleConcurrentClients(t *testing.T) {
c := newTestRuntimeWithDifficulty(8)
for range 20 {
r, ts := freshChallenge(t)
body := buildValidBody(t, c.powDifficulty, r, ts)
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
req.Header.Set("User-Agent", "test-agent")
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
require.NoError(t, err)
}
}
func TestValidateChallengeResponse_InvalidPoW(t *testing.T) {
c := newTestRuntimeWithDifficulty(8)
r, ts := freshChallenge(t)
salt := mustGeneratePowPrefix(t)
powMAC := c.computePowMAC(salt, r, ts, 8)
body := url.Values{
"f": {"dGVzdA=="},
"r": {r},
"ts": {ts},
"sig": {"deadbeef"},
"n": {"invalid-nonce"},
"p": {salt},
"m": {powMAC},
"d": {"8"},
"u": {"/"},
}.Encode()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
assert.ErrorContains(t, err, "invalid proof-of-work")
}
func TestValidateChallengeResponse_ImpossibleDifficulty(t *testing.T) {
// A submission that would otherwise pass at low difficulty is rejected
// outright when the runtime is set to impossible.
c := newTestRuntimeWithDifficulty(PowDifficultyImpossible)
r, ts := freshChallenge(t)
salt := mustGeneratePowPrefix(t)
// A ticket issued (and MAC-bound) at impossible difficulty: the hard-block
// fires before the unsolvable PoW is ever checked, so a dummy nonce is fine.
powMAC := c.computePowMAC(salt, r, ts, PowDifficultyImpossible)
body := url.Values{
"f": {"dGVzdA=="},
"r": {r},
"ts": {ts},
"sig": {"deadbeef"},
"n": {"0"},
"p": {salt},
"m": {powMAC},
"d": {strconv.Itoa(PowDifficultyImpossible)},
"u": {"/"},
}.Encode()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
req.Header.Set("User-Agent", "test-agent")
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
assert.ErrorContains(t, err, "impossible")
}
func TestValidateChallengeResponse_ExpiredTimestamp(t *testing.T) {
c := newTestRuntimeWithDifficulty(8)
r, _ := freshChallenge(t)
oldTS := fmt.Sprintf("%d", time.Now().Add(-3*ticketAgeBackstop).UnixNano())
body := buildValidBody(t, c.powDifficulty, r, oldTS)
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
req.Header.Set("User-Agent", "test-agent")
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
assert.ErrorContains(t, err, "invalid ticket")
}
// TestValidateChallengeResponse_TamperedR sends a different `r` than the one
// the PoW MAC was issued for, so verifyChallenge's MAC check fails.
func TestValidateChallengeResponse_TamperedR(t *testing.T) {
c := newTestRuntimeWithDifficulty(8)
r, ts := freshChallenge(t)
salt := mustGeneratePowPrefix(t)
powMAC := c.computePowMAC(salt, r, ts, 8) // bound to r, not to the tampered value
body := url.Values{
"f": {"dGVzdA=="},
"r": {"ffffffffffffffffffffffffffffffff"},
"ts": {ts},
"sig": {"deadbeef"},
"n": {"0"},
"p": {salt},
"m": {powMAC},
"d": {"8"},
"u": {"/"},
}.Encode()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
assert.ErrorContains(t, err, "invalid ticket")
}
func TestValidateChallengeResponse_ForgedMAC(t *testing.T) {
c := newTestRuntimeWithDifficulty(8)
r, ts := freshChallenge(t)
salt := mustGeneratePowPrefix(t)
body := url.Values{
"f": {"dGVzdA=="},
"r": {r},
"ts": {ts},
"sig": {"deadbeef"},
"n": {"0"},
"p": {salt},
"m": {"forged-mac-value"},
"d": {"8"},
"u": {"/"},
}.Encode()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
assert.ErrorContains(t, err, "invalid ticket")
}
// TestValidateChallengeResponse_TamperedDifficulty submits a lower difficulty
// than the MAC was issued for; verifyChallenge's MAC check fails because the
// MAC binds the difficulty (the per-request escalation-bypass guard).
func TestValidateChallengeResponse_TamperedDifficulty(t *testing.T) {
c := newTestRuntimeWithDifficulty(8)
r, ts := freshChallenge(t)
salt := mustGeneratePowPrefix(t)
powMAC := c.computePowMAC(salt, r, ts, PowDifficultyHigh) // issued at high
nonce := solvePoWGo(salt, 8)
body := url.Values{
"f": {"dGVzdA=="},
"r": {r},
"ts": {ts},
"sig": {"deadbeef"},
"n": {nonce},
"p": {salt},
"m": {powMAC},
"d": {"8"}, // client claims a lower difficulty than was bound
"u": {"/"},
}.Encode()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
assert.ErrorContains(t, err, "invalid ticket")
}
func TestValidateChallengeResponse_InvalidSig(t *testing.T) {
c := newTestRuntimeWithDifficulty(8)
r, ts := freshChallenge(t)
salt := mustGeneratePowPrefix(t)
powMAC := c.computePowMAC(salt, r, ts, 8)
nonce := solvePoWGo(salt, c.powDifficulty)
body := url.Values{
"f": {"dGVzdA=="},
"r": {r},
"ts": {ts},
"sig": {"0000000000000000000000000000000000000000000000000000000000000000"},
"n": {nonce},
"p": {salt},
"m": {powMAC},
"d": {"8"},
"u": {"/"},
}.Encode()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
assert.ErrorContains(t, err, "invalid HMAC")
}
func TestValidateChallengeResponse_MissingFields(t *testing.T) {
c := &ChallengeRuntime{}
body := url.Values{
"f": {"dGVzdA=="},
"r": {"abcd"},
}.Encode()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
assert.ErrorContains(t, err, "missing required fields")
}
func TestValidateChallengeResponse_MalformedR(t *testing.T) {
c := &ChallengeRuntime{}
// All fields present (passes the missing-field guard) but `r` is not a
// well-formed 32-char hex nonce, so the shape check rejects it before any
// spent-set burn. Error is the generic "invalid ticket" so the stage stays
// indistinguishable from a verifyChallenge failure.
for name, badR := range map[string]string{
"too short": strings.Repeat("a", 31),
"too long": strings.Repeat("a", 33),
"right len non-hex": strings.Repeat("z", 32),
} {
t.Run(name, func(t *testing.T) {
body := url.Values{
"f": {"dGVzdA=="},
"r": {badR},
"ts": {"1"},
"sig": {"x"},
"n": {"x"},
"p": {"x"},
"m": {"x"},
"d": {"12"},
"u": {"/"},
}.Encode()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
_, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
assert.ErrorContains(t, err, "invalid ticket")
})
}
}
+151
View File
@@ -0,0 +1,151 @@
// config.go defines the YAML-facing challenge configuration. Multiple
// appsec-config files can each contribute a disjoint subset; MergeFrom
// composes them without one wiping the others. Once merged, BuildOptions
// translates the Config into the runtime's functional-option list
// (NewChallengeRuntime(...)).
package challenge
import (
"fmt"
"time"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/logging"
)
// Config carries the YAML-configurable challenge runtime settings.
type Config struct {
// MasterSecret is the long-lived secret all per-epoch HMAC keys and the
// cookie-sealing AES key derive from. In a distributed deployment every
// instance MUST share the same value to sign/verify each other's
// challenges. If unset, the runtime generates an ephemeral random secret
// at startup — fine for a single instance, but restarts then invalidate
// outstanding cookies.
MasterSecret *string `yaml:"master_secret"`
// KeyRotationInterval is the per-epoch key advance period. All instances
// in a distributed setup MUST agree on it to derive identical keys.
// Defaults to 5m.
KeyRotationInterval *time.Duration `yaml:"key_rotation_interval"`
// MaxLiveEpochs is how many past epochs (besides the current one) the
// keyring keeps accepting, so in-flight submissions aren't invalidated at
// a rotation boundary. Bounds ticket-forgery exposure to
// MaxLiveEpochs × KeyRotationInterval. Defaults to 3.
MaxLiveEpochs *int `yaml:"max_live_epochs"`
// CookieTTL is how long a successful-challenge cookie stays valid.
// Decoupled from the keyring window (enforced by a not_after stamp inside
// the sealed cookie, not key eviction) so cookies can be long-lived while
// per-epoch keys rotate tightly. Defaults to 12h.
CookieTTL *time.Duration `yaml:"cookie_ttl"`
// MaxCookieSize caps the challenge cookie's encoded size, enforced on both
// seal and open. It bounds the memory allocated from the (attacker-supplied)
// fingerprint envelope, closing an over-allocation DoS. Defaults to
// MaxCookieLen (4096, the per-cookie size browsers guarantee); raise it only
// if a non-browser client tolerates larger cookies.
MaxCookieSize *int `yaml:"max_cookie_size"`
// CryptoObfuscationPoolSize is how many obfuscations of the per-epoch
// sign-key module to keep per live epoch. Each variant embeds the same key
// with different byte layout, giving per-visitor variance. Defaults to 1.
CryptoObfuscationPoolSize *int `yaml:"crypto_obfuscation_pool_size"`
// SpentSetMaxEntries caps the replay-protection LRU. A deep DoS backstop;
// steady-state stays far below it. Defaults to spentSetDefaultMaxEntries.
SpentSetMaxEntries *int `yaml:"spent_set_max_entries"`
// LogLevel sets the challenge runtime's own log verbosity, independent of
// the global level. Note: `panic` is not supported — logrus.PanicLevel is 0,
// which SubLogger (pkg/logging/sublogger.go) treats as "inherit the parent level".
LogLevel *log.Level `yaml:"log_level,omitempty"`
}
// MergeFrom overlays the non-nil fields of other onto c, field by field, so
// multiple appsec-configs can each contribute a disjoint subset (last non-nil
// wins). A nil receiver or argument is a no-op.
func (c *Config) MergeFrom(other *Config) {
if c == nil || other == nil {
return
}
if other.MasterSecret != nil {
c.MasterSecret = other.MasterSecret
}
if other.KeyRotationInterval != nil {
c.KeyRotationInterval = other.KeyRotationInterval
}
if other.MaxLiveEpochs != nil {
c.MaxLiveEpochs = other.MaxLiveEpochs
}
if other.CookieTTL != nil {
c.CookieTTL = other.CookieTTL
}
if other.MaxCookieSize != nil {
c.MaxCookieSize = other.MaxCookieSize
}
if other.CryptoObfuscationPoolSize != nil {
c.CryptoObfuscationPoolSize = other.CryptoObfuscationPoolSize
}
if other.SpentSetMaxEntries != nil {
c.SpentSetMaxEntries = other.SpentSetMaxEntries
}
if other.LogLevel != nil {
c.LogLevel = other.LogLevel
}
}
// BuildOptions translates a (possibly nil) merged Config into the WithXxx
// Option list for NewChallengeRuntime; unset fields are omitted so the runtime
// uses its built-in defaults. Returns an error if MasterSecret is set but
// invalid. parent (may be nil) is the logger the "challenge" sublogger derives
// from, at the configured log_level or parent's level.
func BuildOptions(c *Config, parent *log.Entry) ([]Option, error) {
// Always give the runtime its own component sublogger.
base := log.StandardLogger()
if parent != nil {
base = parent.Logger
}
var lvl log.Level
if c != nil && c.LogLevel != nil {
lvl = *c.LogLevel
}
opts := []Option{WithLogger(logging.SubLogger(base, "challenge", lvl))}
if c == nil {
return opts, nil
}
if c.MasterSecret != nil && *c.MasterSecret != "" {
secret, err := ParseConfiguredSecret(*c.MasterSecret)
if err != nil {
return nil, fmt.Errorf("invalid challenge master_secret: %w", err)
}
opts = append(opts, WithMasterSecret(secret))
}
if c.KeyRotationInterval != nil {
opts = append(opts, WithRotationInterval(*c.KeyRotationInterval))
}
if c.MaxLiveEpochs != nil && *c.MaxLiveEpochs > 0 {
opts = append(opts, WithMaxLiveEpochs(*c.MaxLiveEpochs))
}
if c.CookieTTL != nil {
opts = append(opts, WithCookieTTL(*c.CookieTTL))
}
if c.MaxCookieSize != nil && *c.MaxCookieSize > 0 {
opts = append(opts, WithMaxCookieLen(*c.MaxCookieSize))
}
if c.CryptoObfuscationPoolSize != nil && *c.CryptoObfuscationPoolSize > 0 {
opts = append(opts, WithCryptoObfuscationPoolSize(*c.CryptoObfuscationPoolSize))
}
if c.SpentSetMaxEntries != nil && *c.SpentSetMaxEntries > 0 {
opts = append(opts, WithSpentSetMaxEntries(*c.SpentSetMaxEntries))
}
return opts, nil
}
+169
View File
@@ -0,0 +1,169 @@
package challenge
import (
"net/http"
"net/url"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestConfigMergeFromNilReceiverAndOther covers the two no-op edge cases:
// merging into a nil receiver and merging a nil source. Both must not panic
// and must not change anything.
func TestConfigMergeFromNilReceiverAndOther(t *testing.T) {
var nilCfg *Config
require.NotPanics(t, func() { nilCfg.MergeFrom(&Config{CookieTTL: new(time.Hour)}) })
dst := &Config{CookieTTL: new(time.Hour)}
dst.MergeFrom(nil)
require.NotNil(t, dst.CookieTTL)
assert.Equal(t, time.Hour, *dst.CookieTTL, "nil source must leave existing fields untouched")
}
// TestConfigMergeFromOverlaysOnlyNonNilFields confirms the per-field "last
// wins" semantics: each set field in `other` overrides `c`, but unset fields
// leave `c`'s existing values intact. This is what lets multiple appsec-
// configs each contribute a disjoint subset without one wiping the others.
func TestConfigMergeFromOverlaysOnlyNonNilFields(t *testing.T) {
dst := &Config{
MasterSecret: new("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"),
KeyRotationInterval: new(5 * time.Minute),
MaxLiveEpochs: new(3),
CookieTTL: new(12 * time.Hour),
CryptoObfuscationPoolSize: new(1),
}
src := &Config{
// Override CookieTTL and CryptoObfuscationPoolSize; leave others
// nil so the dst values must survive.
CookieTTL: new(1 * time.Hour),
CryptoObfuscationPoolSize: new(4),
// New fields not present on dst.
SpentSetMaxEntries: new(500_000),
MaxCookieSize: new(8192),
}
dst.MergeFrom(src)
// Overridden fields take src values.
assert.Equal(t, 1*time.Hour, *dst.CookieTTL)
assert.Equal(t, 4, *dst.CryptoObfuscationPoolSize)
// Untouched-by-src fields keep dst values.
require.NotNil(t, dst.MasterSecret)
assert.Equal(t, "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", *dst.MasterSecret)
assert.Equal(t, 5*time.Minute, *dst.KeyRotationInterval)
assert.Equal(t, 3, *dst.MaxLiveEpochs)
// New-from-src fields appear on dst.
require.NotNil(t, dst.SpentSetMaxEntries)
assert.Equal(t, 500_000, *dst.SpentSetMaxEntries)
assert.Equal(t, 8192, *dst.MaxCookieSize)
}
// TestBuildOptionsNilOrEmptyConfig confirms a nil or fully-empty Config emits
// only the always-present component-logger option; the runtime is otherwise
// constructed with its built-in defaults.
func TestBuildOptionsNilOrEmptyConfig(t *testing.T) {
opts, err := BuildOptions(nil, nil)
require.NoError(t, err)
assert.Len(t, opts, 1, "nil config still emits the component-logger option")
opts, err = BuildOptions(&Config{}, nil)
require.NoError(t, err)
assert.Len(t, opts, 1, "empty config still emits the component-logger option")
}
// TestBuildOptionsTranslatesFieldsToRuntimeBehavior wires a populated Config
// through BuildOptions and NewChallengeRuntime, then asserts the runtime's
// observable fields match the configured values. This is the integration
// guard for "appsec-config values reach the running challenge engine".
func TestBuildOptionsTranslatesFieldsToRuntimeBehavior(t *testing.T) {
cfg := &Config{
MasterSecret: new("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"),
KeyRotationInterval: new(3 * time.Minute),
MaxLiveEpochs: new(4),
CookieTTL: new(2 * time.Hour),
CryptoObfuscationPoolSize: new(2),
SpentSetMaxEntries: new(250_000),
MaxCookieSize: new(8192),
}
opts, err := BuildOptions(cfg, nil)
require.NoError(t, err)
require.Len(t, opts, 8, "every populated field + the component logger must emit an option")
rt, err := NewChallengeRuntime(t.Context(), opts...)
require.NoError(t, err)
assert.Equal(t, 2*time.Hour, rt.cookieTTL, "CookieTTL must reach the runtime")
assert.Equal(t, 2, rt.cryptoPoolSize, "CryptoObfuscationPoolSize must reach the runtime")
assert.Equal(t, 250_000, rt.spent.maxEntries, "SpentSetMaxEntries must reach the runtime")
assert.Equal(t, 8192, rt.maxCookieLen, "MaxCookieSize must reach the runtime")
}
// TestBuildOptionsInvalidMasterSecret confirms a malformed master_secret
// surfaces as a configuration error rather than silently falling back to a
// random secret — that would be a footgun in distributed deployments.
func TestBuildOptionsInvalidMasterSecret(t *testing.T) {
cfg := &Config{
MasterSecret: new("too-short"),
}
_, err := BuildOptions(cfg, nil)
require.Error(t, err)
assert.Contains(t, err.Error(), "master_secret")
}
// TestSealAllowlistCookieTTLOverride exercises the per-call TTL plumbing on
// SealAllowlistCookie: when ttlOverride is non-nil and positive, the
// generated cookie's Max-Age reflects the override rather than the
// runtime-global cookie_ttl. Nil falls back to the runtime default.
func TestSealAllowlistCookieTTLOverride(t *testing.T) {
rt, err := NewChallengeRuntime(t.Context(),
WithMasterSecret([]byte("0123456789abcdef0123456789abcdef")),
WithCookieTTL(12*time.Hour),
)
require.NoError(t, err)
req := &http.Request{
Header: http.Header{"User-Agent": []string{"test-agent"}},
URL: mustURL("https://example.test/protected"),
}
// Override → short TTL.
override := 30 * time.Minute
before := time.Now().Unix()
ck, err := rt.SealAllowlistCookie(req, "explicit-short", &override)
require.NoError(t, err)
require.NotNil(t, ck)
assert.InDelta(t, before+int64(override.Seconds()), ck.Expiration, 2,
"override TTL must drive the cookie expiration")
// No override → runtime default (12h).
before = time.Now().Unix()
ck, err = rt.SealAllowlistCookie(req, "fallback", nil)
require.NoError(t, err)
require.NotNil(t, ck)
assert.InDelta(t, before+int64((12*time.Hour).Seconds()), ck.Expiration, 2,
"nil override must fall back to runtime cookie_ttl")
// Zero/negative override → treated as "no override", uses runtime default.
before = time.Now().Unix()
zero := time.Duration(0)
ck, err = rt.SealAllowlistCookie(req, "zero-fallback", &zero)
require.NoError(t, err)
require.NotNil(t, ck)
assert.InDelta(t, before+int64((12*time.Hour).Seconds()), ck.Expiration, 2,
"non-positive override must fall back to runtime cookie_ttl")
}
func mustURL(s string) *url.URL {
u, err := url.Parse(s)
if err != nil {
panic(err)
}
return u
}
+277
View File
@@ -0,0 +1,277 @@
// crypto.go implements AES-GCM cookie sealing/unsealing for the challenge
// cookie envelope: a versioned, integrity-protected blob carrying the
// fingerprint proto bytes plus a not_after expiration timestamp. The
// encryption key comes from the keyring (cookie-master derivation, long
// lifetime); rotation is handled by the keyring, not by this file. All
// helpers here are pure functions over byte slices — no IO, no globals.
package challenge
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"errors"
"fmt"
"time"
"golang.org/x/crypto/hkdf"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/pb"
"google.golang.org/protobuf/proto"
)
var (
ErrCookieMalformed = errors.New("malformed cookie")
ErrCookieSignature = errors.New("invalid cookie signature")
ErrCookiePayload = errors.New("invalid cookie payload")
ErrCookieExpired = errors.New("cookie expired")
ErrCookieVersion = errors.New("unknown cookie version")
ErrAllowlistReasonSize = errors.New("allowlist reason exceeds maximum length")
ErrCookieTooLarge = errors.New("cookie exceeds maximum size")
)
const hkdfInfo = "crowdsec-challenge-cookie"
// MaxAllowlistReasonLen caps the reason string operators pass to
// GrantChallengeCookie. The reason travels inside every Set-Cookie + Cookie
// header round-trip until the cookie expires; bounding it keeps the cookie
// well under the 4 KB browser limit even with the AES-GCM tag + base64
// expansion.
const MaxAllowlistReasonLen = 256
// MaxCookieLen is the DEFAULT per-cookie size (RFC 6265 §6.1: 4096 bytes).
// Can be configured via Config.MaxCookieSize and we reject anything bigger.
const MaxCookieLen = 4096
// Cookie wire format. A single version byte at offset 0 lets us evolve the
// format without flag-day-style cookie invalidation. New formats add a new
// case in openCookie's switch.
//
// v0: cookieVersionV0 || nonce(12) || ciphertext
//
// Sealed under KeyRing.MasterCookieKey() — a long-lived AES-GCM key
// derived from master_secret with no epoch dependency. AAD is the
// User-Agent.
//
// Plaintext layout (inside the AEAD envelope):
//
// not_after_be8 (8 bytes BE, unix seconds)
// flags_byte (1 byte; bit 0 = allowlisted, bits 1-7 reserved)
// reason_len_be (2 bytes BE, length of reason_bytes)
// reason_bytes (0..MaxAllowlistReasonLen UTF-8 bytes)
// protobuf(envelope)
//
// All four header fields are inside the AEAD envelope, so any
// tampering with them invalidates the GCM tag.
//
// Decoupling the cookie key from the epoch keyring is what lets us
// have e.g. 12h cookies while keeping ticket-signing keys rotating
// every few minutes — the two security windows are independent.
const cookieVersionV0 byte = 0x00
// cookiePlaintextFixedHeaderLen is the size of the fixed-layout portion of
// the plaintext header that precedes the protobuf envelope:
//
// not_after_be8 (8) + flags_byte (1) + reason_len_be (2) = 11
//
// followed by reason_bytes (variable, 0..MaxAllowlistReasonLen).
const cookiePlaintextFixedHeaderLen = 8 + 1 + 2
// cookieFlagAllowlisted marks a cookie minted by GrantChallengeCookie
// (operator allowlist bypass) rather than by a real challenge submission.
const cookieFlagAllowlisted byte = 0x01
func deriveKey(secret []byte) ([]byte, error) {
h := hkdf.New(sha256.New, secret, nil, []byte(hkdfInfo))
key := make([]byte, 32) // AES-256
if _, err := h.Read(key); err != nil {
return nil, fmt.Errorf("failed to derive key: %w", err)
}
return key, nil
}
// sealCookieV0 produces a v0 cookie sealed under the long-lived master
// cookie key. notAfter is the unix-seconds expiration; flags carries the
// allowlist bit set by GrantChallengeCookie (0 for normal cookies);
// reason is the operator-supplied allowlist reason (empty for normal
// cookies). All three are prepended to the marshaled proto BEFORE
// encryption so they are both confidential (not observable from the wire)
// and authenticated (any tamper attempt invalidates the GCM tag).
//
// Returns ErrAllowlistReasonSize if reason exceeds MaxAllowlistReasonLen.
func sealCookieV0(envelope *pb.ChallengeCookie, masterCookieKey []byte, notAfter int64, flags byte, reason string, aad []byte, maxCookieLen int) (string, error) {
if maxCookieLen <= 0 {
maxCookieLen = MaxCookieLen
}
if len(reason) > MaxAllowlistReasonLen {
return "", fmt.Errorf("%w: %d > %d", ErrAllowlistReasonSize, len(reason), MaxAllowlistReasonLen)
}
key, err := deriveKey(masterCookieKey)
if err != nil {
return "", err
}
block, err := aes.NewCipher(key)
if err != nil {
return "", fmt.Errorf("failed to create cipher: %w", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", fmt.Errorf("failed to create GCM: %w", err)
}
// Reject an over-limit envelope before marshaling it.
maxPlaintext := maxCookieLen/4*3 - 1 - gcm.NonceSize() - gcm.Overhead()
if plaintextLen := cookiePlaintextFixedHeaderLen + len(reason) + proto.Size(envelope); plaintextLen > maxPlaintext {
return "", fmt.Errorf("%w: plaintext=%d > %d", ErrCookieTooLarge, plaintextLen, maxPlaintext)
}
envelopeBytes, err := proto.Marshal(envelope)
if err != nil {
return "", fmt.Errorf("failed to marshal challenge cookie proto: %w", err)
}
nonce := make([]byte, gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return "", fmt.Errorf("failed to generate nonce: %w", err)
}
// Build the plaintext: not_after_be8 || flags || reason_len_be || reason || envelope
plaintext := make([]byte, 0, cookiePlaintextFixedHeaderLen+len(reason)+len(envelopeBytes))
var notAfterBytes [8]byte
binary.BigEndian.PutUint64(notAfterBytes[:], uint64(notAfter))
plaintext = append(plaintext, notAfterBytes[:]...)
plaintext = append(plaintext, flags)
var reasonLenBytes [2]byte
binary.BigEndian.PutUint16(reasonLenBytes[:], uint16(len(reason)))
plaintext = append(plaintext, reasonLenBytes[:]...)
plaintext = append(plaintext, []byte(reason)...)
plaintext = append(plaintext, envelopeBytes...)
ciphertext := gcm.Seal(nonce, nonce, plaintext, aad)
// Wire form: version || nonce || ciphertext
out := make([]byte, 0, 1+len(ciphertext))
out = append(out, cookieVersionV0)
out = append(out, ciphertext...)
return base64.RawURLEncoding.EncodeToString(out), nil
}
// CookieEnvelope bundles the proto payload with the header fields that
// openCookie pulls out of the AEAD-sealed plaintext: allowlist marker,
// allowlist reason, expiration. Returned by openCookie so callers can
// route allowlist cookies differently from real-submission ones without
// re-parsing the plaintext.
type CookieEnvelope struct {
Envelope *pb.ChallengeCookie
Allowlisted bool
AllowlistReason string
NotAfter int64
}
// openCookie decodes a sealed cookie, dispatching on the version byte.
// Unknown versions are rejected with ErrCookieVersion. Expired cookies
// (notAfter <= now) are rejected with ErrCookieExpired.
func openCookie(encoded string, masterCookieKey []byte, aad []byte, maxCookieLen int) (*CookieEnvelope, error) {
if maxCookieLen <= 0 {
maxCookieLen = MaxCookieLen
}
if len(encoded) > maxCookieLen {
return nil, fmt.Errorf("%w: %d > %d", ErrCookieTooLarge, len(encoded), maxCookieLen)
}
raw, err := base64.RawURLEncoding.DecodeString(encoded)
if err != nil {
return nil, fmt.Errorf("%w: failed to decode: %w", ErrCookieMalformed, err)
}
if len(raw) < 1 {
return nil, fmt.Errorf("%w: empty cookie", ErrCookieMalformed)
}
switch raw[0] {
case cookieVersionV0:
return openCookieV0Bytes(raw[1:], masterCookieKey, aad, time.Now())
default:
return nil, fmt.Errorf("%w: 0x%02x", ErrCookieVersion, raw[0])
}
}
func openCookieV0Bytes(body []byte, masterCookieKey []byte, aad []byte, now time.Time) (*CookieEnvelope, error) {
key, err := deriveKey(masterCookieKey)
if err != nil {
return nil, err
}
block, err := aes.NewCipher(key)
if err != nil {
return nil, fmt.Errorf("failed to create cipher: %w", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, fmt.Errorf("failed to create GCM: %w", err)
}
nonceSize := gcm.NonceSize()
if len(body) < nonceSize {
return nil, fmt.Errorf("%w: ciphertext too short", ErrCookieMalformed)
}
nonce, ciphertext := body[:nonceSize], body[nonceSize:]
plaintext, err := gcm.Open(nil, nonce, ciphertext, aad)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrCookieSignature, err)
}
if len(plaintext) < cookiePlaintextFixedHeaderLen {
return nil, fmt.Errorf("%w: plaintext shorter than fixed header", ErrCookieMalformed)
}
notAfter := int64(binary.BigEndian.Uint64(plaintext[:8]))
flags := plaintext[8]
reasonLen := int(binary.BigEndian.Uint16(plaintext[9:11]))
if reasonLen > MaxAllowlistReasonLen {
return nil, fmt.Errorf("%w: reason_len=%d", ErrCookieMalformed, reasonLen)
}
if len(plaintext) < cookiePlaintextFixedHeaderLen+reasonLen {
return nil, fmt.Errorf("%w: plaintext shorter than declared reason_len", ErrCookieMalformed)
}
if notAfter <= now.Unix() {
return nil, fmt.Errorf("%w: not_after=%d now=%d", ErrCookieExpired, notAfter, now.Unix())
}
reasonStart := cookiePlaintextFixedHeaderLen
reasonEnd := reasonStart + reasonLen
reason := string(plaintext[reasonStart:reasonEnd])
envelope := &pb.ChallengeCookie{}
if err := proto.Unmarshal(plaintext[reasonEnd:], envelope); err != nil {
return nil, fmt.Errorf("%w: %w", ErrCookiePayload, err)
}
return &CookieEnvelope{
Envelope: envelope,
Allowlisted: flags&cookieFlagAllowlisted != 0,
AllowlistReason: reason,
NotAfter: notAfter,
}, nil
}
+231
View File
@@ -0,0 +1,231 @@
// Package challenge dynamic_module.go handles the **sensitive** per-epoch
// sign-key module — the small JS that embeds the current epoch's HMAC
// key (~30 lines, expanded to ~10-30KB after obfuscation). This is the
// actual cryptographic material protected by the challenge runtime;
// obfuscation here is doing real work, not cosmetic byte variance.
//
// The module is rebuilt whenever the keyring advances to a new epoch
// (default cadence: 5 minutes via keyringDefaultRotation). To give
// per-visitor variance in how the same key is embedded, the runtime
// keeps a small pool of cryptoPoolSize variants per epoch — each
// variant is an independent obfuscation of the same input. Default
// pool size is 1, preserving the historical single-variant-per-epoch
// behavior; operators that want per-visitor variance for the
// sensitive code can raise WithCryptoObfuscationPoolSize.
//
// For the **non-sensitive** challenge code (the build-time-obfuscated
// crypto/glue served inline) and the separately-served, unobfuscated
// fpscanner bundle, see static_bundle.go.
package challenge
import (
"context"
_ "embed"
"encoding/hex"
"fmt"
"math/rand/v2"
"strconv"
"strings"
"text/template"
"time"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/metrics"
)
//go:embed dynamic_module.js.tmpl
var dynamicModuleTemplate string
// preWarmLeadTime returns how far before the next rotation boundary we want
// to start obfuscating the upcoming epoch's dynamic module. Capped at 30s
// (no point starting much earlier — the module is unused until the
// boundary), floored at 1s so very small rotation intervals don't make the
// ticker degenerate into a tight loop.
func (c *ChallengeRuntime) preWarmLeadTime() time.Duration {
return max(min(c.keys.rotationInterval/4, 30*time.Second), time.Second)
}
// dynamicModulePreWarmer runs in the background and obfuscates the next
// epoch's dynamic key module (all cryptoPoolSize variants) shortly before
// the rotation boundary, so the first request after a rotation finds the
// module already cached and pays no obfuscation latency on the
// request-serving path.
//
// The ticker goroutine calls dynamicModuleForEpoch synchronously. The
// singleflight inside that call coalesces a real request that arrives
// slightly after the rotation, and time.Ticker drops ticks while the
// receiver is busy — so successive triggers during a single obfuscation
// are naturally absorbed.
func (c *ChallengeRuntime) dynamicModulePreWarmer(ctx context.Context) {
leadTime := c.preWarmLeadTime()
tick := time.NewTicker(leadTime)
defer tick.Stop()
intervalSecs := int64(c.keys.rotationInterval / time.Second)
for {
select {
case <-ctx.Done():
return
case <-tick.C:
now := c.keys.now()
current := c.keys.CurrentEpoch()
nextBoundary := time.Unix((current+1)*intervalSecs, 0)
// Skip ticks that are not yet inside the lead-time window
// before the next boundary — pre-warming a far-future epoch
// would waste CPU on a module that might never be served.
if nextBoundary.Sub(now) > leadTime {
continue
}
nextEpoch := current + 1
c.log().WithFields(log.Fields{
"epoch": nextEpoch,
"current_epoch": current,
"seconds_to_boundary": int64(nextBoundary.Sub(now).Seconds()),
"pool_size": c.cryptoPoolSize,
}).Debug("pre-warming dynamic key module for upcoming epoch")
if _, err := c.dynamicModuleForEpoch(ctx, nextEpoch); err != nil {
c.log().WithError(err).WithField("epoch", nextEpoch).
Warn("pre-warm of next epoch failed; first request after rotation will pay obfuscation cost")
}
}
}
}
// dynamicModuleForEpoch returns the cryptoPoolSize obfuscated variants of the
// per-epoch key module, deriving the key from the keyring on demand. Results
// are cached; concurrent calls for the same epoch coalesce via singleflight so
// only one batch obfuscation runs under a thundering-herd at a rotation
// boundary. The cache mutex guards only the map, never the obfuscation.
func (c *ChallengeRuntime) dynamicModuleForEpoch(ctx context.Context, epoch int64) ([]string, error) {
// Fast path: cached.
c.dynamicModuleCacheMu.RLock()
if cached, ok := c.dynamicModuleCache[epoch]; ok {
c.dynamicModuleCacheMu.RUnlock()
return cached, nil
}
c.dynamicModuleCacheMu.RUnlock()
// Slow path: deduplicate concurrent obfuscation calls for the same
// epoch. Only one goroutine runs the obfuscator batch; the others
// block on the singleflight result. The key is the epoch itself,
// formatted as a string.
key := strconv.FormatInt(epoch, 10)
v, err, _ := c.dynamicModuleSF.Do(key, func() (interface{}, error) {
// Re-check the cache inside the singleflight callback in case
// another goroutine populated it between the fast-path read and
// our entry into Do.
c.dynamicModuleCacheMu.RLock()
if cached, ok := c.dynamicModuleCache[epoch]; ok {
c.dynamicModuleCacheMu.RUnlock()
return cached, nil
}
c.dynamicModuleCacheMu.RUnlock()
signKey, ok := c.keys.SignKey(epoch)
if !ok {
return nil, fmt.Errorf("epoch %d is outside the keyring live window", epoch)
}
tmpl, err := template.New("dynamic-module").Parse(dynamicModuleTemplate)
if err != nil {
return nil, fmt.Errorf("parse dynamic module template: %w", err)
}
var rendered strings.Builder
if err := tmpl.Execute(&rendered, map[string]interface{}{
"Key": hex.EncodeToString(signKey),
"Epoch": epoch,
}); err != nil {
return nil, fmt.Errorf("render dynamic module template: %w", err)
}
input := rendered.String()
inputSize := len(input)
poolSize := max(c.cryptoPoolSize, 1)
variants := make([]string, 0, poolSize)
batchStart := time.Now()
for i := range poolSize {
obfuscateStart := time.Now()
obfuscated, err := c.ObfuscateJS(ctx, input)
obfuscateDuration := time.Since(obfuscateStart)
if err != nil {
return nil, fmt.Errorf("obfuscate dynamic module variant %d/%d: %w", i+1, poolSize, err)
}
if obfuscated == "" {
return nil, fmt.Errorf("obfuscator produced empty dynamic module output (variant %d/%d)", i+1, poolSize)
}
metrics.AppsecChallengeReobfuscation.WithLabelValues("dynamic").Inc()
c.log().WithFields(log.Fields{
"epoch": epoch,
"variant": i,
"input_bytes": inputSize,
"output_bytes": len(obfuscated),
"duration_ms": obfuscateDuration.Milliseconds(),
}).Debug("obfuscated dynamic key module variant")
variants = append(variants, obfuscated)
}
c.log().WithFields(log.Fields{
"epoch": epoch,
"variants": poolSize,
"batch_ms": time.Since(batchStart).Milliseconds(),
"input_bytes": inputSize,
}).Debug("obfuscated dynamic key module batch for new epoch")
c.dynamicModuleCacheMu.Lock()
// Prune any cached modules whose epoch has fallen out of the
// live window before inserting the new one.
live := make(map[int64]bool, len(c.dynamicModuleCache))
for _, e := range c.keys.LiveEpochs() {
live[e] = true
}
evicted := 0
for e := range c.dynamicModuleCache {
if !live[e] {
delete(c.dynamicModuleCache, e)
evicted++
}
}
c.dynamicModuleCache[epoch] = variants
c.dynamicModuleCacheMu.Unlock()
if evicted > 0 {
metrics.AppsecChallengeDynamicModuleEvicted.Add(float64(evicted))
}
return variants, nil
})
if err != nil {
return nil, err
}
return v.([]string), nil
}
// currentDynamicModule returns one of the obfuscated dynamic-module
// variants for the keyring's current epoch, selected uniformly at
// random. When the pool size is 1 (default), every render returns the
// same variant; when raised via WithCryptoObfuscationPoolSize, each
// render picks one of the N variants of the same epoch key.
func (c *ChallengeRuntime) currentDynamicModule(ctx context.Context) (string, error) {
epoch, _ := c.keys.Current()
variants, err := c.dynamicModuleForEpoch(ctx, epoch)
if err != nil {
return "", err
}
if len(variants) == 0 {
return "", fmt.Errorf("dynamic module pool is empty for epoch %d", epoch)
}
if len(variants) == 1 {
return variants[0], nil
}
return variants[rand.IntN(len(variants))], nil
}
@@ -0,0 +1,23 @@
// dynamic_module.js.tmpl — the per-epoch module, obfuscated at runtime on each
// keyring rotation. Small by design (the heavy fpscanner/crypto lives in the
// pre-obfuscated static bundle). {{.Key}} = hex per-epoch key, {{.Epoch}} =
// epoch id. The hook name must match CSEC_HOOK_NAME in challenge.js and is kept
// literal via reservedStrings in obfuscate.js; the key string is deliberately
// left for the obfuscator's string-array transform to encode.
(function () {
var hookName = "__CSEC_CHALLENGE_HOOK_v1__";
var key = "{{.Key}}";
var epoch = {{.Epoch}};
// The static bundle registers the hook synchronously at module load. If
// for some reason it is not present yet, retry on a microtask.
function invoke() {
var hook = globalThis[hookName];
if (typeof hook === "function") {
hook({ key: key, epoch: epoch });
return;
}
setTimeout(invoke, 10);
}
invoke();
})();
+721
View File
@@ -0,0 +1,721 @@
// fingerprint.go holds the wire-shape structs the JS fingerprint bundle
// returns: FlexInt / FlexBool tolerant primitives and the nested
// FingerprintData / fingerprintBotAlias structures that mirror the JSON
// payload one-to-one. Handwritten accessors live in fingerprint_helpers.go,
// proto ↔ struct conversion in fingerprint_proto.go, and mismatch detection
// methods in fingerprint_mismatch.go.
package challenge
import (
"encoding/json"
"strconv"
)
// FlexInt handles JSON values that may be either a number or a string.
// The fingerprint JS library returns a string (error message) instead of
// a number when it fails to collect a value.
type FlexInt int
// FlexBool handles JSON values that may be either a boolean or a string.
type FlexBool bool
func (fb *FlexBool) UnmarshalJSON(data []byte) error {
var b bool
if err := json.Unmarshal(data, &b); err == nil {
*fb = FlexBool(b)
return nil
}
var s string
if err := json.Unmarshal(data, &s); err != nil {
return err
}
*fb = FlexBool(s == "true" || s == "1")
return nil
}
func (fb FlexBool) Bool() bool {
return bool(fb)
}
func (fi *FlexInt) UnmarshalJSON(data []byte) error {
// Try number first (most common case).
var n int
if err := json.Unmarshal(data, &n); err == nil {
*fi = FlexInt(n)
return nil
}
// Fall back to string → parse as int, or default to 0.
var s string
if err := json.Unmarshal(data, &s); err != nil {
return err
}
parsed, err := strconv.Atoi(s)
if err != nil {
*fi = 0
return nil //nolint:nilerr // tolerant by design: non-numeric strings (JS-side error messages) default to 0
}
*fi = FlexInt(parsed)
return nil
}
func (fi FlexInt) Int() int {
return int(fi)
}
/*
{
"signals": {
"automation": {
"webdriver": false,
"webdriverWritable": false,
"selenium": false,
"cdp": true,
"playwright": false,
"navigatorPropertyDescriptors": "00000"
},
"device": {
"cpuCount": 14,
"memory": 8,
"platform": "MacIntel",
"screenResolution": {
"width": 2304,
"height": 1296,
"pixelDepth": 30,
"colorDepth": 30,
"availableWidth": 2304,
"availableHeight": 1265,
"innerWidth": 1444,
"innerHeight": 1144,
"hasMultipleDisplays": true
},
"multimediaDevices": {
"speakers": 1,
"microphones": 1,
"webcams": 1
},
"mediaQueries": {
"prefersColorScheme": "dark",
"prefersReducedMotion": false,
"prefersReducedTransparency": false,
"colorGamut": "p3",
"pointer": "fine",
"anyPointer": "fine",
"hover": true,
"anyHover": true,
"colorDepth": 10
}
},
"browser": {
"userAgent": "Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_15_7)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/144.0.0.0+Safari/537.36",
"features": {
"bitmask": "1000111100",
"chrome": true,
"brave": false,
"applePaySupport": false,
"opera": false,
"serial": true,
"attachShadow": true,
"caches": true,
"webAssembly": true,
"buffer": false,
"showModalDialog": false
},
"plugins": {
"isValidPluginArray": true,
"pluginCount": 5,
"pluginNamesHash": "-2cdc5c8b",
"pluginConsistency1": true,
"pluginOverflow": false
},
"extensions": {
"bitmask": "00000000",
"extensions": []
},
"highEntropyValues": {
"architecture": "arm",
"bitness": "64",
"brands": [
{
"brand": "Not(A:Brand",
"version": "8"
},
{
"brand": "Chromium",
"version": "144"
},
{
"brand": "Google+Chrome",
"version": "144"
}
],
"mobile": false,
"model": "",
"platform": "macOS",
"platformVersion": "26.2.0",
"uaFullVersion": "144.0.7559.133"
},
"etsl": 33,
"maths": "67d0b556",
"toSourceError": {
"toSourceError": "TypeError:+Cannot+read+properties+of+null+(reading+usdfsh)",
"hasToSource": false
}
},
"graphics": {
"webGL": {
"vendor": "Google+Inc.+(Apple)",
"renderer": "ANGLE+(Apple,+ANGLE+Metal+Renderer:+Apple+M4+Max,+Unspecified+Version)"
},
"webgpu": {
"vendor": "apple",
"architecture": "metal-3",
"device": "",
"description": ""
},
"canvas": {
"hasModifiedCanvas": false,
"canvasFingerprint": "-421c84e0"
}
},
"codecs": {
"audioCanPlayTypeHash": "688c7345",
"videoCanPlayTypeHash": "-126cde82",
"audioMediaSourceHash": "-3cbc04a4",
"videoMediaSourceHash": "-48c15d34",
"rtcAudioCapabilitiesHash": "26a15cc5",
"rtcVideoCapabilitiesHash": "4f24a817",
"hasMediaSource": true
},
"locale": {
"internationalization": {
"timezone": "Europe/Paris",
"localeLanguage": "en-US"
},
"languages": {
"languages": [
"en",
"en-US",
"fr-FR",
"fr"
],
"language": "en"
}
},
"contexts": {
"iframe": {
"webdriver": false,
"userAgent": "Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_15_7)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/144.0.0.0+Safari/537.36",
"platform": "MacIntel",
"memory": 8,
"cpuCount": 14,
"language": "en"
},
"webWorker": {
"vendor": "Google+Inc.+(Apple)",
"renderer": "ANGLE+(Apple,+ANGLE+Metal+Renderer:+Apple+M4+Max,+Unspecified+Version)",
"userAgent": "Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_15_7)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/144.0.0.0+Safari/537.36",
"language": "en",
"platform": "MacIntel",
"memory": 8,
"cpuCount": 14
}
}
},
"fsid": "FS1_00001000000000_00010h02ba_2304x1296c14m08b10011h-5db59_f1000111100e00000000p1100h-7ce12_0h06653c_1h-36e6b_en4h720e_0100h-156f8",
"nonce": "jb8t2jfm839",
"time": 1770669806462,
"url": "http://localhost/",
"fastBotDetection": true,
"fastBotDetectionDetails": {
"headlessChromeScreenResolution": {
"detected": false,
"severity": "high"
},
"hasWebdriver": {
"detected": false,
"severity": "high"
},
"hasWebdriverWritable": {
"detected": false,
"severity": "high"
},
"hasSeleniumProperty": {
"detected": false,
"severity": "high"
},
"hasCDP": {
"detected": true,
"severity": "high"
},
"hasPlaywright": {
"detected": false,
"severity": "high"
},
"hasImpossibleDeviceMemory": {
"detected": false,
"severity": "high"
},
"hasHighCPUCount": {
"detected": false,
"severity": "high"
},
"hasMissingChromeObject": {
"detected": false,
"severity": "high"
},
"hasWebdriverIframe": {
"detected": false,
"severity": "high"
},
"hasWebdriverWorker": {
"detected": false,
"severity": "high"
},
"hasMismatchWebGLInWorker": {
"detected": false,
"severity": "high"
},
"hasMismatchPlatformIframe": {
"detected": false,
"severity": "high"
},
"hasMismatchPlatformWorker": {
"detected": false,
"severity": "high"
},
"hasSwiftshaderRenderer": {
"detected": false,
"severity": "low"
},
"hasUTCTimezone": {
"detected": false,
"severity": "medium"
}
}
}
*/
// FingerprintData is the deserialized payload produced by the JS
// fingerprint bundle and (after a successful challenge) carried inside the
// sealed challenge cookie. It is the value exposed to rule authors via the
// `fingerprint` variable in expr environments. The struct mirrors the JSON
// wire shape one-to-one; FlexInt/FlexBool primitives tolerate the bundle's
// occasional "error string instead of value" outputs without aborting the
// whole submission.
type FingerprintData struct {
Signals fingerprintSignals `json:"signals"`
FSID string `json:"fsid"`
Nonce string `json:"nonce"`
Time int64 `json:"time"`
URL string `json:"url"`
FastBotDetection FlexBool `json:"fastBotDetection"`
FastBotDetectionDetails fingerprintFastBotDetectionDetails `json:"fastBotDetectionDetails"`
Bot fingerprintBotAlias `json:"-"`
// Allowlisted is true on cookies minted by GrantChallengeCookie (operator
// bypass for trusted bots like Googlebot) — these cookies never went
// through a real challenge submission and carry no measured signals.
// AllowlistReason is the operator-supplied free-form string identifying
// why the bypass was granted, exposed to on_challenge expressions so
// per-route policy can distinguish bypass categories.
//
// Both fields live in the cookie's AEAD plaintext header (see
// crypto.go), NOT in the protobuf envelope. They are populated by
// ValidCookie / GrantChallengeCookie and zero for normal cookies.
Allowlisted bool `json:"-"`
AllowlistReason string `json:"-"`
}
type fingerprintSignals struct {
Automation fingerprintAutomation `json:"automation"`
Device fingerprintDevice `json:"device"`
Browser fingerprintBrowser `json:"browser"`
Graphics fingerprintGraphics `json:"graphics"`
Codecs fingerprintCodecs `json:"codecs"`
Locale fingerprintLocale `json:"locale"`
Contexts fingerprintContexts `json:"contexts"`
}
type fingerprintAutomation struct {
Webdriver FlexBool `json:"webdriver"`
WebdriverWritable FlexBool `json:"webdriverWritable"`
Selenium FlexBool `json:"selenium"`
CDP FlexBool `json:"cdp"`
Playwright FlexBool `json:"playwright"`
NavigatorPropertyDescriptors string `json:"navigatorPropertyDescriptors"`
}
type fingerprintDevice struct {
CPUCount FlexInt `json:"cpuCount"`
Memory FlexInt `json:"memory"`
Platform string `json:"platform"`
ScreenResolution fingerprintScreenResolution `json:"screenResolution"`
MultimediaDevices fingerprintMultimediaDevices `json:"multimediaDevices"`
MediaQueries fingerprintDeviceMediaQueries `json:"mediaQueries"`
}
type fingerprintScreenResolution struct {
Width FlexInt `json:"width"`
Height FlexInt `json:"height"`
PixelDepth FlexInt `json:"pixelDepth"`
ColorDepth FlexInt `json:"colorDepth"`
AvailableWidth FlexInt `json:"availableWidth"`
AvailableHeight FlexInt `json:"availableHeight"`
InnerWidth FlexInt `json:"innerWidth"`
InnerHeight FlexInt `json:"innerHeight"`
HasMultipleDisplays FlexBool `json:"hasMultipleDisplays"`
}
type fingerprintMultimediaDevices struct {
Speakers FlexInt `json:"speakers"`
Microphones FlexInt `json:"microphones"`
Webcams FlexInt `json:"webcams"`
}
type fingerprintDeviceMediaQueries struct {
PrefersColorScheme string `json:"prefersColorScheme"`
PrefersReducedMotion FlexBool `json:"prefersReducedMotion"`
PrefersReducedTransparency FlexBool `json:"prefersReducedTransparency"`
ColorGamut string `json:"colorGamut"`
Pointer string `json:"pointer"`
AnyPointer string `json:"anyPointer"`
Hover FlexBool `json:"hover"`
AnyHover FlexBool `json:"anyHover"`
ColorDepth FlexInt `json:"colorDepth"`
}
type fingerprintBrowser struct {
UserAgent string `json:"userAgent"`
Features fingerprintBrowserFeatures `json:"features"`
Plugins fingerprintBrowserPlugins `json:"plugins"`
Extensions fingerprintBrowserExtensions `json:"extensions"`
HighEntropyValues fingerprintBrowserHighEntropyValues `json:"highEntropyValues"`
ETSL FlexInt `json:"etsl"`
Maths string `json:"maths"`
ToSourceError fingerprintBrowserToSourceError `json:"toSourceError"`
}
type fingerprintBrowserFeatures struct {
Bitmask string `json:"bitmask"`
Chrome FlexBool `json:"chrome"`
Brave FlexBool `json:"brave"`
ApplePaySupport FlexBool `json:"applePaySupport"`
Opera FlexBool `json:"opera"`
Serial FlexBool `json:"serial"`
AttachShadow FlexBool `json:"attachShadow"`
Caches FlexBool `json:"caches"`
WebAssembly FlexBool `json:"webAssembly"`
Buffer FlexBool `json:"buffer"`
ShowModalDialog FlexBool `json:"showModalDialog"`
Safari FlexBool `json:"safari"`
WebkitPrefixedFunction FlexBool `json:"webkitPrefixedFunction"`
MozPrefixedFunction FlexBool `json:"mozPrefixedFunction"`
USB FlexBool `json:"usb"`
BrowserCapture FlexBool `json:"browserCapture"`
PaymentRequestUpdateEvent FlexBool `json:"paymentRequestUpdateEvent"`
PressureObserver FlexBool `json:"pressureObserver"`
AudioSession FlexBool `json:"audioSession"`
SelectAudioOutput FlexBool `json:"selectAudioOutput"`
BarcodeDetector FlexBool `json:"barcodeDetector"`
Battery FlexBool `json:"battery"`
DevicePosture FlexBool `json:"devicePosture"`
DocumentPictureInPicture FlexBool `json:"documentPictureInPicture"`
EyeDropper FlexBool `json:"eyeDropper"`
EditContext FlexBool `json:"editContext"`
FencedFrame FlexBool `json:"fencedFrame"`
Sanitizer FlexBool `json:"sanitizer"`
OTPCredential FlexBool `json:"otpCredential"`
}
type fingerprintBrowserPlugins struct {
IsValidPluginArray FlexBool `json:"isValidPluginArray"`
PluginCount FlexInt `json:"pluginCount"`
PluginNamesHash string `json:"pluginNamesHash"`
PluginConsistency1 FlexBool `json:"pluginConsistency1"`
PluginOverflow FlexBool `json:"pluginOverflow"`
}
type fingerprintBrowserExtensions struct {
Bitmask string `json:"bitmask"`
Extensions []string `json:"extensions"`
}
type fingerprintBrowserHighEntropyValues struct {
Architecture string `json:"architecture"`
Bitness string `json:"bitness"`
Brands FlexBrandVersions `json:"brands"`
Mobile FlexBool `json:"mobile"`
Model string `json:"model"`
Platform string `json:"platform"`
PlatformVersion string `json:"platformVersion"`
UAFullVersion string `json:"uaFullVersion"`
}
type fingerprintBrandVersion struct {
Brand string `json:"brand"`
Version string `json:"version"`
}
// FlexBrandVersions handles a JSON value that should be []fingerprintBrandVersion
// but may be a string (error) from the fingerprint library.
type FlexBrandVersions []fingerprintBrandVersion
func (f *FlexBrandVersions) UnmarshalJSON(data []byte) error {
var arr []fingerprintBrandVersion
if err := json.Unmarshal(data, &arr); err == nil {
*f = arr
return nil
}
// Probably a string error — ignore it, leave empty.
*f = nil
return nil
}
type fingerprintBrowserToSourceError struct {
ToSourceError string `json:"toSourceError"`
HasToSource FlexBool `json:"hasToSource"`
}
type fingerprintGraphics struct {
WebGL fingerprintGraphicsWebGL `json:"webGL"`
WebGPU fingerprintGraphicsWebGPU `json:"webgpu"`
Canvas fingerprintGraphicsCanvas `json:"canvas"`
}
type fingerprintGraphicsWebGL struct {
Vendor string `json:"vendor"`
Renderer string `json:"renderer"`
}
type fingerprintGraphicsWebGPU struct {
Vendor string `json:"vendor"`
Architecture string `json:"architecture"`
Device string `json:"device"`
Description string `json:"description"`
}
type fingerprintGraphicsCanvas struct {
HasModifiedCanvas FlexBool `json:"hasModifiedCanvas"`
CanvasFingerprint string `json:"canvasFingerprint"`
}
type fingerprintCodecs struct {
AudioCanPlayTypeHash string `json:"audioCanPlayTypeHash"`
VideoCanPlayTypeHash string `json:"videoCanPlayTypeHash"`
AudioMediaSourceHash string `json:"audioMediaSourceHash"`
VideoMediaSourceHash string `json:"videoMediaSourceHash"`
RTCAudioCapabilitiesHash string `json:"rtcAudioCapabilitiesHash"`
RTCVideoCapabilitiesHash string `json:"rtcVideoCapabilitiesHash"`
HasMediaSource FlexBool `json:"hasMediaSource"`
}
type fingerprintLocale struct {
Internationalization fingerprintLocaleInternationalization `json:"internationalization"`
Languages fingerprintLocaleLanguages `json:"languages"`
}
type fingerprintLocaleInternationalization struct {
Timezone string `json:"timezone"`
LocaleLanguage string `json:"localeLanguage"`
}
type fingerprintLocaleLanguages struct {
Languages []string `json:"languages"`
Language string `json:"language"`
}
type fingerprintContexts struct {
Iframe fingerprintIframeContext `json:"iframe"`
WebWorker fingerprintWebWorkerContext `json:"webWorker"`
}
type fingerprintIframeContext struct {
Webdriver FlexBool `json:"webdriver"`
UserAgent string `json:"userAgent"`
Platform string `json:"platform"`
Memory FlexInt `json:"memory"`
CPUCount FlexInt `json:"cpuCount"`
Language string `json:"language"`
}
type fingerprintWebWorkerContext struct {
Vendor string `json:"vendor"`
Renderer string `json:"renderer"`
UserAgent string `json:"userAgent"`
Language string `json:"language"`
Platform string `json:"platform"`
Memory FlexInt `json:"memory"`
CPUCount FlexInt `json:"cpuCount"`
}
type fingerprintFastBotDetectionDetails struct {
HeadlessChromeScreenResolution fingerprintDetectionResult `json:"headlessChromeScreenResolution"`
HasWebdriver fingerprintDetectionResult `json:"hasWebdriver"`
HasWebdriverWritable fingerprintDetectionResult `json:"hasWebdriverWritable"`
HasSeleniumProperty fingerprintDetectionResult `json:"hasSeleniumProperty"`
HasCDP fingerprintDetectionResult `json:"hasCDP"`
HasPlaywright fingerprintDetectionResult `json:"hasPlaywright"`
HasImpossibleDeviceMemory fingerprintDetectionResult `json:"hasImpossibleDeviceMemory"`
HasHighCPUCount fingerprintDetectionResult `json:"hasHighCPUCount"`
HasMissingChromeObject fingerprintDetectionResult `json:"hasMissingChromeObject"`
HasWebdriverIframe fingerprintDetectionResult `json:"hasWebdriverIframe"`
HasWebdriverWorker fingerprintDetectionResult `json:"hasWebdriverWorker"`
HasMismatchWebGLInWorker fingerprintDetectionResult `json:"hasMismatchWebGLInWorker"`
HasMismatchPlatformIframe fingerprintDetectionResult `json:"hasMismatchPlatformIframe"`
HasMismatchPlatformWorker fingerprintDetectionResult `json:"hasMismatchPlatformWorker"`
HasSwiftshaderRenderer fingerprintDetectionResult `json:"hasSwiftshaderRenderer"`
HasUTCTimezone fingerprintDetectionResult `json:"hasUTCTimezone"`
HasMismatchLanguages fingerprintDetectionResult `json:"hasMismatchLanguages"`
HasInconsistentEtsl fingerprintDetectionResult `json:"hasInconsistentEtsl"`
HasBotUserAgent fingerprintDetectionResult `json:"hasBotUserAgent"`
HasGPUMismatch fingerprintDetectionResult `json:"hasGPUMismatch"`
HasPlatformMismatch fingerprintDetectionResult `json:"hasPlatformMismatch"`
}
type fingerprintDetectionResult struct {
Detected FlexBool `json:"detected"`
}
type fingerprintBotAlias struct {
HeadlessChromeScreenResolution bool `json:"headlessChromeScreenResolution"`
Webdriver bool `json:"webdriver"`
WebdriverWritable bool `json:"webdriverWritable"`
Selenium bool `json:"selenium"`
CDP bool `json:"cdp"`
Playwright bool `json:"playwright"`
ImpossibleDeviceMemory bool `json:"impossibleDeviceMemory"`
HighCPUCount bool `json:"highCPUCount"`
MissingChromeObject bool `json:"missingChromeObject"`
WebdriverIframe bool `json:"webdriverIframe"`
WebdriverWorker bool `json:"webdriverWorker"`
MismatchWebGLInWorker bool `json:"mismatchWebGLInWorker"`
MismatchPlatformIframe bool `json:"mismatchPlatformIframe"`
MismatchPlatformWorker bool `json:"mismatchPlatformWorker"`
SwiftshaderRenderer bool `json:"swiftshaderRenderer"`
UTCTimezone bool `json:"utcTimezone"`
MismatchLanguages bool `json:"mismatchLanguages"`
InconsistentEtsl bool `json:"inconsistentEtsl"`
BotUserAgent bool `json:"botUserAgent"`
GPUMismatch bool `json:"gpuMismatch"`
PlatformMismatch bool `json:"platformMismatch"`
AnyDetected bool `json:"anyDetected"`
DetectedCount FlexInt `json:"detectedCount"`
}
func (f *FingerprintData) UnmarshalJSON(data []byte) error {
type rawFingerprintData FingerprintData
var raw rawFingerprintData
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
*f = FingerprintData(raw)
f.Bot = newFingerprintBotAlias(f.FastBotDetectionDetails)
return nil
}
func newFingerprintBotAlias(details fingerprintFastBotDetectionDetails) fingerprintBotAlias {
alias := fingerprintBotAlias{
HeadlessChromeScreenResolution: bool(details.HeadlessChromeScreenResolution.Detected),
Webdriver: bool(details.HasWebdriver.Detected),
WebdriverWritable: bool(details.HasWebdriverWritable.Detected),
Selenium: bool(details.HasSeleniumProperty.Detected),
CDP: bool(details.HasCDP.Detected),
Playwright: bool(details.HasPlaywright.Detected),
ImpossibleDeviceMemory: bool(details.HasImpossibleDeviceMemory.Detected),
HighCPUCount: bool(details.HasHighCPUCount.Detected),
MissingChromeObject: bool(details.HasMissingChromeObject.Detected),
WebdriverIframe: bool(details.HasWebdriverIframe.Detected),
WebdriverWorker: bool(details.HasWebdriverWorker.Detected),
MismatchWebGLInWorker: bool(details.HasMismatchWebGLInWorker.Detected),
MismatchPlatformIframe: bool(details.HasMismatchPlatformIframe.Detected),
MismatchPlatformWorker: bool(details.HasMismatchPlatformWorker.Detected),
SwiftshaderRenderer: bool(details.HasSwiftshaderRenderer.Detected),
UTCTimezone: bool(details.HasUTCTimezone.Detected),
MismatchLanguages: bool(details.HasMismatchLanguages.Detected),
InconsistentEtsl: bool(details.HasInconsistentEtsl.Detected),
BotUserAgent: bool(details.HasBotUserAgent.Detected),
GPUMismatch: bool(details.HasGPUMismatch.Detected),
PlatformMismatch: bool(details.HasPlatformMismatch.Detected),
}
if alias.HeadlessChromeScreenResolution {
alias.DetectedCount++
}
if alias.Webdriver {
alias.DetectedCount++
}
if alias.WebdriverWritable {
alias.DetectedCount++
}
if alias.Selenium {
alias.DetectedCount++
}
if alias.CDP {
alias.DetectedCount++
}
if alias.Playwright {
alias.DetectedCount++
}
if alias.ImpossibleDeviceMemory {
alias.DetectedCount++
}
if alias.HighCPUCount {
alias.DetectedCount++
}
if alias.MissingChromeObject {
alias.DetectedCount++
}
if alias.WebdriverIframe {
alias.DetectedCount++
}
if alias.WebdriverWorker {
alias.DetectedCount++
}
if alias.MismatchWebGLInWorker {
alias.DetectedCount++
}
if alias.MismatchPlatformIframe {
alias.DetectedCount++
}
if alias.MismatchPlatformWorker {
alias.DetectedCount++
}
if alias.SwiftshaderRenderer {
alias.DetectedCount++
}
if alias.UTCTimezone {
alias.DetectedCount++
}
if alias.MismatchLanguages {
alias.DetectedCount++
}
if alias.InconsistentEtsl {
alias.DetectedCount++
}
if alias.BotUserAgent {
alias.DetectedCount++
}
if alias.GPUMismatch {
alias.DetectedCount++
}
if alias.PlatformMismatch {
alias.DetectedCount++
}
alias.AnyDetected = alias.DetectedCount > 0
return alias
}
+417
View File
@@ -0,0 +1,417 @@
package challenge
import (
log "github.com/sirupsen/logrus"
)
// Handwritten helpers on *FingerprintData. Kept in a dedicated file (not
// fingerprint.go) so a future fpscanner library bump — which rewrites the
// data-shape structs — doesn't clobber these methods.
// IsBot returns the library's fast-bot verdict as a native bool, so rules can write
// `fingerprint.IsBot()` instead of `fingerprint.FastBotDetection.Bool() == true`.
func (f *FingerprintData) IsBot() bool {
if f == nil {
return false
}
return bool(f.FastBotDetection)
}
// BotSignalCount returns the number of fast-bot-detection signals that fired.
func (f *FingerprintData) BotSignalCount() int {
if f == nil {
return 0
}
return f.Bot.DetectedCount.Int()
}
// HasBotSignal returns true if any fast-bot-detection signal fired.
func (f *FingerprintData) HasBotSignal() bool {
if f == nil {
return false
}
return f.Bot.AnyDetected
}
// HasAutomationSignal returns true if any automation-framework signal fired
// (webdriver, selenium, CDP, playwright, bot user-agent regex).
func (f *FingerprintData) HasAutomationSignal() bool {
if f == nil {
return false
}
b := f.Bot
return b.Webdriver ||
b.WebdriverWritable ||
b.Selenium ||
b.CDP ||
b.Playwright ||
b.BotUserAgent
}
// HasHeadlessSignal returns true if any headless-browser signal fired. Also
// folds in inconsistent-etsl, which fires when the TLS-level `etsl` integer
// disagrees with the claimed browser family — characteristic of patched /
// forged headless environments.
func (f *FingerprintData) HasHeadlessSignal() bool {
if f == nil {
return false
}
b := f.Bot
return b.HeadlessChromeScreenResolution ||
b.MissingChromeObject ||
b.SwiftshaderRenderer ||
b.InconsistentEtsl
}
// HasMismatchSignal returns true if any cross-context or cross-API mismatch
// signal fired (iframe/worker webdriver, platform, WebGL in worker, UA-vs-
// navigator platform, GPU, languages).
func (f *FingerprintData) HasMismatchSignal() bool {
if f == nil {
return false
}
b := f.Bot
return b.MismatchWebGLInWorker ||
b.MismatchPlatformIframe ||
b.MismatchPlatformWorker ||
b.WebdriverIframe ||
b.WebdriverWorker ||
b.PlatformMismatch ||
b.GPUMismatch ||
b.MismatchLanguages
}
// HasImpossibleDeviceSignal returns true if the reported device specs are outside
// plausible bounds (memory / CPU count).
func (f *FingerprintData) HasImpossibleDeviceSignal() bool {
if f == nil {
return false
}
b := f.Bot
return b.ImpossibleDeviceMemory || b.HighCPUCount
}
// UserAgent returns the user-agent reported by the browser.
func (f *FingerprintData) UserAgent() string {
if f == nil {
return ""
}
return f.Signals.Browser.UserAgent
}
// Platform returns the browser-reported platform, preferring the high-entropy
// client-hint value and falling back to navigator.platform.
func (f *FingerprintData) Platform() string {
if f == nil {
return ""
}
if p := f.Signals.Browser.HighEntropyValues.Platform; p != "" {
return p
}
return f.Signals.Device.Platform
}
// Timezone returns the browser-reported IANA timezone.
func (f *FingerprintData) Timezone() string {
if f == nil {
return ""
}
return f.Signals.Locale.Internationalization.Timezone
}
// Language returns the browser's primary language.
func (f *FingerprintData) Language() string {
if f == nil {
return ""
}
return f.Signals.Locale.Languages.Language
}
// IsMobile returns true if the browser advertises a mobile form factor
// (via UA client hints).
func (f *FingerprintData) IsMobile() bool {
if f == nil {
return false
}
return bool(f.Signals.Browser.HighEntropyValues.Mobile)
}
// CPUCount returns navigator.hardwareConcurrency as a native int.
func (f *FingerprintData) CPUCount() int {
if f == nil {
return 0
}
return f.Signals.Device.CPUCount.Int()
}
// Memory returns navigator.deviceMemory as a native int.
func (f *FingerprintData) Memory() int {
if f == nil {
return 0
}
return f.Signals.Device.Memory.Int()
}
// BotSignals returns the names of every library-native bot-detection signal
// that fired on this fingerprint (e.g. "webdriver", "cdp",
// "headless_screen_resolution", "platform_mismatch"). The slice is empty
// when nothing fired. Ordering matches libDetections and is therefore
// stable across calls, which keeps log output deterministic.
//
// Only signals carried directly by f.Bot are returned: the custom
// CrowdSec mismatches in customDetections need request/geo context that
// is not on FingerprintData and are surfaced separately via
// MismatchReport.
func (f *FingerprintData) BotSignals() []string {
if f == nil {
return nil
}
out := make([]string, 0, len(libDetections))
for _, d := range libDetections {
if d.Read(f.Bot) {
out = append(out, d.Key)
}
}
return out
}
// FingerprintLogVerbosity controls how much of a fingerprint is included
// in accept/reject log entries.
type FingerprintLogVerbosity int
const (
// FingerprintLogMinimal: source, fsid, ua, platform, is_bot, signals,
// allowlisted (and allowlist_reason when allowlisted). This is the
// default when verbosity is omitted.
FingerprintLogMinimal FingerprintLogVerbosity = iota
// FingerprintLogInfo: minimal + is_mobile and the category roll-ups
// (automation/headless/mismatch/impossible_device) when they fired.
// This is the default tier surfaced to expr helpers as the string
// "info".
FingerprintLogInfo
// FingerprintLogVerbose: info + timezone, language, cpu_count,
// memory, url, nonce, fp_time.
FingerprintLogVerbose
)
// resolveVerbosity returns the verbosity to use, defaulting to
// FingerprintLogMinimal when the variadic argument is omitted.
func resolveVerbosity(verbosity []FingerprintLogVerbosity) FingerprintLogVerbosity {
if len(verbosity) == 0 {
return FingerprintLogMinimal
}
return verbosity[0]
}
// LogAccepted emits a single structured log line for a fingerprint we
// are accepting (valid cookie, successful submission, allowlist grant).
//
// clientIP is the real visitor address (typically request.ClientIP, set
// by the bouncer via X-Forwarded-For or equivalent) and is logged as
// "source". bouncerIP is the connection-level peer of the appsec listener
// (typically request.RemoteAddrNormalized) and is logged as "bouncer".
// Both are needed: operators correlate visitor behavior on "source", but
// "bouncer" is what they use to debug which gateway forwarded the request
// (multi-WAF setups, misconfigured X-Forwarded-For chains, etc.).
//
// The caller picks the level — typically Debug for per-request sites and
// Info for rarer events. verbosity defaults to FingerprintLogMinimal.
func (f *FingerprintData) LogAccepted(logger *log.Entry, level log.Level, clientIP, bouncerIP, msg string, verbosity ...FingerprintLogVerbosity) {
if f == nil || logger == nil {
return
}
emitFingerprintLog(logger.WithFields(f.logFieldsAccept(clientIP, bouncerIP, resolveVerbosity(verbosity))), level, msg)
}
// LogRejected emits a single structured log line for a fingerprint we
// are rejecting.
//
// reason is the operator-facing rejection cause and is always included
// regardless of verbosity. clientIP is logged as "source", bouncerIP as
// "bouncer" — see LogAccepted for the rationale. Only positive
// information is included — negative facts ("not headless", "0 mismatches")
// are omitted because they are noise on a reject log.
func (f *FingerprintData) LogRejected(logger *log.Entry, level log.Level, clientIP, bouncerIP, reason, msg string, verbosity ...FingerprintLogVerbosity) {
if f == nil || logger == nil {
return
}
emitFingerprintLog(logger.WithFields(f.logFieldsReject(clientIP, bouncerIP, reason, resolveVerbosity(verbosity))), level, msg)
}
// emitFingerprintLog dispatches a message to the right logrus method for
// the requested level. Unknown levels fall back to Info so a typo can
// never silently drop the log.
func emitFingerprintLog(entry *log.Entry, level log.Level, msg string) {
switch level {
case log.PanicLevel, log.FatalLevel, log.ErrorLevel:
entry.Error(msg)
case log.WarnLevel:
entry.Warn(msg)
case log.DebugLevel:
entry.Debug(msg)
case log.TraceLevel:
entry.Trace(msg)
default:
entry.Info(msg)
}
}
// logFieldsAccept builds the structured field set for an accept log.
// Empty-string fields are omitted so partial fingerprints stay compact.
func (f *FingerprintData) logFieldsAccept(clientIP, bouncerIP string, v FingerprintLogVerbosity) log.Fields {
fields := log.Fields{
"is_bot": f.IsBot(),
"signals": f.BotSignals(),
"allowlisted": f.Allowlisted,
}
addCommonFingerprintFields(fields, f, clientIP, bouncerIP)
if f.Allowlisted && f.AllowlistReason != "" {
fields["allowlist_reason"] = f.AllowlistReason
}
if v >= FingerprintLogInfo {
fields["is_mobile"] = f.IsMobile()
}
if v >= FingerprintLogVerbose {
addVerboseFingerprintFields(fields, f)
}
return fields
}
// logFieldsReject builds the structured field set for a reject log. Only
// positive signals/flags are included; negative facts are dropped.
func (f *FingerprintData) logFieldsReject(clientIP, bouncerIP, reason string, v FingerprintLogVerbosity) log.Fields {
fields := log.Fields{
"reason": reason,
"signals": f.BotSignals(),
}
addCommonFingerprintFields(fields, f, clientIP, bouncerIP)
if f.IsBot() {
fields["is_bot"] = true
}
if f.Allowlisted {
fields["allowlisted"] = true
if f.AllowlistReason != "" {
fields["allowlist_reason"] = f.AllowlistReason
}
}
if v >= FingerprintLogInfo {
if f.IsMobile() {
fields["is_mobile"] = true
}
if f.HasAutomationSignal() {
fields["automation"] = true
}
if f.HasHeadlessSignal() {
fields["headless"] = true
}
if f.HasMismatchSignal() {
fields["mismatch"] = true
}
if f.HasImpossibleDeviceSignal() {
fields["impossible_device"] = true
}
}
if v >= FingerprintLogVerbose {
addVerboseFingerprintFields(fields, f)
}
return fields
}
// addCommonFingerprintFields adds the identity fields shared by accept
// and reject logs. Empty strings are omitted so partial / synthetic
// fingerprints (e.g. allowlist cookies with no measured signals) don't
// log noisy empty keys.
func addCommonFingerprintFields(fields log.Fields, f *FingerprintData, clientIP, bouncerIP string) {
if clientIP != "" {
fields["source"] = clientIP
}
if bouncerIP != "" {
fields["bouncer"] = bouncerIP
}
if f.FSID != "" {
fields["fsid"] = f.FSID
}
if ua := f.UserAgent(); ua != "" {
fields["ua"] = ua
}
if p := f.Platform(); p != "" {
fields["platform"] = p
}
}
// addVerboseFingerprintFields adds the verbose-only fields shared by
// accept and reject logs. Zero / empty values are omitted.
func addVerboseFingerprintFields(fields log.Fields, f *FingerprintData) {
if tz := f.Timezone(); tz != "" {
fields["timezone"] = tz
}
if lang := f.Language(); lang != "" {
fields["language"] = lang
}
if cpu := f.CPUCount(); cpu != 0 {
fields["cpu_count"] = cpu
}
if mem := f.Memory(); mem != 0 {
fields["memory"] = mem
}
if f.URL != "" {
fields["url"] = f.URL
}
if f.Nonce != "" {
fields["nonce"] = f.Nonce
}
if f.Time != 0 {
fields["fp_time"] = f.Time
}
}
@@ -0,0 +1,158 @@
package challenge
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const sampleFingerprintJSON = `{
"signals": {
"device": {
"cpuCount": 14,
"memory": 8,
"platform": "MacIntel"
},
"browser": {
"userAgent": "Mozilla/5.0 Chrome",
"highEntropyValues": {
"platform": "macOS",
"mobile": false
}
},
"locale": {
"internationalization": {"timezone": "Europe/Paris"},
"languages": {"language": "en", "languages": ["en","fr"]}
}
},
"fsid": "FS1_abc",
"nonce": "n1",
"time": 1770669806462,
"url": "http://localhost/",
"fastBotDetection": true,
"fastBotDetectionDetails": {
"hasCDP": {"detected": true, "severity": "high"},
"headlessChromeScreenResolution": {"detected": false, "severity": "high"},
"hasWebdriver": {"detected": false, "severity": "high"},
"hasWebdriverWritable": {"detected": false, "severity": "high"},
"hasSeleniumProperty": {"detected": false, "severity": "high"},
"hasPlaywright": {"detected": false, "severity": "high"},
"hasImpossibleDeviceMemory":{"detected": false, "severity": "high"},
"hasHighCPUCount": {"detected": false, "severity": "high"},
"hasMissingChromeObject": {"detected": false, "severity": "high"},
"hasWebdriverIframe": {"detected": false, "severity": "high"},
"hasWebdriverWorker": {"detected": false, "severity": "high"},
"hasMismatchWebGLInWorker": {"detected": false, "severity": "high"},
"hasMismatchPlatformIframe":{"detected": false, "severity": "high"},
"hasMismatchPlatformWorker":{"detected": false, "severity": "high"},
"hasSwiftshaderRenderer": {"detected": false, "severity": "low"},
"hasUTCTimezone": {"detected": false, "severity": "medium"}
}
}`
func TestFingerprintHelpers_Getters(t *testing.T) {
var fp FingerprintData
require.NoError(t, json.Unmarshal([]byte(sampleFingerprintJSON), &fp))
assert.True(t, fp.IsBot())
assert.True(t, fp.HasBotSignal())
assert.Equal(t, 1, fp.BotSignalCount())
// only CDP is set in the sample, so only HasAutomationSignal should flip
assert.True(t, fp.HasAutomationSignal())
assert.False(t, fp.HasHeadlessSignal())
assert.False(t, fp.HasMismatchSignal())
assert.False(t, fp.HasImpossibleDeviceSignal())
assert.Equal(t, "Mozilla/5.0 Chrome", fp.UserAgent())
assert.Equal(t, "macOS", fp.Platform()) // prefers high-entropy value
assert.Equal(t, "Europe/Paris", fp.Timezone())
assert.Equal(t, "en", fp.Language())
assert.False(t, fp.IsMobile())
assert.Equal(t, 14, fp.CPUCount())
assert.Equal(t, 8, fp.Memory())
}
func TestFingerprintHelpers_PlatformFallback(t *testing.T) {
// no high-entropy platform -> falls back to navigator.platform
fp := FingerprintData{}
fp.Signals.Device.Platform = "Linux x86_64"
assert.Equal(t, "Linux x86_64", fp.Platform())
}
func TestFingerprintHelpers_NilReceiver(t *testing.T) {
var fp *FingerprintData
assert.False(t, fp.IsBot())
assert.False(t, fp.HasBotSignal())
assert.Equal(t, 0, fp.BotSignalCount())
assert.False(t, fp.HasAutomationSignal())
assert.False(t, fp.HasHeadlessSignal())
assert.False(t, fp.HasMismatchSignal())
assert.False(t, fp.HasImpossibleDeviceSignal())
assert.Empty(t, fp.UserAgent())
assert.Empty(t, fp.Platform())
assert.Empty(t, fp.Timezone())
assert.Empty(t, fp.Language())
assert.False(t, fp.IsMobile())
assert.Equal(t, 0, fp.CPUCount())
assert.Equal(t, 0, fp.Memory())
}
func TestFingerprintHelpers_CategoryAggregates(t *testing.T) {
const (
catAutomation = "automation"
catHeadless = "headless"
catMismatch = "mismatch"
catImpossibleDevice = "impossible_device"
)
cases := []struct {
name string
set func(*fingerprintBotAlias)
want string
}{
{"webdriver", func(b *fingerprintBotAlias) { b.Webdriver = true }, catAutomation},
{"webdriverWritable", func(b *fingerprintBotAlias) { b.WebdriverWritable = true }, catAutomation},
{"selenium", func(b *fingerprintBotAlias) { b.Selenium = true }, catAutomation},
{"cdp", func(b *fingerprintBotAlias) { b.CDP = true }, catAutomation},
{"playwright", func(b *fingerprintBotAlias) { b.Playwright = true }, catAutomation},
{"headlessChromeScreenResolution", func(b *fingerprintBotAlias) { b.HeadlessChromeScreenResolution = true }, catHeadless},
{"missingChromeObject", func(b *fingerprintBotAlias) { b.MissingChromeObject = true }, catHeadless},
{"swiftshaderRenderer", func(b *fingerprintBotAlias) { b.SwiftshaderRenderer = true }, catHeadless},
{"mismatchWebGLInWorker", func(b *fingerprintBotAlias) { b.MismatchWebGLInWorker = true }, catMismatch},
{"mismatchPlatformIframe", func(b *fingerprintBotAlias) { b.MismatchPlatformIframe = true }, catMismatch},
{"mismatchPlatformWorker", func(b *fingerprintBotAlias) { b.MismatchPlatformWorker = true }, catMismatch},
{"webdriverIframe", func(b *fingerprintBotAlias) { b.WebdriverIframe = true }, catMismatch},
{"webdriverWorker", func(b *fingerprintBotAlias) { b.WebdriverWorker = true }, catMismatch},
{"impossibleDeviceMemory", func(b *fingerprintBotAlias) { b.ImpossibleDeviceMemory = true }, catImpossibleDevice},
{"highCPUCount", func(b *fingerprintBotAlias) { b.HighCPUCount = true }, catImpossibleDevice},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
fp := &FingerprintData{}
tc.set(&fp.Bot)
got := map[string]bool{
catAutomation: fp.HasAutomationSignal(),
catHeadless: fp.HasHeadlessSignal(),
catMismatch: fp.HasMismatchSignal(),
catImpossibleDevice: fp.HasImpossibleDeviceSignal(),
}
for cat, fired := range got {
if cat == tc.want {
assert.True(t, fired, "expected category %q to fire", cat)
} else {
assert.False(t, fired, "expected category %q to stay false", cat)
}
}
})
}
}
@@ -0,0 +1,258 @@
// fingerprint_mismatch.go implements the *atomic* mismatch checks: small,
// composable predicates on FingerprintData (UA-vs-viewport, UA-language vs
// Accept-Language, timezone vs GeoIP country, etc.) that rules can call
// directly. The aggregate report and the lookup tables that map these
// signals to stable reason keys + severities live in
// fingerprint_mismatch_data.go.
package challenge
import (
"net/http"
"slices"
"strings"
"golang.org/x/text/language"
)
// UAMobileMismatch reports whether the fingerprint's user-agent claims a
// mobile form factor while the reported inner viewport width is implausibly
// wide for a real device. Catches the "UA switcher set to Android while
// actually on a desktop" pattern.
//
// Returns false (no signal) when either the UA doesn't claim mobile or the
// inner width is zero/missing.
func (f *FingerprintData) UAMobileMismatch() bool {
if f == nil {
return false
}
ua := f.UserAgent()
if ua == "" {
return false
}
if !mobileUARegex.MatchString(ua) {
return false
}
innerWidth := f.Signals.Device.ScreenResolution.InnerWidth.Int()
if innerWidth <= 0 {
return false
}
return innerWidth >= mobileViewportThreshold
}
// AcceptLanguageMismatch reports whether the request's Accept-Language
// header disagrees with the fingerprint's navigator.language. Both are
// derived from the same browser preference, so on a real browser they
// should agree at the base-language level.
//
// Returns false when either side is empty or unparseable — never a false
// positive on missing data.
//
// Deliberately different from the library's MismatchLanguages detection,
// which only compares navigator.languages[0] to navigator.language inside
// the browser; this helper brings the HTTP header into the comparison.
func (f *FingerprintData) AcceptLanguageMismatch(req *http.Request) bool {
if f == nil || req == nil {
return false
}
fpLang := f.Language()
if fpLang == "" {
return false
}
header := req.Header.Get("Accept-Language")
if header == "" {
return false
}
tags, _, err := language.ParseAcceptLanguage(header)
if err != nil || len(tags) == 0 {
return false
}
headerBase, _ := tags[0].Base()
fpTag, err := language.Parse(fpLang)
if err != nil {
return false
}
fpBase, _ := fpTag.Base()
return headerBase.String() != fpBase.String()
}
// TimezoneCountryMismatch reports whether the fingerprint's timezone
// disagrees with the client's country of origin (typically resolved via
// IPToCountry from the client IP).
//
// SOFT SIGNAL. Travelers whose OS timezone hasn't auto-adjusted and VPN
// users will legitimately trigger this. Rule authors should combine it
// with other signals (High()/Medium() counts, or reason-specific checks)
// before using it alone to ban.
//
// Returns false when either the timezone is unknown to our IANA table or
// the country is empty.
func (f *FingerprintData) TimezoneCountryMismatch(country string) bool {
if f == nil || country == "" {
return false
}
tz := f.Timezone()
if tz == "" {
return false
}
countries := lookupTimezoneCountries(tz)
if len(countries) == 0 {
return false
}
return !slices.Contains(countries, country)
}
// MismatchSignal is a single fired reason/severity pair on the report.
type MismatchSignal struct {
Reason string
Severity string
}
// MismatchReport aggregates the fired mismatch signals for a single
// fingerprint evaluation. Returned by ComputeMismatchReport; typically
// accessed through the cached EvaluateMismatches closure in the
// on_challenge expr env.
type MismatchReport struct {
Signals []MismatchSignal
}
// Count returns the total number of fired signals.
func (r *MismatchReport) Count() int {
if r == nil {
return 0
}
return len(r.Signals)
}
// Empty reports whether no signal fired.
func (r *MismatchReport) Empty() bool {
return r.Count() == 0
}
// Has reports whether a signal with the given reason key is present.
func (r *MismatchReport) Has(reason string) bool {
if r == nil {
return false
}
for _, s := range r.Signals {
if s.Reason == reason {
return true
}
}
return false
}
// Reasons returns the stable-ordered list of fired reason keys.
func (r *MismatchReport) Reasons() []string {
if r == nil {
return nil
}
out := make([]string, len(r.Signals))
for i, s := range r.Signals {
out[i] = s.Reason
}
return out
}
// High returns the count of signals tagged with the "high" severity.
func (r *MismatchReport) High() int {
return r.BySeverity(SeverityHigh)
}
// Medium returns the count of signals tagged with the "medium" severity.
func (r *MismatchReport) Medium() int {
return r.BySeverity(SeverityMedium)
}
// Low returns the count of signals tagged with the "low" severity.
func (r *MismatchReport) Low() int {
return r.BySeverity(SeverityLow)
}
// BySeverity returns the count of signals at the requested severity level.
func (r *MismatchReport) BySeverity(sev string) int {
if r == nil {
return 0
}
n := 0
for _, s := range r.Signals {
if s.Severity == sev {
n++
}
}
return n
}
// String renders the report as "reason1(sev),reason2(sev)" for log lines.
func (r *MismatchReport) String() string {
if r == nil || len(r.Signals) == 0 {
return ""
}
var b strings.Builder
for i, s := range r.Signals {
if i > 0 {
b.WriteByte(',')
}
b.WriteString(s.Reason)
b.WriteByte('(')
b.WriteString(s.Severity)
b.WriteByte(')')
}
return b.String()
}
// ComputeMismatchReport walks the library-native bot alias plus the
// CrowdSec custom helpers and returns every fired signal in a stable
// order. It is the pure computation backing EvaluateMismatches; callers
// wanting caching + observability should use the env closure registered
// in GetOnChallengeEnv instead.
//
// `country` is the ISO-3166 alpha-2 code of the client's geolocated
// country, or "" when unknown. Pass `exprhelpers.IPToCountry(...)`'s
// return value (or equivalent) — resolved once at the call site so this
// method stays free of side-effects.
//
// Iteration order is libDetections first (in the order declared there),
// then customDetections. Reason / severity / accessor binding for every
// check lives in one place — fingerprint_mismatch_data.go.
func (f *FingerprintData) ComputeMismatchReport(req *http.Request, country string) *MismatchReport {
report := &MismatchReport{}
if f == nil {
return report
}
for _, d := range libDetections {
if d.Read(f.Bot) {
report.Signals = append(report.Signals, MismatchSignal{
Reason: d.Key,
Severity: d.Severity,
})
}
}
for _, d := range customDetections {
if d.Fire(f, req, country) {
report.Signals = append(report.Signals, MismatchSignal{
Reason: d.Key,
Severity: d.Severity,
})
}
}
return report
}
@@ -0,0 +1,170 @@
// fingerprint_mismatch_data.go is the static lookup data backing the
// mismatch report: stable reason keys, severity labels, and the slice that
// binds each fpscanner-native signal to the bool it reads from the
// FingerprintData wire shape. Atomic detection methods live in
// fingerprint_mismatch.go; the aggregate report builder lives in
// fingerprint_helpers.go.
package challenge
import (
"net/http"
"regexp"
)
// Severity labels. Mirror the fpscanner library's fastBotDetectionDetails[*].severity
// strings so the vocabulary is consistent end-to-end.
const (
SeverityHigh = "high"
SeverityMedium = "medium"
SeverityLow = "low"
)
// Stable reason keys. One per entry in libDetections / customDetections
// below. Kept as exported constants so rule authors can
// `Has("platform_mismatch")` without typos.
const (
ReasonCDP = "cdp"
ReasonWebdriver = "webdriver"
ReasonWebdriverWritable = "webdriver_writable"
ReasonSelenium = "selenium"
ReasonPlaywright = "playwright"
ReasonWebdriverIframe = "webdriver_iframe"
ReasonWebdriverWorker = "webdriver_worker"
ReasonHeadlessScreenResolution = "headless_screen_resolution"
ReasonMissingChromeObject = "missing_chrome_object"
ReasonImpossibleMemory = "impossible_memory"
ReasonHighCPUCount = "high_cpu_count"
ReasonMismatchWebGLWorker = "mismatch_webgl_worker"
ReasonMismatchPlatformIframe = "mismatch_platform_iframe"
ReasonMismatchPlatformWorker = "mismatch_platform_worker"
ReasonPlatformMismatch = "platform_mismatch"
ReasonGPUMismatch = "gpu_mismatch"
ReasonBotUserAgent = "bot_user_agent"
ReasonInconsistentEtsl = "inconsistent_etsl"
ReasonUAMobile = "ua_mobile"
ReasonUTCTimezone = "utc_timezone"
ReasonAcceptLanguage = "accept_language"
ReasonSwiftshaderRenderer = "swiftshader_renderer"
ReasonMismatchLanguages = "mismatch_languages"
ReasonTimezoneCountry = "timezone_country"
)
// libDetection binds an fpscanner-native signal to the bot-alias bool that
// carries its `detected` flag, together with our severity label for it.
//
// Note: the fpscanner library ships each check with its own severity in the
// JSON payload, but `fingerprintDetectionResult` only unmarshals `detected`
// today, so the severity values below mirror the library defaults. The day
// we extend the struct with a Severity field, this column can be dropped
// and read from the payload instead.
type libDetection struct {
Key string
Severity string
Read func(fingerprintBotAlias) bool
}
// libDetections is the single source of truth for every library-native bot
// signal we surface: the reason key, its severity, and how to read its
// `detected` value from the bot alias. Append-only and ordering-stable so
// `MismatchReport.Reasons()` has a deterministic output.
var libDetections = []libDetection{
{ReasonCDP, SeverityHigh, func(b fingerprintBotAlias) bool { return b.CDP }},
{ReasonWebdriver, SeverityHigh, func(b fingerprintBotAlias) bool { return b.Webdriver }},
{ReasonWebdriverWritable, SeverityHigh, func(b fingerprintBotAlias) bool { return b.WebdriverWritable }},
{ReasonSelenium, SeverityHigh, func(b fingerprintBotAlias) bool { return b.Selenium }},
{ReasonPlaywright, SeverityHigh, func(b fingerprintBotAlias) bool { return b.Playwright }},
{ReasonWebdriverIframe, SeverityHigh, func(b fingerprintBotAlias) bool { return b.WebdriverIframe }},
{ReasonWebdriverWorker, SeverityHigh, func(b fingerprintBotAlias) bool { return b.WebdriverWorker }},
{ReasonHeadlessScreenResolution, SeverityHigh, func(b fingerprintBotAlias) bool { return b.HeadlessChromeScreenResolution }},
{ReasonMissingChromeObject, SeverityHigh, func(b fingerprintBotAlias) bool { return b.MissingChromeObject }},
{ReasonImpossibleMemory, SeverityHigh, func(b fingerprintBotAlias) bool { return b.ImpossibleDeviceMemory }},
{ReasonHighCPUCount, SeverityHigh, func(b fingerprintBotAlias) bool { return b.HighCPUCount }},
{ReasonMismatchWebGLWorker, SeverityHigh, func(b fingerprintBotAlias) bool { return b.MismatchWebGLInWorker }},
{ReasonMismatchPlatformIframe, SeverityHigh, func(b fingerprintBotAlias) bool { return b.MismatchPlatformIframe }},
{ReasonMismatchPlatformWorker, SeverityHigh, func(b fingerprintBotAlias) bool { return b.MismatchPlatformWorker }},
{ReasonPlatformMismatch, SeverityHigh, func(b fingerprintBotAlias) bool { return b.PlatformMismatch }},
{ReasonGPUMismatch, SeverityHigh, func(b fingerprintBotAlias) bool { return b.GPUMismatch }},
{ReasonBotUserAgent, SeverityHigh, func(b fingerprintBotAlias) bool { return b.BotUserAgent }},
{ReasonInconsistentEtsl, SeverityHigh, func(b fingerprintBotAlias) bool { return b.InconsistentEtsl }},
{ReasonUTCTimezone, SeverityMedium, func(b fingerprintBotAlias) bool { return b.UTCTimezone }},
{ReasonSwiftshaderRenderer, SeverityLow, func(b fingerprintBotAlias) bool { return b.SwiftshaderRenderer }},
{ReasonMismatchLanguages, SeverityLow, func(b fingerprintBotAlias) bool { return b.MismatchLanguages }},
}
// customDetection binds a CrowdSec-authored mismatch helper (which may need
// request or geoip data) to its reason key and severity. The Fire function
// wraps whichever method-on-fp is the actual check — unified signature so
// customDetections can be iterated like libDetections.
type customDetection struct {
Key string
Severity string
Fire func(fp *FingerprintData, req *http.Request, country string) bool
}
// customDetections lists every check we run on top of the library's own.
// Ordering is stable and comes after libDetections in the final report.
var customDetections = []customDetection{
{
Key: ReasonUAMobile,
Severity: SeverityMedium,
Fire: func(fp *FingerprintData, _ *http.Request, _ string) bool { return fp.UAMobileMismatch() },
},
{
Key: ReasonAcceptLanguage,
Severity: SeverityMedium,
Fire: func(fp *FingerprintData, req *http.Request, _ string) bool { return fp.AcceptLanguageMismatch(req) },
},
{
Key: ReasonTimezoneCountry,
Severity: SeverityLow,
Fire: func(fp *FingerprintData, _ *http.Request, country string) bool { return fp.TimezoneCountryMismatch(country) },
},
}
// severityByReason is derived once at init from the two detection tables
// above. It exists so callers that only need the severity for a reason key
// (metric labels, tests, SeverityFor) don't have to walk the tables.
var severityByReason = buildSeverityByReason()
func buildSeverityByReason() map[string]string {
m := make(map[string]string, len(libDetections)+len(customDetections))
for _, d := range libDetections {
m[d.Key] = d.Severity
}
for _, d := range customDetections {
m[d.Key] = d.Severity
}
return m
}
// SeverityFor returns the severity tagged to a reason key, or "" for
// unknown reasons.
func SeverityFor(reason string) string {
return severityByReason[reason]
}
// KnownReasons returns the full set of reason keys the aggregator may emit.
// Order is not guaranteed; callers that need deterministic ordering should
// walk libDetections / customDetections directly.
func KnownReasons() []string {
out := make([]string, 0, len(severityByReason))
for k := range severityByReason {
out = append(out, k)
}
return out
}
// mobileUARegex matches UAs that claim a PHONE form factor. Deliberately
// does not match bare "Android", because per Google's UA convention the
// `Mobile` / `Mobi` token is what phones add and tablets omit. Matching
// Android alone would FP on every Android tablet in landscape and on
// Samsung DeX / Android desktop mode (UA keeps "Android", viewport is
// ≥1000px).
var mobileUARegex = regexp.MustCompile(`(?i)Mobi|iPhone|iPod`)
// mobileViewportThreshold is the innerWidth cutoff above which we treat a
// mobile-claiming UA as inconsistent. Real phones report CSS pixel widths
// well below this (typical phones: 360-430). Tablets run wider but don't
// match the UA regex above, so this threshold only applies to phone-UAs.
const mobileViewportThreshold = 1000
@@ -0,0 +1,334 @@
package challenge
import (
"encoding/json"
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// androidOnLinuxSample is the user-provided fingerprint with a spoofed
// Android/Firefox UA on a real Chromium-on-Linux environment. It's kept
// verbatim as a permanent regression fixture.
const androidOnLinuxSample = `{
"signals": {
"automation": {"webdriver": false, "cdp": true},
"device": {
"cpuCount": 16,
"memory": 8,
"platform": "Linux x86_64",
"screenResolution": {
"width": 1920, "height": 1080,
"innerWidth": 1920, "innerHeight": 993
}
},
"browser": {
"userAgent": "Mozilla/5.0 (Android 4.4; Mobile; rv:70.0) Gecko/70.0 Firefox/70.0",
"features": {"chrome": true},
"highEntropyValues": {"platform": "", "mobile": false}
},
"locale": {
"internationalization": {"timezone": "Europe/Paris"},
"languages": {"language": "en-US", "languages": ["en-US"]}
}
},
"fsid": "FS_SAMPLE",
"nonce": "n",
"time": 1776669341379,
"url": "https://example/",
"fastBotDetection": true,
"fastBotDetectionDetails": {
"hasCDP": {"detected": true, "severity": "high"},
"hasInconsistentEtsl": {"detected": true, "severity": "high"},
"hasPlatformMismatch": {"detected": true, "severity": "high"},
"headlessChromeScreenResolution": {"detected": false, "severity": "high"},
"hasWebdriver": {"detected": false, "severity": "high"},
"hasWebdriverWritable": {"detected": false, "severity": "high"},
"hasSeleniumProperty": {"detected": false, "severity": "high"},
"hasPlaywright": {"detected": false, "severity": "high"},
"hasImpossibleDeviceMemory":{"detected": false, "severity": "high"},
"hasHighCPUCount": {"detected": false, "severity": "high"},
"hasMissingChromeObject": {"detected": false, "severity": "high"},
"hasWebdriverIframe": {"detected": false, "severity": "high"},
"hasWebdriverWorker": {"detected": false, "severity": "high"},
"hasMismatchWebGLInWorker": {"detected": false, "severity": "high"},
"hasMismatchPlatformIframe":{"detected": false, "severity": "high"},
"hasMismatchPlatformWorker":{"detected": false, "severity": "high"},
"hasSwiftshaderRenderer": {"detected": false, "severity": "low"},
"hasUTCTimezone": {"detected": false, "severity": "medium"},
"hasMismatchLanguages": {"detected": false, "severity": "low"},
"hasBotUserAgent": {"detected": false, "severity": "high"},
"hasGPUMismatch": {"detected": false, "severity": "high"}
}
}`
func mustUnmarshal(t *testing.T, raw string) *FingerprintData {
t.Helper()
fp := &FingerprintData{}
require.NoError(t, json.Unmarshal([]byte(raw), fp))
return fp
}
func TestUAMobileMismatch(t *testing.T) {
cases := []struct {
name string
ua string
innerWidth int
want bool
}{
{"android phone UA desktop viewport", "Mozilla/5.0 (Android 4.4; Mobile; rv:70.0)", 1920, true},
{"iphone UA huge viewport", "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0)", 1440, true},
{"real android phone small viewport", "Mozilla/5.0 (Linux; Android 14; Pixel 7) Mobile", 412, false},
{"desktop UA desktop viewport", "Mozilla/5.0 (X11; Linux x86_64) Chrome/120", 1920, false},
{"mobile UA missing innerWidth", "Mozilla/5.0 (Android 10; Mobile)", 0, false},
{"empty UA", "", 1920, false},
// Regression: a real Android tablet UA lacks the "Mobile" token by
// Google's convention. Its landscape viewport is wider than 1000px.
// Must NOT fire — previously a FP when the regex matched bare "Android".
{"android tablet landscape", "Mozilla/5.0 (Linux; Android 13; SM-X710) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36", 1280, false},
// Regression: Samsung DeX / Android desktop mode keeps "Android" in
// the UA but the user is on a desktop-sized screen. Must NOT fire.
{"samsung dex desktop viewport", "Mozilla/5.0 (Linux; Android 13; SM-S908U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36", 1920, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
fp := &FingerprintData{}
fp.Signals.Browser.UserAgent = tc.ua
fp.Signals.Device.ScreenResolution.InnerWidth = FlexInt(tc.innerWidth)
assert.Equal(t, tc.want, fp.UAMobileMismatch())
})
}
var nilFP *FingerprintData
assert.False(t, nilFP.UAMobileMismatch())
}
func TestAcceptLanguageMismatch(t *testing.T) {
cases := []struct {
name string
header string
fpLang string
want bool
}{
{"match en simple", "en-US,fr;q=0.8", "en", false},
{"match fr simple", "fr-FR,fr;q=0.9,en;q=0.5", "fr", false},
{"mismatch de vs en", "de-DE,de;q=0.9", "en", true},
{"mismatch zh vs en", "zh-CN", "en-US", true},
{"missing header", "", "en", false},
{"missing fp lang", "en-US", "", false},
{"garbage header", "@@@", "en", false},
{"garbage fp lang", "en-US", "@@@", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
fp := &FingerprintData{}
fp.Signals.Locale.Languages.Language = tc.fpLang
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
if tc.header != "" {
req.Header.Set("Accept-Language", tc.header)
}
assert.Equal(t, tc.want, fp.AcceptLanguageMismatch(req))
})
}
var nilFP *FingerprintData
assert.False(t, nilFP.AcceptLanguageMismatch(nil))
}
func TestTimezoneCountryMismatch(t *testing.T) {
cases := []struct {
name string
tz string
country string
want bool
}{
{"paris in france", "Europe/Paris", "FR", false},
{"paris for monaco too", "Europe/Paris", "MC", false},
{"paris claimed from US", "Europe/Paris", "US", true},
{"new york in us", "America/New_York", "US", false},
{"new york claimed from fr", "America/New_York", "FR", true},
{"alias calcutta", "Asia/Calcutta", "IN", false},
{"alias kiev", "Europe/Kiev", "UA", false},
{"unknown tz", "Made/Up", "FR", false},
{"empty country", "Europe/Paris", "", false},
{"empty tz", "", "FR", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
fp := &FingerprintData{}
fp.Signals.Locale.Internationalization.Timezone = tc.tz
assert.Equal(t, tc.want, fp.TimezoneCountryMismatch(tc.country))
})
}
var nilFP *FingerprintData
assert.False(t, nilFP.TimezoneCountryMismatch("FR"))
}
func TestMismatchReport_Helpers(t *testing.T) {
r := &MismatchReport{Signals: []MismatchSignal{
{Reason: ReasonCDP, Severity: SeverityHigh},
{Reason: ReasonPlatformMismatch, Severity: SeverityHigh},
{Reason: ReasonAcceptLanguage, Severity: SeverityMedium},
{Reason: ReasonTimezoneCountry, Severity: SeverityLow},
}}
assert.Equal(t, 4, r.Count())
assert.False(t, r.Empty())
assert.Equal(t, 2, r.High())
assert.Equal(t, 1, r.Medium())
assert.Equal(t, 1, r.Low())
assert.Equal(t, 2, r.BySeverity(SeverityHigh))
assert.True(t, r.Has(ReasonCDP))
assert.False(t, r.Has(ReasonBotUserAgent))
assert.Equal(t, []string{ReasonCDP, ReasonPlatformMismatch, ReasonAcceptLanguage, ReasonTimezoneCountry}, r.Reasons())
assert.Equal(t,
"cdp(high),platform_mismatch(high),accept_language(medium),timezone_country(low)",
r.String(),
)
}
func TestMismatchReport_NilReceiver(t *testing.T) {
var r *MismatchReport
assert.Equal(t, 0, r.Count())
assert.True(t, r.Empty())
assert.False(t, r.Has(ReasonCDP))
assert.Nil(t, r.Reasons())
assert.Equal(t, 0, r.High())
assert.Equal(t, 0, r.Medium())
assert.Equal(t, 0, r.Low())
assert.Empty(t, r.String())
}
func TestComputeMismatchReport_AndroidOnLinuxSample(t *testing.T) {
fp := mustUnmarshal(t, androidOnLinuxSample)
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
// Request originates from France; timezone is also Europe/Paris, so
// timezone_country should NOT fire. The Android-UA + 1920 viewport
// should fire ua_mobile, and the library-native detections fire cdp,
// inconsistent_etsl, platform_mismatch.
report := fp.ComputeMismatchReport(req, "FR")
assert.Equal(t, []string{
ReasonCDP,
ReasonPlatformMismatch,
ReasonInconsistentEtsl,
ReasonUAMobile,
}, report.Reasons())
assert.Equal(t, 3, report.High())
assert.Equal(t, 1, report.Medium(), "ua_mobile is medium severity")
assert.Equal(t, 0, report.Low())
}
func TestComputeMismatchReport_BaselineConsistent(t *testing.T) {
// Minimal consistent fingerprint — no library detections, consistent
// UA/platform, en Accept-Language, FR timezone, from FR.
raw := `{
"signals": {
"device": {"platform": "MacIntel"},
"browser": {
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120",
"highEntropyValues": {"platform": "macOS", "mobile": false}
},
"locale": {
"internationalization": {"timezone": "Europe/Paris"},
"languages": {"language": "en"}
}
},
"fsid": "FS_OK", "nonce": "n", "time": 1, "url": "http://x/",
"fastBotDetection": false,
"fastBotDetectionDetails": {}
}`
fp := mustUnmarshal(t, raw)
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
report := fp.ComputeMismatchReport(req, "FR")
assert.True(t, report.Empty())
}
func TestComputeMismatchReport_AcceptLanguageOnly(t *testing.T) {
raw := `{
"signals": {
"device": {"platform": "MacIntel"},
"browser": {
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X) Chrome/120",
"highEntropyValues": {"platform": "macOS"}
},
"locale": {
"internationalization": {"timezone": "Europe/Paris"},
"languages": {"language": "en"}
}
},
"fsid": "X", "nonce": "n", "time": 1, "url": "http://x/",
"fastBotDetection": false,
"fastBotDetectionDetails": {}
}`
fp := mustUnmarshal(t, raw)
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9")
report := fp.ComputeMismatchReport(req, "FR")
assert.Equal(t, []string{ReasonAcceptLanguage}, report.Reasons())
assert.Equal(t, 0, report.High())
assert.Equal(t, 1, report.Medium())
assert.Equal(t, 0, report.Low())
}
func TestComputeMismatchReport_TimezoneCountryOnly(t *testing.T) {
raw := `{
"signals": {
"device": {"platform": "MacIntel"},
"browser": {
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X) Chrome/120",
"highEntropyValues": {"platform": "macOS"}
},
"locale": {
"internationalization": {"timezone": "Europe/Paris"},
"languages": {"language": "en"}
}
},
"fsid": "X", "nonce": "n", "time": 1, "url": "http://x/",
"fastBotDetection": false,
"fastBotDetectionDetails": {}
}`
fp := mustUnmarshal(t, raw)
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
req.Header.Set("Accept-Language", "en-US")
// Europe/Paris but client IP geolocated in the US.
report := fp.ComputeMismatchReport(req, "US")
assert.Equal(t, []string{ReasonTimezoneCountry}, report.Reasons())
assert.Equal(t, 0, report.High())
assert.Equal(t, 0, report.Medium())
assert.Equal(t, 1, report.Low())
}
func TestComputeMismatchReport_NilFingerprint(t *testing.T) {
report := ((*FingerprintData)(nil)).ComputeMismatchReport(nil, "")
assert.True(t, report.Empty())
}
func TestSeverityFor(t *testing.T) {
assert.Equal(t, SeverityHigh, SeverityFor(ReasonCDP))
assert.Equal(t, SeverityMedium, SeverityFor(ReasonAcceptLanguage))
assert.Equal(t, SeverityLow, SeverityFor(ReasonTimezoneCountry))
assert.Empty(t, SeverityFor("not_a_reason"))
}
+568
View File
@@ -0,0 +1,568 @@
// fingerprint_proto.go bridges the protobuf-encoded fingerprint payload
// (pb.FingerprintData, the wire format used inside the encrypted cookie) and
// the FingerprintData struct used at runtime. Unexported helpers only; the
// conversion is mechanical field-by-field and intentionally verbose to keep
// it auditable. One helper per signal section, in the same order on both the
// from-proto and to-proto sides.
package challenge
import (
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/pb"
)
func fingerprintDataFromProto(p *pb.FingerprintData) FingerprintData {
f := FingerprintData{
FSID: p.GetFsid(),
Nonce: p.GetNonce(),
Time: p.GetTime(),
URL: p.GetUrl(),
FastBotDetection: FlexBool(p.GetFastBotDetection()),
}
if s := p.GetSignals(); s != nil {
f.Signals = signalsFromProto(s)
}
if det := p.GetFastBotDetectionDetails(); det != nil {
f.FastBotDetectionDetails = fastBotDetectionDetailsFromProto(det)
}
f.Bot = newFingerprintBotAlias(f.FastBotDetectionDetails)
return f
}
func signalsFromProto(s *pb.FingerprintSignals) fingerprintSignals {
var out fingerprintSignals
if a := s.GetAutomation(); a != nil {
out.Automation = automationFromProto(a)
}
if d := s.GetDevice(); d != nil {
out.Device = deviceFromProto(d)
}
if b := s.GetBrowser(); b != nil {
out.Browser = browserFromProto(b)
}
if g := s.GetGraphics(); g != nil {
out.Graphics = graphicsFromProto(g)
}
if c := s.GetCodecs(); c != nil {
out.Codecs = codecsFromProto(c)
}
if l := s.GetLocale(); l != nil {
out.Locale = localeFromProto(l)
}
if c := s.GetContexts(); c != nil {
out.Contexts = contextsFromProto(c)
}
return out
}
func automationFromProto(a *pb.FingerprintAutomation) fingerprintAutomation {
return fingerprintAutomation{
Webdriver: FlexBool(a.GetWebdriver()),
WebdriverWritable: FlexBool(a.GetWebdriverWritable()),
Selenium: FlexBool(a.GetSelenium()),
CDP: FlexBool(a.GetCdp()),
Playwright: FlexBool(a.GetPlaywright()),
NavigatorPropertyDescriptors: a.GetNavigatorPropertyDescriptors(),
}
}
func deviceFromProto(d *pb.FingerprintDevice) fingerprintDevice {
out := fingerprintDevice{
CPUCount: FlexInt(d.GetCpuCount()),
Memory: FlexInt(d.GetMemory()),
Platform: d.GetPlatform(),
}
if sr := d.GetScreenResolution(); sr != nil {
out.ScreenResolution = fingerprintScreenResolution{
Width: FlexInt(sr.GetWidth()),
Height: FlexInt(sr.GetHeight()),
PixelDepth: FlexInt(sr.GetPixelDepth()),
ColorDepth: FlexInt(sr.GetColorDepth()),
AvailableWidth: FlexInt(sr.GetAvailableWidth()),
AvailableHeight: FlexInt(sr.GetAvailableHeight()),
InnerWidth: FlexInt(sr.GetInnerWidth()),
InnerHeight: FlexInt(sr.GetInnerHeight()),
HasMultipleDisplays: FlexBool(sr.GetHasMultipleDisplays()),
}
}
if mm := d.GetMultimediaDevices(); mm != nil {
out.MultimediaDevices = fingerprintMultimediaDevices{
Speakers: FlexInt(mm.GetSpeakers()),
Microphones: FlexInt(mm.GetMicrophones()),
Webcams: FlexInt(mm.GetWebcams()),
}
}
if mq := d.GetMediaQueries(); mq != nil {
out.MediaQueries = fingerprintDeviceMediaQueries{
PrefersColorScheme: mq.GetPrefersColorScheme(),
PrefersReducedMotion: FlexBool(mq.GetPrefersReducedMotion()),
PrefersReducedTransparency: FlexBool(mq.GetPrefersReducedTransparency()),
ColorGamut: mq.GetColorGamut(),
Pointer: mq.GetPointer(),
AnyPointer: mq.GetAnyPointer(),
Hover: FlexBool(mq.GetHover()),
AnyHover: FlexBool(mq.GetAnyHover()),
ColorDepth: FlexInt(mq.GetColorDepth()),
}
}
return out
}
func browserFromProto(b *pb.FingerprintBrowser) fingerprintBrowser {
out := fingerprintBrowser{
UserAgent: b.GetUserAgent(),
ETSL: FlexInt(b.GetEtsl()),
Maths: b.GetMaths(),
}
if ft := b.GetFeatures(); ft != nil {
out.Features = fingerprintBrowserFeatures{
Bitmask: ft.GetBitmask(),
Chrome: FlexBool(ft.GetChrome()),
Brave: FlexBool(ft.GetBrave()),
ApplePaySupport: FlexBool(ft.GetApplePaySupport()),
Opera: FlexBool(ft.GetOpera()),
Serial: FlexBool(ft.GetSerial()),
AttachShadow: FlexBool(ft.GetAttachShadow()),
Caches: FlexBool(ft.GetCaches()),
WebAssembly: FlexBool(ft.GetWebAssembly()),
Buffer: FlexBool(ft.GetBuffer()),
ShowModalDialog: FlexBool(ft.GetShowModalDialog()),
Safari: FlexBool(ft.GetSafari()),
WebkitPrefixedFunction: FlexBool(ft.GetWebkitPrefixedFunction()),
MozPrefixedFunction: FlexBool(ft.GetMozPrefixedFunction()),
USB: FlexBool(ft.GetUsb()),
BrowserCapture: FlexBool(ft.GetBrowserCapture()),
PaymentRequestUpdateEvent: FlexBool(ft.GetPaymentRequestUpdateEvent()),
PressureObserver: FlexBool(ft.GetPressureObserver()),
AudioSession: FlexBool(ft.GetAudioSession()),
SelectAudioOutput: FlexBool(ft.GetSelectAudioOutput()),
BarcodeDetector: FlexBool(ft.GetBarcodeDetector()),
Battery: FlexBool(ft.GetBattery()),
DevicePosture: FlexBool(ft.GetDevicePosture()),
DocumentPictureInPicture: FlexBool(ft.GetDocumentPictureInPicture()),
EyeDropper: FlexBool(ft.GetEyeDropper()),
EditContext: FlexBool(ft.GetEditContext()),
FencedFrame: FlexBool(ft.GetFencedFrame()),
Sanitizer: FlexBool(ft.GetSanitizer()),
OTPCredential: FlexBool(ft.GetOtpCredential()),
}
}
if pl := b.GetPlugins(); pl != nil {
out.Plugins = fingerprintBrowserPlugins{
IsValidPluginArray: FlexBool(pl.GetIsValidPluginArray()),
PluginCount: FlexInt(pl.GetPluginCount()),
PluginNamesHash: pl.GetPluginNamesHash(),
PluginConsistency1: FlexBool(pl.GetPluginConsistency1()),
PluginOverflow: FlexBool(pl.GetPluginOverflow()),
}
}
if ext := b.GetExtensions(); ext != nil {
out.Extensions = fingerprintBrowserExtensions{
Bitmask: ext.GetBitmask(),
Extensions: ext.GetExtensions(),
}
}
if hev := b.GetHighEntropyValues(); hev != nil {
out.HighEntropyValues = fingerprintBrowserHighEntropyValues{
Architecture: hev.GetArchitecture(),
Bitness: hev.GetBitness(),
Mobile: FlexBool(hev.GetMobile()),
Model: hev.GetModel(),
Platform: hev.GetPlatform(),
PlatformVersion: hev.GetPlatformVersion(),
UAFullVersion: hev.GetUaFullVersion(),
}
for _, bv := range hev.GetBrands() {
out.HighEntropyValues.Brands = append(out.HighEntropyValues.Brands, fingerprintBrandVersion{
Brand: bv.GetBrand(),
Version: bv.GetVersion(),
})
}
}
if tse := b.GetToSourceError(); tse != nil {
out.ToSourceError = fingerprintBrowserToSourceError{
ToSourceError: tse.GetToSourceError(),
HasToSource: FlexBool(tse.GetHasToSource()),
}
}
return out
}
func graphicsFromProto(g *pb.FingerprintGraphics) fingerprintGraphics {
var out fingerprintGraphics
if gl := g.GetWebGL(); gl != nil {
out.WebGL = fingerprintGraphicsWebGL{
Vendor: gl.GetVendor(),
Renderer: gl.GetRenderer(),
}
}
if gpu := g.GetWebgpu(); gpu != nil {
out.WebGPU = fingerprintGraphicsWebGPU{
Vendor: gpu.GetVendor(),
Architecture: gpu.GetArchitecture(),
Device: gpu.GetDevice(),
Description: gpu.GetDescription(),
}
}
if cv := g.GetCanvas(); cv != nil {
out.Canvas = fingerprintGraphicsCanvas{
HasModifiedCanvas: FlexBool(cv.GetHasModifiedCanvas()),
CanvasFingerprint: cv.GetCanvasFingerprint(),
}
}
return out
}
func codecsFromProto(c *pb.FingerprintCodecs) fingerprintCodecs {
return fingerprintCodecs{
AudioCanPlayTypeHash: c.GetAudioCanPlayTypeHash(),
VideoCanPlayTypeHash: c.GetVideoCanPlayTypeHash(),
AudioMediaSourceHash: c.GetAudioMediaSourceHash(),
VideoMediaSourceHash: c.GetVideoMediaSourceHash(),
RTCAudioCapabilitiesHash: c.GetRtcAudioCapabilitiesHash(),
RTCVideoCapabilitiesHash: c.GetRtcVideoCapabilitiesHash(),
HasMediaSource: FlexBool(c.GetHasMediaSource()),
}
}
func localeFromProto(l *pb.FingerprintLocale) fingerprintLocale {
var out fingerprintLocale
if i18n := l.GetInternationalization(); i18n != nil {
out.Internationalization = fingerprintLocaleInternationalization{
Timezone: i18n.GetTimezone(),
LocaleLanguage: i18n.GetLocaleLanguage(),
}
}
if lang := l.GetLanguages(); lang != nil {
out.Languages = fingerprintLocaleLanguages{
Languages: lang.GetLanguages(),
Language: lang.GetLanguage(),
}
}
return out
}
func contextsFromProto(c *pb.FingerprintContexts) fingerprintContexts {
var out fingerprintContexts
if iframe := c.GetIframe(); iframe != nil {
out.Iframe = fingerprintIframeContext{
Webdriver: FlexBool(iframe.GetWebdriver()),
UserAgent: iframe.GetUserAgent(),
Platform: iframe.GetPlatform(),
Memory: FlexInt(iframe.GetMemory()),
CPUCount: FlexInt(iframe.GetCpuCount()),
Language: iframe.GetLanguage(),
}
}
if ww := c.GetWebWorker(); ww != nil {
out.WebWorker = fingerprintWebWorkerContext{
Vendor: ww.GetVendor(),
Renderer: ww.GetRenderer(),
UserAgent: ww.GetUserAgent(),
Language: ww.GetLanguage(),
Platform: ww.GetPlatform(),
Memory: FlexInt(ww.GetMemory()),
CPUCount: FlexInt(ww.GetCpuCount()),
}
}
return out
}
func fastBotDetectionDetailsFromProto(det *pb.FingerprintFastBotDetectionDetails) fingerprintFastBotDetectionDetails {
return fingerprintFastBotDetectionDetails{
HeadlessChromeScreenResolution: fingerprintDetectionResult{Detected: FlexBool(det.GetHeadlessChromeScreenResolution().GetDetected())},
HasWebdriver: fingerprintDetectionResult{Detected: FlexBool(det.GetHasWebdriver().GetDetected())},
HasWebdriverWritable: fingerprintDetectionResult{Detected: FlexBool(det.GetHasWebdriverWritable().GetDetected())},
HasSeleniumProperty: fingerprintDetectionResult{Detected: FlexBool(det.GetHasSeleniumProperty().GetDetected())},
HasCDP: fingerprintDetectionResult{Detected: FlexBool(det.GetHasCDP().GetDetected())},
HasPlaywright: fingerprintDetectionResult{Detected: FlexBool(det.GetHasPlaywright().GetDetected())},
HasImpossibleDeviceMemory: fingerprintDetectionResult{Detected: FlexBool(det.GetHasImpossibleDeviceMemory().GetDetected())},
HasHighCPUCount: fingerprintDetectionResult{Detected: FlexBool(det.GetHasHighCPUCount().GetDetected())},
HasMissingChromeObject: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMissingChromeObject().GetDetected())},
HasWebdriverIframe: fingerprintDetectionResult{Detected: FlexBool(det.GetHasWebdriverIframe().GetDetected())},
HasWebdriverWorker: fingerprintDetectionResult{Detected: FlexBool(det.GetHasWebdriverWorker().GetDetected())},
HasMismatchWebGLInWorker: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMismatchWebGLInWorker().GetDetected())},
HasMismatchPlatformIframe: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMismatchPlatformIframe().GetDetected())},
HasMismatchPlatformWorker: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMismatchPlatformWorker().GetDetected())},
HasSwiftshaderRenderer: fingerprintDetectionResult{Detected: FlexBool(det.GetHasSwiftshaderRenderer().GetDetected())},
HasUTCTimezone: fingerprintDetectionResult{Detected: FlexBool(det.GetHasUTCTimezone().GetDetected())},
HasMismatchLanguages: fingerprintDetectionResult{Detected: FlexBool(det.GetHasMismatchLanguages().GetDetected())},
HasInconsistentEtsl: fingerprintDetectionResult{Detected: FlexBool(det.GetHasInconsistentEtsl().GetDetected())},
HasBotUserAgent: fingerprintDetectionResult{Detected: FlexBool(det.GetHasBotUserAgent().GetDetected())},
HasGPUMismatch: fingerprintDetectionResult{Detected: FlexBool(det.GetHasGPUMismatch().GetDetected())},
HasPlatformMismatch: fingerprintDetectionResult{Detected: FlexBool(det.GetHasPlatformMismatch().GetDetected())},
}
}
func (f *FingerprintData) ToProto() *pb.FingerprintData {
return &pb.FingerprintData{
Signals: &pb.FingerprintSignals{
Automation: f.Signals.Automation.toProto(),
Device: f.Signals.Device.toProto(),
Browser: f.Signals.Browser.toProto(),
Graphics: f.Signals.Graphics.toProto(),
Codecs: f.Signals.Codecs.toProto(),
Locale: f.Signals.Locale.toProto(),
Contexts: f.Signals.Contexts.toProto(),
},
Fsid: f.FSID,
Nonce: f.Nonce,
Time: f.Time,
Url: f.URL,
FastBotDetection: bool(f.FastBotDetection),
FastBotDetectionDetails: f.FastBotDetectionDetails.toProto(),
Bot: f.Bot.toProto(),
}
}
func (a fingerprintAutomation) toProto() *pb.FingerprintAutomation {
return &pb.FingerprintAutomation{
Webdriver: bool(a.Webdriver),
WebdriverWritable: bool(a.WebdriverWritable),
Selenium: bool(a.Selenium),
Cdp: bool(a.CDP),
Playwright: bool(a.Playwright),
NavigatorPropertyDescriptors: a.NavigatorPropertyDescriptors,
}
}
func (d fingerprintDevice) toProto() *pb.FingerprintDevice {
return &pb.FingerprintDevice{
CpuCount: int32(d.CPUCount),
Memory: int32(d.Memory),
Platform: d.Platform,
ScreenResolution: &pb.FingerprintScreenResolution{
Width: int32(d.ScreenResolution.Width),
Height: int32(d.ScreenResolution.Height),
PixelDepth: int32(d.ScreenResolution.PixelDepth),
ColorDepth: int32(d.ScreenResolution.ColorDepth),
AvailableWidth: int32(d.ScreenResolution.AvailableWidth),
AvailableHeight: int32(d.ScreenResolution.AvailableHeight),
InnerWidth: int32(d.ScreenResolution.InnerWidth),
InnerHeight: int32(d.ScreenResolution.InnerHeight),
HasMultipleDisplays: bool(d.ScreenResolution.HasMultipleDisplays),
},
MultimediaDevices: &pb.FingerprintMultimediaDevices{
Speakers: int32(d.MultimediaDevices.Speakers),
Microphones: int32(d.MultimediaDevices.Microphones),
Webcams: int32(d.MultimediaDevices.Webcams),
},
MediaQueries: &pb.FingerprintDeviceMediaQueries{
PrefersColorScheme: d.MediaQueries.PrefersColorScheme,
PrefersReducedMotion: bool(d.MediaQueries.PrefersReducedMotion),
PrefersReducedTransparency: bool(d.MediaQueries.PrefersReducedTransparency),
ColorGamut: d.MediaQueries.ColorGamut,
Pointer: d.MediaQueries.Pointer,
AnyPointer: d.MediaQueries.AnyPointer,
Hover: bool(d.MediaQueries.Hover),
AnyHover: bool(d.MediaQueries.AnyHover),
ColorDepth: int32(d.MediaQueries.ColorDepth),
},
}
}
func (b fingerprintBrowser) toProto() *pb.FingerprintBrowser {
brands := make([]*pb.FingerprintBrandVersion, len(b.HighEntropyValues.Brands))
for i, bv := range b.HighEntropyValues.Brands {
brands[i] = &pb.FingerprintBrandVersion{
Brand: bv.Brand,
Version: bv.Version,
}
}
return &pb.FingerprintBrowser{
UserAgent: b.UserAgent,
Features: &pb.FingerprintBrowserFeatures{
Bitmask: b.Features.Bitmask,
Chrome: bool(b.Features.Chrome),
Brave: bool(b.Features.Brave),
ApplePaySupport: bool(b.Features.ApplePaySupport),
Opera: bool(b.Features.Opera),
Serial: bool(b.Features.Serial),
AttachShadow: bool(b.Features.AttachShadow),
Caches: bool(b.Features.Caches),
WebAssembly: bool(b.Features.WebAssembly),
Buffer: bool(b.Features.Buffer),
ShowModalDialog: bool(b.Features.ShowModalDialog),
Safari: bool(b.Features.Safari),
WebkitPrefixedFunction: bool(b.Features.WebkitPrefixedFunction),
MozPrefixedFunction: bool(b.Features.MozPrefixedFunction),
Usb: bool(b.Features.USB),
BrowserCapture: bool(b.Features.BrowserCapture),
PaymentRequestUpdateEvent: bool(b.Features.PaymentRequestUpdateEvent),
PressureObserver: bool(b.Features.PressureObserver),
AudioSession: bool(b.Features.AudioSession),
SelectAudioOutput: bool(b.Features.SelectAudioOutput),
BarcodeDetector: bool(b.Features.BarcodeDetector),
Battery: bool(b.Features.Battery),
DevicePosture: bool(b.Features.DevicePosture),
DocumentPictureInPicture: bool(b.Features.DocumentPictureInPicture),
EyeDropper: bool(b.Features.EyeDropper),
EditContext: bool(b.Features.EditContext),
FencedFrame: bool(b.Features.FencedFrame),
Sanitizer: bool(b.Features.Sanitizer),
OtpCredential: bool(b.Features.OTPCredential),
},
Plugins: &pb.FingerprintBrowserPlugins{
IsValidPluginArray: bool(b.Plugins.IsValidPluginArray),
PluginCount: int32(b.Plugins.PluginCount),
PluginNamesHash: b.Plugins.PluginNamesHash,
PluginConsistency1: bool(b.Plugins.PluginConsistency1),
PluginOverflow: bool(b.Plugins.PluginOverflow),
},
Extensions: &pb.FingerprintBrowserExtensions{
Bitmask: b.Extensions.Bitmask,
Extensions: b.Extensions.Extensions,
},
HighEntropyValues: &pb.FingerprintBrowserHighEntropyValues{
Architecture: b.HighEntropyValues.Architecture,
Bitness: b.HighEntropyValues.Bitness,
Brands: brands,
Mobile: bool(b.HighEntropyValues.Mobile),
Model: b.HighEntropyValues.Model,
Platform: b.HighEntropyValues.Platform,
PlatformVersion: b.HighEntropyValues.PlatformVersion,
UaFullVersion: b.HighEntropyValues.UAFullVersion,
},
Etsl: int32(b.ETSL),
Maths: b.Maths,
ToSourceError: &pb.FingerprintBrowserToSourceError{
ToSourceError: b.ToSourceError.ToSourceError,
HasToSource: bool(b.ToSourceError.HasToSource),
},
}
}
func (g fingerprintGraphics) toProto() *pb.FingerprintGraphics {
return &pb.FingerprintGraphics{
WebGL: &pb.FingerprintGraphicsWebGL{
Vendor: g.WebGL.Vendor,
Renderer: g.WebGL.Renderer,
},
Webgpu: &pb.FingerprintGraphicsWebGPU{
Vendor: g.WebGPU.Vendor,
Architecture: g.WebGPU.Architecture,
Device: g.WebGPU.Device,
Description: g.WebGPU.Description,
},
Canvas: &pb.FingerprintGraphicsCanvas{
HasModifiedCanvas: bool(g.Canvas.HasModifiedCanvas),
CanvasFingerprint: g.Canvas.CanvasFingerprint,
},
}
}
func (c fingerprintCodecs) toProto() *pb.FingerprintCodecs {
return &pb.FingerprintCodecs{
AudioCanPlayTypeHash: c.AudioCanPlayTypeHash,
VideoCanPlayTypeHash: c.VideoCanPlayTypeHash,
AudioMediaSourceHash: c.AudioMediaSourceHash,
VideoMediaSourceHash: c.VideoMediaSourceHash,
RtcAudioCapabilitiesHash: c.RTCAudioCapabilitiesHash,
RtcVideoCapabilitiesHash: c.RTCVideoCapabilitiesHash,
HasMediaSource: bool(c.HasMediaSource),
}
}
func (l fingerprintLocale) toProto() *pb.FingerprintLocale {
return &pb.FingerprintLocale{
Internationalization: &pb.FingerprintLocaleInternationalization{
Timezone: l.Internationalization.Timezone,
LocaleLanguage: l.Internationalization.LocaleLanguage,
},
Languages: &pb.FingerprintLocaleLanguages{
Languages: l.Languages.Languages,
Language: l.Languages.Language,
},
}
}
func (c fingerprintContexts) toProto() *pb.FingerprintContexts {
return &pb.FingerprintContexts{
Iframe: &pb.FingerprintIframeContext{
Webdriver: bool(c.Iframe.Webdriver),
UserAgent: c.Iframe.UserAgent,
Platform: c.Iframe.Platform,
Memory: int32(c.Iframe.Memory),
CpuCount: int32(c.Iframe.CPUCount),
Language: c.Iframe.Language,
},
WebWorker: &pb.FingerprintWebWorkerContext{
Vendor: c.WebWorker.Vendor,
Renderer: c.WebWorker.Renderer,
UserAgent: c.WebWorker.UserAgent,
Language: c.WebWorker.Language,
Platform: c.WebWorker.Platform,
Memory: int32(c.WebWorker.Memory),
CpuCount: int32(c.WebWorker.CPUCount),
},
}
}
func (d fingerprintFastBotDetectionDetails) toProto() *pb.FingerprintFastBotDetectionDetails {
return &pb.FingerprintFastBotDetectionDetails{
HeadlessChromeScreenResolution: &pb.FingerprintDetectionResult{Detected: bool(d.HeadlessChromeScreenResolution.Detected)},
HasWebdriver: &pb.FingerprintDetectionResult{Detected: bool(d.HasWebdriver.Detected)},
HasWebdriverWritable: &pb.FingerprintDetectionResult{Detected: bool(d.HasWebdriverWritable.Detected)},
HasSeleniumProperty: &pb.FingerprintDetectionResult{Detected: bool(d.HasSeleniumProperty.Detected)},
HasCDP: &pb.FingerprintDetectionResult{Detected: bool(d.HasCDP.Detected)},
HasPlaywright: &pb.FingerprintDetectionResult{Detected: bool(d.HasPlaywright.Detected)},
HasImpossibleDeviceMemory: &pb.FingerprintDetectionResult{Detected: bool(d.HasImpossibleDeviceMemory.Detected)},
HasHighCPUCount: &pb.FingerprintDetectionResult{Detected: bool(d.HasHighCPUCount.Detected)},
HasMissingChromeObject: &pb.FingerprintDetectionResult{Detected: bool(d.HasMissingChromeObject.Detected)},
HasWebdriverIframe: &pb.FingerprintDetectionResult{Detected: bool(d.HasWebdriverIframe.Detected)},
HasWebdriverWorker: &pb.FingerprintDetectionResult{Detected: bool(d.HasWebdriverWorker.Detected)},
HasMismatchWebGLInWorker: &pb.FingerprintDetectionResult{Detected: bool(d.HasMismatchWebGLInWorker.Detected)},
HasMismatchPlatformIframe: &pb.FingerprintDetectionResult{Detected: bool(d.HasMismatchPlatformIframe.Detected)},
HasMismatchPlatformWorker: &pb.FingerprintDetectionResult{Detected: bool(d.HasMismatchPlatformWorker.Detected)},
HasSwiftshaderRenderer: &pb.FingerprintDetectionResult{Detected: bool(d.HasSwiftshaderRenderer.Detected)},
HasUTCTimezone: &pb.FingerprintDetectionResult{Detected: bool(d.HasUTCTimezone.Detected)},
HasMismatchLanguages: &pb.FingerprintDetectionResult{Detected: bool(d.HasMismatchLanguages.Detected)},
HasInconsistentEtsl: &pb.FingerprintDetectionResult{Detected: bool(d.HasInconsistentEtsl.Detected)},
HasBotUserAgent: &pb.FingerprintDetectionResult{Detected: bool(d.HasBotUserAgent.Detected)},
HasGPUMismatch: &pb.FingerprintDetectionResult{Detected: bool(d.HasGPUMismatch.Detected)},
HasPlatformMismatch: &pb.FingerprintDetectionResult{Detected: bool(d.HasPlatformMismatch.Detected)},
}
}
func (b fingerprintBotAlias) toProto() *pb.FingerprintBotAlias {
return &pb.FingerprintBotAlias{
HeadlessChromeScreenResolution: b.HeadlessChromeScreenResolution,
Webdriver: b.Webdriver,
WebdriverWritable: b.WebdriverWritable,
Selenium: b.Selenium,
Cdp: b.CDP,
Playwright: b.Playwright,
ImpossibleDeviceMemory: b.ImpossibleDeviceMemory,
HighCPUCount: b.HighCPUCount,
MissingChromeObject: b.MissingChromeObject,
WebdriverIframe: b.WebdriverIframe,
WebdriverWorker: b.WebdriverWorker,
MismatchWebGLInWorker: b.MismatchWebGLInWorker,
MismatchPlatformIframe: b.MismatchPlatformIframe,
MismatchPlatformWorker: b.MismatchPlatformWorker,
SwiftshaderRenderer: b.SwiftshaderRenderer,
UtcTimezone: b.UTCTimezone,
MismatchLanguages: b.MismatchLanguages,
InconsistentEtsl: b.InconsistentEtsl,
BotUserAgent: b.BotUserAgent,
GpuMismatch: b.GPUMismatch,
PlatformMismatch: b.PlatformMismatch,
AnyDetected: b.AnyDetected,
DetectedCount: int32(b.DetectedCount),
}
}
@@ -0,0 +1,11 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Redirecting…</title>
<meta name="robots" content="noindex,nofollow">
</head>
<body>
<p>You are being redirected. If your browser does not redirect automatically, reload this page.</p>
</body>
</html>
Binary file not shown.
@@ -0,0 +1,84 @@
package challenge
import (
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
challengejs "github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/js"
)
// TestNewChallengeRuntimeStartupBudget ensures NewChallengeRuntime returns
// quickly (seeded from the baked-in bundle), without paying the synchronous
// obfuscation it would otherwise pay.
func TestNewChallengeRuntimeStartupBudget(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("timing is not reliable on windows CI (wazero/WASM obfuscation runs ~2.5x slower)")
}
const budget = 10 * time.Second
start := time.Now()
rt, err := NewChallengeRuntime(t.Context())
dur := time.Since(start)
require.NoError(t, err)
require.NotNil(t, rt)
t.Logf("NewChallengeRuntime returned in %s", dur)
if dur > budget {
t.Fatalf("NewChallengeRuntime took %s, exceeds budget of %s; baked-in initial bundle should make this near-instant", dur, budget)
}
// The challenge code must be loaded so the very first request can be served.
require.NotEmpty(t, rt.getChallengeCode(), "challenge code should be seeded from the baked-in initial bundle")
}
// TestSeedCacheFromInitialBundle directly exercises the seeding helper and
// asserts the bundle decompresses to non-empty obfuscated JS that holds the
// challenge crypto code — and crucially NOT the public fpscanner (which is now
// served separately and unobfuscated).
func TestSeedCacheFromInitialBundle(t *testing.T) {
rt := &ChallengeRuntime{}
require.NoError(t, rt.seedCacheFromInitialBundle())
code := rt.getChallengeCode()
require.NotEmpty(t, code)
// The obfuscated challenge code inflates well beyond its tiny source.
t.Logf("seeded challenge code size: %d bytes", len(code))
require.Greater(t, len(code), 50_000,
"seeded challenge code is suspiciously small; was it actually obfuscated?")
// Path placeholders must have been substituted before obfuscation.
require.NotContains(t, code, "__CROWDSEC_SUBMIT_PATH__",
"submit path placeholder still present; substitution missed at build time")
require.NotContains(t, code, "__CROWDSEC_POW_WORKER_PATH__",
"pow worker path placeholder still present; substitution missed at build time")
// The fpscanner sources must NOT be baked into the obfuscated challenge
// code anymore — they live in the separately-served fpscanner bundle.
// fpscanner-only detection strings are a reliable tell.
require.NotContains(t, strings.ToLower(code), "swiftshader",
"fpscanner code leaked into the obfuscated challenge bundle (it should be served separately)")
require.NotContains(t, strings.ToLower(code), "playwright",
"fpscanner code leaked into the obfuscated challenge bundle (it should be served separately)")
}
// TestFPScannerBundleServedRaw asserts the public fpscanner bundle is the
// unobfuscated, separately-served asset: it exposes the scanner on globalThis
// and still contains the readable fpscanner detection logic.
func TestFPScannerBundleServedRaw(t *testing.T) {
require.NotEmpty(t, challengejs.FPScannerJS)
require.Equal(t, challengejs.FPScannerJS, FPScannerJS,
"challenge.FPScannerJS must re-export the js subpackage bundle verbatim")
require.Contains(t, FPScannerJS, "globalThis.CrowdsecFingerprintScanner",
"served fpscanner bundle must expose the scanner on globalThis for the challenge code")
require.Contains(t, strings.ToLower(FPScannerJS), "playwright",
"served fpscanner bundle is minified but NOT obfuscated; detection strings should remain readable")
}
+120
View File
@@ -0,0 +1,120 @@
# WAF challenge JS pipeline
This directory holds the JavaScript sources that the WAF challenge runtime
serves to clients, and the build-time tools that turn them into the
artifacts embedded in the crowdsec binary.
End users running `make build` do **not** need any of the tools below —
the generated artifacts are committed to the repository and pulled in via
`//go:embed`. The pipeline is only exercised when someone intentionally
changes the JS sources.
## What gets generated
Three pipeline steps, all driven by `go:generate` directives in
`generate.go`:
| Step | Tool used | Output | Used at runtime by |
|---|---|---|---|
| `cmd/bundle` | esbuild (Go module) | `fpscanner/fpscanner.js` **and** `challenge_code.js` | `fpscanner.js` is served as-is; `challenge_code.js` is the obfuscator input |
| `cmd/obfuscate` | esbuild + `javy` | `obfuscate/index.wasm.gz` | `wazero` to obfuscate the dynamic key module per epoch |
| `cmd/initialbundle` | `wazero` running the obfuscator wasm | `../initial_bundle.js.gz` | Seeded into the challenge cache at startup so the service is ready instantly |
The client JS is split into **two** bundles:
- **`fpscanner/fpscanner.js`** — the public fingerprint scanner
(`fpscanner/src/*`, entered via `fpscanner/global.js`), built as a minified
IIFE that assigns `globalThis.CrowdsecFingerprintScanner`. It is **served
unobfuscated** at `ChallengeFPScannerPath` via a plain `<script src>` tag
(cacheable across challenge pages). fpscanner is public open-source code, so
there is nothing to hide and no reason to spend CPU obfuscating it.
- **`challenge_code.js`** — the crypto/glue (`../challenge.js`: SHA-256, HMAC,
PoW, fingerprint obfuscation, submission, and the hook registration). This is
the **obfuscation input**: `cmd/initialbundle` substitutes the internal-path
placeholders and obfuscates it into `../initial_bundle.js.gz`, which the
runtime injects inline on the challenge page. The challenge code reads the
scanner from `globalThis.CrowdsecFingerprintScanner` at use time.
There is no runtime re-obfuscation of the challenge code — the build-time
`initial_bundle.js.gz` is the single static variant served. Only the sensitive
per-epoch key module is re-obfuscated at runtime (see `../dynamic_module.go`).
`obfuscate/obfuscate.js` is a small driver around the vendored
`obfuscate/javascript-obfuscator.js` (≈1.6 MB, kept in tree so we don't
pull from npm at build time). The driver reads source code from stdin,
runs `JavaScriptObfuscator.obfuscate(...)` with the `high-obfuscation`
preset (plus a `reservedStrings` entry for the `__CSEC_CHALLENGE_HOOK_v1__`
sentinel that bridges the static and dynamic bundles), and writes the
obfuscated code to stdout. `javy` then compiles that driver to a WASM
module.
## How to regenerate
```sh
make generate-challenge-js
```
Or directly:
```sh
go generate ./pkg/appsec/challenge/js/...
```
Both invocations require `javy` (the Bytecode Alliance JS-to-WASM
compiler) to be on `PATH`. There is no apt/brew package; install via
the upstream releases:
```sh
curl -sL -o javy.gz \
https://github.com/bytecodealliance/javy/releases/download/v8.1.1/javy-x86_64-linux-v8.1.1.gz
gunzip javy.gz && chmod +x javy && mv javy /usr/local/bin/
```
(Substitute the platform suffix for macOS/Windows.) `javy` is
**build-time only** — it is not a runtime dependency of crowdsec.
Regeneration is fast now: only the small `challenge_code.js` (~4 KB) goes
through the obfuscator's `high-obfuscation` preset; the bulk of the client
JS (fpscanner, ~33 KB) is served raw and is never obfuscated.
> `javy` is only needed by `cmd/obfuscate` (it rebuilds `index.wasm.gz`). If
> you only changed `challenge.js` or the fpscanner sources, you can skip javy
> and regenerate just the two bundles:
>
> ```sh
> cd pkg/appsec/challenge/js && go run ./cmd/bundle && go run ./cmd/initialbundle
> ```
## When to regenerate
Run `make generate-challenge-js` (or the two-step command above) after
changing any of:
- `pkg/appsec/challenge/challenge.js`
- `pkg/appsec/challenge/dynamic_module.js.tmpl`
- `pkg/appsec/challenge/js/fpscanner/global.js`
- `pkg/appsec/challenge/js/fpscanner/src/**/*.ts`
- `pkg/appsec/challenge/js/obfuscate/obfuscate.js`
- The vendored `pkg/appsec/challenge/js/obfuscate/javascript-obfuscator.js`
(only when intentionally upgrading)
The pipeline output bytes (`fpscanner/fpscanner.js`, `challenge_code.js`,
`index.wasm.gz`, `initial_bundle.js.gz`) are committed so a clean `git clone`
builds and runs without javy.
## Sentinel survival
The `__CSEC_CHALLENGE_HOOK_v1__` literal must appear verbatim in both
the obfuscated challenge code (`initial_bundle.js.gz`) and any dynamic key
module generated at runtime, so the two bundles meet at the same `globalThis`
key after independent obfuscation passes. The
`reservedStrings: ["__CSEC_CHALLENGE_HOOK_v1__"]` option in
`obfuscate/obfuscate.js` is what guarantees this. If you change the
sentinel string, change it in **all three** places:
- `obfuscate/obfuscate.js` — `reservedStrings`
- `pkg/appsec/challenge/challenge.js` — `CSEC_HOOK_NAME`
- `pkg/appsec/challenge/dynamic_module.js.tmpl` — `hookName`
The Go-side regression test
`TestSplitBundle_HookSentinelInBakedBundle` will fire on a mismatch.
@@ -0,0 +1 @@
var G=[1116352408,1899447441,3049323471,3921009573,961987163,1508970993,2453635748,2870763221,3624381080,310598401,607225278,1426881987,1925078388,2162078206,2614888103,3248222580,3835390401,4022224774,264347078,604807628,770255983,1249150122,1555081692,1996064986,2554220882,2821834349,2952996808,3210313671,3336571891,3584528711,113926993,338241895,666307205,773529912,1294757372,1396182291,1695183700,1986661051,2177026350,2456956037,2730485921,2820302411,3259730800,3345764771,3516065817,3600352804,4094571909,275423344,430227734,506948616,659060556,883997877,958139571,1322822218,1537002063,1747873779,1955562222,2024104815,2227730452,2361852424,2428436474,2756734187,3204031479,3329325298];function S(e){let n=1779033703,t=3144134277,s=1013904242,r=2773480762,f=1359893119,a=2600822924,i=528734635,c=1541459225,h=e.length,F=h*8,T=64-(h+9)%64,U=h+1+(T===64?0:T)+8,A=new Uint8Array(U);A.set(e),A[h]=128;let v=new DataView(A.buffer);v.setUint32(U-4,F,!1);let x=new Int32Array(64);for(let C=0;C<U;C+=64){for(let o=0;o<16;o++)x[o]=v.getInt32(C+o*4,!1);for(let o=16;o<64;o++){let E=(x[o-15]>>>7|x[o-15]<<25)^(x[o-15]>>>18|x[o-15]<<14)^x[o-15]>>>3|0,O=(x[o-2]>>>17|x[o-2]<<15)^(x[o-2]>>>19|x[o-2]<<13)^x[o-2]>>>10|0;x[o]=x[o-16]+E+x[o-7]+O|0}let l=n,w=t,g=s,P=r,d=f,p=a,_=i,H=c;for(let o=0;o<64;o++){let E=(d>>>6|d<<26)^(d>>>11|d<<21)^(d>>>25|d<<7)|0,O=d&p^~d&_|0,k=H+E+O+G[o]+x[o]|0,N=(l>>>2|l<<30)^(l>>>13|l<<19)^(l>>>22|l<<10)|0,j=l&w^l&g^w&g|0,V=N+j|0;H=_,_=p,p=d,d=P+k|0,P=g,g=w,w=l,l=k+V|0}n=n+l|0,t=t+w|0,s=s+g|0,r=r+P|0,f=f+d|0,a=a+p|0,i=i+_|0,c=c+H|0}let W=new Uint8Array(32),u=new DataView(W.buffer);return u.setUint32(0,n,!1),u.setUint32(4,t,!1),u.setUint32(8,s,!1),u.setUint32(12,r,!1),u.setUint32(16,f,!1),u.setUint32(20,a,!1),u.setUint32(24,i,!1),u.setUint32(28,c,!1),W}function R(e){let n="";for(let t=0;t<e.length;t++)n+=(e[t]>>4).toString(16)+(e[t]&15).toString(16);return n}function b(e){return new TextEncoder().encode(e)}function I(e,n){e.length>64&&(e=S(e));let t=new Uint8Array(64);t.set(e);let s=new Uint8Array(64),r=new Uint8Array(64);for(let c=0;c<64;c++)s[c]=t[c]^54,r[c]=t[c]^92;let f=new Uint8Array(64+n.length);f.set(s),f.set(n,64);let a=S(f),i=new Uint8Array(96);return i.set(r),i.set(a,64),S(i)}function D(e,n){return R(I(b(e),b(n)))}function J(e){let n=new Uint8Array(e.length/2);for(let t=0;t<n.length;t++)n[t]=parseInt(e.substr(t*2,2),16);return n}function q(e,n){return R(I(J(e),b(n)))}var z="__CROWDSEC_POW_WORKER_PATH__";function Q(e,n){if(n<=0)return Promise.resolve("0");try{let t=new Worker(z);return new Promise(s=>{t.onmessage=r=>{t.terminate(),s(r.data)},t.onerror=()=>{t.terminate(),s(K(e,n))},t.postMessage({p:e,d:n})})}catch{return Promise.resolve(K(e,n))}}function K(e,n){let t=0;for(;;){let s=e+t.toString(36),r=S(b(s)),f=n>>3,a=n&7,i=!0;for(let c=0;c<f;c++)if(r[c]!==0){i=!1;break}if(i&&a>0){let c=255<<8-a&255;(r[f]&c)!==0&&(i=!1)}if(i)return t.toString(36);t++}}function X(e,n){let t=b(e),s=b(n),r=new Uint8Array(s.length);for(let a=0;a<s.length;a++)r[a]=s[a]^t[a%t.length];let f=String.fromCharCode(...r);return btoa(f)}var M=typeof _ts<"u"?_ts:"",B=typeof _powP<"u"?_powP:"",Y=typeof _powM<"u"?_powM:"",L=typeof _powD<"u"?_powD:12,y=typeof _r<"u"?_r:"",Z="__CROWDSEC_SUBMIT_PATH__";function m(e){if(typeof window.crowdsecSetChallengeStatus=="function"){window.crowdsecSetChallengeStatus(e);return}window.dispatchEvent(new CustomEvent("crowdsec-challenge-status",{detail:{status:e}}))}var $="__CSEC_CHALLENGE_HOOK_v1__";async function ee(e){let n=globalThis.CrowdsecFingerprintScanner;if(typeof n!="function"){m("fail");return}let[t,s]=await Promise.all([Q(B,L),new n().collectFingerprint({encrypt:!1})]),r=q(e,y),f=D(r,"fpenc"+y),a=X(f,JSON.stringify(s)),i=D(r,y+M+t+a);return fetch(Z,{method:"POST",credentials:"same-origin",body:new URLSearchParams({f:a,r:y,ts:M,sig:i,n:t,p:B,m:Y,d:L,u:window.location.pathname})}).then(c=>c.json()).then(c=>{let h=typeof c?.status=="string"?c.status:"fail";m(h)}).catch(()=>{m("fail")})}globalThis[$]=function(e){if(!e||typeof e.key!="string"||e.key.length===0){m("fail");return}ee(e.key)};
@@ -0,0 +1,68 @@
package main
import (
"errors"
"fmt"
"os"
esbuildapi "github.com/evanw/esbuild/pkg/api"
)
// The challenge JS is shipped as two separate bundles:
//
// - fpscanner.js — the public fingerprint scanner, served UNOBFUSCATED via a
// plain <script src> tag (cacheable). Built as a minified IIFE that assigns
// globalThis.CrowdsecFingerprintScanner (see fpscanner/global.js).
// - challenge_code.js — the crypto/glue (SHA-256, HMAC, PoW, submission, hook
// registration). This is the obfuscation input consumed by ./cmd/initialbundle.
type bundleTarget struct {
entryPoint string
outFile string
format esbuildapi.Format
}
var targets = []bundleTarget{
{entryPoint: "fpscanner/global.js", outFile: "fpscanner/fpscanner.js", format: esbuildapi.FormatIIFE},
{entryPoint: "../challenge.js", outFile: "challenge_code.js", format: esbuildapi.FormatESModule},
}
func main() {
for _, t := range targets {
if err := build(t); err != nil {
fmt.Fprintf(os.Stderr, "bundle %s: %v\n", t.outFile, err)
os.Exit(1)
}
}
}
func build(t bundleTarget) error {
result := esbuildapi.Build(esbuildapi.BuildOptions{
EntryPoints: []string{t.entryPoint},
Bundle: true,
Write: false,
Format: t.format,
Platform: esbuildapi.PlatformBrowser,
Target: esbuildapi.ES2022,
MinifyWhitespace: true,
MinifyIdentifiers: true,
MinifySyntax: true,
Sourcemap: esbuildapi.SourceMapNone,
})
if len(result.Errors) > 0 {
for _, msg := range result.Errors {
fmt.Fprintln(os.Stderr, msg.Text)
}
return fmt.Errorf("esbuild failed with %d error(s)", len(result.Errors))
}
if len(result.OutputFiles) == 0 {
return errors.New("esbuild returned no output files")
}
if err := os.WriteFile(t.outFile, result.OutputFiles[0].Contents, 0o644); err != nil {
return fmt.Errorf("write %s: %w", t.outFile, err)
}
return nil
}
@@ -0,0 +1,174 @@
// initialbundle is a build-time tool that produces a pre-obfuscated initial
// challenge bundle, embedded into the Go binary. This eliminates the ~1 minute
// of synchronous obfuscation that NewChallengeRuntime would otherwise have to
// pay at startup before the service is ready to serve challenges.
//
// Pipeline:
//
// challenge_code.js ──substitute placeholders──▶ source JS
// obfuscate/index.wasm.gz ──decompress──▶ obfuscator WASM
// wazero(obfuscator).Run(stdin=source) ──▶ obfuscated JS
// gzip ──▶ ../initial_bundle.js.gz
//
// The output `initial_bundle.js.gz` is committed and embedded via go:embed in
// pkg/appsec/challenge/challenge.go.
package main
import (
"bytes"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"os"
"strings"
"time"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
)
// These must match the ChallengeSubmitPath / ChallengePowWorkerPath constants
// in pkg/appsec/challenge/challenge.go. Kept as plain strings here to avoid an
// import cycle (pkg/appsec/challenge depends on this js subpackage).
const (
submitPath = "/crowdsec-internal/challenge/submit"
powWorkerPath = "/crowdsec-internal/challenge/pow-worker.js"
)
const (
// Inputs (relative to pkg/appsec/challenge/js where this tool runs).
challengeCodePath = "challenge_code.js"
obfuscatorWasmGz = "obfuscate/index.wasm.gz"
// Output (relative to pkg/appsec/challenge/js).
outputPath = "../initial_bundle.js.gz"
)
func main() {
start := time.Now()
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "initialbundle: %v\n", err)
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "initialbundle: wrote %s in %s\n", outputPath, time.Since(start).Round(time.Second))
}
func run() error {
source, err := buildSourceBundle()
if err != nil {
return fmt.Errorf("build source bundle: %w", err)
}
wasmBytes, err := loadObfuscatorWasm()
if err != nil {
return fmt.Errorf("load obfuscator wasm: %w", err)
}
ctx := context.Background()
r := wazero.NewRuntime(ctx)
defer r.Close(ctx)
if _, err := wasi_snapshot_preview1.Instantiate(ctx, r); err != nil {
return fmt.Errorf("instantiate wasi: %w", err)
}
compiled, err := r.CompileModule(ctx, wasmBytes)
if err != nil {
return fmt.Errorf("compile obfuscator wasm: %w", err)
}
obfuscated, err := obfuscate(ctx, r, compiled, source)
if err != nil {
return fmt.Errorf("obfuscate: %w", err)
}
if obfuscated == "" {
return errors.New("obfuscator produced empty output")
}
if err := writeGzip(outputPath, obfuscated); err != nil {
return fmt.Errorf("write %s: %w", outputPath, err)
}
fmt.Fprintf(os.Stderr, "initialbundle: input %d bytes -> obfuscated %d bytes -> compressed file written\n",
len(source), len(obfuscated))
return nil
}
func buildSourceBundle() (string, error) {
raw, err := os.ReadFile(challengeCodePath)
if err != nil {
return "", fmt.Errorf("read %s: %w", challengeCodePath, err)
}
// Same substitution as ChallengeRuntime.buildChallengeBundle in challenge.go.
source := strings.NewReplacer(
"__CROWDSEC_SUBMIT_PATH__", submitPath,
"__CROWDSEC_POW_WORKER_PATH__", powWorkerPath,
).Replace(string(raw))
return source, nil
}
func loadObfuscatorWasm() ([]byte, error) {
f, err := os.Open(obfuscatorWasmGz)
if err != nil {
return nil, err
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("gzip reader: %w", err)
}
defer gz.Close()
return io.ReadAll(gz)
}
func obfuscate(ctx context.Context, r wazero.Runtime, compiled wazero.CompiledModule, source string) (string, error) {
stdin := bytes.NewReader([]byte(source))
var stdout, stderr bytes.Buffer
cfg := wazero.NewModuleConfig().
WithStdin(stdin).
WithStdout(&stdout).
WithStderr(&stderr)
mod, err := r.InstantiateModule(ctx, compiled, cfg)
if err != nil {
if stderr.Len() > 0 {
return "", fmt.Errorf("wasm runtime error: %v | stderr: %s", err, stderr.String())
}
return "", fmt.Errorf("wasm instantiation: %w", err)
}
mod.Close(ctx)
return stdout.String(), nil
}
func writeGzip(path string, data string) error {
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
zw, err := gzip.NewWriterLevel(f, gzip.BestCompression)
if err != nil {
return fmt.Errorf("gzip writer: %w", err)
}
zw.Name = "initial_bundle.js"
zw.ModTime = time.Unix(0, 0)
if _, err := zw.Write([]byte(data)); err != nil {
zw.Close()
return err
}
return zw.Close()
}
@@ -0,0 +1,157 @@
package main
import (
"compress/gzip"
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
"os/exec"
"strings"
"time"
esbuildapi "github.com/evanw/esbuild/pkg/api"
)
const (
entryPoint = "obfuscate/obfuscate.js"
bundleFile = "obfuscate/bundle.js"
wasmFile = "obfuscate/index.wasm"
wasmGzFile = "obfuscate/index.wasm.gz"
)
func main() {
bundleData, err := buildObfuscatorBundle()
if err != nil {
fmt.Fprintf(os.Stderr, "failed to build obfuscator bundle: %v\n", err)
os.Exit(1)
}
// javy output is not byte-reproducible, so rebuilding from unchanged
// sources would always dirty index.wasm.gz. The inputs (bundle content,
// javy version) are recorded in the gzip header comment; skip the rebuild
// when they match the existing file.
buildKey, err := computeBuildKey(bundleData)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to compute build key: %v\n", err)
os.Exit(1)
}
if currentBuildKey() == buildKey {
fmt.Fprintf(os.Stderr, "obfuscate: %s is up to date, skipping javy build\n", wasmGzFile)
return
}
if err := os.WriteFile(bundleFile, bundleData, 0o644); err != nil {
fmt.Fprintf(os.Stderr, "failed to write %s: %v\n", bundleFile, err)
os.Exit(1)
}
if err := buildWASM(); err != nil {
fmt.Fprintf(os.Stderr, "failed to build obfuscator wasm: %v\n", err)
os.Exit(1)
}
if err := gzipWASM(buildKey); err != nil {
fmt.Fprintf(os.Stderr, "failed to compress obfuscator wasm: %v\n", err)
os.Exit(1)
}
}
func buildObfuscatorBundle() ([]byte, error) {
result := esbuildapi.Build(esbuildapi.BuildOptions{
EntryPoints: []string{entryPoint},
Bundle: true,
Write: false,
Format: esbuildapi.FormatIIFE,
Platform: esbuildapi.PlatformBrowser,
Target: esbuildapi.ES2022,
Sourcemap: esbuildapi.SourceMapNone,
// JS obfuscator expects window/global to be available, but in WASM we don't have them, so we alias them to globalThis
Banner: map[string]string{"js": "var self=globalThis; var window=globalThis; var global=globalThis;"},
})
if len(result.Errors) > 0 {
return nil, fmt.Errorf("esbuild failed with %d error(s): %s", len(result.Errors), result.Errors[0].Text)
}
if len(result.OutputFiles) == 0 {
return nil, errors.New("esbuild returned no output files")
}
return result.OutputFiles[0].Contents, nil
}
func computeBuildKey(bundleData []byte) (string, error) {
out, err := exec.CommandContext(context.Background(), "javy", "--version").Output()
if err != nil {
return "", fmt.Errorf("failed to run 'javy --version' (is javy in PATH?): %w", err)
}
return fmt.Sprintf("bundle-sha256=%x %s", sha256.Sum256(bundleData), strings.TrimSpace(string(out))), nil
}
// currentBuildKey returns the build key recorded in the existing wasm.gz
// header, or "" if the file is missing or unreadable.
func currentBuildKey() string {
f, err := os.Open(wasmGzFile)
if err != nil {
return ""
}
defer f.Close()
zr, err := gzip.NewReader(f)
if err != nil {
return ""
}
defer zr.Close()
return zr.Comment
}
func buildWASM() error {
cmd := exec.CommandContext(context.Background(), "javy", "build", "-o", wasmFile, bundleFile)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("javy build failed: %w", err)
}
return nil
}
func gzipWASM(buildKey string) error {
wasmData, err := os.ReadFile(wasmFile)
if err != nil {
return fmt.Errorf("failed to read %s: %w", wasmFile, err)
}
f, err := os.Create(wasmGzFile)
if err != nil {
return fmt.Errorf("failed to create %s: %w", wasmGzFile, err)
}
defer f.Close()
zw, err := gzip.NewWriterLevel(f, gzip.BestCompression)
if err != nil {
return fmt.Errorf("failed to create gzip writer: %w", err)
}
zw.Name = "index.wasm"
zw.ModTime = time.Unix(0, 0)
zw.Comment = buildKey
if _, err := zw.Write(wasmData); err != nil {
zw.Close()
return fmt.Errorf("failed to write gzip data: %w", err)
}
if err := zw.Close(); err != nil {
return fmt.Errorf("failed to close gzip writer: %w", err)
}
os.Remove(wasmFile)
os.Remove(bundleFile)
return nil
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,5 @@
// Entry point for the fpscanner bundle: exposes the scanner on globalThis.
// Built by ./cmd/bundle into ./fpscanner.js (minified IIFE, served as-is).
import FingerprintScanner from "./fpscanner/src/index.ts";
globalThis.CrowdsecFingerprintScanner = FingerprintScanner;
+5
View File
@@ -0,0 +1,5 @@
package js
//go:generate go run ./cmd/bundle
//go:generate go run ./cmd/obfuscate
//go:generate go run ./cmd/initialbundle
+27
View File
@@ -0,0 +1,27 @@
// This package contains all JavaScript-related code for the WAF challenge and fingerprinting.
// fpscanner is https://github.com/antoinevastel/fpscanner/
// obfuscate is a custom wrapper around javascript-obfuscator.
// It uses a vendored bundle of javascript-obfuscator to avoid any dependency on npm at build time.
// The goal is to build a WASM module that can dynamically obfuscate the challenge code at runtime.
//
// The client JS is split into two bundles (built by ./cmd/bundle):
// - FPScannerJS: the public fingerprint scanner, served UNOBFUSCATED via its
// own <script src> tag (cacheable across challenge pages).
// - ChallengeCode: the crypto/glue, obfuscated at build time (./cmd/initialbundle)
// into ../initial_bundle.js.gz and injected inline on the challenge page.
package js
import _ "embed"
// FPScannerJS is the standalone, unobfuscated fpscanner bundle served as-is.
//
//go:embed fpscanner/fpscanner.js
var FPScannerJS string
// ChallengeCode is the minified (not-yet-obfuscated) challenge crypto/glue code.
// It still carries the __CROWDSEC_*_PATH__ placeholders; the runtime substitutes
// them before obfuscation (see buildChallengeBundle in static_bundle.go).
//
//go:embed challenge_code.js
var ChallengeCode string
Binary file not shown.
File diff suppressed because one or more lines are too long
@@ -0,0 +1,56 @@
// Downloaded from https://cdn.jsdelivr.net/npm/javascript-obfuscator/dist/index.browser.js
import JavaScriptObfuscator from "./javascript-obfuscator.js";
function readStdin() {
const buffer = new Uint8Array(1024);
const input = [];
while (true) {
const bytesRead = Javy.IO.readSync(0, buffer);
if (bytesRead === 0) break;
const chunk = buffer.subarray(0, bytesRead);
for (let i = 0; i < bytesRead; i++) {
input.push(chunk[i]);
}
}
return new TextDecoder().decode(new Uint8Array(input));
}
function writeStdout(str) {
const buffer = new TextEncoder().encode(str);
Javy.IO.writeSync(1, buffer);
}
try {
const sourceCode = readStdin();
// Spread the high-obfuscation preset, then add reservedStrings so the
// sentinel that bridges the static bundle and the per-epoch dynamic key
// module (CSEC_CHALLENGE_HOOK_v1) survives the string-array transform
// identically in both bundles. Without this, the two independently-
// obfuscated artifacts wouldn't agree on the globalThis symbol they
// need to meet at.
//
// disableConsoleOutput is forced off because fpscanner's CDP detection
// (signals/cdp.ts) relies on `console.log(err)` triggering DevTools'
// eager access of `err.stack`, which in turn fires our overridden
// `Error.prepareStackTrace`. The preset's default `true` rewrites the
// call into a no-op stub and the detection silently always reports
// false. Leave it off so the side-channel works.
const opts = Object.assign(
{},
JavaScriptObfuscator.getOptionsByPreset("high-obfuscation"),
{
reservedStrings: ["__CSEC_CHALLENGE_HOOK_v1__"],
disableConsoleOutput: false,
},
);
const result = JavaScriptObfuscator.obfuscate(sourceCode, opts);
writeStdout(result.getObfuscatedCode());
} catch (e) {
const msg = (e && e.message) ? e.message : String(e);
const buffer = new TextEncoder().encode(`obfuscate: ${msg}\n`);
Javy.IO.writeSync(2, buffer);
throw e;
}
+259
View File
@@ -0,0 +1,259 @@
// keyring.go derives and rotates the per-epoch HMAC signing keys (and the
// long-lived AES cookie-sealing key) from the master secret using HKDF.
// Epoch advances on a fixed cadence (keyringDefaultRotation, 5m); past
// epochs stay valid for the configured live window so in-flight tickets
// don't get invalidated at the boundary. The cookie key is derived once and
// reused — cookie expiration is enforced by an explicit not_after stamp
// inside the sealed envelope (see crypto.go), not by key rotation.
package challenge
import (
"crypto/sha256"
"encoding/binary"
"fmt"
"sync"
"time"
log "github.com/sirupsen/logrus"
"golang.org/x/crypto/hkdf"
"github.com/crowdsecurity/crowdsec/pkg/metrics"
)
// keyringHKDFSalt and the per-context info strings define the HKDF
// derivations used to turn the long-lived master secret into operational
// keys. They are part of the wire protocol — changing them silently would
// invalidate every outstanding ticket and cookie across a fleet upgrade.
// Bump the version suffix when intentionally rotating.
//
// Two independent derivation contexts are used:
//
// - "epoch-sign": per-epoch HMAC key for ticket / PoW MAC signing.
// Rotates with the keyring (tight forgery window).
// - "cookie-master": single long-lived AES key for cookie sealing.
// Lifetime equals master_secret's lifetime; cookie
// expiration is enforced by an explicit not_after
// timestamp inside the sealed envelope (see crypto.go).
//
// The two contexts produce cryptographically independent keys: leaking
// one tells the attacker nothing about the other.
const (
keyringHKDFSalt = "crowdsec-challenge-keyring-v1"
keyringInfoSign = "epoch-sign"
keyringInfoMasterCookie = "cookie-master"
keyringDerivedKeyBytes = 32 // 256-bit keys throughout
keyringClockSkew = 1 // accept currentEpoch + 1
keyringDefaultMaxLive = 3 // currentEpoch and 2 prior
keyringDefaultRotation = 5 * time.Minute
keyringMinRotation = 30 * time.Second
)
// KeyRing produces keys deterministically from a shared master secret. Two
// instances configured with the same masterSecret derive bit-identical
// keys — the property that lets distributed (multi-WAF) deployments sign
// and verify each other's challenges and cookies without coordination.
//
// Two key families with different lifetimes:
//
// - Per-epoch signing key (rotates on rotationInterval). Used for ticket
// HMAC and PoW MAC. Live window:
// [currentEpoch - maxLive + 1 ... currentEpoch + clockSkew]
// Epochs outside the window are rejected, bounding ticket forgery
// exposure to maxLive * rotationInterval.
//
// - Long-lived master cookie key (no rotation). Used for AES-GCM cookie
// sealing. Cookie expiration is enforced by an explicit not_after
// timestamp inside the sealed envelope, NOT by key eviction — so
// cookie TTL can be much larger than the ticket window without
// widening ticket forgery exposure.
type KeyRing struct {
masterSecret []byte
rotationInterval time.Duration
maxLive int
clockSkew int
// masterCookieKey is the long-lived cookie-sealing key; derived once
// at construction. Pointer-receiver methods read it without locking
// because it's never mutated after NewKeyRing returns.
masterCookieKey []byte
now func() time.Time // overridable for tests
logger *log.Entry
mu sync.RWMutex
cache map[int64][]byte // epoch -> per-epoch sign key
}
// log returns the component logger (never nil; see the logger field).
func (k *KeyRing) log() *log.Entry {
return k.logger
}
// NewKeyRing constructs a KeyRing. masterSecret must be at least minSecretBytes
// long (callers should already have validated this via WithMasterSecret); the
// rotation interval must be at least keyringMinRotation. maxLive defaults to
// keyringDefaultMaxLive when zero.
func NewKeyRing(masterSecret []byte, rotationInterval time.Duration, maxLive int) (*KeyRing, error) {
if len(masterSecret) < minSecretBytes {
return nil, fmt.Errorf("keyring master secret is %d bytes; minimum is %d", len(masterSecret), minSecretBytes)
}
if rotationInterval < keyringMinRotation {
return nil, fmt.Errorf("keyring rotation interval %s is below the floor %s", rotationInterval, keyringMinRotation)
}
if maxLive <= 0 {
maxLive = keyringDefaultMaxLive
}
return &KeyRing{
masterSecret: masterSecret,
rotationInterval: rotationInterval,
maxLive: maxLive,
clockSkew: keyringClockSkew,
masterCookieKey: deriveMasterCookieKey(masterSecret),
logger: log.StandardLogger().WithField("module", "challenge"),
now: time.Now,
cache: make(map[int64][]byte),
}, nil
}
// CurrentEpoch returns the epoch identifier for the current wall-clock time.
// Equal across all instances with synchronized clocks.
func (k *KeyRing) CurrentEpoch() int64 {
return k.now().Unix() / int64(k.rotationInterval/time.Second)
}
// Current returns the epoch and signing key that should be used to sign new
// outbound material right now.
func (k *KeyRing) Current() (int64, []byte) {
epoch := k.CurrentEpoch()
return epoch, k.deriveOrCache(epoch)
}
// SignKey returns the HMAC key for an epoch if it's within the live window;
// returns (nil, false) otherwise.
func (k *KeyRing) SignKey(epoch int64) ([]byte, bool) {
if !k.isLive(epoch) {
return nil, false
}
return k.deriveOrCache(epoch), true
}
// MasterCookieKey returns the long-lived AES-key-input for cookie sealing.
// Does not depend on epoch; same value for the lifetime of the master
// secret. Cookie expiration is enforced via an explicit not_after timestamp
// inside the sealed envelope (see crypto.go).
func (k *KeyRing) MasterCookieKey() []byte {
return k.masterCookieKey
}
// LiveEpochs returns every epoch currently in the live window, oldest first.
// Used by callers that need to enumerate the acceptable ticket-signing epochs.
func (k *KeyRing) LiveEpochs() []int64 {
current := k.CurrentEpoch()
out := make([]int64, 0, k.maxLive+k.clockSkew)
for e := current - int64(k.maxLive-1); e <= current+int64(k.clockSkew); e++ {
out = append(out, e)
}
return out
}
func (k *KeyRing) isLive(epoch int64) bool {
current := k.CurrentEpoch()
return epoch >= current-int64(k.maxLive-1) && epoch <= current+int64(k.clockSkew)
}
// deriveOrCache returns the cached per-epoch signing key or derives and
// stores it. Pure function of (masterSecret, epoch) so two instances
// always agree.
func (k *KeyRing) deriveOrCache(epoch int64) []byte {
k.mu.RLock()
if cached, ok := k.cache[epoch]; ok {
k.mu.RUnlock()
return cached
}
k.mu.RUnlock()
k.mu.Lock()
defer k.mu.Unlock()
// Re-check after acquiring the write lock.
if cached, ok := k.cache[epoch]; ok {
return cached
}
start := time.Now()
derived := deriveEpochKey(k.masterSecret, epoch, keyringInfoSign)
derivationDuration := time.Since(start)
// Bound the cache so it can't grow without limit if the clock jumps.
// Anything outside the live window is safe to evict.
evicted := 0
for cachedEpoch := range k.cache {
if !k.isLive(cachedEpoch) {
delete(k.cache, cachedEpoch)
evicted++
}
}
k.cache[epoch] = derived
metrics.AppsecChallengeKepochGenerated.Inc()
if evicted > 0 {
metrics.AppsecChallengeKepochEvicted.Add(float64(evicted))
}
k.log().WithFields(log.Fields{
"epoch": epoch,
"current_epoch": k.CurrentEpoch(),
"derivation_us": derivationDuration.Microseconds(),
"cache_evicted": evicted,
"cache_size": len(k.cache),
}).Debug("derived per-epoch key")
return derived
}
// deriveEpochKey performs HKDF-SHA256 with a stable salt and a context-
// specific info value: <context bytes> || ':' || <epoch_be8>. Two instances
// with the same masterSecret always derive the same bytes for the same epoch
// and context — this is what makes distributed agreement work.
//
// The colon separator prevents context-prefix collisions (e.g. avoids "epoch"
// + "sign" colliding with "epochs" + "ign").
func deriveEpochKey(masterSecret []byte, epoch int64, context string) []byte {
info := make([]byte, 0, len(context)+1+8)
info = append(info, context...)
info = append(info, ':')
var epochBytes [8]byte
binary.BigEndian.PutUint64(epochBytes[:], uint64(epoch))
info = append(info, epochBytes[:]...)
return hkdfExtract(masterSecret, info)
}
// deriveMasterCookieKey performs HKDF-SHA256 with no epoch component, so
// the output depends only on the master secret. Identical across all
// instances sharing the master, stable for the lifetime of the master.
// Cookie expiration is enforced separately by a not_after timestamp
// inside the sealed envelope (see crypto.go).
func deriveMasterCookieKey(masterSecret []byte) []byte {
return hkdfExtract(masterSecret, []byte(keyringInfoMasterCookie))
}
// hkdfExtract is the shared HKDF-SHA256 call used by both derivation helpers
// (fixed keyringHKDFSalt, caller-supplied info). It panics on a Read error
// because a 32-byte SHA-256 output is infallible (HKDF-Expand only fails above
// 255*HashLen bytes); an error here means the crypto stack itself is broken,
// not a recoverable condition, so threading an error through every call site
// would be handling an impossible case.
func hkdfExtract(masterSecret []byte, info []byte) []byte {
r := hkdf.New(sha256.New, masterSecret, []byte(keyringHKDFSalt), info)
out := make([]byte, keyringDerivedKeyBytes)
if _, err := r.Read(out); err != nil {
panic(fmt.Sprintf("keyring: hkdf read failed (should be unreachable): %v", err))
}
return out
}
@@ -0,0 +1,474 @@
package challenge
import (
"bytes"
"encoding/base64"
"net/http"
"strconv"
"strings"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/pb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestCookieV0_RoundTrip is the basic positive case: seal under the master
// cookie key with an explicit not_after one hour in the future, open
// immediately, fingerprint comes back.
func TestCookieV0_RoundTrip(t *testing.T) {
keys := testKeyRing()
envelope := &pb.ChallengeCookie{PowDifficulty: 12}
notAfter := time.Now().Add(time.Hour).Unix()
encoded, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.NoError(t, err)
assert.Equal(t, int32(12), got.Envelope.GetPowDifficulty())
}
// TestCookieV0_ExpiredRejected confirms the not_after timestamp embedded
// inside the encrypted envelope is enforced on open.
func TestCookieV0_ExpiredRejected(t *testing.T) {
keys := testKeyRing()
// notAfter is one hour in the past — the cookie was already expired
// when issued (this is what a stale cookie sent by a slow client would
// look like on the server).
pastNotAfter := time.Now().Add(-time.Hour).Unix()
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), pastNotAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.Error(t, err)
assert.ErrorIs(t, err, ErrCookieExpired,
"expired cookie must produce ErrCookieExpired, got %v", err)
}
// TestCookieV0_TamperedExpirationRejected is the security-critical test for
// the design choice of putting not_after INSIDE the encrypted envelope. Any
// flip of a byte in the ciphertext (including bytes covering the not_after
// header) must invalidate the AEAD tag, not just shift the apparent
// expiration time.
func TestCookieV0_TamperedExpirationRejected(t *testing.T) {
keys := testKeyRing()
notAfter := time.Now().Add(time.Hour).Unix()
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
raw, err := base64.RawURLEncoding.DecodeString(encoded)
require.NoError(t, err)
// Wire layout: version(1) || nonce(12) || ciphertext(...)
// The first byte of ciphertext corresponds to the first byte of
// plaintext, which is the high byte of not_after_be8. Flipping it
// MUST invalidate the AEAD tag.
tampered := append([]byte(nil), raw...)
tampered[1+12] ^= 0x80 // flip MSB of not_after's high byte
tamperedEncoded := base64.RawURLEncoding.EncodeToString(tampered)
_, err = openCookie(tamperedEncoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.Error(t, err)
assert.ErrorIs(t, err, ErrCookieSignature,
"tampering with the not_after region must produce ErrCookieSignature (AEAD tag failure)")
}
// TestCookieV0_UAMismatchRejected confirms the User-Agent AAD binding.
func TestCookieV0_UAMismatchRejected(t *testing.T) {
keys := testKeyRing()
notAfter := time.Now().Add(time.Hour).Unix()
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua-A"), MaxCookieLen)
require.NoError(t, err)
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua-B"), MaxCookieLen)
assert.ErrorIs(t, err, ErrCookieSignature)
}
// TestCookieV0_SurvivesArbitraryRotation is the headline test for the v0
// design: a cookie sealed under the long-lived master cookie key remains
// valid across as many ticket-signing-key rotations as the keyring goes
// through. The previous design tied cookie validity to the same live
// window as ticket signing; this one decouples them completely.
//
// We advance the keyring's clock (forcing it to derive a stream of new
// epoch signing keys) and confirm that openCookie — which depends only
// on the master cookie key, not on any epoch — keeps working.
func TestCookieV0_SurvivesArbitraryRotation(t *testing.T) {
t0 := time.Now()
keys := newTestKeyRing(t, testSecret, time.Minute, t0)
// notAfter is well in the future from real wall-clock time, so the
// in-envelope expiration check (which uses real time.Now()) won't
// reject the cookie.
notAfter := t0.Add(24 * time.Hour).Unix()
encoded, err := sealCookieV0(&pb.ChallengeCookie{PowDifficulty: 9}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
// Roll the keyring's clock forward — this triggers per-epoch sign-key
// derivations and (with maxLive=3) evictions. None of that affects
// the master cookie key, so the cookie keeps validating.
for _, advance := range []time.Duration{
1 * time.Minute,
10 * time.Minute,
1 * time.Hour,
6 * time.Hour,
23 * time.Hour,
} {
keys.now = func() time.Time { return t0.Add(advance) }
// Touching keys.Current() forces a derivation under the new
// keyring time, simulating real rotation activity.
_, _ = keys.Current()
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.NoError(t, err,
"cookie should still validate after %s of keyring rotation; got %v", advance, err)
assert.Equal(t, int32(9), got.Envelope.GetPowDifficulty())
}
}
// TestCookieV0_ExpiryEnforcedAgainstWallClock is the counterpart to the
// rotation test: cookie expiration uses real wall-clock time (not the
// keyring's overridable now), so we seal a cookie with notAfter already
// in the past and confirm rejection.
func TestCookieV0_ExpiryEnforcedAgainstWallClock(t *testing.T) {
keys := testKeyRing()
pastNotAfter := time.Now().Add(-time.Minute).Unix()
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), pastNotAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.Error(t, err)
assert.ErrorIs(t, err, ErrCookieExpired)
}
// TestCookieV0_MasterSecretChangeInvalidates is the expected counter-property
// to SurvivesArbitraryRotation: if the operator rotates the master secret
// (a manual config change), every outstanding cookie is invalidated.
func TestCookieV0_MasterSecretChangeInvalidates(t *testing.T) {
notAfter := time.Now().Add(time.Hour).Unix()
keysA, err := NewKeyRing(bytes.Repeat([]byte{0xaa}, 32), time.Minute, 3)
require.NoError(t, err)
keysB, err := NewKeyRing(bytes.Repeat([]byte{0xbb}, 32), time.Minute, 3)
require.NoError(t, err)
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keysA.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
_, err = openCookie(encoded, keysB.MasterCookieKey(), []byte("ua"), MaxCookieLen)
assert.ErrorIs(t, err, ErrCookieSignature,
"cookie sealed under secret A must not decrypt under secret B")
}
// TestCookieV0_DistributedSetup_TwoInstances confirms cookies issued by
// instance A validate against instance B as long as both share the same
// master secret. This is the multi-WAF deployment invariant.
func TestCookieV0_DistributedSetup_TwoInstances(t *testing.T) {
secret := bytes.Repeat([]byte{0xcc}, 32)
a, err := NewKeyRing(secret, time.Minute, 3)
require.NoError(t, err)
b, err := NewKeyRing(secret, time.Minute, 3)
require.NoError(t, err)
assert.True(t, bytes.Equal(a.MasterCookieKey(), b.MasterCookieKey()),
"two instances with the same master_secret must derive the same master cookie key")
notAfter := time.Now().Add(time.Hour).Unix()
encoded, err := sealCookieV0(&pb.ChallengeCookie{PowDifficulty: 14}, a.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
got, err := openCookie(encoded, b.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.NoError(t, err, "cookie issued by A must validate against B (same master_secret)")
assert.Equal(t, int32(14), got.Envelope.GetPowDifficulty())
}
// TestCookie_UnknownVersionRejected makes the version-byte dispatch
// explicit: a cookie with an unknown leading byte is rejected with the
// dedicated ErrCookieVersion sentinel. This is the extension point for
// any future cookie schema bump.
func TestCookie_UnknownVersionRejected(t *testing.T) {
keys := testKeyRing()
// Forge: version byte 0xFE followed by random-looking junk.
raw := append([]byte{0xFE}, []byte("0123456789abcdefghijklmnop")...)
encoded := base64.RawURLEncoding.EncodeToString(raw)
_, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.Error(t, err)
assert.ErrorIs(t, err, ErrCookieVersion, "unknown version byte must produce ErrCookieVersion")
}
// TestRotation_TicketSurvives_WithinLiveWindow exercises the most important
// guarantee for in-flight requests: a ticket signed under epoch N validates
// after the keyring rolls to epoch N+1, as long as N is still within the
// live window. Without this, every key rotation would invalidate every
// challenge currently in flight.
//
// verifyChallenge uses time.Since on the wall clock (not the keyring's
// overridable now), so we anchor t0 at the real current time and only
// override the keyring's clock — small enough offsets stay inside the
// ticketAgeBackstop freshness check.
func TestRotation_TicketSurvives_WithinLiveWindow(t *testing.T) {
t0 := time.Now()
keys := newTestKeyRing(t, testSecret, time.Minute, t0)
c := &ChallengeRuntime{keys: keys}
tsStr := strconvI64(t0.UnixNano())
r, err := generateChallengeNonce()
require.NoError(t, err)
salt := mustGeneratePowPrefix(t)
mac := c.computePowMAC(salt, r, tsStr, PowDifficultyMedium)
// Roll the keyring clock forward one rotation interval — the challenge's
// epoch is now "previous" but still in the live window.
keys.now = func() time.Time { return t0.Add(70 * time.Second) }
_, ok := c.verifyChallenge(r, tsStr, salt, mac, PowDifficultyMedium)
assert.True(t, ok, "challenge from previous epoch must still validate within the live window")
}
// TestRotation_TicketRejected_OutOfWindow proves the negative side of the
// previous test: once the keyring's view of time has advanced enough to
// evict the challenge's epoch, validation fails. Anchored at real-time t0 so
// verifyChallenge's own freshness check (time.Since) doesn't conflate with
// the keyring's eviction logic.
func TestRotation_TicketRejected_OutOfWindow(t *testing.T) {
t0 := time.Now()
keys := newTestKeyRing(t, testSecret, time.Minute, t0)
c := &ChallengeRuntime{keys: keys}
tsStr := strconvI64(t0.UnixNano())
r, err := generateChallengeNonce()
require.NoError(t, err)
salt := mustGeneratePowPrefix(t)
mac := c.computePowMAC(salt, r, tsStr, PowDifficultyMedium)
// Jump the keyring's clock past the live window (maxLive=3 + skew=1 =
// any epoch >4 minutes ahead in the keyring's view evicts the original).
keys.now = func() time.Time { return t0.Add(5 * time.Minute) }
_, ok := c.verifyChallenge(r, tsStr, salt, mac, PowDifficultyMedium)
assert.False(t, ok, "challenge from an evicted epoch must not validate")
}
// TestEndToEnd_ValidateChallengeResponse walks the full validation path
// (matches challenge, opens cookie). Regression guard for keyring +
// cookie-v0 wired together.
func TestEndToEnd_ValidateChallengeResponse(t *testing.T) {
keys := testKeyRing()
c := &ChallengeRuntime{keys: keys, powDifficulty: 8, cookieTTL: time.Hour, spent: newSpentSet(spentSetDefaultMaxEntries)}
r, ts := freshChallenge(t)
body := buildValidBody(t, c.powDifficulty, r, ts)
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
require.NoError(t, err)
req.Header.Set("User-Agent", "test-agent")
ck, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
require.NoError(t, err)
require.NotNil(t, ck)
// Round-trip the cookie through the public ValidCookie API to confirm
// the full Seal → parse Set-Cookie → Open path is correct.
parsed, err := http.ParseSetCookie(ck.String())
require.NoError(t, err)
got, err := c.ValidCookie(parsed, "test-agent")
require.NoError(t, err)
assert.Equal(t, 8, got.PowDifficulty)
}
// TestCookieV0_BrowserTTLMatchesServerTTL guards the gotcha mentioned in
// the v0 design discussion: if the browser drops the cookie before the
// server stops accepting it (or vice versa), users get unexpected
// re-challenges. The Set-Cookie Max-Age MUST match the server-side
// cookieTTL.
func TestCookieV0_BrowserTTLMatchesServerTTL(t *testing.T) {
keys := testKeyRing()
c := &ChallengeRuntime{keys: keys, powDifficulty: 8, cookieTTL: 90 * time.Minute, spent: newSpentSet(spentSetDefaultMaxEntries)}
r, ts := freshChallenge(t)
body := buildValidBody(t, c.powDifficulty, r, ts)
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com/submit", strings.NewReader(body))
require.NoError(t, err)
req.Header.Set("User-Agent", "test-agent")
ck, _, _, err := c.ValidateChallengeResponse(req, []byte(body))
require.NoError(t, err)
parsed, err := http.ParseSetCookie(ck.String())
require.NoError(t, err)
// MaxAge is a count of seconds; compare to our cookieTTL.
// The cookie helper computes MaxAge as
// int(time.Until(Expires).Seconds())
// where Expires was set from time.Now()+TTL on a slightly earlier
// time.Now() reading. Sub-second elapsed time between the two reads
// can shave a full second off after truncation, so we allow ±2s.
expected := int((90 * time.Minute).Seconds())
assert.InDelta(t, expected, parsed.MaxAge, 2,
"browser-side Max-Age (%d) should be within 2s of server-side cookieTTL (%d)",
parsed.MaxAge, expected)
}
func strconvI64(n int64) string {
return strconv.FormatInt(n, 10)
}
// TestCookieV0_AllowlistRoundTrip seals an allowlist cookie (Allowlisted bit
// + reason) and confirms the open path returns both fields.
func TestCookieV0_AllowlistRoundTrip(t *testing.T) {
keys := testKeyRing()
notAfter := time.Now().Add(time.Hour).Unix()
reason := "Googlebot/2.1 (compatible)"
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, cookieFlagAllowlisted, reason, []byte("ua"), MaxCookieLen)
require.NoError(t, err)
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.NoError(t, err)
assert.True(t, got.Allowlisted, "allowlist flag should round-trip true")
assert.Equal(t, reason, got.AllowlistReason, "reason should round-trip verbatim")
}
// TestCookieV0_AllowlistFlagDefaultsFalse confirms a regular (non-allowlist)
// cookie reports Allowlisted=false and empty reason after open.
func TestCookieV0_AllowlistFlagDefaultsFalse(t *testing.T) {
keys := testKeyRing()
notAfter := time.Now().Add(time.Hour).Unix()
encoded, err := sealCookieV0(&pb.ChallengeCookie{PowDifficulty: 12}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.NoError(t, err)
assert.False(t, got.Allowlisted, "default flag must be false")
assert.Empty(t, got.AllowlistReason)
}
// TestCookieV0_AllowlistReasonTooLong asserts the seal path rejects reasons
// larger than MaxAllowlistReasonLen with the typed error.
func TestCookieV0_AllowlistReasonTooLong(t *testing.T) {
keys := testKeyRing()
notAfter := time.Now().Add(time.Hour).Unix()
oversize := bytes.Repeat([]byte("x"), MaxAllowlistReasonLen+1)
_, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, cookieFlagAllowlisted, string(oversize), []byte("ua"), MaxCookieLen)
require.Error(t, err)
assert.ErrorIs(t, err, ErrAllowlistReasonSize)
}
// TestCookieV0_SealEnvelopeTooLarge confirms the seal path rejects an envelope
// (built from the client-submitted fingerprint) that would produce an
// over-limit cookie, before allocating for it. Without this bound a crafted
// submission forces a large allocation for a cookie the browser would refuse.
func TestCookieV0_SealEnvelopeTooLarge(t *testing.T) {
keys := testKeyRing()
notAfter := time.Now().Add(time.Hour).Unix()
// A fingerprint whose marshaled size blows past the plaintext budget.
envelope := &pb.ChallengeCookie{
Fingerprint: &pb.FingerprintData{Fsid: strings.Repeat("x", MaxCookieLen)},
}
_, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
require.Error(t, err)
assert.ErrorIs(t, err, ErrCookieTooLarge)
}
// TestCookieV0_OpenValueTooLarge confirms the open path rejects an over-limit
// cookie value before base64-decoding or AEAD-opening it.
func TestCookieV0_OpenValueTooLarge(t *testing.T) {
keys := testKeyRing()
oversized := strings.Repeat("A", MaxCookieLen+1)
_, err := openCookie(oversized, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.Error(t, err)
assert.ErrorIs(t, err, ErrCookieTooLarge)
}
// TestCookieV0_RealCookieUnderLimit guards against the cap being set too low:
// a normal sealed cookie must stay under MaxCookieLen and still round-trip.
func TestCookieV0_RealCookieUnderLimit(t *testing.T) {
keys := testKeyRing()
notAfter := time.Now().Add(time.Hour).Unix()
envelope := &pb.ChallengeCookie{
Fingerprint: &pb.FingerprintData{Fsid: "abc123", Url: "https://example.com/"},
PowDifficulty: 12,
}
encoded, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
assert.LessOrEqual(t, len(encoded), MaxCookieLen)
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.NoError(t, err)
}
// TestCookieV0_ConfigurableLimit confirms the seal/open ceiling honors a
// caller-supplied bound (Config.MaxCookieSize → runtime.maxCookieLen) and that
// a non-positive bound falls back to the MaxCookieLen default.
func TestCookieV0_ConfigurableLimit(t *testing.T) {
keys := testKeyRing()
notAfter := time.Now().Add(time.Hour).Unix()
// A ~4 KB fingerprint: seals to a cookie that exceeds the 4096 default but
// fits a raised 8192 ceiling.
envelope := &pb.ChallengeCookie{
Fingerprint: &pb.FingerprintData{Fsid: strings.Repeat("x", 4000)},
}
// A tight limit rejects it on seal, before allocating.
_, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), 1024)
require.Error(t, err)
require.ErrorIs(t, err, ErrCookieTooLarge)
// A raised limit accepts the same envelope.
encoded, err := sealCookieV0(envelope, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), 8192)
require.NoError(t, err)
require.Greater(t, len(encoded), MaxCookieLen, "sanity: this cookie exceeds the default ceiling")
// The default ceiling (0 → MaxCookieLen) then rejects it on open...
_, err = openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), 0)
require.Error(t, err)
require.ErrorIs(t, err, ErrCookieTooLarge)
// ...while the matching raised ceiling opens it successfully.
got, err := openCookie(encoded, keys.MasterCookieKey(), []byte("ua"), 8192)
require.NoError(t, err)
assert.NotNil(t, got)
}
// TestCookieV0_TamperedFlagsRejected confirms the flags byte is INSIDE the
// AEAD envelope: any modification invalidates the GCM tag.
func TestCookieV0_TamperedFlagsRejected(t *testing.T) {
keys := testKeyRing()
notAfter := time.Now().Add(time.Hour).Unix()
encoded, err := sealCookieV0(&pb.ChallengeCookie{}, keys.MasterCookieKey(), notAfter, 0, "", []byte("ua"), MaxCookieLen)
require.NoError(t, err)
raw, err := base64.RawURLEncoding.DecodeString(encoded)
require.NoError(t, err)
// Wire layout: version(1) || nonce(12) || ciphertext(...). The 9th
// byte of ciphertext (offset 1+12+8 = 21) covers the flags_byte of
// the plaintext. Flipping any bit there should fail the AEAD tag.
tampered := append([]byte(nil), raw...)
tampered[1+12+8] ^= 0x01
tamperedEncoded := base64.RawURLEncoding.EncodeToString(tampered)
_, err = openCookie(tamperedEncoded, keys.MasterCookieKey(), []byte("ua"), MaxCookieLen)
require.Error(t, err)
assert.ErrorIs(t, err, ErrCookieSignature)
}
+172
View File
@@ -0,0 +1,172 @@
package challenge
import (
"bytes"
"encoding/hex"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newTestKeyRing(t testing.TB, secret []byte, interval time.Duration, fixed time.Time) *KeyRing {
t.Helper()
if secret == nil {
secret = bytes.Repeat([]byte{0xab}, 32)
}
if interval == 0 {
interval = time.Minute
}
k, err := NewKeyRing(secret, interval, 3)
require.NoError(t, err)
if !fixed.IsZero() {
k.now = func() time.Time { return fixed }
}
return k
}
func TestNewKeyRing_RejectsShortSecret(t *testing.T) {
_, err := NewKeyRing([]byte("too-short"), time.Minute, 3)
require.Error(t, err)
}
func TestNewKeyRing_RejectsTinyInterval(t *testing.T) {
_, err := NewKeyRing(bytes.Repeat([]byte{1}, 32), time.Second, 3)
require.Error(t, err)
}
func TestKeyRing_DeterministicAcrossInstances(t *testing.T) {
// Two KeyRings with the same secret + same interval, evaluated at the
// same wall-clock moment, MUST produce bit-identical keys for the same
// epoch. Regression guard for distributed (multi-WAF) deployments.
secret := bytes.Repeat([]byte{0x42}, 32)
fixed := time.Date(2026, 5, 7, 12, 0, 0, 0, time.UTC)
a := newTestKeyRing(t, secret, time.Minute, fixed)
b := newTestKeyRing(t, secret, time.Minute, fixed)
epochA, signA := a.Current()
epochB, signB := b.Current()
assert.Equal(t, epochA, epochB)
assert.True(t, bytes.Equal(signA, signB), "sign keys diverge across instances")
// The master cookie key is independent of epoch and must also agree
// across instances configured with the same master secret.
assert.True(t, bytes.Equal(a.MasterCookieKey(), b.MasterCookieKey()),
"master cookie keys diverge across instances")
// Sign and master cookie keys derive from the same secret with
// different HKDF info strings, so they MUST differ. If they didn't,
// a leaked sign key would also forge cookies (and vice versa).
assert.False(t, bytes.Equal(signA, a.MasterCookieKey()),
"sign and master cookie keys must differ — domain separation is the whole point")
}
func TestKeyRing_RotatesAtIntervalBoundary(t *testing.T) {
secret := bytes.Repeat([]byte{0x99}, 32)
t0 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
k := newTestKeyRing(t, secret, time.Minute, t0)
e0, key0 := k.Current()
// Cross the 60-second boundary — current epoch must increment.
k.now = func() time.Time { return t0.Add(61 * time.Second) }
e1, key1 := k.Current()
assert.Equal(t, e0+1, e1)
assert.False(t, bytes.Equal(key0, key1), "key did not change across rotation")
// And the previous epoch's key is still derivable while in the live window.
prev, ok := k.SignKey(e0)
require.True(t, ok)
assert.True(t, bytes.Equal(key0, prev), "previous epoch key changed when re-derived")
}
func TestKeyRing_LiveWindow(t *testing.T) {
t0 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
k := newTestKeyRing(t, nil, time.Minute, t0)
// maxLive=3 + clockSkew=1 → live window is current-2 .. current+1.
current, _ := k.Current()
// In window.
for _, e := range []int64{current - 2, current - 1, current, current + 1} {
_, ok := k.SignKey(e)
assert.True(t, ok, "epoch %d should be live", e)
}
// Out of window.
for _, e := range []int64{current - 3, current + 2} {
_, ok := k.SignKey(e)
assert.False(t, ok, "epoch %d should NOT be live", e)
}
}
func TestKeyRing_LiveEpochsOrdered(t *testing.T) {
t0 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
k := newTestKeyRing(t, nil, time.Minute, t0)
current, _ := k.Current()
live := k.LiveEpochs()
require.Len(t, live, 4) // 3 past/current + 1 future skew
// Ordered ascending, contiguous.
for i, e := range live {
assert.Equal(t, current-2+int64(i), e)
}
}
func TestKeyRing_CacheEvictsStale(t *testing.T) {
t0 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
k := newTestKeyRing(t, nil, time.Minute, t0)
// Touch the current epoch to seed the cache.
startEpoch, _ := k.Current()
// Jump far into the future so startEpoch falls out of the live window;
// the next derivation must evict it.
k.now = func() time.Time { return t0.Add(time.Hour) }
_, _ = k.Current()
k.mu.RLock()
_, present := k.cache[startEpoch]
k.mu.RUnlock()
assert.False(t, present, "stale epoch %d should have been evicted from the cache", startEpoch)
}
// TestKeyRing_KnownVectors locks down the HKDF derivation. Any change to the
// salt / info strings / serialization that breaks distributed agreement on a
// fleet upgrade will make these vectors fail. If you intentionally change the
// scheme, regenerate these vectors AND rotate the keyring version constant.
func TestKeyRing_KnownVectors(t *testing.T) {
secret, err := hex.DecodeString("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f")
require.NoError(t, err)
k, err := NewKeyRing(secret, time.Minute, 3)
require.NoError(t, err)
// Sign key for epoch 0 must derive deterministically.
want := hex.EncodeToString(deriveEpochKey(secret, 0, keyringInfoSign))
got := hex.EncodeToString(deriveEpochKey(k.masterSecret, 0, keyringInfoSign))
assert.Equal(t, want, got, "epoch 0 sign key derivation must be deterministic")
// Master cookie key (no epoch component) must also be deterministic.
wantCookie := hex.EncodeToString(deriveMasterCookieKey(secret))
gotCookie := hex.EncodeToString(k.MasterCookieKey())
assert.Equal(t, wantCookie, gotCookie, "master cookie key derivation must be deterministic")
// Cross-context separation: sign key for any epoch must NOT equal the
// master cookie key. Domain separation is the whole point of using
// distinct HKDF info strings.
signKey := deriveEpochKey(secret, 42, keyringInfoSign)
cookieKey := deriveMasterCookieKey(secret)
assert.NotEqual(t, hex.EncodeToString(signKey), hex.EncodeToString(cookieKey),
"sign and master-cookie contexts must produce different keys")
// Cross-epoch separation: same context, different epoch → different bytes.
e0 := deriveEpochKey(secret, 0, keyringInfoSign)
e1 := deriveEpochKey(secret, 1, keyringInfoSign)
assert.NotEqual(t, hex.EncodeToString(e0), hex.EncodeToString(e1),
"adjacent epochs must produce different sign keys")
}
@@ -0,0 +1,92 @@
package challenge
import (
"slices"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestChallengeCodeSeededStatically asserts that the default
// NewChallengeRuntime path loads the build-time-obfuscated challenge code
// once from initial_bundle.js.gz and serves it verbatim — there is no
// runtime re-obfuscation or pool for the (public) library path anymore.
func TestChallengeCodeSeededStatically(t *testing.T) {
rt, err := NewChallengeRuntime(t.Context())
require.NoError(t, err)
code := rt.getChallengeCode()
require.NotEmpty(t, code, "challenge code must be seeded from the baked-in initial bundle")
// The hook sentinel must survive obfuscation so the dynamic key module
// can find the registered hook (reservedStrings in obfuscate.js).
require.Contains(t, code, hookSentinel,
"obfuscated challenge code must preserve the hook sentinel verbatim")
// The accessor is a stable read of static state: repeated calls return
// the identical string (no pool, no random selection).
assert.Equal(t, code, rt.getChallengeCode(), "challenge code must be static across calls")
}
// TestCryptoObfuscationDefaultPoolSize confirms that with no
// WithCryptoObfuscationPoolSize option, the per-epoch dynamic module
// cache holds exactly 1 variant per epoch — preserving the historical
// single-variant-per-epoch behavior.
func TestCryptoObfuscationDefaultPoolSize(t *testing.T) {
rt, err := NewChallengeRuntime(t.Context())
require.NoError(t, err)
require.Equal(t, cryptoObfuscationPoolDefaultSize, rt.cryptoPoolSize,
"default crypto pool size must be cryptoObfuscationPoolDefaultSize")
// The pre-warmer runs currentDynamicModule at construction; the
// cache for the current epoch must hold exactly 1 variant.
currentEpoch, _ := rt.keys.Current()
rt.dynamicModuleCacheMu.RLock()
variants, ok := rt.dynamicModuleCache[currentEpoch]
rt.dynamicModuleCacheMu.RUnlock()
require.True(t, ok, "current epoch must be cached after construction (pre-warmer ran)")
require.Len(t, variants, 1, "default crypto pool keeps 1 variant per epoch")
}
// TestCryptoObfuscationPoolSize asserts that when
// WithCryptoObfuscationPoolSize is raised, the per-epoch cache holds
// the configured number of distinct obfuscated variants of the SAME
// epoch key, and currentDynamicModule picks from them.
func TestCryptoObfuscationPoolSize(t *testing.T) {
if testing.Short() {
t.Skip("each crypto pool variant costs ~3s of obfuscator CPU; skipped in -short")
}
const poolSize = 3
rt, err := NewChallengeRuntime(t.Context(),
WithCryptoObfuscationPoolSize(poolSize),
)
require.NoError(t, err)
require.Equal(t, poolSize, rt.cryptoPoolSize, "crypto pool size option must propagate")
currentEpoch, _ := rt.keys.Current()
rt.dynamicModuleCacheMu.RLock()
variants, ok := rt.dynamicModuleCache[currentEpoch]
rt.dynamicModuleCacheMu.RUnlock()
require.True(t, ok, "current epoch must be cached after construction")
require.Len(t, variants, poolSize, "crypto pool must hold poolSize variants")
// Each variant must be a distinct obfuscation. javascript-obfuscator
// with mangled identifier-names produces different byte output each
// run, so duplicates are an obfuscator regression.
for i := 1; i < poolSize; i++ {
assert.NotEqual(t, variants[0], variants[i],
"variants %d and 0 are byte-identical; obfuscator should produce distinct output per pass", i)
}
// currentDynamicModule must return one of the cached variants.
got, err := rt.currentDynamicModule(t.Context())
require.NoError(t, err)
found := slices.Contains(variants, got)
assert.True(t, found, "currentDynamicModule returned a value not in the cached pool")
}
+56
View File
@@ -0,0 +1,56 @@
// obfuscator.go embeds the wasm build of `javascript-obfuscator` and exposes
// a single thread-safe entry point (ObfuscateJS) used by both the static and
// dynamic bundle paths. The compiled wazero module is shared across calls;
// each ObfuscateJS invocation instantiates a fresh module instance with its
// own stdin/stdout buffers so concurrent obfuscations don't collide.
package challenge
import (
"bytes"
"context"
crand "crypto/rand"
_ "embed"
"fmt"
"sync"
"github.com/tetratelabs/wazero"
)
//go:embed js/obfuscate/index.wasm.gz
var obfuscatorWasmGz []byte
var (
obfuscatorWasm []byte
obfuscatorWasmOnce sync.Once
)
// ObfuscateJS runs the input source through the embedded `javascript-obfuscator`
// wasm module and returns the obfuscated output. Thread-safe: wazero allows
// concurrent module instantiations from the same compiled module, which is
// what makes the dynamic-module singleflight pattern work.
func (c *ChallengeRuntime) ObfuscateJS(ctx context.Context, inputJS string) (string, error) {
stdin := bytes.NewReader([]byte(inputJS))
var stdout bytes.Buffer
var stderr bytes.Buffer
config := wazero.NewModuleConfig().
WithStdin(stdin).
WithStdout(&stdout).
WithStderr(&stderr).
// Give wazero a real PRNG source and real wall/nano time so output is non-deterministic.
WithRandSource(crand.Reader).
WithSysWalltime().
WithSysNanotime()
mod, err := c.r.InstantiateModule(ctx, c.obfuscatorMod, config)
if err != nil {
if stderr.Len() > 0 {
return "", fmt.Errorf("wasm runtime error: %v | stderr: %s", err, stderr.String())
}
return "", fmt.Errorf("wasm instantiation error: %v", err)
}
mod.Close(ctx)
return stdout.String(), nil
}
File diff suppressed because it is too large Load Diff
+289
View File
@@ -0,0 +1,289 @@
syntax = "proto3";
package fingerprint;
option go_package = ".;pb";
message FingerprintData {
fingerprintSignals signals = 1;
string fsid = 2;
string nonce = 3;
int64 time = 4;
string url = 5;
bool fastBotDetection = 6;
fingerprintFastBotDetectionDetails fastBotDetectionDetails = 7;
fingerprintBotAlias bot = 8;
}
// ChallengeCookie is the sealed cookie payload. It wraps the validated
// fingerprint together with metadata that influences re-challenge decisions
// (e.g. the PoW difficulty the client proved). Keeping this separate from
// FingerprintData lets us evolve the cookie envelope without polluting the
// fingerprint model exposed to hook expressions.
message ChallengeCookie {
FingerprintData fingerprint = 1;
// pow_difficulty is the PoW difficulty (in leading zero bits) the client
// proved when this cookie was sealed.
int32 pow_difficulty = 2;
}
message fingerprintSignals {
fingerprintAutomation automation = 1;
fingerprintDevice device = 2;
fingerprintBrowser browser = 3;
fingerprintGraphics graphics = 4;
fingerprintCodecs codecs = 5;
fingerprintLocale locale = 6;
fingerprintContexts contexts = 7;
}
message fingerprintAutomation {
bool webdriver = 1;
bool webdriverWritable = 2;
bool selenium = 3;
bool cdp = 4;
bool playwright = 5;
string navigatorPropertyDescriptors = 6;
}
message fingerprintDevice {
int32 cpuCount = 1;
int32 memory = 2;
string platform = 3;
fingerprintScreenResolution screenResolution = 4;
fingerprintMultimediaDevices multimediaDevices = 5;
fingerprintDeviceMediaQueries mediaQueries = 6;
}
message fingerprintScreenResolution {
int32 width = 1;
int32 height = 2;
int32 pixelDepth = 3;
int32 colorDepth = 4;
int32 availableWidth = 5;
int32 availableHeight = 6;
int32 innerWidth = 7;
int32 innerHeight = 8;
bool hasMultipleDisplays = 9;
}
message fingerprintMultimediaDevices {
int32 speakers = 1;
int32 microphones = 2;
int32 webcams = 3;
}
message fingerprintDeviceMediaQueries {
string prefersColorScheme = 1;
bool prefersReducedMotion = 2;
bool prefersReducedTransparency = 3;
string colorGamut = 4;
string pointer = 5;
string anyPointer = 6;
bool hover = 7;
bool anyHover = 8;
int32 colorDepth = 9;
}
message fingerprintBrowser {
string userAgent = 1;
fingerprintBrowserFeatures features = 2;
fingerprintBrowserPlugins plugins = 3;
fingerprintBrowserExtensions extensions = 4;
fingerprintBrowserHighEntropyValues highEntropyValues = 5;
int32 etsl = 6;
string maths = 7;
fingerprintBrowserToSourceError toSourceError = 8;
}
message fingerprintBrowserFeatures {
string bitmask = 1;
bool chrome = 2;
bool brave = 3;
bool applePaySupport = 4;
bool opera = 5;
bool serial = 6;
bool attachShadow = 7;
bool caches = 8;
bool webAssembly = 9;
bool buffer = 10;
bool showModalDialog = 11;
bool safari = 12;
bool webkitPrefixedFunction = 13;
bool mozPrefixedFunction = 14;
bool usb = 15;
bool browserCapture = 16;
bool paymentRequestUpdateEvent = 17;
bool pressureObserver = 18;
bool audioSession = 19;
bool selectAudioOutput = 20;
bool barcodeDetector = 21;
bool battery = 22;
bool devicePosture = 23;
bool documentPictureInPicture = 24;
bool eyeDropper = 25;
bool editContext = 26;
bool fencedFrame = 27;
bool sanitizer = 28;
bool otpCredential = 29;
}
message fingerprintBrowserPlugins {
bool isValidPluginArray = 1;
int32 pluginCount = 2;
string pluginNamesHash = 3;
bool pluginConsistency1 = 4;
bool pluginOverflow = 5;
}
message fingerprintBrowserExtensions {
string bitmask = 1;
repeated string extensions = 2;
}
message fingerprintBrowserHighEntropyValues {
string architecture = 1;
string bitness = 2;
repeated fingerprintBrandVersion brands = 3;
bool mobile = 4;
string model = 5;
string platform = 6;
string platformVersion = 7;
string uaFullVersion = 8;
}
message fingerprintBrandVersion {
string brand = 1;
string version = 2;
}
message fingerprintBrowserToSourceError {
string toSourceError = 1;
bool hasToSource = 2;
}
message fingerprintGraphics {
fingerprintGraphicsWebGL webGL = 1;
fingerprintGraphicsWebGPU webgpu = 2;
fingerprintGraphicsCanvas canvas = 3;
}
message fingerprintGraphicsWebGL {
string vendor = 1;
string renderer = 2;
}
message fingerprintGraphicsWebGPU {
string vendor = 1;
string architecture = 2;
string device = 3;
string description = 4;
}
message fingerprintGraphicsCanvas {
bool hasModifiedCanvas = 1;
string canvasFingerprint = 2;
}
message fingerprintCodecs {
string audioCanPlayTypeHash = 1;
string videoCanPlayTypeHash = 2;
string audioMediaSourceHash = 3;
string videoMediaSourceHash = 4;
string rtcAudioCapabilitiesHash = 5;
string rtcVideoCapabilitiesHash = 6;
bool hasMediaSource = 7;
}
message fingerprintLocale {
fingerprintLocaleInternationalization internationalization = 1;
fingerprintLocaleLanguages languages = 2;
}
message fingerprintLocaleInternationalization {
string timezone = 1;
string localeLanguage = 2;
}
message fingerprintLocaleLanguages {
repeated string languages = 1;
string language = 2;
}
message fingerprintContexts {
fingerprintIframeContext iframe = 1;
fingerprintWebWorkerContext webWorker = 2;
}
message fingerprintIframeContext {
bool webdriver = 1;
string userAgent = 2;
string platform = 3;
int32 memory = 4;
int32 cpuCount = 5;
string language = 6;
}
message fingerprintWebWorkerContext {
string vendor = 1;
string renderer = 2;
string userAgent = 3;
string language = 4;
string platform = 5;
int32 memory = 6;
int32 cpuCount = 7;
}
message fingerprintFastBotDetectionDetails {
fingerprintDetectionResult headlessChromeScreenResolution = 1;
fingerprintDetectionResult hasWebdriver = 2;
fingerprintDetectionResult hasWebdriverWritable = 3;
fingerprintDetectionResult hasSeleniumProperty = 4;
fingerprintDetectionResult hasCDP = 5;
fingerprintDetectionResult hasPlaywright = 6;
fingerprintDetectionResult hasImpossibleDeviceMemory = 7;
fingerprintDetectionResult hasHighCPUCount = 8;
fingerprintDetectionResult hasMissingChromeObject = 9;
fingerprintDetectionResult hasWebdriverIframe = 10;
fingerprintDetectionResult hasWebdriverWorker = 11;
fingerprintDetectionResult hasMismatchWebGLInWorker = 12;
fingerprintDetectionResult hasMismatchPlatformIframe = 13;
fingerprintDetectionResult hasMismatchPlatformWorker = 14;
fingerprintDetectionResult hasSwiftshaderRenderer = 15;
fingerprintDetectionResult hasUTCTimezone = 16;
fingerprintDetectionResult hasMismatchLanguages = 17;
fingerprintDetectionResult hasInconsistentEtsl = 18;
fingerprintDetectionResult hasBotUserAgent = 19;
fingerprintDetectionResult hasGPUMismatch = 20;
fingerprintDetectionResult hasPlatformMismatch = 21;
}
message fingerprintDetectionResult {
bool detected = 1;
}
message fingerprintBotAlias {
bool headlessChromeScreenResolution = 1;
bool webdriver = 2;
bool webdriverWritable = 3;
bool selenium = 4;
bool cdp = 5;
bool playwright = 6;
bool impossibleDeviceMemory = 7;
bool highCPUCount = 8;
bool missingChromeObject = 9;
bool webdriverIframe = 10;
bool webdriverWorker = 11;
bool mismatchWebGLInWorker = 12;
bool mismatchPlatformIframe = 13;
bool mismatchPlatformWorker = 14;
bool swiftshaderRenderer = 15;
bool utcTimezone = 16;
bool anyDetected = 17;
int32 detectedCount = 18;
bool mismatchLanguages = 19;
bool inconsistentEtsl = 20;
bool botUserAgent = 21;
bool gpuMismatch = 22;
bool platformMismatch = 23;
}
+14
View File
@@ -0,0 +1,14 @@
package pb
// Dependencies:
//
// apt install protobuf-compiler
//
// keep this in sync with go.mod
// go install google.golang.org/protobuf/cmd/[email protected]
//
// Not the same versions as google.golang.org/grpc
// go list -m -versions google.golang.org/grpc/cmd/protoc-gen-go-grpc
// go install google.golang.org/grpc/cmd/[email protected]
//go:generate protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative fingerprint.proto
+77
View File
@@ -0,0 +1,77 @@
"use strict";
var K = [
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
];
function sha256(data) {
var h0 = 0x6a09e667, h1 = 0xbb67ae85, h2 = 0x3c6ef372, h3 = 0xa54ff53a,
h4 = 0x510e527f, h5 = 0x9b05688c, h6 = 0x1f83d9ab, h7 = 0x5be0cd19;
var len = data.length, bitLen = len * 8;
var padLen = 64 - ((len + 9) % 64);
var totalLen = len + 1 + (padLen === 64 ? 0 : padLen) + 8;
var msg = new Uint8Array(totalLen);
msg.set(data); msg[len] = 0x80;
var dv = new DataView(msg.buffer);
dv.setUint32(totalLen - 4, bitLen, false);
var w = new Int32Array(64);
for (var offset = 0; offset < totalLen; offset += 64) {
for (var i = 0; i < 16; i++) w[i] = dv.getInt32(offset + i * 4, false);
for (var i = 16; i < 64; i++) {
var s0 = (((w[i-15] >>> 7) | (w[i-15] << 25)) ^ ((w[i-15] >>> 18) | (w[i-15] << 14)) ^ (w[i-15] >>> 3)) | 0;
var s1 = (((w[i-2] >>> 17) | (w[i-2] << 15)) ^ ((w[i-2] >>> 19) | (w[i-2] << 13)) ^ (w[i-2] >>> 10)) | 0;
w[i] = (w[i-16] + s0 + w[i-7] + s1) | 0;
}
var a = h0, b = h1, c = h2, d = h3, e = h4, f = h5, g = h6, h = h7;
for (var i = 0; i < 64; i++) {
var S1 = (((e >>> 6) | (e << 26)) ^ ((e >>> 11) | (e << 21)) ^ ((e >>> 25) | (e << 7))) | 0;
var ch = ((e & f) ^ (~e & g)) | 0;
var t1 = (h + S1 + ch + K[i] + w[i]) | 0;
var S0 = (((a >>> 2) | (a << 30)) ^ ((a >>> 13) | (a << 19)) ^ ((a >>> 22) | (a << 10))) | 0;
var maj = ((a & b) ^ (a & c) ^ (b & c)) | 0;
var t2 = (S0 + maj) | 0;
h = g; g = f; f = e; e = (d + t1) | 0; d = c; c = b; b = a; a = (t1 + t2) | 0;
}
h0 = (h0 + a) | 0; h1 = (h1 + b) | 0; h2 = (h2 + c) | 0; h3 = (h3 + d) | 0;
h4 = (h4 + e) | 0; h5 = (h5 + f) | 0; h6 = (h6 + g) | 0; h7 = (h7 + h) | 0;
}
var out = new Uint8Array(32);
var ov = new DataView(out.buffer);
ov.setUint32(0, h0, false); ov.setUint32(4, h1, false);
ov.setUint32(8, h2, false); ov.setUint32(12, h3, false);
ov.setUint32(16, h4, false); ov.setUint32(20, h5, false);
ov.setUint32(24, h6, false); ov.setUint32(28, h7, false);
return out;
}
self.onmessage = function(e) {
var prefix = e.data.p, difficulty = e.data.d, nonce = 0;
var enc = new TextEncoder();
while (true) {
var candidate = prefix + nonce.toString(36);
var hash = sha256(enc.encode(candidate));
var fullBytes = difficulty >> 3, remainBits = difficulty & 7, ok = true;
for (var i = 0; i < fullBytes; i++) { if (hash[i] !== 0) { ok = false; break; } }
if (ok && remainBits > 0) {
var mask = (0xff << (8 - remainBits)) & 0xff;
if ((hash[fullBytes] & mask) !== 0) ok = false;
}
if (ok) { self.postMessage(nonce.toString(36)); return; }
nonce++;
}
};
+75
View File
@@ -0,0 +1,75 @@
// secret.go parses and validates the master-secret used by the keyring, and
// generates a random fallback when none is configured. The secret is the
// trust anchor for every per-epoch HMAC key and the cookie-sealing AES key,
// so any change here is wire-protocol-affecting.
package challenge
import (
crand "crypto/rand"
"encoding/hex"
"errors"
"fmt"
)
// minSecretBytes is the smallest accepted master-secret length. 32 bytes is
// the natural minimum for an HMAC-SHA256 key with full security margin; we
// reject anything shorter as a configuration error rather than silently
// padding it.
const minSecretBytes = 32
// generateRandomSecret returns a fresh 32-byte secret suitable for use when
// no master_secret is configured. Single-instance deployments are fine with
// this; distributed deployments MUST configure a shared secret because each
// instance generates an independent random one here.
func generateRandomSecret() ([]byte, error) {
buf := make([]byte, 32)
if _, err := crand.Read(buf); err != nil {
return nil, fmt.Errorf("generate random master secret: %w", err)
}
return buf, nil
}
// ParseConfiguredSecret accepts a configured master secret as either a hex
// string (preferred — encodes raw bytes unambiguously) or a raw passphrase
// (fallback for human-edited configs). The result is at least minSecretBytes.
func ParseConfiguredSecret(value string) ([]byte, error) {
if value == "" {
return nil, errors.New("empty master secret")
}
// Hex form: even length, hex digits only.
if isHex(value) {
raw, err := hex.DecodeString(value)
if err == nil {
if len(raw) < minSecretBytes {
return nil, fmt.Errorf("hex master secret decodes to %d bytes; minimum is %d", len(raw), minSecretBytes)
}
return raw, nil
}
// Fall through to passphrase handling on hex parse failure — defensive.
}
if len(value) < minSecretBytes {
return nil, fmt.Errorf("passphrase master secret is %d bytes; minimum is %d", len(value), minSecretBytes)
}
return []byte(value), nil
}
func isHex(s string) bool {
if s == "" || len(s)%2 != 0 {
return false
}
for i := range len(s) {
c := s[i]
switch {
case c >= '0' && c <= '9':
case c >= 'a' && c <= 'f':
case c >= 'A' && c <= 'F':
default:
return false
}
}
return true
}
+63
View File
@@ -0,0 +1,63 @@
package challenge
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseConfiguredSecret_Hex(t *testing.T) {
// 32 raw bytes = 64 hex chars
hex := strings.Repeat("ab", 32)
got, err := ParseConfiguredSecret(hex)
require.NoError(t, err)
assert.Len(t, got, 32)
}
func TestParseConfiguredSecret_HexTooShort(t *testing.T) {
// 31 raw bytes = 62 hex chars → must be rejected
hex := strings.Repeat("ab", 31)
_, err := ParseConfiguredSecret(hex)
require.Error(t, err)
assert.Contains(t, err.Error(), "minimum is 32")
}
func TestParseConfiguredSecret_Passphrase(t *testing.T) {
// "long enough passphrase, definitely > 32 bytes" — 47 bytes
pw := "long enough passphrase, definitely > 32 bytes"
got, err := ParseConfiguredSecret(pw)
require.NoError(t, err)
assert.Equal(t, []byte(pw), got)
}
func TestParseConfiguredSecret_PassphraseTooShort(t *testing.T) {
_, err := ParseConfiguredSecret("short")
require.Error(t, err)
}
func TestParseConfiguredSecret_Empty(t *testing.T) {
_, err := ParseConfiguredSecret("")
require.Error(t, err)
}
func TestGenerateRandomSecret(t *testing.T) {
a, err := generateRandomSecret()
require.NoError(t, err)
b, err := generateRandomSecret()
require.NoError(t, err)
assert.Len(t, a, 32)
assert.Len(t, b, 32)
assert.NotEqual(t, a, b, "successive calls must yield different secrets")
}
func TestNewChallengeRuntime_RejectsShortSecret(t *testing.T) {
// Bypassing ParseConfiguredSecret to feed the runtime a too-short secret
// directly via the option — the runtime itself must defend against this.
short := []byte("too-short")
_, err := NewChallengeRuntime(t.Context(), WithMasterSecret(short))
require.Error(t, err)
assert.Contains(t, err.Error(), "minimum")
}
+48
View File
@@ -0,0 +1,48 @@
// spent_set.go is the single-use store that eliminates challenge replay: each
// validated submission burns its per-challenge nonce `r`, so a replay fails.
package challenge
import (
"sync"
"time"
"github.com/bluele/gcache"
)
// spentSetDefaultMaxEntries is a deep DoS backstop. Growth is sig+PoW-gated and
// TTL-bounded (ticketAgeBackstop), so steady-state stays far below this. If the
// cap is ever hit, LRU evicts the oldest (maybe still-live) `r`, letting that
// one submission replay once — acceptable at this size.
const spentSetDefaultMaxEntries = 1_000_000
// spentSet records consumed per-challenge nonces. Safe for concurrent use.
type spentSet struct {
// mu makes the Has/Set pair in checkAndInsert atomic (gcache locks each op
// independently, which alone wouldn't stop two concurrent replays winning).
mu sync.Mutex
cache gcache.Cache
// maxEntries is the configured LRU cap, retained for introspection.
maxEntries int
}
func newSpentSet(maxEntries int) *spentSet {
return &spentSet{cache: gcache.New(maxEntries).LRU().Build(), maxEntries: maxEntries}
}
// checkAndInsert atomically records `r` as spent, returning true if it was
// fresh and false if already present (replay). TTL matches the freshness
// window, so an expired (absent) entry is also one the caller rejects anyway.
func (s *spentSet) checkAndInsert(r string, ttl time.Duration) bool {
s.mu.Lock()
defer s.mu.Unlock()
if s.cache.Has(r) {
return false
}
// SetWithExpire can't fail here: we configure no serialization/eviction hook.
_ = s.cache.SetWithExpire(r, struct{}{}, ttl)
return true
}
+63
View File
@@ -0,0 +1,63 @@
package challenge
import (
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestSpentSet_CheckAndInsert(t *testing.T) {
s := newSpentSet(spentSetDefaultMaxEntries)
// First insert of a fresh r succeeds.
assert.True(t, s.checkAndInsert("r1", time.Minute))
// Replay of the same r is rejected.
assert.False(t, s.checkAndInsert("r1", time.Minute))
// A different r is independent.
assert.True(t, s.checkAndInsert("r2", time.Minute))
}
// TestSpentSet_ConcurrentSameKey asserts the check-and-insert pair is atomic:
// when N goroutines race to burn the same r, exactly one wins. Run with -race.
func TestSpentSet_ConcurrentSameKey(t *testing.T) {
s := newSpentSet(spentSetDefaultMaxEntries)
const goroutines = 64
var wins int64
var wg sync.WaitGroup
start := make(chan struct{})
for range goroutines {
wg.Add(1)
go func() {
defer wg.Done()
<-start
if s.checkAndInsert("same-r", time.Minute) {
atomic.AddInt64(&wins, 1)
}
}()
}
close(start)
wg.Wait()
assert.Equal(t, int64(1), wins, "exactly one goroutine must win the single-use race")
}
// TestSpentSet_Expiry asserts an entry is re-accepted after its TTL lapses.
// (In production the freshness check rejects such an aged r anyway; this only
// guards the store's own expiry behavior.)
func TestSpentSet_Expiry(t *testing.T) {
s := newSpentSet(spentSetDefaultMaxEntries)
assert.True(t, s.checkAndInsert("r-exp", 20*time.Millisecond))
assert.False(t, s.checkAndInsert("r-exp", 20*time.Millisecond))
time.Sleep(40 * time.Millisecond)
assert.True(t, s.checkAndInsert("r-exp", 20*time.Millisecond),
"entry must be re-accepted after its TTL expires")
}
+231
View File
@@ -0,0 +1,231 @@
package challenge
import (
"encoding/hex"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestSplitBundle_HookSentinelInBakedBundle is the smallest possible
// regression guard for the split-bundle protocol: the static obfuscated
// bundle that we ship in initial_bundle.js.gz MUST contain a literal
// occurrence of the hook sentinel. Without it, the dynamic key module
// would never find globalThis[hookName] and the challenge would silently
// fail in every browser.
//
// The sentinel survives the high-obfuscation preset only because
// obfuscate.js explicitly registers it via reservedStrings. If someone
// removes that registration this test fires before any user notices.
func TestSplitBundle_HookSentinelInBakedBundle(t *testing.T) {
rt := &ChallengeRuntime{}
require.NoError(t, rt.seedCacheFromInitialBundle())
bundle := rt.getChallengeCode()
require.NotEmpty(t, bundle)
count := strings.Count(bundle, hookSentinel)
assert.GreaterOrEqual(t, count, 1,
"hook sentinel %q must survive obfuscation in the challenge code (was reservedStrings dropped from obfuscate.js?)",
hookSentinel)
}
// TestSplitBundle_DynamicModuleEmbedsCurrentEpochKey verifies the dynamic
// module that GetChallengePage will serve actually contains the per-epoch
// key. The output is heavily obfuscated, but the key bytes (hex-encoded)
// MUST be present — if they aren't, the client can't HMAC the ticket.
//
// We do not assert the literal hex string survives intact (the obfuscator
// transforms most string literals); instead we run the obfuscated module
// through a JS engine and check it tries to call our hook with the right
// key. That requires running JS — out of scope here, so we settle for: the
// dynamic module is non-empty, references the hook sentinel, and doesn't
// accidentally leak the hex key as a plain literal.
func TestSplitBundle_DynamicModuleObfuscatesKey(t *testing.T) {
keys := testKeyRing()
rt := newChallengeRuntimeForSplitTest(t, keys)
got, err := rt.currentDynamicModule(t.Context())
require.NoError(t, err)
require.NotEmpty(t, got)
epoch, signKey := keys.Current()
keyHex := hex.EncodeToString(signKey)
// The hook sentinel must appear (reservedStrings keeps it intact).
assert.Contains(t, got, hookSentinel,
"dynamic module is missing the hook sentinel %q", hookSentinel)
// The key MUST NOT appear in plain hex form. The obfuscator's
// string-array transform should have encoded it. If the literal hex
// shows up the obfuscation strength has dropped, defeating the whole
// point of the split-bundle design.
assert.NotContains(t, got, keyHex,
"per-epoch key leaked into the dynamic module in plain hex (obfuscator regression?)")
// Sanity: epoch number can leak as a literal — it's not a secret.
t.Logf("dynamic module size for epoch %d: %d bytes", epoch, len(got))
}
// TestSplitBundle_DynamicModuleCachedPerEpoch confirms repeated calls in
// the same epoch share the obfuscation cost (single-flight via cache).
//
// We pin the keyring's clock so the two calls are guaranteed to see the
// same epoch — without this, the ~12s obfuscation cost can straddle a
// rotation boundary on testKeyRing's 1-minute interval and the second
// call ends up generating a different module under a different key.
func TestSplitBundle_DynamicModuleCachedPerEpoch(t *testing.T) {
keys := testKeyRing()
pinned := time.Now()
keys.now = func() time.Time { return pinned }
rt := newChallengeRuntimeForSplitTest(t, keys)
first, err := rt.currentDynamicModule(t.Context())
require.NoError(t, err)
second, err := rt.currentDynamicModule(t.Context())
require.NoError(t, err)
assert.Equal(t, first, second, "second call must return the cached module byte-for-byte")
}
// TestSplitBundle_DynamicModuleRebuildsOnEpochAdvance asserts that when the
// keyring rolls to a new epoch, currentDynamicModule produces a
// fresh module embedding the new key.
func TestSplitBundle_DynamicModuleRebuildsOnEpochAdvance(t *testing.T) {
keys := testKeyRing()
rt := newChallengeRuntimeForSplitTest(t, keys)
first, err := rt.currentDynamicModule(t.Context())
require.NoError(t, err)
firstEpoch, _ := keys.Current()
// Advance the keyring's clock past the rotation interval.
keys.now = func() time.Time { return time.Now().Add(2 * time.Minute) }
second, err := rt.currentDynamicModule(t.Context())
require.NoError(t, err)
secondEpoch, _ := keys.Current()
assert.NotEqual(t, firstEpoch, secondEpoch, "epoch did not advance")
assert.NotEqual(t, first, second, "dynamic module did not change after epoch advance")
}
// TestSplitBundle_DynamicModuleConcurrentSingleflight asserts that concurrent
// callers for the same epoch share a single obfuscation pass via singleflight.
// N concurrent goroutines call currentDynamicModule for the same epoch; only
// ONE must actually run the obfuscator (the others share its result via
// singleflight). If we ever hold the cache mutex across the obfuscation
// pass, this test will time out OR show every caller doing its own
// obfuscation.
func TestSplitBundle_DynamicModuleConcurrentSingleflight(t *testing.T) {
keys := testKeyRing()
pinned := time.Now()
keys.now = func() time.Time { return pinned }
rt := newChallengeRuntimeForSplitTest(t, keys)
const N = 8
results := make([]string, N)
errs := make([]error, N)
var wg sync.WaitGroup
wg.Add(N)
start := time.Now()
for i := range N {
go func(idx int) {
defer wg.Done()
out, err := rt.currentDynamicModule(t.Context())
results[idx] = out
errs[idx] = err
}(i)
}
wg.Wait()
elapsed := time.Since(start)
for i, err := range errs {
require.NoError(t, err, "goroutine %d failed", i)
}
// All callers must have observed the same obfuscation output. If any
// caller diverged, singleflight wasn't doing its job.
for i := 1; i < N; i++ {
require.Equal(t, results[0], results[i],
"goroutines %d and 0 saw different module bytes; singleflight is broken", i)
}
// Tighter bound: with N=8 goroutines and a single ~5s obfuscation,
// the wall-clock should be roughly one obfuscation, not N. We use
// 2x as the threshold to absorb scheduler noise; a regression where
// every caller runs its own obfuscation would push elapsed past
// N * 5s = 40s, way above the threshold.
t.Logf("8 concurrent currentDynamicModule calls completed in %s", elapsed)
require.Less(t, elapsed, 15*time.Second,
"concurrent calls took %s; singleflight likely not coalescing (would expect 1 obfuscation, got serial)",
elapsed)
}
// TestSplitBundle_HTMLDoesNotContainSecret is the core security invariant
// of the split-bundle design: the HTML challenge page MUST NOT contain the
// per-epoch sign key in plain (hex) form. If it does, an attacker can
// scrape it and forge tickets without running the obfuscated bundle —
// exactly the failure mode the split-bundle protocol was designed to fix.
func TestSplitBundle_HTMLDoesNotContainSecret(t *testing.T) {
keys := testKeyRing()
rt := newChallengeRuntimeForSplitTest(t, keys)
html, err := rt.GetChallengePage(t.Context(), "test-agent", 8)
require.NoError(t, err)
require.NotEmpty(t, html)
_, signKey := keys.Current()
keyHex := hex.EncodeToString(signKey)
assert.NotContains(t, html, keyHex,
"the per-epoch sign key leaked into plain HTML \u2014 split-bundle invariant violated")
}
// TestSplitBundle_HTMLLoadsFPScannerViaScriptTag asserts the rendered page
// pulls the public fpscanner in via a <script src> tag (so it is cacheable and
// served unobfuscated) rather than inlining it. fpscanner-only source strings
// must NOT appear inline in the page.
func TestSplitBundle_HTMLLoadsFPScannerViaScriptTag(t *testing.T) {
keys := testKeyRing()
rt := newChallengeRuntimeForSplitTest(t, keys)
html, err := rt.GetChallengePage(t.Context(), "test-agent", 8)
require.NoError(t, err)
require.Contains(t, html, `<script src="`+ChallengeFPScannerPath+`">`,
"challenge page must load fpscanner via a <script src> tag")
require.NotContains(t, strings.ToLower(html), "swiftshader",
"fpscanner source must be served separately, not inlined in the challenge page")
}
// hookSentinel must match the constant used by both challenge.js (as
// CSEC_HOOK_NAME) and obfuscate.js (in reservedStrings). Hardcoded here on
// purpose so the test breaks if the JS-side constant is renamed without a
// matching update.
const hookSentinel = "__CSEC_CHALLENGE_HOOK_v1__"
// newChallengeRuntimeForSplitTest builds a runtime with WASM + the baked-in
// initial bundle but a deterministic keyring, so split-bundle assertions
// can be made against a known key.
func newChallengeRuntimeForSplitTest(t *testing.T, keys *KeyRing) *ChallengeRuntime {
t.Helper()
rt, err := NewChallengeRuntime(t.Context(),
WithMasterSecret(testSecret),
)
require.NoError(t, err)
rt.keys = keys
// Reset the dynamic module cache because the constructor pre-warmed it
// with rt.keys (which we just swapped). The next call regenerates.
rt.dynamicModuleCacheMu.Lock()
rt.dynamicModuleCache = make(map[int64][]string)
rt.dynamicModuleCacheMu.Unlock()
return rt
}
+119
View File
@@ -0,0 +1,119 @@
// Package challenge static_bundle.go handles the **challenge code** — the
// crypto/glue JavaScript injected inline on the challenge page. It is obfuscated
// once at build time (initial_bundle.js.gz) and loaded verbatim at startup; no
// runtime re-obfuscation. The public fpscanner is served separately and
// unobfuscated at ChallengeFPScannerPath. For the sensitive per-epoch HMAC sign
// key, see dynamic_module.go.
package challenge
import (
"bytes"
"compress/gzip"
"context"
_ "embed"
"errors"
"fmt"
"io"
"strings"
"sync"
"time"
challengejs "github.com/crowdsecurity/crowdsec/pkg/appsec/challenge/js"
log "github.com/sirupsen/logrus"
)
// initialBundleGz is the build-time obfuscated challenge code (via
// `go generate ./pkg/appsec/challenge/js/...`), embedded so the runtime can
// serve it immediately without paying the obfuscation cost at startup.
//
//go:embed initial_bundle.js.gz
var initialBundleGz []byte
var (
initialBundle string
initialBundleOnce sync.Once
initialBundleErr error
)
// FPScannerJS is the public, unobfuscated fpscanner bundle served at
// ChallengeFPScannerPath. Re-exported so the dispatcher can serve it via the
// challenge package alongside PowWorkerJS.
var FPScannerJS = challengejs.FPScannerJS
// seedCacheFromInitialBundle decompresses the build-time obfuscated challenge
// code (initial_bundle.js.gz) and stores it as the static code served on every
// challenge page. Cheap (~ms) — eliminates the obfuscation that startup would
// otherwise pay.
func (c *ChallengeRuntime) seedCacheFromInitialBundle() error {
initialBundleOnce.Do(func() {
decompressStart := time.Now()
if len(initialBundleGz) == 0 {
initialBundleErr = errors.New("baked-in initial_bundle.js.gz is empty (was `go generate` run?)")
return
}
gz, err := gzip.NewReader(bytes.NewReader(initialBundleGz))
if err != nil {
initialBundleErr = fmt.Errorf("gzip reader for initial bundle: %w", err)
return
}
defer gz.Close()
decoded, err := io.ReadAll(gz)
if err != nil {
initialBundleErr = fmt.Errorf("decompress initial bundle: %w", err)
return
}
initialBundle = string(decoded)
c.log().WithFields(log.Fields{
"compressed_bytes": len(initialBundleGz),
"decompressed_bytes": len(initialBundle),
"duration_ms": time.Since(decompressStart).Milliseconds(),
}).Debug("decompressed baked-in obfuscated challenge code")
})
if initialBundleErr != nil {
return initialBundleErr
}
if initialBundle == "" {
return errors.New("initial bundle is empty after decompression")
}
c.challengeCode = initialBundle
return nil
}
// buildChallengeBundle substitutes the internal-path placeholders into the
// (minified, not-yet-obfuscated) challenge code. Used only by the synchronous
// fallback below — the normal path serves the pre-obfuscated initial bundle.
func (*ChallengeRuntime) buildChallengeBundle() string {
return strings.NewReplacer(
"__CROWDSEC_SUBMIT_PATH__", ChallengeSubmitPath,
"__CROWDSEC_POW_WORKER_PATH__", ChallengePowWorkerPath,
).Replace(challengejs.ChallengeCode)
}
// generateAndCacheChallengeJS is the synchronous fallback used when the
// baked-in initial bundle is missing or corrupt at startup. It obfuscates the
// challenge code in the foreground — undesirable but preferable to starting up
// with no code to serve when `go generate` was not run. Steady-state operation
// never hits this path.
func (c *ChallengeRuntime) generateAndCacheChallengeJS(ctx context.Context) error {
obfuscated, err := c.ObfuscateJS(ctx, c.buildChallengeBundle())
if err != nil {
return err
}
c.challengeCode = obfuscated
return nil
}
// getChallengeCode returns the static, build-time-obfuscated challenge code
// injected inline on the challenge page.
func (c *ChallengeRuntime) getChallengeCode() string {
return c.challengeCode
}
+192
View File
@@ -0,0 +1,192 @@
// ticket.go holds the per-challenge crypto handed to the browser and verified
// on submit: a random per-challenge nonce `r`, the PoW salt + its MAC, and the
// helpers to derive the per-challenge secret `s = HMAC(K_epoch, r)`. The browser
// solves the PoW (leading-zero-bits) and signs its submission with `s`;
// ValidateChallengeResponse (in challenge.go) re-derives `s`, checks the
// signature and PoW, then burns `r` (single-use). PoW difficulty levels are
// tuned for pure-JS SHA-256 through the obfuscated runtime; see the
// PowDifficulty* constants below.
package challenge
import (
"crypto/hmac"
crand "crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"strconv"
"time"
)
// PoW difficulty levels in leading zero bits. Pure JS SHA-256 through the
// obfuscator runs ~500-5000 ops/sec, so keep these conservative.
const (
PowDifficultyDisabled = 0 // no PoW required, nonce "0" always valid
PowDifficultyLow = 10 // ~1024 avg iterations ≈ 0.2-2s
PowDifficultyMedium = 12 // ~4096 avg iterations ≈ 1-8s
PowDifficultyHigh = 15 // ~32768 avg iterations ≈ 7-60s
PowDifficultyImpossible = 256 // full SHA-256 width: clients cannot solve, server always rejects
defaultPowDifficulty = PowDifficultyMedium
)
// ticketAgeBackstop is a loose ceiling on accepted submission age in
// verifyChallenge. The actual freshness gate is the keyring live
// window (rotationInterval × maxLiveEpochs); this is a separate ceiling
// that protects against operators configuring an unusually wide live
// window. Loose enough not to interfere with real submissions on slow
// clients (high-difficulty PoW) but tight enough to bound replay
// surface in pathological configurations.
const ticketAgeBackstop = 20 * time.Minute
// generateChallengeNonce returns a fresh 16-byte random per-challenge nonce
// (`r`) as hex. `r` keys single-use bookkeeping (spent_set.go) and seeds the
// per-challenge secret `s = HMAC(K_epoch, r)`. Error (not panic) on entropy
// failure so only the current request fails.
func generateChallengeNonce() (string, error) {
buf := make([]byte, 16)
if _, err := crand.Read(buf); err != nil {
return "", fmt.Errorf("generate challenge nonce: %w", err)
}
return hex.EncodeToString(buf), nil
}
// deriveChallengeSecret computes the per-challenge signing secret
// `s = HMAC(K_epoch, r)` (hex). `s` is never transmitted; client and server
// derive it independently from the same per-epoch key.
func deriveChallengeSecret(signKey []byte, r string) string {
return hmacSHA256Hex(signKey, []byte(r))
}
// epochForTimestamp converts a nanosecond UnixNano string (the format used in
// challenge.go's ts) into the keyring's epoch identifier. Uses the same
// rotation interval as the keyring so two instances always agree.
func (c *ChallengeRuntime) epochForTimestamp(ts string) int64 {
tsVal, err := strconv.ParseInt(ts, 10, 64)
if err != nil || tsVal <= 0 {
// Caller will reject the request via the liveness check anyway; return
// an out-of-window sentinel epoch.
return -1
}
return tsVal / int64(time.Second) / int64(c.keys.rotationInterval/time.Second)
}
// generatePowPrefix returns a freshly-generated 16-byte random PoW salt
// rendered as a hex string. Errors from crypto/rand.Read indicate a broken
// kernel entropy pool, which is recoverable at the request layer (we can
// reject the current challenge and let the client retry) — returning the
// error rather than panicking keeps a single failing request from taking
// down the whole WAF.
func generatePowPrefix() (string, error) {
buf := make([]byte, 16)
if _, err := crand.Read(buf); err != nil {
return "", fmt.Errorf("generate PoW prefix: %w", err)
}
return hex.EncodeToString(buf), nil
}
// computePowMAC authenticates a PoW salt as server-generated and bound to a
// specific challenge (`r`) + timestamp + difficulty, signed with the per-epoch
// key.
func (c *ChallengeRuntime) computePowMAC(salt, r, ts string, difficulty int) string {
epoch := c.epochForTimestamp(ts)
signKey, ok := c.keys.SignKey(epoch)
if !ok {
_, signKey = c.keys.Current()
}
h := hmac.New(sha256.New, signKey)
h.Write([]byte(salt))
h.Write([]byte(r))
h.Write([]byte(ts))
h.Write([]byte(strconv.Itoa(difficulty)))
return fmt.Sprintf("%x", h.Sum(nil))
}
func hasLeadingZeroBits(hash []byte, bits int) bool {
fullBytes := bits / 8
remainBits := bits % 8
for i := range fullBytes {
if hash[i] != 0 {
return false
}
}
if remainBits > 0 {
mask := byte(0xFF << (8 - remainBits))
if hash[fullBytes]&mask != 0 {
return false
}
}
return true
}
// deriveFingerprintObfKey returns the keystream key for the fingerprint
// payload: `HMAC(s, "fpenc"||r)`.
func deriveFingerprintObfKey(s, r string) string {
return hmacSHA256Hex([]byte(s), []byte("fpenc"+r))
}
// deobfuscateFingerprint reverses the client-side repeating-key XOR + base64
// applied to the fingerprint JSON, using the key from deriveFingerprintObfKey.
func deobfuscateFingerprint(obfKey string, payload string) (string, error) {
payloadBytes, err := base64.StdEncoding.DecodeString(payload)
if err != nil {
return "", fmt.Errorf("failed to decode obfuscated fingerprint: %w", err)
}
out := make([]byte, len(payloadBytes))
for i := range payloadBytes {
out[i] = payloadBytes[i] ^ obfKey[i%len(obfKey)]
}
return string(out), nil
}
// verifyChallenge gates timestamp freshness and authenticates the PoW-salt
// binding, returning the per-epoch sign key (for deriving `s`) on success.
// Stateless — any instance sharing the master secret can verify. Knowledge of
// the per-epoch key is proven separately by the caller's `sig` check.
func (c *ChallengeRuntime) verifyChallenge(clientR, clientTS, clientPowSalt, clientPowMAC string, clientDifficulty int) ([]byte, bool) {
tsVal, err := strconv.ParseInt(clientTS, 10, 64)
if err != nil || tsVal <= 0 {
return nil, false
}
age := time.Since(time.Unix(0, tsVal))
if age < 0 || age > ticketAgeBackstop {
return nil, false
}
epoch := c.epochForTimestamp(clientTS)
signKey, ok := c.keys.SignKey(epoch)
if !ok {
// Epoch fell out of the live window — reject without leaking a usable
// "signature mismatch" vs "stale epoch" distinction via timing.
return nil, false
}
// MAC is recomputed over the client-submitted difficulty: a tampered `d`
// won't match, so this is what authenticates the difficulty.
expectedMAC := c.computePowMAC(clientPowSalt, clientR, clientTS, clientDifficulty)
if !hmac.Equal([]byte(clientPowMAC), []byte(expectedMAC)) {
return nil, false
}
return signKey, true
}
func hmacSHA256Hex(key, msg []byte) string {
h := hmac.New(sha256.New, key)
h.Write(msg)
return fmt.Sprintf("%x", h.Sum(nil))
}
+354
View File
@@ -0,0 +1,354 @@
package challenge
// Generated from IANA tzdb zone1970.tab (public domain). Maps an IANA
// timezone name to the set of ISO-3166 alpha-2 country codes associated
// with it. Regenerate with:
//
// awk -F'\t' '!/^#/ && NF>=3 { ... }' /usr/share/zoneinfo/zone1970.tab
//
// see the plan / repo docs for the full script.
//
// Historical aliases (Asia/Calcutta, Europe/Kiev, etc.) are in tzAliases
// below so older browsers that still send those names resolve correctly.
var tzCountry = map[string][]string{
"Europe/Andorra": {"AD"},
"Asia/Dubai": {"AE", "OM", "RE", "SC", "TF"},
"Asia/Kabul": {"AF"},
"Europe/Tirane": {"AL"},
"Asia/Yerevan": {"AM"},
"Antarctica/Casey": {"AQ"},
"Antarctica/Davis": {"AQ"},
"Antarctica/Mawson": {"AQ"},
"Antarctica/Palmer": {"AQ"},
"Antarctica/Rothera": {"AQ"},
"Antarctica/Troll": {"AQ"},
"Antarctica/Vostok": {"AQ"},
"America/Argentina/Buenos_Aires": {"AR"},
"America/Argentina/Cordoba": {"AR"},
"America/Argentina/Salta": {"AR"},
"America/Argentina/Jujuy": {"AR"},
"America/Argentina/Tucuman": {"AR"},
"America/Argentina/Catamarca": {"AR"},
"America/Argentina/La_Rioja": {"AR"},
"America/Argentina/San_Juan": {"AR"},
"America/Argentina/Mendoza": {"AR"},
"America/Argentina/San_Luis": {"AR"},
"America/Argentina/Rio_Gallegos": {"AR"},
"America/Argentina/Ushuaia": {"AR"},
"Pacific/Pago_Pago": {"AS", "UM"},
"Europe/Vienna": {"AT"},
"Australia/Lord_Howe": {"AU"},
"Antarctica/Macquarie": {"AU"},
"Australia/Hobart": {"AU"},
"Australia/Melbourne": {"AU"},
"Australia/Sydney": {"AU"},
"Australia/Broken_Hill": {"AU"},
"Australia/Brisbane": {"AU"},
"Australia/Lindeman": {"AU"},
"Australia/Adelaide": {"AU"},
"Australia/Darwin": {"AU"},
"Australia/Perth": {"AU"},
"Australia/Eucla": {"AU"},
"Asia/Baku": {"AZ"},
"America/Barbados": {"BB"},
"Asia/Dhaka": {"BD"},
"Europe/Brussels": {"BE", "LU", "NL"},
"Europe/Sofia": {"BG"},
"Atlantic/Bermuda": {"BM"},
"America/La_Paz": {"BO"},
"America/Noronha": {"BR"},
"America/Belem": {"BR"},
"America/Fortaleza": {"BR"},
"America/Recife": {"BR"},
"America/Araguaina": {"BR"},
"America/Maceio": {"BR"},
"America/Bahia": {"BR"},
"America/Sao_Paulo": {"BR"},
"America/Campo_Grande": {"BR"},
"America/Cuiaba": {"BR"},
"America/Santarem": {"BR"},
"America/Porto_Velho": {"BR"},
"America/Boa_Vista": {"BR"},
"America/Manaus": {"BR"},
"America/Eirunepe": {"BR"},
"America/Rio_Branco": {"BR"},
"Asia/Thimphu": {"BT"},
"Europe/Minsk": {"BY"},
"America/Belize": {"BZ"},
"America/St_Johns": {"CA"},
"America/Halifax": {"CA"},
"America/Glace_Bay": {"CA"},
"America/Moncton": {"CA"},
"America/Goose_Bay": {"CA"},
"America/Toronto": {"CA", "BS"},
"America/Iqaluit": {"CA"},
"America/Winnipeg": {"CA"},
"America/Resolute": {"CA"},
"America/Rankin_Inlet": {"CA"},
"America/Regina": {"CA"},
"America/Swift_Current": {"CA"},
"America/Edmonton": {"CA"},
"America/Cambridge_Bay": {"CA"},
"America/Inuvik": {"CA"},
"America/Dawson_Creek": {"CA"},
"America/Fort_Nelson": {"CA"},
"America/Whitehorse": {"CA"},
"America/Dawson": {"CA"},
"America/Vancouver": {"CA"},
"Europe/Zurich": {"CH", "DE", "LI"},
"Africa/Abidjan": {"CI", "BF", "GH", "GM", "GN", "IS", "ML", "MR", "SH", "SL", "SN", "TG"},
"Pacific/Rarotonga": {"CK"},
"America/Santiago": {"CL"},
"America/Coyhaique": {"CL"},
"America/Punta_Arenas": {"CL"},
"Pacific/Easter": {"CL"},
"Asia/Shanghai": {"CN"},
"Asia/Urumqi": {"CN"},
"America/Bogota": {"CO"},
"America/Costa_Rica": {"CR"},
"America/Havana": {"CU"},
"Atlantic/Cape_Verde": {"CV"},
"Asia/Nicosia": {"CY"},
"Asia/Famagusta": {"CY"},
"Europe/Prague": {"CZ", "SK"},
"Europe/Berlin": {"DE", "DK", "NO", "SE", "SJ"},
"America/Santo_Domingo": {"DO"},
"Africa/Algiers": {"DZ"},
"America/Guayaquil": {"EC"},
"Pacific/Galapagos": {"EC"},
"Europe/Tallinn": {"EE"},
"Africa/Cairo": {"EG"},
"Africa/El_Aaiun": {"EH"},
"Europe/Madrid": {"ES"},
"Africa/Ceuta": {"ES"},
"Atlantic/Canary": {"ES"},
"Europe/Helsinki": {"FI", "AX"},
"Pacific/Fiji": {"FJ"},
"Atlantic/Stanley": {"FK"},
"Pacific/Kosrae": {"FM"},
"Atlantic/Faroe": {"FO"},
"Europe/Paris": {"FR", "MC"},
"Europe/London": {"GB", "GG", "IM", "JE"},
"Asia/Tbilisi": {"GE"},
"America/Cayenne": {"GF"},
"Europe/Gibraltar": {"GI"},
"America/Nuuk": {"GL"},
"America/Danmarkshavn": {"GL"},
"America/Scoresbysund": {"GL"},
"America/Thule": {"GL"},
"Europe/Athens": {"GR"},
"Atlantic/South_Georgia": {"GS"},
"America/Guatemala": {"GT"},
"Pacific/Guam": {"GU", "MP"},
"Africa/Bissau": {"GW"},
"America/Guyana": {"GY"},
"Asia/Hong_Kong": {"HK"},
"America/Tegucigalpa": {"HN"},
"America/Port-au-Prince": {"HT"},
"Europe/Budapest": {"HU"},
"Asia/Jakarta": {"ID"},
"Asia/Pontianak": {"ID"},
"Asia/Makassar": {"ID"},
"Asia/Jayapura": {"ID"},
"Europe/Dublin": {"IE"},
"Asia/Jerusalem": {"IL"},
"Asia/Kolkata": {"IN"},
"Indian/Chagos": {"IO"},
"Asia/Baghdad": {"IQ"},
"Asia/Tehran": {"IR"},
"Europe/Rome": {"IT", "SM", "VA"},
"America/Jamaica": {"JM"},
"Asia/Amman": {"JO"},
"Asia/Tokyo": {"JP", "AU"},
"Africa/Nairobi": {"KE", "DJ", "ER", "ET", "KM", "MG", "SO", "TZ", "UG", "YT"},
"Asia/Bishkek": {"KG"},
"Pacific/Tarawa": {"KI", "MH", "TV", "UM", "WF"},
"Pacific/Kanton": {"KI"},
"Pacific/Kiritimati": {"KI"},
"Asia/Pyongyang": {"KP"},
"Asia/Seoul": {"KR"},
"Asia/Almaty": {"KZ"},
"Asia/Qyzylorda": {"KZ"},
"Asia/Qostanay": {"KZ"},
"Asia/Aqtobe": {"KZ"},
"Asia/Aqtau": {"KZ"},
"Asia/Atyrau": {"KZ"},
"Asia/Oral": {"KZ"},
"Asia/Beirut": {"LB"},
"Asia/Colombo": {"LK"},
"Africa/Monrovia": {"LR"},
"Europe/Vilnius": {"LT"},
"Europe/Riga": {"LV"},
"Africa/Tripoli": {"LY"},
"Africa/Casablanca": {"MA"},
"Europe/Chisinau": {"MD"},
"Pacific/Kwajalein": {"MH"},
"Asia/Yangon": {"MM", "CC"},
"Asia/Ulaanbaatar": {"MN"},
"Asia/Hovd": {"MN"},
"Asia/Macau": {"MO"},
"America/Martinique": {"MQ"},
"Europe/Malta": {"MT"},
"Indian/Mauritius": {"MU"},
"Indian/Maldives": {"MV", "TF"},
"America/Mexico_City": {"MX"},
"America/Cancun": {"MX"},
"America/Merida": {"MX"},
"America/Monterrey": {"MX"},
"America/Matamoros": {"MX"},
"America/Chihuahua": {"MX"},
"America/Ciudad_Juarez": {"MX"},
"America/Ojinaga": {"MX"},
"America/Mazatlan": {"MX"},
"America/Bahia_Banderas": {"MX"},
"America/Hermosillo": {"MX"},
"America/Tijuana": {"MX"},
"Asia/Kuching": {"MY", "BN"},
"Africa/Maputo": {"MZ", "BI", "BW", "CD", "MW", "RW", "ZM", "ZW"},
"Africa/Windhoek": {"NA"},
"Pacific/Noumea": {"NC"},
"Pacific/Norfolk": {"NF"},
"Africa/Lagos": {"NG", "AO", "BJ", "CD", "CF", "CG", "CM", "GA", "GQ", "NE"},
"America/Managua": {"NI"},
"Asia/Kathmandu": {"NP"},
"Pacific/Nauru": {"NR"},
"Pacific/Niue": {"NU"},
"Pacific/Auckland": {"NZ", "AQ"},
"Pacific/Chatham": {"NZ"},
"America/Panama": {"PA", "CA", "KY"},
"America/Lima": {"PE"},
"Pacific/Tahiti": {"PF"},
"Pacific/Marquesas": {"PF"},
"Pacific/Gambier": {"PF"},
"Pacific/Port_Moresby": {"PG", "AQ", "FM"},
"Pacific/Bougainville": {"PG"},
"Asia/Manila": {"PH"},
"Asia/Karachi": {"PK"},
"Europe/Warsaw": {"PL"},
"America/Miquelon": {"PM"},
"Pacific/Pitcairn": {"PN"},
"America/Puerto_Rico": {"PR", "AG", "CA", "AI", "AW", "BL", "BQ", "CW", "DM", "GD", "GP", "KN", "LC", "MF", "MS", "SX", "TT", "VC", "VG", "VI"},
"Asia/Gaza": {"PS"},
"Asia/Hebron": {"PS"},
"Europe/Lisbon": {"PT"},
"Atlantic/Madeira": {"PT"},
"Atlantic/Azores": {"PT"},
"Pacific/Palau": {"PW"},
"America/Asuncion": {"PY"},
"Asia/Qatar": {"QA", "BH"},
"Europe/Bucharest": {"RO"},
"Europe/Belgrade": {"RS", "BA", "HR", "ME", "MK", "SI"},
"Europe/Kaliningrad": {"RU"},
"Europe/Moscow": {"RU"},
"Europe/Simferopol": {"RU", "UA"},
"Europe/Kirov": {"RU"},
"Europe/Volgograd": {"RU"},
"Europe/Astrakhan": {"RU"},
"Europe/Saratov": {"RU"},
"Europe/Ulyanovsk": {"RU"},
"Europe/Samara": {"RU"},
"Asia/Yekaterinburg": {"RU"},
"Asia/Omsk": {"RU"},
"Asia/Novosibirsk": {"RU"},
"Asia/Barnaul": {"RU"},
"Asia/Tomsk": {"RU"},
"Asia/Novokuznetsk": {"RU"},
"Asia/Krasnoyarsk": {"RU"},
"Asia/Irkutsk": {"RU"},
"Asia/Chita": {"RU"},
"Asia/Yakutsk": {"RU"},
"Asia/Khandyga": {"RU"},
"Asia/Vladivostok": {"RU"},
"Asia/Ust-Nera": {"RU"},
"Asia/Magadan": {"RU"},
"Asia/Sakhalin": {"RU"},
"Asia/Srednekolymsk": {"RU"},
"Asia/Kamchatka": {"RU"},
"Asia/Anadyr": {"RU"},
"Asia/Riyadh": {"SA", "AQ", "KW", "YE"},
"Pacific/Guadalcanal": {"SB", "FM"},
"Africa/Khartoum": {"SD"},
"Asia/Singapore": {"SG", "AQ", "MY"},
"America/Paramaribo": {"SR"},
"Africa/Juba": {"SS"},
"Africa/Sao_Tome": {"ST"},
"America/El_Salvador": {"SV"},
"Asia/Damascus": {"SY"},
"America/Grand_Turk": {"TC"},
"Africa/Ndjamena": {"TD"},
"Asia/Bangkok": {"TH", "CX", "KH", "LA", "VN"},
"Asia/Dushanbe": {"TJ"},
"Pacific/Fakaofo": {"TK"},
"Asia/Dili": {"TL"},
"Asia/Ashgabat": {"TM"},
"Africa/Tunis": {"TN"},
"Pacific/Tongatapu": {"TO"},
"Europe/Istanbul": {"TR"},
"Asia/Taipei": {"TW"},
"Europe/Kyiv": {"UA"},
"America/New_York": {"US"},
"America/Detroit": {"US"},
"America/Kentucky/Louisville": {"US"},
"America/Kentucky/Monticello": {"US"},
"America/Indiana/Indianapolis": {"US"},
"America/Indiana/Vincennes": {"US"},
"America/Indiana/Winamac": {"US"},
"America/Indiana/Marengo": {"US"},
"America/Indiana/Petersburg": {"US"},
"America/Indiana/Vevay": {"US"},
"America/Chicago": {"US"},
"America/Indiana/Tell_City": {"US"},
"America/Indiana/Knox": {"US"},
"America/Menominee": {"US"},
"America/North_Dakota/Center": {"US"},
"America/North_Dakota/New_Salem": {"US"},
"America/North_Dakota/Beulah": {"US"},
"America/Denver": {"US"},
"America/Boise": {"US"},
"America/Phoenix": {"US", "CA"},
"America/Los_Angeles": {"US"},
"America/Anchorage": {"US"},
"America/Juneau": {"US"},
"America/Sitka": {"US"},
"America/Metlakatla": {"US"},
"America/Yakutat": {"US"},
"America/Nome": {"US"},
"America/Adak": {"US"},
"Pacific/Honolulu": {"US"},
"America/Montevideo": {"UY"},
"Asia/Samarkand": {"UZ"},
"Asia/Tashkent": {"UZ"},
"America/Caracas": {"VE"},
"Asia/Ho_Chi_Minh": {"VN"},
"Pacific/Efate": {"VU"},
"Pacific/Apia": {"WS"},
"Africa/Johannesburg": {"ZA", "LS", "SZ"},
}
// tzAliases maps deprecated / legacy timezone names that older browsers or
// operating systems still emit to their canonical equivalents in tzCountry.
// Keeping the aliases small and explicit rather than generated; pull more in
// as we observe them in production traffic.
var tzAliases = map[string]string{
"Asia/Calcutta": "Asia/Kolkata",
"Asia/Saigon": "Asia/Ho_Chi_Minh",
"Asia/Katmandu": "Asia/Kathmandu",
"Asia/Istanbul": "Europe/Istanbul",
"Europe/Kiev": "Europe/Kyiv",
"America/Buenos_Aires": "America/Argentina/Buenos_Aires",
"America/Godthab": "America/Nuuk",
}
// lookupTimezoneCountries returns the ISO country codes associated with an
// IANA timezone, resolving aliases first. Returns nil for unknown zones —
// callers treat nil as "no data" and decline to emit a mismatch signal.
func lookupTimezoneCountries(tz string) []string {
if tz == "" {
return nil
}
if canonical, ok := tzAliases[tz]; ok {
tz = canonical
}
return tzCountry[tz]
}
+156
View File
@@ -0,0 +1,156 @@
// Package cookie holds the AppsecCookie value type and its builder helpers.
// The AppsecCookie shape is what WAF rules manipulate via the expr
// environment (`AppsecCookie(name).Secure().HttpOnly()...`) and what the
// challenge runtime hands back to the bouncer; encoding to/from
// http.SetCookie strings is also done here.
package cookie
import (
"net/http"
"strings"
"time"
)
const (
defaultCookiePath = "/"
defaultCookieSameSite = "Lax"
)
const (
SameSiteLax = "Lax"
SameSiteStrict = "Strict"
SameSiteNone = "None"
)
// AppsecCookie is the value-type passed around through expr rules to build
// up a Set-Cookie header. Rule authors construct it via cookie.AppsecCookie(name)
// and chain the builder methods below; the final String() call renders it
// to a Set-Cookie-compatible string.
type AppsecCookie struct {
Name string
Expiration int64
Val string
PathVal string
DomainName string
SecureFlag bool
HttpOnlyFlag bool
SameSiteMode string
}
// NewAppsecCookie returns a cookie pre-populated with sensible defaults.
// Defaults: Path "/", SameSite=Lax, session lifetime, not secure/httponly unless set.
func NewAppsecCookie(name string) *AppsecCookie {
return (&AppsecCookie{Name: name}).withDefaults()
}
// Cookie is a short alias for NewAppsecCookie.
func Cookie(name string) *AppsecCookie {
return NewAppsecCookie(name)
}
func (c *AppsecCookie) withDefaults() *AppsecCookie {
if c.PathVal == "" {
c.PathVal = defaultCookiePath
}
if c.SameSiteMode == "" {
c.SameSiteMode = defaultCookieSameSite
}
return c
}
// Value sets the cookie's value.
func (c *AppsecCookie) Value(value string) *AppsecCookie {
c.Val = value
return c
}
// Path sets the cookie Path attribute; empty input restores the default ("/").
func (c *AppsecCookie) Path(path string) *AppsecCookie {
if path == "" {
path = defaultCookiePath
}
c.PathVal = path
return c
}
// Domain sets the cookie Domain attribute. Leave unset to scope the cookie
// to the request host.
func (c *AppsecCookie) Domain(domain string) *AppsecCookie {
c.DomainName = domain
return c
}
// Secure marks the cookie Secure. Required when SameSite=None.
func (c *AppsecCookie) Secure() *AppsecCookie {
c.SecureFlag = true
return c
}
// HttpOnly marks the cookie HttpOnly so it cannot be read from JS.
func (c *AppsecCookie) HttpOnly() *AppsecCookie {
c.HttpOnlyFlag = true
return c
}
// SameSite sets the SameSite mode (Lax / Strict / None). Unknown values are
// passed through verbatim; validation happens on String() rendering.
func (c *AppsecCookie) SameSite(mode string) *AppsecCookie {
c.SameSiteMode = mode
return c
}
// ExpiresAt sets an absolute expiration time (unix seconds).
func (c *AppsecCookie) ExpiresAt(t time.Time) *AppsecCookie {
c.Expiration = t.Unix()
return c
}
// ExpiresIn sets the expiration relative to now.
func (c *AppsecCookie) ExpiresIn(d time.Duration) *AppsecCookie {
c.Expiration = time.Now().Add(d).Unix()
return c
}
// String formats the cookie into a Set-Cookie compatible string.
func (c *AppsecCookie) String() string {
c.withDefaults()
cookie := http.Cookie{
Name: c.Name,
Value: c.Val,
Path: c.PathVal,
Domain: c.DomainName,
Secure: c.SecureFlag,
HttpOnly: c.HttpOnlyFlag,
SameSite: toSameSite(c.SameSiteMode),
}
if c.Expiration > 0 {
cookie.Expires = time.Unix(c.Expiration, 0).UTC()
cookie.MaxAge = int(time.Until(cookie.Expires).Seconds())
}
// SameSite=None requires Secure=true per modern browsers; enforce to avoid invalid cookies.
if cookie.SameSite == http.SameSiteNoneMode && !cookie.Secure {
cookie.Secure = true
}
return cookie.String()
}
func toSameSite(mode string) http.SameSite {
switch strings.ToLower(mode) {
case "lax":
return http.SameSiteLaxMode
case "strict":
return http.SameSiteStrictMode
case "none":
return http.SameSiteNoneMode
default:
return http.SameSiteDefaultMode
}
}
+112
View File
@@ -0,0 +1,112 @@
package appsec
import (
"strconv"
"time"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
)
const (
// ModuleName is the acquisition module name carried on events produced by
// the appsec datasource.
ModuleName = "appsec"
// SourceWAF is the `source` field set on events generated by a WAF rule match.
SourceWAF = "crowdsec-appsec"
// SourceChallenge is the `source` field set on events generated by the
// challenge lifecycle. It is intentionally distinct from SourceWAF so that
// challenge scenarios can filter on it without colliding with WAF scenarios.
SourceChallenge = "crowdsec-appsec-challenge"
)
// ChallengeReason identifies a challenge lifecycle moment. Its string value is
// set as the `challenge_event` field on events produced by
// ChallengeEventFromRequest.
type ChallengeReason string
const (
ChallengeReasonRequested ChallengeReason = "requested" // challenge page served to the client
ChallengeReasonSubmitted ChallengeReason = "submitted" // a challenge validation attempt was received
ChallengeReasonFailed ChallengeReason = "failed" // submission failed crypto/PoW validation
ChallengeReasonRejected ChallengeReason = "rejected" // submission decoded fine but an on_challenge_submit hook called RejectSubmission
ChallengeReasonSolved ChallengeReason = "solved" // validation passed, cookie issued
)
// baseEventFromRequest builds the common pipeline.Event skeleton shared by WAF
// and challenge events: the request metadata in Parsed and the acquisition Line.
func baseEventFromRequest(r *ParsedRequest, labels map[string]string, txUuid, source string) pipeline.Event {
evt := pipeline.MakeEvent(false, pipeline.LOG, true)
evt.Stage = "s00-raw"
evt.Parsed = map[string]string{
"source_ip": r.ClientIP,
"target_host": r.Host,
"target_uri": r.URI,
"method": r.Method,
"req_uuid": txUuid,
"source": source,
"remediation_cmpt_ip": r.RemoteAddrNormalized,
}
evt.Line = pipeline.Line{
Time: time.Now(),
Labels: labels,
Process: true,
Module: ModuleName,
Src: ModuleName,
Raw: "dummy-appsec-data", // we discard empty Line.Raw items :)
}
evt.Appsec = pipeline.AppsecEvent{}
return evt
}
func EventFromRequest(r *ParsedRequest, labels map[string]string, txUuid string) (pipeline.Event, error) {
return baseEventFromRequest(r, labels, txUuid, SourceWAF), nil
}
// ChallengeEventInfo describes a single challenge lifecycle moment to be turned
// into a pipeline.Event by ChallengeEventFromRequest.
type ChallengeEventInfo struct {
Reason ChallengeReason // requested | submitted | failed | rejected | solved
FailReason string // set on failed (raw error message) or rejected (operator-supplied reason)
FailErr error // set on failed; carries the sentinel-wrapped error so metrics can classify via errors.Is.
Difficulty int // target PoW difficulty for this moment
Fingerprint *challenge.FingerprintData // nil when none available (e.g. requested w/o cookie)
}
// ChallengeEventFromRequest builds a LOG event for a challenge lifecycle moment.
// It carries SourceChallenge so it never collides with WAF scenarios, plus the
// reason, difficulty and (when available) fingerprint scalars for filtering.
func ChallengeEventFromRequest(r *ParsedRequest, labels map[string]string, txUuid string, info ChallengeEventInfo) pipeline.Event {
evt := baseEventFromRequest(r, labels, txUuid, SourceChallenge)
if r.HTTPRequest != nil {
evt.Parsed["user_agent"] = r.HTTPRequest.UserAgent()
}
evt.Parsed["challenge_event"] = string(info.Reason)
evt.Parsed["challenge_difficulty"] = strconv.Itoa(info.Difficulty)
if info.FailReason != "" {
evt.Parsed["challenge_fail_reason"] = info.FailReason
}
if fp := info.Fingerprint; fp != nil {
// Expose the full fingerprint (all nested signals, bot-detection details,
// etc.) so scenarios and alert-context can traverse it as
// evt.Unmarshaled.fingerprint.Signals... Stored as a pointer so expr can
// also call the *FingerprintData helper methods (Platform(), BotSignals(),
// IsBot(), ...). It stays out of LAPI/CAPI unless explicitly mapped into Meta.
evt.Unmarshaled["fingerprint"] = fp
// Flat shortcuts for cheap scenario filtering.
evt.Parsed["fsid"] = fp.FSID
evt.Parsed["fingerprint_bot"] = strconv.FormatBool(fp.FastBotDetection.Bool())
evt.Parsed["fingerprint_allowlisted"] = strconv.FormatBool(fp.Allowlisted)
if fp.AllowlistReason != "" {
evt.Parsed["fingerprint_allowlist_reason"] = fp.AllowlistReason
}
evt.Parsed["platform"] = fp.Signals.Device.Platform
}
return evt
}
+122
View File
@@ -0,0 +1,122 @@
package appsec
import (
"net/http"
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
)
func testChallengeRequest() *ParsedRequest {
return &ParsedRequest{
ClientIP: "1.2.3.4",
Host: "example.com",
URI: "/protected",
Method: "GET",
UUID: "test-uuid",
RemoteAddrNormalized: "10.0.0.1",
IsInBand: true,
HTTPRequest: &http.Request{Header: http.Header{"User-Agent": []string{"test-agent"}}},
}
}
func TestChallengeEventFromRequest(t *testing.T) {
labels := map[string]string{"type": "appsec"}
tests := []struct {
name string
info ChallengeEventInfo
assert func(t *testing.T, evt pipeline.Event)
}{
{
name: "requested carries the distinct source and reason",
info: ChallengeEventInfo{Reason: ChallengeReasonRequested, Difficulty: 4},
assert: func(t *testing.T, evt pipeline.Event) {
require.Equal(t, pipeline.LOG, evt.Type)
require.Equal(t, SourceChallenge, evt.Parsed["source"])
require.Equal(t, string(ChallengeReasonRequested), evt.Parsed["challenge_event"])
require.Equal(t, "4", evt.Parsed["challenge_difficulty"])
// client/request metadata
require.Equal(t, "1.2.3.4", evt.Parsed["source_ip"])
require.Equal(t, "example.com", evt.Parsed["target_host"])
require.Equal(t, "test-uuid", evt.Parsed["req_uuid"])
require.Equal(t, "10.0.0.1", evt.Parsed["remediation_cmpt_ip"])
require.Equal(t, "test-agent", evt.Parsed["user_agent"])
require.Equal(t, ModuleName, evt.Line.Module)
// no fingerprint → no fingerprint fields
require.NotContains(t, evt.Parsed, "fsid")
},
},
{
name: "submitted has no fingerprint and no fail reason",
info: ChallengeEventInfo{Reason: ChallengeReasonSubmitted},
assert: func(t *testing.T, evt pipeline.Event) {
require.Equal(t, string(ChallengeReasonSubmitted), evt.Parsed["challenge_event"])
require.NotContains(t, evt.Parsed, "challenge_fail_reason")
},
},
{
name: "failed surfaces the fail reason",
info: ChallengeEventInfo{Reason: ChallengeReasonFailed, FailReason: "invalid proof-of-work"},
assert: func(t *testing.T, evt pipeline.Event) {
require.Equal(t, string(ChallengeReasonFailed), evt.Parsed["challenge_event"])
require.Equal(t, "invalid proof-of-work", evt.Parsed["challenge_fail_reason"])
},
},
{
name: "solved surfaces fingerprint scalars",
info: ChallengeEventInfo{
Reason: ChallengeReasonSolved,
Difficulty: 5,
Fingerprint: &challenge.FingerprintData{
FSID: "FS1_abc",
FastBotDetection: challenge.FlexBool(true),
Allowlisted: true,
AllowlistReason: "googlebot",
},
},
assert: func(t *testing.T, evt pipeline.Event) {
require.Equal(t, string(ChallengeReasonSolved), evt.Parsed["challenge_event"])
require.Equal(t, "FS1_abc", evt.Parsed["fsid"])
require.Equal(t, "true", evt.Parsed["fingerprint_bot"])
require.Equal(t, "true", evt.Parsed["fingerprint_allowlisted"])
require.Equal(t, "googlebot", evt.Parsed["fingerprint_allowlist_reason"])
// the full fingerprint struct must be available for traversal
fp, ok := evt.Unmarshaled["fingerprint"].(*challenge.FingerprintData)
require.True(t, ok, "fingerprint should be attached to Unmarshaled")
require.Equal(t, "FS1_abc", fp.FSID)
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
evt := ChallengeEventFromRequest(testChallengeRequest(), labels, "test-uuid", tc.info)
tc.assert(t, evt)
})
}
}
// emitChallengeEvent must be a no-op (and not panic) when no output channel is wired.
func TestEmitChallengeEventNoChannel(t *testing.T) {
w := &AppsecRuntimeConfig{}
require.NotPanics(t, func() {
w.emitChallengeEvent(testChallengeRequest(), ChallengeEventInfo{Reason: ChallengeReasonRequested})
})
}
// emitChallengeEvent sends the built event on the wired channel.
func TestEmitChallengeEventSendsOnChannel(t *testing.T) {
out := make(chan pipeline.Event, 1)
w := &AppsecRuntimeConfig{OutChan: out, Labels: map[string]string{"type": "appsec"}}
w.emitChallengeEvent(testChallengeRequest(), ChallengeEventInfo{Reason: ChallengeReasonFailed, FailReason: "boom"})
evt := <-out
require.Equal(t, SourceChallenge, evt.Parsed["source"])
require.Equal(t, string(ChallengeReasonFailed), evt.Parsed["challenge_event"])
require.Equal(t, "boom", evt.Parsed["challenge_fail_reason"])
}
+120
View File
@@ -0,0 +1,120 @@
package appsec
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
)
// Per-output-path mutexes as there can be concurrency. Outer mutex only guards the map
// itself
var (
fpDumpMapMu sync.Mutex
fpDumpLocks = map[string]*sync.Mutex{}
)
func fpDumpLockFor(path string) *sync.Mutex {
fpDumpMapMu.Lock()
defer fpDumpMapMu.Unlock()
m, ok := fpDumpLocks[path]
if !ok {
m = &sync.Mutex{}
fpDumpLocks[path] = m
}
return m
}
var fpLabelSanitizer = regexp.MustCompile(`[^a-z0-9]+`)
// sanitizeFpLabel returns a filesystem-safe slug for the supplied label.
func sanitizeFpLabel(label string) string {
s := fpLabelSanitizer.ReplaceAllString(strings.ToLower(label), "_")
s = strings.Trim(s, "_")
if s == "" {
return "unlabeled"
}
if len(s) > 40 {
s = strings.Trim(s[:40], "_")
if s == "" {
return "unlabeled"
}
}
return s
}
// fpDumpEntry is the on-disk record shape: one per line in the
// labeled JSONL file.
type fpDumpEntry struct {
Label string `json:"label"`
Timestamp time.Time `json:"timestamp"`
ClientIP string `json:"client_ip,omitempty"`
RemoteAddr string `json:"remote_addr,omitempty"`
RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"`
UserAgent string `json:"user_agent,omitempty"`
Host string `json:"host,omitempty"`
URI string `json:"uri,omitempty"`
Method string `json:"method,omitempty"`
Fingerprint *challenge.FingerprintData `json:"fingerprint"`
}
// DumpFingerprint allows to dump the fingerprint + some context (ip, host, timestamp etc.)
// to as JSONL file for later analysis.
func DumpFingerprint(dir, label string, fp *challenge.FingerprintData, req *ParsedRequest) string {
if fp == nil {
log.Warnf("DumpFingerprint(%q) called with nil fingerprint, skipping", label)
return ""
}
if dir == "" {
log.Warnf("DumpFingerprint(%q): no dump directory configured, skipping", label)
return ""
}
path := filepath.Join(dir, fmt.Sprintf("crowdsec_fp_dump_%s.jsonl", sanitizeFpLabel(label)))
entry := fpDumpEntry{
Label: label,
Timestamp: time.Now().UTC(),
Fingerprint: fp,
}
if req != nil {
entry.ClientIP = req.ClientIP
entry.RemoteAddr = req.RemoteAddr
entry.RemoteAddrNormalized = req.RemoteAddrNormalized
entry.UserAgent = req.Headers.Get("User-Agent")
entry.Host = req.Host
entry.URI = req.URI
entry.Method = req.Method
}
line, err := json.Marshal(entry)
if err != nil {
log.Errorf("DumpFingerprint(%q): marshal: %s", label, err)
return ""
}
mu := fpDumpLockFor(path)
mu.Lock()
defer mu.Unlock()
fd, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
if err != nil {
log.Errorf("DumpFingerprint(%q): open %s: %s", label, path, err)
return ""
}
defer fd.Close()
if _, err := fd.Write(append(line, '\n')); err != nil {
log.Errorf("DumpFingerprint(%q): write %s: %s", label, path, err)
return ""
}
log.Infof("fingerprint dumped (label=%q) to %s", label, path)
return path
}
+272
View File
@@ -0,0 +1,272 @@
package appsec
import (
"bufio"
"encoding/json"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
)
// fakeFingerprint returns a minimal but distinguishable *challenge.FingerprintData
// suitable for asserting that DumpFingerprint preserved the value end-to-end.
func fakeFingerprint(fsid string) *challenge.FingerprintData {
return &challenge.FingerprintData{
FSID: fsid,
Nonce: "test-nonce",
Time: 1717000000,
URL: "https://example.test/path",
}
}
func fakeRequest() *ParsedRequest {
return &ParsedRequest{
// Distinct visitor vs. bouncer addresses so the test can prove
// DumpFingerprint records the *real* source IP (ClientIP) and
// not just the bouncer's TCP peer (RemoteAddr).
ClientIP: "203.0.113.7",
RemoteAddr: "127.0.0.1:54321",
RemoteAddrNormalized: "127.0.0.1",
Host: "example.test",
URI: "/login",
Method: "POST",
Headers: http.Header{"User-Agent": []string{"Mozilla/5.0 (test)"}},
}
}
// readJSONL reads every line of the given path as a JSON object and returns them.
func readJSONL(t *testing.T, path string) []map[string]any {
t.Helper()
fd, err := os.Open(path)
if err != nil {
t.Fatalf("open %s: %s", path, err)
}
defer fd.Close()
var out []map[string]any
sc := bufio.NewScanner(fd)
// Fingerprints with full signals can be large; bump the buffer to avoid
// false negatives on real-shape payloads, even though our test data is tiny.
sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
for sc.Scan() {
var m map[string]any
if err := json.Unmarshal(sc.Bytes(), &m); err != nil {
t.Fatalf("invalid JSON line %q: %s", sc.Text(), err)
}
out = append(out, m)
}
if err := sc.Err(); err != nil {
t.Fatalf("scan: %s", err)
}
return out
}
func TestSanitizeFpLabel(t *testing.T) {
cases := []struct {
in string
want string
}{
{"bot", "bot"},
{"human – Chrome on macOS!", "human_chrome_on_macos"},
{" ", "unlabeled"},
{"", "unlabeled"},
{"!!! @@@ ", "unlabeled"},
{"Bot-1", "bot_1"},
{"Bot_1", "bot_1"},
// 50-char alpha label gets truncated to 40, trim of trailing underscore is a no-op.
{strings.Repeat("a", 50), strings.Repeat("a", 40)},
}
for _, tc := range cases {
got := sanitizeFpLabel(tc.in)
if got != tc.want {
t.Errorf("sanitizeFpLabel(%q) = %q, want %q", tc.in, got, tc.want)
}
}
}
func TestDumpFingerprint_AppendsAndPreservesFields(t *testing.T) {
dir := t.TempDir()
fp := fakeFingerprint("fsid-abc")
req := fakeRequest()
path1 := DumpFingerprint(dir, "test-human", fp, req)
if path1 == "" {
t.Fatal("first DumpFingerprint returned empty path")
}
wantPath := filepath.Join(dir, "crowdsec_fp_dump_test_human.jsonl")
if path1 != wantPath {
t.Errorf("path = %q, want %q", path1, wantPath)
}
// Second call with the same label must append, not truncate.
path2 := DumpFingerprint(dir, "test-human", fakeFingerprint("fsid-def"), req)
if path2 != path1 {
t.Errorf("second call wrote to %q, want same file %q", path2, path1)
}
lines := readJSONL(t, path1)
if len(lines) != 2 {
t.Fatalf("got %d JSONL lines, want 2 (append failed?)", len(lines))
}
// Both lines must carry the operator-supplied label verbatim.
for i, line := range lines {
if line["label"] != "test-human" {
t.Errorf("line %d label = %v, want %q", i, line["label"], "test-human")
}
// Timestamp must parse as RFC3339.
ts, ok := line["timestamp"].(string)
if !ok {
t.Errorf("line %d timestamp not a string: %T", i, line["timestamp"])
} else if _, err := time.Parse(time.RFC3339Nano, ts); err != nil {
t.Errorf("line %d timestamp %q not RFC3339: %s", i, ts, err)
}
// Minimal request context preserved.
// client_ip is the real visitor address — the field a SOC
// operator actually wants when triaging a sample. Verify it
// is distinct from remote_addr (which is the bouncer).
if line["client_ip"] != "203.0.113.7" {
t.Errorf("line %d client_ip = %v, want %q", i, line["client_ip"], "203.0.113.7")
}
if line["remote_addr"] != "127.0.0.1:54321" {
t.Errorf("line %d remote_addr = %v", i, line["remote_addr"])
}
if line["normalized_remote_addr"] != "127.0.0.1" {
t.Errorf("line %d normalized_remote_addr = %v", i, line["normalized_remote_addr"])
}
if line["user_agent"] != "Mozilla/5.0 (test)" {
t.Errorf("line %d user_agent = %v", i, line["user_agent"])
}
if line["host"] != "example.test" {
t.Errorf("line %d host = %v", i, line["host"])
}
if line["uri"] != "/login" {
t.Errorf("line %d uri = %v", i, line["uri"])
}
if line["method"] != "POST" {
t.Errorf("line %d method = %v", i, line["method"])
}
}
// Fingerprint payload must be preserved per-line (fsid differs across the two calls).
wantFSIDs := []string{"fsid-abc", "fsid-def"}
for i, line := range lines {
fp, ok := line["fingerprint"].(map[string]any)
if !ok {
t.Fatalf("line %d: fingerprint not an object: %T", i, line["fingerprint"])
}
if fp["fsid"] != wantFSIDs[i] {
t.Errorf("line %d fingerprint.fsid = %v, want %q", i, fp["fsid"], wantFSIDs[i])
}
}
}
func TestDumpFingerprint_DifferentLabelsGoToDifferentFiles(t *testing.T) {
dir := t.TempDir()
pHuman := DumpFingerprint(dir, "human", fakeFingerprint("a"), fakeRequest())
pBot := DumpFingerprint(dir, "bot", fakeFingerprint("b"), fakeRequest())
if pHuman == "" || pBot == "" {
t.Fatalf("paths empty: %q %q", pHuman, pBot)
}
if pHuman == pBot {
t.Fatalf("expected distinct files for distinct labels, got %q", pHuman)
}
if got := len(readJSONL(t, pHuman)); got != 1 {
t.Errorf("human file has %d lines, want 1", got)
}
if got := len(readJSONL(t, pBot)); got != 1 {
t.Errorf("bot file has %d lines, want 1", got)
}
}
func TestDumpFingerprint_NilFingerprintIsNoop(t *testing.T) {
dir := t.TempDir()
got := DumpFingerprint(dir, "anything", nil, fakeRequest())
if got != "" {
t.Errorf("nil fingerprint returned path %q, want empty", got)
}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
if strings.HasPrefix(e.Name(), "crowdsec_fp_dump_") {
t.Errorf("nil fingerprint wrote a file: %s", e.Name())
}
}
}
func TestDumpFingerprint_EmptyDirIsNoop(t *testing.T) {
got := DumpFingerprint("", "anything", fakeFingerprint("xx"), fakeRequest())
if got != "" {
t.Errorf("empty dir returned path %q, want empty", got)
}
}
func TestDumpFingerprint_NilRequestStillDumps(t *testing.T) {
dir := t.TempDir()
path := DumpFingerprint(dir, "nofreq", fakeFingerprint("xx"), nil)
if path == "" {
t.Fatal("expected dump to succeed with nil request")
}
lines := readJSONL(t, path)
if len(lines) != 1 {
t.Fatalf("got %d lines, want 1", len(lines))
}
// Request fields are omitempty, so they should be absent from the JSON.
for _, k := range []string{"client_ip", "remote_addr", "normalized_remote_addr", "user_agent", "host", "uri", "method"} {
if _, ok := lines[0][k]; ok {
t.Errorf("nil request still wrote %s=%v", k, lines[0][k])
}
}
}
func TestDumpFingerprint_EmptyAndJunkLabelsCollapse(t *testing.T) {
dir := t.TempDir()
p1 := DumpFingerprint(dir, "", fakeFingerprint("a"), nil)
p2 := DumpFingerprint(dir, "!!! @@@ ", fakeFingerprint("b"), nil)
if p1 == "" || p2 == "" {
t.Fatalf("paths empty: %q %q", p1, p2)
}
wantPath := filepath.Join(dir, "crowdsec_fp_dump_unlabeled.jsonl")
if p1 != wantPath || p2 != wantPath {
t.Errorf("expected both junk labels to land in %q, got %q and %q", wantPath, p1, p2)
}
if got := len(readJSONL(t, p1)); got != 2 {
t.Errorf("unlabeled file has %d lines, want 2", got)
}
}
func TestDumpFingerprint_ConcurrentAppendIsLineSafe(t *testing.T) {
dir := t.TempDir()
const goroutines = 32
const perGoroutine = 20
var wg sync.WaitGroup
wg.Add(goroutines)
for range goroutines {
go func() {
defer wg.Done()
for range perGoroutine {
if path := DumpFingerprint(dir, "race", fakeFingerprint("fsid"), fakeRequest()); path == "" {
t.Error("DumpFingerprint returned empty path under contention")
return
}
}
}()
}
wg.Wait()
path := filepath.Join(dir, "crowdsec_fp_dump_race.jsonl")
lines := readJSONL(t, path) // this would fatal on any torn line
if len(lines) != goroutines*perGoroutine {
t.Errorf("got %d lines, want %d", len(lines), goroutines*perGoroutine)
}
}
@@ -0,0 +1,259 @@
package appsec
import (
"encoding/json"
"net/http"
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/metrics"
)
// Well-known MaxMind test vectors that exist in the test mmdb under
// pkg/parser/testdata. Used to exercise the geoip-backed country
// resolution inside EvaluateMismatches without mocks.
const (
testIPUS = "216.160.83.56" // Milton, US
testIPGB = "81.2.69.142" // London, GB
)
// setupGeoIP loads the same test mmdb the parser tests use so that
// IPToCountryString has a working reader to query. Safe to call more than
// once in a single test run.
func setupGeoIP(t *testing.T) {
t.Helper()
require.NoError(t, exprhelpers.GeoIPInit("../parser/testdata/"), "geoip init failed")
}
// fpEuropeParisCDP returns a fingerprint with Europe/Paris timezone and a
// single fired library signal (CDP), so EvaluateMismatches can be driven
// predictably.
func fpEuropeParisCDP(t *testing.T) *challenge.FingerprintData {
t.Helper()
raw := `{
"signals": {
"device": {"platform": "MacIntel"},
"browser": {
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120",
"highEntropyValues": {"platform": "macOS"}
},
"locale": {
"internationalization": {"timezone": "Europe/Paris"},
"languages": {"language": "en"}
}
},
"fsid": "FS_RUNTIME", "nonce": "n", "time": 1, "url": "http://x/",
"fastBotDetection": true,
"fastBotDetectionDetails": {
"hasCDP": {"detected": true, "severity": "high"}
}
}`
fp := &challenge.FingerprintData{}
require.NoError(t, json.Unmarshal([]byte(raw), fp))
return fp
}
func makeRuntime() *AppsecRuntimeConfig {
return &AppsecRuntimeConfig{Logger: log.NewEntry(log.StandardLogger())}
}
func TestEvaluateMismatches_GeoIPTimezoneMismatch_Fires(t *testing.T) {
setupGeoIP(t)
metrics.AppsecFingerprintMismatch.Reset()
w := makeRuntime()
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
state := &AppsecRequestState{Fingerprint: fpEuropeParisCDP(t)}
req := &ParsedRequest{
HTTPRequest: httpReq,
ClientIP: testIPUS, // geoip resolves to "US"
AppsecEngine: "test-engine",
}
report := w.EvaluateMismatches(state, req)
// Europe/Paris fingerprint timezone but request geoips to US → the
// soft timezone_country signal must fire. CDP already fires from the
// library side.
assert.True(t, report.Has(challenge.ReasonTimezoneCountry),
"expected timezone_country to fire for Europe/Paris + US geoip")
assert.True(t, report.Has(challenge.ReasonCDP))
assert.Equal(t, 1, report.High(), "only CDP is high severity here")
assert.Equal(t, 1, report.Low(), "timezone_country is low severity")
// Prometheus counters must reflect one bump per fired signal.
assert.InDelta(t, 1.0, testutil.ToFloat64(
metrics.AppsecFingerprintMismatch.With(prometheus.Labels{
"reason": challenge.ReasonTimezoneCountry,
"severity": challenge.SeverityLow,
"appsec_engine": "test-engine",
}),
), 0)
assert.InDelta(t, 1.0, testutil.ToFloat64(
metrics.AppsecFingerprintMismatch.With(prometheus.Labels{
"reason": challenge.ReasonCDP,
"severity": challenge.SeverityHigh,
"appsec_engine": "test-engine",
}),
), 0)
}
func TestEvaluateMismatches_GeoIPConsistent_NoTimezoneSignal(t *testing.T) {
setupGeoIP(t)
metrics.AppsecFingerprintMismatch.Reset()
w := makeRuntime()
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
// Browser reports Europe/London tz; client IP geolocates to GB.
// No timezone_country mismatch should fire.
raw := `{
"signals": {
"device": {"platform": "Win32"},
"browser": {
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120",
"highEntropyValues": {"platform": "Windows"}
},
"locale": {
"internationalization": {"timezone": "Europe/London"},
"languages": {"language": "en"}
}
},
"fsid": "FS_OK_GB", "nonce": "n", "time": 1, "url": "http://x/",
"fastBotDetection": false,
"fastBotDetectionDetails": {}
}`
fp := &challenge.FingerprintData{}
require.NoError(t, json.Unmarshal([]byte(raw), fp))
state := &AppsecRequestState{Fingerprint: fp}
req := &ParsedRequest{
HTTPRequest: httpReq,
ClientIP: testIPGB, // geoip resolves to "GB"
AppsecEngine: "test-engine",
}
report := w.EvaluateMismatches(state, req)
assert.False(t, report.Has(challenge.ReasonTimezoneCountry),
"Europe/London + GB geoip must not fire timezone_country")
assert.True(t, report.Empty(), "no signals expected at all")
}
func TestEvaluateMismatches_CachesAcrossCalls(t *testing.T) {
setupGeoIP(t)
metrics.AppsecFingerprintMismatch.Reset()
w := makeRuntime()
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
state := &AppsecRequestState{Fingerprint: fpEuropeParisCDP(t)}
req := &ParsedRequest{
HTTPRequest: httpReq,
ClientIP: testIPUS,
AppsecEngine: "test-engine",
}
first := w.EvaluateMismatches(state, req)
second := w.EvaluateMismatches(state, req)
third := w.EvaluateMismatches(state, req)
// All three calls must return the exact same pointer — state cache.
assert.Same(t, first, second)
assert.Same(t, second, third)
assert.Same(t, first, state.LastMismatchReport)
// Observability fires once total (log + metric) despite three calls.
// Verifies the emitMismatchObservability gate at state.LastMismatchReport.
assert.InDelta(t, 1.0, testutil.ToFloat64(
metrics.AppsecFingerprintMismatch.With(prometheus.Labels{
"reason": challenge.ReasonTimezoneCountry,
"severity": challenge.SeverityLow,
"appsec_engine": "test-engine",
}),
), 0)
}
func TestEvaluateMismatches_UnresolvableClientIP_NoTimezoneSignal(t *testing.T) {
setupGeoIP(t)
metrics.AppsecFingerprintMismatch.Reset()
w := makeRuntime()
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
state := &AppsecRequestState{Fingerprint: fpEuropeParisCDP(t)}
req := &ParsedRequest{
HTTPRequest: httpReq,
ClientIP: "203.0.113.1", // not in the test mmdb
AppsecEngine: "test-engine",
}
report := w.EvaluateMismatches(state, req)
// With country == "" the timezone_country helper declines to decide,
// so only the library CDP signal fires.
assert.False(t, report.Has(challenge.ReasonTimezoneCountry))
assert.True(t, report.Has(challenge.ReasonCDP))
}
func TestEvaluateMismatches_EmptyReport_NoObservability(t *testing.T) {
setupGeoIP(t)
metrics.AppsecFingerprintMismatch.Reset()
w := makeRuntime()
httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://x/", http.NoBody)
require.NoError(t, err)
// Consistent fingerprint — no library signals, no custom helper fires.
raw := `{
"signals": {
"device": {"platform": "Win32"},
"browser": {
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120",
"highEntropyValues": {"platform": "Windows"}
},
"locale": {
"internationalization": {"timezone": "Europe/London"},
"languages": {"language": "en"}
}
},
"fsid": "X", "nonce": "n", "time": 1, "url": "http://x/",
"fastBotDetection": false,
"fastBotDetectionDetails": {}
}`
fp := &challenge.FingerprintData{}
require.NoError(t, json.Unmarshal([]byte(raw), fp))
state := &AppsecRequestState{Fingerprint: fp}
req := &ParsedRequest{
HTTPRequest: httpReq,
ClientIP: testIPGB,
AppsecEngine: "test-engine",
}
report := w.EvaluateMismatches(state, req)
assert.True(t, report.Empty())
// No metric bump for any reason when the report is empty.
for _, reason := range challenge.KnownReasons() {
sev := challenge.SeverityFor(reason)
got := testutil.ToFloat64(metrics.AppsecFingerprintMismatch.With(prometheus.Labels{
"reason": reason,
"severity": sev,
"appsec_engine": "test-engine",
}))
assert.InDelta(t, 0.0, got, 0, "no counter bump expected for %q", reason)
}
}
+26
View File
@@ -0,0 +1,26 @@
package appsec
import "github.com/expr-lang/expr/ast"
// This is not an actual patcher: we just walk the AST to check if we need to create a WASM VM for the challenge mode.
type appsecExprPatcher struct {
NeedWASMVM bool
}
// challengeRuntimeCallees is the set of expr helper names whose presence in
// any compiled rule body implies that the challenge runtime (WASM VM,
// obfuscator, keyring) must be initialized.
var challengeRuntimeCallees = map[string]struct{}{
"SendChallenge": {},
"GrantChallengeCookie": {},
"RejectSubmission": {},
"LogAccepted": {},
}
func (p *appsecExprPatcher) Visit(node *ast.Node) { //nolint:gocritic // signature fixed by expr-lang ast.Visitor interface
if n, ok := (*node).(*ast.CallNode); ok {
if _, needs := challengeRuntimeCallees[n.Callee.String()]; needs {
p.NeedWASMVM = true
}
}
}
+48
View File
@@ -0,0 +1,48 @@
package appsec
import (
"io"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAppsecConfigBuildDetectsRequireValidChallenge(t *testing.T) {
logger := log.New()
logger.SetOutput(io.Discard)
// SendChallenge is only exposed in post_eval / on_challenge envs; the patcher
// detects it from any stage and flags NeedWASMVM.
cfg := AppsecConfig{
Logger: log.NewEntry(logger),
PostEval: []Hook{
{
Apply: []string{"SendChallenge()"},
},
},
}
runtimeCfg, err := cfg.Build(t.Context(), nil)
require.NoError(t, err)
assert.True(t, runtimeCfg.NeedWASMVM)
}
func TestAppsecConfigBuildDoesNotDetectRequireValidChallengeWhenUnused(t *testing.T) {
logger := log.New()
logger.SetOutput(io.Discard)
cfg := AppsecConfig{
Logger: log.NewEntry(logger),
PreEval: []Hook{
{
Apply: []string{"SetRemediation(\"ban\")"},
},
},
}
runtimeCfg, err := cfg.Build(t.Context(), nil)
require.NoError(t, err)
assert.False(t, runtimeCfg.NeedWASMVM)
}
+34
View File
@@ -0,0 +1,34 @@
package appsec
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/appsec/cookie"
)
// TestAppsecTempResponse_Clone asserts the clone's reference-typed fields have
// independent backing: mutating the original (as the out-of-band phase does)
// must not be visible through the clone the HTTP handler holds.
func TestAppsecTempResponse_Clone(t *testing.T) {
orig := AppsecTempResponse{
Action: ChallengeRemediation,
UserHeaders: map[string][]string{"X-Test": {"a"}},
UserHTTPCookies: []cookie.AppsecCookie{{Name: "c1"}},
}
clone := orig.Clone()
// Out-of-band-style mutations on the original.
orig.UserHeaders["X-New"] = []string{"z"} // new key
orig.UserHeaders["X-Test"][0] = "MUT" // in-place inner-slice write
orig.UserHTTPCookies = append(orig.UserHTTPCookies, cookie.AppsecCookie{Name: "c2"})
assert.NotContains(t, clone.UserHeaders, "X-New", "new keys must not leak into the clone")
assert.Equal(t, "a", clone.UserHeaders["X-Test"][0], "inner slices must be copied, not aliased")
require.Len(t, clone.UserHTTPCookies, 1, "cookie slice must be copied")
assert.Equal(t, "c1", clone.UserHTTPCookies[0].Name)
assert.Equal(t, ChallengeRemediation, clone.Action)
}
+244 -14
View File
@@ -2,10 +2,83 @@ package appsec
import (
"context"
"fmt"
"strings"
"time"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/appsec/challenge"
"github.com/crowdsecurity/crowdsec/pkg/appsec/cookie"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
)
// isLegitimateBotHelper wraps exprhelpers.IsLegitimateBot with the
// per-request escape hatch: once ExemptFromChallenge was called, the verdict
// is true without datafile or DNS checks. The standard rules gate
// SendChallenge() on IsLegitimateBot(), so forcing the verdict true is what
// whitelists the request from the challenge.
func isLegitimateBotHelper(state *AppsecRequestState) func(string, string, string) bool {
return func(ip string, ua string, path string) bool {
return state.ChallengeExempt || exprhelpers.IsLegitimateBot(ip, ua, path)
}
}
// parseLogVerbosity maps an optional expr-side verbosity argument
// ("minimal", "info", "verbose") to a FingerprintLogVerbosity. Empty /
// missing returns FingerprintLogInfo (the default tier). Unknown values
// emit a warning through the supplied logger and also fall back to
// FingerprintLogInfo so the surrounding action (reject / log) still
// takes effect — a typo in the verbosity arg shouldn't silently swallow
// the operator's reject.
func parseLogVerbosity(logger *log.Entry, verbosity []string) challenge.FingerprintLogVerbosity {
if len(verbosity) == 0 {
return challenge.FingerprintLogInfo
}
switch strings.ToLower(strings.TrimSpace(verbosity[0])) {
case "", "info":
return challenge.FingerprintLogInfo
case "minimal":
return challenge.FingerprintLogMinimal
case "verbose":
return challenge.FingerprintLogVerbose
default:
if logger != nil {
logger.Warnf("unknown fingerprint log verbosity %q; falling back to info", verbosity[0])
}
return challenge.FingerprintLogInfo
}
}
// parseChallengeCookieTTLArg interprets the optional TTL argument to the
// GrantChallengeCookie expr helper. Zero variadic args means "use the
// runtime default" and yields a nil override. A single non-empty string is
// parsed with time.ParseDuration (e.g. "1h", "30m"). More than one TTL
// argument or an unparseable value is reported as an error so hook authors
// get a precise diagnostic at evaluation time rather than a silent fallback.
func parseChallengeCookieTTLArg(ttl []string) (*time.Duration, error) {
if len(ttl) == 0 {
return nil, nil
}
if len(ttl) > 1 {
return nil, fmt.Errorf("GrantChallengeCookie accepts at most one TTL argument, got %d", len(ttl))
}
if ttl[0] == "" {
return nil, nil
}
d, err := time.ParseDuration(ttl[0])
if err != nil {
return nil, fmt.Errorf("invalid GrantChallengeCookie TTL %q: %w", ttl[0], err)
}
if d <= 0 {
return nil, fmt.Errorf("GrantChallengeCookie TTL must be positive, got %s", d)
}
return &d, nil
}
func GetOnLoadEnv(w *AppsecRuntimeConfig) map[string]interface{} {
return map[string]interface{}{
"RemoveInBandRuleByID": w.DisableInBandRuleByID,
@@ -17,6 +90,7 @@ func GetOnLoadEnv(w *AppsecRuntimeConfig) map[string]interface{} {
"SetRemediationByTag": w.SetActionByTag,
"SetRemediationByID": w.SetActionByID,
"SetRemediationByName": w.SetActionByName,
"SetChallengeDifficulty": w.SetChallengeDifficulty,
"LoadAPISchemaWithName": w.LoadAPISchemaWithName,
"LoadAPISchemaWithOptions": w.LoadAPISchemaWithOptions,
"RegisterAPISchemaBodyDecoder": w.RegisterAPISchemaBodyDecoder,
@@ -38,6 +112,8 @@ func GetPreEvalEnv(ctx context.Context, w *AppsecRuntimeConfig, state *AppsecReq
"RemoveOutBandRuleByTag": func(tag string) error { return w.RemoveOutbandRuleByTag(state, tag) },
"RemoveOutBandRuleByName": func(name string) error { return w.RemoveOutbandRuleByName(state, name) },
"DropRequest": func(reason string) error { return w.DropRequest(state, request, reason) },
"SetChallengeBody": func(body string) error { return w.SetChallengeBody(state, body) },
"SetChallengeCookie": func(cookie cookie.AppsecCookie) error { return w.SetChallengeCookie(state, cookie) },
"SetRemediationByTag": w.SetActionByTag,
"SetRemediationByID": w.SetActionByID,
"SetRemediationByName": w.SetActionByName,
@@ -49,36 +125,190 @@ func GetPreEvalEnv(ctx context.Context, w *AppsecRuntimeConfig, state *AppsecReq
state.PendingHTTPCode = &code
return nil
},
"AppsecCookie": cookie.NewAppsecCookie,
// SendChallenge is intentionally not exposed in pre_eval: a challenge must
// only be issued after in-band WAF evaluation. Referencing it here fails to
// compile at config load.
"SetChallengeDifficulty": func(level string) error {
return w.SetChallengeDifficultyPerRequest(state, level)
},
"GrantChallengeCookie": func(reason string, ttl ...string) error {
ttlOverride, err := parseChallengeCookieTTLArg(ttl)
if err != nil {
return err
}
return w.GrantChallengeCookie(state, request, reason, ttlOverride)
},
"fingerprint": state.Fingerprint,
"ValidateRequestWithSchema": func(ref string) bool {
return w.ValidateRequestWithSchema(ctx, state, request, ref)
},
"DisableBodyInspection": func() error { return w.DisableBodyInspection(state) },
"IsLegitimateBot": isLegitimateBotHelper(state),
"ExemptFromChallenge": func() error { state.ChallengeExempt = true; return nil },
}
}
func GetPostEvalEnv(w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest) map[string]interface{} {
func GetPostEvalEnv(ctx context.Context, w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest) map[string]interface{} {
return map[string]interface{}{
"IsInBand": request.IsInBand,
"IsOutBand": request.IsOutBand,
"DumpRequest": request.DumpRequest,
"req": request.HTTPRequest,
"hook_vars": state.HookVars,
"SendChallenge": func() error {
return w.SendChallenge(ctx, state, request)
},
"SetChallengeDifficulty": func(level string) error {
return w.SetChallengeDifficultyPerRequest(state, level)
},
"GrantChallengeCookie": func(reason string, ttl ...string) error {
ttlOverride, err := parseChallengeCookieTTLArg(ttl)
if err != nil {
return err
}
return w.GrantChallengeCookie(state, request, reason, ttlOverride)
},
"DumpFingerprint": func(label string) string {
return DumpFingerprint(w.FingerprintDumpDir, label, state.Fingerprint, request)
},
"fingerprint": state.Fingerprint,
"hook_vars": state.HookVars,
"IsLegitimateBot": isLegitimateBotHelper(state),
"ExemptFromChallenge": func() error { state.ChallengeExempt = true; return nil },
}
}
func GetOnChallengeEnv(ctx context.Context, w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest) map[string]interface{} {
return map[string]interface{}{
"req": request.HTTPRequest,
"IsInBand": request.IsInBand,
"fingerprint": state.Fingerprint,
"SendChallenge": func() error {
return w.SendChallenge(ctx, state, request)
},
"SetRemediation": func(action string) error {
state.PendingAction = &action
return nil
},
"SetReturnCode": func(code int) error {
state.PendingHTTPCode = &code
return nil
},
"DropRequest": func(reason string) error { return w.DropRequest(state, request, reason) },
"SetChallengeDifficulty": func(level string) error {
return w.SetChallengeDifficultyPerRequest(state, level)
},
// EvaluateMismatches: aggregate fingerprint mismatch report. First
// call per request computes the report, caches it on state, and
// emits one Debug log line + per-signal Prometheus counter bumps;
// subsequent calls return the cached pointer so rules can chain
// `.High() >= 1 && .Has("cdp")` without redoing the work.
//
// Atomic helpers (fingerprint.UAMobileMismatch,
// fingerprint.AcceptLanguageMismatch(req),
// fingerprint.TimezoneCountryMismatch(country)) are methods on
// `fingerprint` and can still be called directly from rules.
"EvaluateMismatches": func() *challenge.MismatchReport {
return w.EvaluateMismatches(state, request)
},
}
}
// GetOnChallengeSubmitEnv is the env exposed to on_challenge_submit hooks.
// Deliberately narrow: the hook fires once during the challenge submission
// JSON response, so anything that would change the response shape
// (SendChallenge, SetRemediation, SetReturnCode, SetChallengeDifficulty,
// DropRequest) is intentionally omitted to avoid breaking the client-side
// JS handler. Operators wanting to escalate or block at the next request
// should do so via pre_eval.
func GetOnChallengeSubmitEnv(w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest) map[string]interface{} {
return map[string]interface{}{
"req": request.HTTPRequest,
"IsInBand": request.IsInBand,
"fingerprint": state.Fingerprint,
"RejectSubmission": func(reason string, verbosity ...string) error {
if err := w.RejectSubmission(state, reason); err != nil {
return err
}
// Emit the reject log immediately with the operator-chosen
// verbosity. state.Fingerprint is populated at submit time
// (set just before processHooks runs); LogRejected is
// nil-safe so an unexpected nil here is a no-op rather than
// a panic.
state.Fingerprint.LogRejected(
w.Logger,
log.InfoLevel,
request.ClientIP,
request.RemoteAddrNormalized,
state.SubmissionRejection.Reason,
"on_challenge_submit rejected",
parseLogVerbosity(w.Logger, verbosity),
)
// Terminal: halt later on_challenge_submit rules so a `filter: "true"`
// `LogAccepted` can't fire for an already-rejected submission.
state.HooksHalted = true
return nil
},
// LogAccepted is intentionally exposed only here: per-request
// cookie-validation acceptance is logged at Debug from the
// internal ProcessOnChallengeRules path, so the only operator-
// authored accept point is on a real challenge submission.
"LogAccepted": func(msg string, verbosity ...string) error {
state.Fingerprint.LogAccepted(
w.Logger,
log.InfoLevel,
request.ClientIP,
request.RemoteAddrNormalized,
msg,
parseLogVerbosity(w.Logger, verbosity),
)
return nil
},
// In on_challenge_submit the response is the challenge-submit JSON
// envelope the client is already awaiting; a 307 redirect would
// break its state machine. Route to the inline variant that
// attaches the cookie to the existing envelope.
"GrantChallengeCookie": func(reason string, ttl ...string) error {
ttlOverride, err := parseChallengeCookieTTLArg(ttl)
if err != nil {
return err
}
if err := w.GrantAllowlistCookieInline(state, request, reason, ttlOverride); err != nil {
return err
}
// Terminal: halt later rules — the grant is final, and a following
// rule could only overwrite the synthetic allowlist fingerprint.
state.HooksHalted = true
return nil
},
"EvaluateMismatches": func() *challenge.MismatchReport {
return w.EvaluateMismatches(state, request)
},
"DumpFingerprint": func(label string) string {
return DumpFingerprint(w.FingerprintDumpDir, label, state.Fingerprint, request)
},
}
}
func GetOnMatchEnv(w *AppsecRuntimeConfig, state *AppsecRequestState, request *ParsedRequest, evt pipeline.Event) map[string]interface{} {
return map[string]interface{}{
"evt": evt,
"req": request.HTTPRequest,
"hook_vars": state.HookVars,
"IsInBand": request.IsInBand,
"IsOutBand": request.IsOutBand,
"SetRemediation": func(action string) error { return w.SetAction(state, action) },
"SetReturnCode": func(code int) error { return w.SetHTTPCode(state, code) },
"CancelEvent": func() error { return w.CancelEvent(state) },
"SendEvent": func() error { return w.SendEvent(state) },
"CancelAlert": func() error { return w.CancelAlert(state) },
"SendAlert": func() error { return w.SendAlert(state) },
"DumpRequest": request.DumpRequest,
"evt": evt,
"req": request.HTTPRequest,
"hook_vars": state.HookVars,
"IsInBand": request.IsInBand,
"IsOutBand": request.IsOutBand,
"SetRemediation": func(action string) error { return w.SetAction(state, action) },
"SetReturnCode": func(code int) error { return w.SetHTTPCode(state, code) },
"CancelEvent": func() error { return w.CancelEvent(state) },
"SendEvent": func() error { return w.SendEvent(state) },
"CancelAlert": func() error { return w.CancelAlert(state) },
"SendAlert": func() error { return w.SendAlert(state) },
"DumpRequest": request.DumpRequest,
"SetChallengeBody": func(body string) error { return w.SetChallengeBody(state, body) },
"SetChallengeCookie": func(cookie cookie.AppsecCookie) error { return w.SetChallengeCookie(state, cookie) },
"AppsecCookie": cookie.NewAppsecCookie,
"IsLegitimateBot": isLegitimateBotHelper(state),
"ExemptFromChallenge": func() error { state.ChallengeExempt = true; return nil },
}
}
+86
View File
@@ -0,0 +1,86 @@
package appsec
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestLegitimateBotHooksCompile guards the env maps: a hook referencing
// IsLegitimateBot/ExemptFromChallenge must compile in every phase exposing them.
func TestLegitimateBotHooksCompile(t *testing.T) {
for _, stage := range []hookStage{hookPreEval, hookPostEval, hookOnMatch} {
h := &Hook{
Filter: `IsLegitimateBot(req.RemoteAddr, req.UserAgent(), req.URL.Path)`,
Apply: []string{`ExemptFromChallenge()`},
}
require.NoError(t, h.Build(t.Context(), stage, nil), "stage %v", stage)
}
}
// TestExemptFromChallengeEscapeHatch verifies the per-request escape hatch:
// once ExemptFromChallenge was called, IsLegitimateBot returns true without
// consulting datafiles (none are loaded here) or DNS.
func TestExemptFromChallengeEscapeHatch(t *testing.T) {
state := &AppsecRequestState{HookVars: map[string]string{}}
env := GetPreEvalEnv(t.Context(), &AppsecRuntimeConfig{}, state, &ParsedRequest{})
isLegit := env["IsLegitimateBot"].(func(string, string, string) bool)
exempt := env["ExemptFromChallenge"].(func() error)
assert.False(t, isLegit("1.2.3.4", "googlebot", "/"))
require.NoError(t, exempt())
assert.True(t, isLegit("1.2.3.4", "googlebot", "/"))
assert.True(t, isLegit("garbage-ip", "", ""), "escape hatch bypasses all checks")
// the flag is per-request state: a fresh state starts clean
freshEnv := GetPreEvalEnv(t.Context(), &AppsecRuntimeConfig{}, &AppsecRequestState{HookVars: map[string]string{}}, &ParsedRequest{})
assert.False(t, freshEnv["IsLegitimateBot"].(func(string, string, string) bool)("1.2.3.4", "googlebot", "/"))
}
// TestParseChallengeCookieTTLArg covers the GrantChallengeCookie optional TTL
// argument parsing: no args / empty string yield a nil override (use runtime
// default), a parseable duration yields a positive pointer, and malformed /
// non-positive / multi-arg inputs surface as errors so hook authors see a
// precise diagnostic instead of silent fallback.
func TestParseChallengeCookieTTLArg(t *testing.T) {
t.Run("no args → nil override", func(t *testing.T) {
got, err := parseChallengeCookieTTLArg(nil)
require.NoError(t, err)
assert.Nil(t, got)
})
t.Run("empty string → nil override", func(t *testing.T) {
got, err := parseChallengeCookieTTLArg([]string{""})
require.NoError(t, err)
assert.Nil(t, got)
})
t.Run("valid duration", func(t *testing.T) {
got, err := parseChallengeCookieTTLArg([]string{"1h30m"})
require.NoError(t, err)
require.NotNil(t, got)
assert.Equal(t, 90*time.Minute, *got)
})
t.Run("malformed duration", func(t *testing.T) {
_, err := parseChallengeCookieTTLArg([]string{"forever"})
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid GrantChallengeCookie TTL")
})
t.Run("non-positive duration", func(t *testing.T) {
_, err := parseChallengeCookieTTLArg([]string{"-5m"})
require.Error(t, err)
assert.Contains(t, err.Error(), "must be positive")
})
t.Run("multiple TTL args rejected", func(t *testing.T) {
_, err := parseChallengeCookieTTLArg([]string{"1h", "2h"})
require.Error(t, err)
assert.Contains(t, err.Error(), "at most one TTL argument")
})
}
+9
View File
@@ -6,6 +6,7 @@ import (
"io/fs"
"os"
"path/filepath"
"time"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
@@ -26,11 +27,19 @@ type CrowdsecServiceCfg struct {
BucketStateFile string `yaml:"state_input_file,omitempty"` // if we need to unserialize buckets at start
BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode
DNSCache *DNSCacheCfg `yaml:"dns_cache,omitempty"`
SimulationFilePath string `yaml:"-"`
ContextToSend map[string][]string `yaml:"-"`
}
// Cache config for DNS lookups (legit bots, rdns PO)
type DNSCacheCfg struct {
TTL *time.Duration `yaml:"ttl,omitempty"`
NegativeTTL *time.Duration `yaml:"negative_ttl,omitempty"`
Size *int `yaml:"size,omitempty"`
}
var ErrNoAcquisitionDefined = errors.New("no acquisition_path or acquisition_dir specified")
func (c *CrowdsecServiceCfg) CollectAcquisitionFiles() ([]string, error) {
+29
View File
@@ -3,8 +3,11 @@ package csconfig
import (
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
@@ -195,3 +198,29 @@ func TestLoadCrowdsec(t *testing.T) {
})
}
}
func TestDNSCacheCfg(t *testing.T) {
yamlConfig := `
acquisition_path: ./testdata/acquis.yaml
dns_cache:
ttl: 2h
negative_ttl: 30s
size: 4096
`
cfg := CrowdsecServiceCfg{}
require.NoError(t, yaml.Unmarshal([]byte(yamlConfig), &cfg))
require.NotNil(t, cfg.DNSCache)
require.NotNil(t, cfg.DNSCache.TTL)
assert.Equal(t, 2*time.Hour, *cfg.DNSCache.TTL)
require.NotNil(t, cfg.DNSCache.NegativeTTL)
assert.Equal(t, 30*time.Second, *cfg.DNSCache.NegativeTTL)
require.NotNil(t, cfg.DNSCache.Size)
assert.Equal(t, 4096, *cfg.DNSCache.Size)
// the section is optional
bare := CrowdsecServiceCfg{}
require.NoError(t, yaml.Unmarshal([]byte("acquisition_path: ./testdata/acquis.yaml"), &bare))
assert.Nil(t, bare.DNSCache)
}
+5
View File
@@ -26,7 +26,12 @@ type Hub struct {
}
// GetDataDir returns the data directory, where data sets are installed.
// Empty for a hub created without local configuration (e.g. zero-value Hub in tests).
func (h *Hub) GetDataDir() string {
if h.local == nil {
return ""
}
return h.local.InstallDataDir
}
+172
View File
@@ -0,0 +1,172 @@
// Package dnscache provides process-wide cached DNS lookups: PTR records,
// forward (A/AAAA) records, and forward-confirmed reverse DNS (FCrDNS)
// composed from the two. Results — including failures — are cached in a
// shared LRU, and concurrent lookups for the same key coalesce via
// singleflight, so a given DNS fact costs at most one query per TTL.
//
// Consumers: the parser's reverse_dns enricher (PTR only) and the
// IsLegitimateBot appsec helper (FCrDNS).
package dnscache
import (
"context"
"net"
"net/netip"
"slices"
"strings"
"sync"
"time"
"github.com/bluele/gcache"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/singleflight"
)
// Resolver is satisfied by *net.Resolver; swapped for a fake in tests.
type Resolver interface {
LookupAddr(ctx context.Context, addr string) ([]string, error)
LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
}
var resolver Resolver = net.DefaultResolver
func SetResolver(r Resolver) {
resolver = r
}
const (
lookupTimeout = 3 * time.Second // per lookup (PTR or forward)
maxPTRNames = 3 // PTR fan-out cap; legit hosts return 1
defaultPositiveTTL = 1 * time.Hour
defaultNegativeTTL = 5 * time.Minute
defaultCacheSize = 16384
)
var (
positiveTTL = defaultPositiveTTL
negativeTTL = defaultNegativeTTL
cacheSize = defaultCacheSize
cacheOnce sync.Once
cache gcache.Cache
sf singleflight.Group
)
func Configure(posTTL time.Duration, negTTL time.Duration, size int) {
if posTTL > 0 {
positiveTTL = posTTL
}
if negTTL > 0 {
negativeTTL = negTTL
}
if size > 0 {
cacheSize = size
}
}
func Purge() {
if cache != nil {
cache.Purge()
}
}
func cachedLookup[T any](key string, lookup func() T, empty func(T) bool) T {
cacheOnce.Do(func() {
cache = gcache.New(cacheSize).LRU().Build()
})
if val, err := cache.Get(key); err == nil {
return val.(T)
}
val, _, _ := sf.Do(key, func() (any, error) {
// Re-check the cache inside the singleflight callback in case
// another goroutine populated it between the fast-path read and
// our entry into Do.
if val, err := cache.Get(key); err == nil {
return val.(T), nil
}
result := lookup()
ttl := positiveTTL
if empty(result) {
ttl = negativeTTL
}
if err := cache.SetWithExpire(key, result, ttl); err != nil {
log.Debugf("failed to cache DNS result for %s: %s", key, err)
}
return result, nil
})
return val.(T)
}
func PTRRecords(addr netip.Addr) []string {
return cachedLookup("ptr:"+addr.String(), func() []string {
return lookupPTR(addr)
}, func(names []string) bool { return len(names) == 0 })
}
func ForwardIPs(host string) []netip.Addr {
host = strings.TrimSuffix(strings.ToLower(host), ".")
return cachedLookup("fwd:"+host, func() []netip.Addr {
return lookupForward(host)
}, func(ips []netip.Addr) bool { return len(ips) == 0 })
}
func ForwardConfirmedNames(addr netip.Addr) []string {
verified := []string{}
names := PTRRecords(addr)
for _, name := range names[:min(len(names), maxPTRNames)] {
name = strings.TrimSuffix(strings.ToLower(name), ".")
if slices.Contains(ForwardIPs(name), addr.Unmap()) {
verified = append(verified, name)
}
}
return verified
}
func lookupPTR(addr netip.Addr) []string {
ctx, cancel := context.WithTimeout(context.Background(), lookupTimeout)
defer cancel()
names, err := resolver.LookupAddr(ctx, addr.String())
if err != nil {
log.Debugf("dnscache: PTR lookup failed for %s: %s", addr, err)
return []string{}
}
return names
}
func lookupForward(host string) []netip.Addr {
ctx, cancel := context.WithTimeout(context.Background(), lookupTimeout)
defer cancel()
ips, err := resolver.LookupIPAddr(ctx, host)
if err != nil {
log.Debugf("dnscache: forward lookup failed for %s: %s", host, err)
return []netip.Addr{}
}
addrs := []netip.Addr{}
for _, ip := range ips {
if addr, ok := netip.AddrFromSlice(ip.IP); ok {
addrs = append(addrs, addr.Unmap())
}
}
return addrs
}
+213
View File
@@ -0,0 +1,213 @@
package dnscache
import (
"context"
"net"
"net/netip"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeResolver implements Resolver from in-memory maps so no test ever hits
// real DNS. ptrCalls/fwdCalls let tests assert how many lookups ran.
type fakeResolver struct {
ptr map[string][]string // ip → PTR names
fwd map[string][]net.IPAddr // hostname → addresses
delay time.Duration
ptrCalls atomic.Int32
fwdCalls atomic.Int32
}
func (f *fakeResolver) LookupAddr(_ context.Context, addr string) ([]string, error) {
f.ptrCalls.Add(1)
time.Sleep(f.delay)
names, ok := f.ptr[addr]
if !ok {
return nil, &net.DNSError{Err: "no such host", Name: addr, IsNotFound: true}
}
return names, nil
}
func (f *fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) {
f.fwdCalls.Add(1)
ips, ok := f.fwd[host]
if !ok {
return nil, &net.DNSError{Err: "no such host", Name: host, IsNotFound: true}
}
return ips, nil
}
// setupDNSTest installs the fake resolver and guarantees an empty cache,
// restoring the default resolver when the test ends.
func setupDNSTest(t *testing.T, r *fakeResolver) {
t.Helper()
SetResolver(r)
Purge()
t.Cleanup(func() {
SetResolver(net.DefaultResolver)
Purge()
})
}
func TestPTRRecordsRawNames(t *testing.T) {
resolver := &fakeResolver{
ptr: map[string][]string{"1.1.1.1": {"One.One.one.one."}},
}
setupDNSTest(t, resolver)
// names come back raw: trailing dot and case preserved (the reverse_dns
// parser enricher exposes them as-is)
assert.Equal(t, []string{"One.One.one.one."}, PTRRecords(netip.MustParseAddr("1.1.1.1")))
// failures return an empty slice, and are cached too
assert.Empty(t, PTRRecords(netip.MustParseAddr("203.0.113.1")))
assert.Empty(t, PTRRecords(netip.MustParseAddr("203.0.113.1")))
assert.Equal(t, int32(2), resolver.ptrCalls.Load(), "failed lookup must be cached as negative")
}
func TestForwardIPsKeyNormalization(t *testing.T) {
resolver := &fakeResolver{
fwd: map[string][]net.IPAddr{"host.example.com": {{IP: net.ParseIP("192.0.2.1")}}},
}
setupDNSTest(t, resolver)
want := []netip.Addr{netip.MustParseAddr("192.0.2.1")}
// trailing dot and case variants hit the same cache entry
assert.Equal(t, want, ForwardIPs("Host.Example.COM."))
assert.Equal(t, want, ForwardIPs("host.example.com"))
assert.Equal(t, int32(1), resolver.fwdCalls.Load(), "normalized variants must share one cache entry")
}
func TestForwardConfirmedNames(t *testing.T) {
tests := []struct {
name string
resolver *fakeResolver
ip string
expected []string
}{
{
name: "happy path, trailing dot and mixed case PTR",
resolver: &fakeResolver{
ptr: map[string][]string{"66.249.66.1": {"Crawl-66-249-66-1.GoogleBot.com."}},
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
},
ip: "66.249.66.1",
expected: []string{"crawl-66-249-66-1.googlebot.com"},
},
{
name: "spoofed PTR, forward lookup does not confirm",
resolver: &fakeResolver{
ptr: map[string][]string{"203.0.113.5": {"crawl.googlebot.com."}},
fwd: map[string][]net.IPAddr{"crawl.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
},
ip: "203.0.113.5",
expected: []string{},
},
{
name: "no PTR record",
resolver: &fakeResolver{},
ip: "203.0.113.7",
expected: []string{},
},
{
name: "PTR fan-out capped, extra names ignored",
resolver: &fakeResolver{
ptr: map[string][]string{"203.0.113.8": {"a.example.com.", "b.example.com.", "c.example.com.", "d.example.com."}},
fwd: map[string][]net.IPAddr{"d.example.com": {{IP: net.ParseIP("203.0.113.8")}}},
},
ip: "203.0.113.8",
expected: []string{},
},
{
name: "IPv4-mapped forward answer still confirms",
resolver: &fakeResolver{
ptr: map[string][]string{"192.0.2.9": {"mapped.example.com."}},
fwd: map[string][]net.IPAddr{"mapped.example.com": {{IP: net.ParseIP("192.0.2.9").To16()}}},
},
ip: "192.0.2.9",
expected: []string{"mapped.example.com"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
setupDNSTest(t, tc.resolver)
assert.Equal(t, tc.expected, ForwardConfirmedNames(netip.MustParseAddr(tc.ip)))
})
}
}
func TestForwardConfirmedNamesSharesPTRCache(t *testing.T) {
resolver := &fakeResolver{
ptr: map[string][]string{"66.249.66.1": {"crawl-66-249-66-1.googlebot.com."}},
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
}
setupDNSTest(t, resolver)
addr := netip.MustParseAddr("66.249.66.1")
// a plain PTR consumer warms the cache for the FCrDNS consumer
require.NotEmpty(t, PTRRecords(addr))
assert.Equal(t, []string{"crawl-66-249-66-1.googlebot.com"}, ForwardConfirmedNames(addr))
assert.Equal(t, int32(1), resolver.ptrCalls.Load(), "FCrDNS must reuse the cached PTR entry")
// and the composed result is fully cached on repeat
assert.Equal(t, []string{"crawl-66-249-66-1.googlebot.com"}, ForwardConfirmedNames(addr))
assert.Equal(t, int32(1), resolver.ptrCalls.Load())
assert.Equal(t, int32(1), resolver.fwdCalls.Load())
}
func TestSingleflight(t *testing.T) {
resolver := &fakeResolver{
ptr: map[string][]string{"66.249.66.1": {"crawl-66-249-66-1.googlebot.com."}},
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
delay: 50 * time.Millisecond,
}
setupDNSTest(t, resolver)
addr := netip.MustParseAddr("66.249.66.1")
var wg sync.WaitGroup
for range 10 {
wg.Go(func() {
assert.Equal(t, []string{"crawl-66-249-66-1.googlebot.com"}, ForwardConfirmedNames(addr))
})
}
wg.Wait()
assert.Equal(t, int32(1), resolver.ptrCalls.Load(), "concurrent cold lookups must coalesce")
}
func TestConfigureZeroValuesKeepDefaults(t *testing.T) {
t.Cleanup(func() {
positiveTTL = defaultPositiveTTL
negativeTTL = defaultNegativeTTL
cacheSize = defaultCacheSize
})
Configure(0, 0, 0)
assert.Equal(t, defaultPositiveTTL, positiveTTL)
assert.Equal(t, defaultNegativeTTL, negativeTTL)
assert.Equal(t, defaultCacheSize, cacheSize)
Configure(2*time.Hour, 0, 0)
assert.Equal(t, 2*time.Hour, positiveTTL)
assert.Equal(t, defaultNegativeTTL, negativeTTL)
assert.Equal(t, defaultCacheSize, cacheSize)
Configure(0, 10*time.Minute, 4096)
assert.Equal(t, 2*time.Hour, positiveTTL)
assert.Equal(t, 10*time.Minute, negativeTTL)
assert.Equal(t, 4096, cacheSize)
}
+215
View File
@@ -0,0 +1,215 @@
package exprhelpers
import (
"encoding/json"
"fmt"
"net"
"net/netip"
"regexp"
"strings"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/dnscache"
)
var dataFileBots map[string][]*botEntry
// botEntry is one bot definition (one JSON line in a "bots" data file).
// Matching semantics: (UA && PATH) && (IP || RANGE || RDNS) — the optional
// user_agent and paths regexes are preconditions, then the source must be
// verified by at least one identity check (exact IP, CIDR range, or
// forward-confirmed reverse DNS). UA-only entries are rejected at load time:
// a User-Agent is trivially spoofable.
//
// rdns patterns are regexes matched against the FCrDNS-verified names
// (lowercase, no trailing dot). Anchor them — `(^|\.)googlebot\.com$` —
// or "evilgooglebot.com.attacker.net" matches `googlebot\.com` too.
type botEntry struct {
Name string `json:"name"`
UserAgent string `json:"user_agent,omitempty"`
Paths []string `json:"paths,omitempty"`
IPs []string `json:"ips,omitempty"`
Ranges []string `json:"ranges,omitempty"`
RDNS []string `json:"rdns,omitempty"`
// compiled at load time
uaRegex *regexp.Regexp
pathRegexes []*regexp.Regexp
ipSet map[netip.Addr]struct{}
prefixes []netip.Prefix
rdnsRegexes []*regexp.Regexp
}
func compileBotRegex(pattern string) (*regexp.Regexp, error) {
return regexp.Compile("(?i)" + pattern) // Force case insensitive match
}
func botFileInit(filename string, line string) error {
entry := &botEntry{}
dec := json.NewDecoder(strings.NewReader(line))
dec.DisallowUnknownFields()
if err := dec.Decode(entry); err != nil {
return fmt.Errorf("failed to parse JSON line in %s: %w", filename, err)
}
if entry.Name == "" {
return fmt.Errorf("missing mandatory 'name' field in %s: %s", filename, line)
}
if len(entry.IPs)+len(entry.Ranges)+len(entry.RDNS) == 0 {
return fmt.Errorf("bot entry '%s' in %s has no identity verification (need at least one of ips/ranges/rdns)", entry.Name, filename)
}
var err error
if entry.UserAgent != "" {
if entry.uaRegex, err = compileBotRegex(entry.UserAgent); err != nil {
return fmt.Errorf("invalid user_agent regex for bot entry '%s' in %s: %w", entry.Name, filename, err)
}
}
for _, p := range entry.Paths {
re, err := compileBotRegex(p)
if err != nil {
return fmt.Errorf("invalid path regex '%s' for bot entry '%s' in %s: %w", p, entry.Name, filename, err)
}
entry.pathRegexes = append(entry.pathRegexes, re)
}
entry.ipSet = make(map[netip.Addr]struct{}, len(entry.IPs))
for _, ip := range entry.IPs {
addr, err := netip.ParseAddr(ip)
if err != nil {
return fmt.Errorf("invalid IP '%s' for bot entry '%s' in %s: %w", ip, entry.Name, filename, err)
}
entry.ipSet[addr.Unmap()] = struct{}{}
}
for _, r := range entry.Ranges {
prefix, err := netip.ParsePrefix(r)
if err != nil {
return fmt.Errorf("invalid CIDR range '%s' for bot entry '%s' in %s: %w", r, entry.Name, filename, err)
}
entry.prefixes = append(entry.prefixes, prefix.Masked())
}
for _, p := range entry.RDNS {
// an empty pattern matches every PTR-confirmed host: almost
// certainly a mistake, reject it
if p == "" {
return fmt.Errorf("empty rdns pattern for bot entry '%s' in %s", entry.Name, filename)
}
re, err := compileBotRegex(p)
if err != nil {
return fmt.Errorf("invalid rdns regex '%s' for bot entry '%s' in %s: %w", p, entry.Name, filename, err)
}
entry.rdnsRegexes = append(entry.rdnsRegexes, re)
}
dataFileBots[filename] = append(dataFileBots[filename], entry)
return nil
}
// parseBotAddr normalizes a source address as found in HTTP contexts:
// bare IP, "ip:port", "[v6]:port". The zone is stripped and IPv4-mapped
// IPv6 is unmapped so comparisons against load-time-parsed IPs/ranges are
// consistent.
func parseBotAddr(s string) (netip.Addr, bool) {
addr, err := netip.ParseAddr(s)
if err != nil {
host, _, splitErr := net.SplitHostPort(s)
if splitErr != nil {
return netip.Addr{}, false
}
if addr, err = netip.ParseAddr(host); err != nil {
return netip.Addr{}, false
}
}
return addr.WithZone("").Unmap(), true
}
// IsLegitimateBot reports whether the request (source address, User-Agent,
// path) matches a bot definition from the loaded "bots" data files:
// (UA && PATH) && (IP || RANGE || RDNS). Fail-closed: an unparseable
// address or a DNS failure means "not a legitimate bot", never an error.
//
// The expensive FCrDNS resolution runs at most once per call — after the
// cheap checks, against all candidate entries at once — and is cached per IP.
func IsLegitimateBot(ip string, ua string, path string) bool {
if len(dataFileBots) == 0 {
return false
}
addr, ok := parseBotAddr(ip)
if !ok {
log.Debugf("IsLegitimateBot: invalid source address '%s'", ip)
return false
}
var rdnsCandidates []*botEntry
for _, entries := range dataFileBots {
for _, entry := range entries {
if entry.uaRegex != nil && !entry.uaRegex.MatchString(ua) {
continue
}
if len(entry.pathRegexes) > 0 && !matchAnyRegex(entry.pathRegexes, path) {
continue
}
if _, found := entry.ipSet[addr]; found {
log.Debugf("IsLegitimateBot: %s verified as '%s' via exact IP", ip, entry.Name)
return true
}
for _, prefix := range entry.prefixes {
if prefix.Contains(addr) {
log.Debugf("IsLegitimateBot: %s verified as '%s' via range %s", ip, entry.Name, prefix)
return true
}
}
if len(entry.rdnsRegexes) > 0 {
rdnsCandidates = append(rdnsCandidates, entry)
}
}
}
if len(rdnsCandidates) == 0 {
return false
}
for _, name := range dnscache.ForwardConfirmedNames(addr) {
for _, entry := range rdnsCandidates {
if matchAnyRegex(entry.rdnsRegexes, name) {
log.Debugf("IsLegitimateBot: %s verified as '%s' via FCrDNS (%s)", ip, entry.Name, name)
return true
}
}
}
return false
}
func matchAnyRegex(regexes []*regexp.Regexp, s string) bool {
for _, re := range regexes {
if re.MatchString(s) {
return true
}
}
return false
}
+336
View File
@@ -0,0 +1,336 @@
package exprhelpers
import (
"context"
"net"
"os"
"path/filepath"
"sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/dnscache"
)
// fakeResolver implements dnscache.Resolver from in-memory maps so no test
// ever hits real DNS. ptrCalls lets tests assert how many lookups ran.
// (The cache/singleflight internals are tested in pkg/dnscache; here the
// fake only backs the end-to-end IsLegitimateBot checks.)
type fakeResolver struct {
ptr map[string][]string // ip → PTR names
fwd map[string][]net.IPAddr // hostname → addresses
ptrCalls atomic.Int32
}
func (f *fakeResolver) LookupAddr(_ context.Context, addr string) ([]string, error) {
f.ptrCalls.Add(1)
names, ok := f.ptr[addr]
if !ok {
return nil, &net.DNSError{Err: "no such host", Name: addr, IsNotFound: true}
}
return names, nil
}
func (f *fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) {
ips, ok := f.fwd[host]
if !ok {
return nil, &net.DNSError{Err: "no such host", Name: host, IsNotFound: true}
}
return ips, nil
}
// setupBotTest resets the datafile globals and, when a fake resolver is
// given, installs it in pkg/dnscache with an empty cache for isolation.
func setupBotTest(t *testing.T, resolver *fakeResolver) {
t.Helper()
err := Init(nil)
require.NoError(t, err)
if resolver != nil {
dnscache.SetResolver(resolver)
dnscache.Purge()
t.Cleanup(func() {
dnscache.SetResolver(net.DefaultResolver)
dnscache.Purge()
})
}
}
func TestBotFileInit(t *testing.T) {
setupBotTest(t, nil)
err := FileInit("testdata", "test_data_bots.json", "bots")
require.NoError(t, err)
entries, ok := dataFileBots["test_data_bots.json"]
require.True(t, ok, "test_data_bots.json should be loaded")
require.Len(t, entries, 4, "should have 4 entries (skipping comment and empty line)")
googlebot := entries[0]
assert.Equal(t, "googlebot", googlebot.Name)
assert.NotNil(t, googlebot.uaRegex)
assert.Empty(t, googlebot.pathRegexes)
assert.Len(t, googlebot.rdnsRegexes, 2)
uptimerobot := entries[1]
assert.Len(t, uptimerobot.pathRegexes, 2)
assert.Len(t, uptimerobot.prefixes, 2)
assert.Len(t, uptimerobot.ipSet, 1)
// double load is a no-op, not an error
err = FileInit("testdata", "test_data_bots.json", "bots")
require.NoError(t, err)
assert.Len(t, dataFileBots["test_data_bots.json"], 4)
}
func TestBotFileInitErrors(t *testing.T) {
tests := []struct {
name string
filename string
expectedErr string
}{
{"invalid JSON", "test_data_bots_invalid.json", "failed to parse JSON line"},
{"missing name", "test_data_bots_no_name.json", "missing mandatory 'name' field"},
{"no identity check", "test_data_bots_no_identity.json", "no identity verification"},
{"bad user_agent regex", "test_data_bots_bad_regex.json", "invalid user_agent regex"},
{"bad rdns regex", "test_data_bots_bad_rdns.json", "invalid rdns regex"},
{"bad CIDR", "test_data_bots_bad_cidr.json", "invalid CIDR range"},
{"bad IP", "test_data_bots_bad_ip.json", "invalid IP"},
{"unknown field", "test_data_bots_unknown_field.json", "unknown field"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
setupBotTest(t, nil)
err := FileInit("testdata", tc.filename, "bots")
require.Error(t, err)
assert.Contains(t, err.Error(), tc.expectedErr)
})
}
}
func TestBotFileUnknownType(t *testing.T) {
setupBotTest(t, nil)
// singular "bot" is a typo, must be rejected by existsInFileMaps
err := FileInit("testdata", "test_data_bots.json", "bot")
require.Error(t, err)
assert.Contains(t, err.Error(), "unknown data type")
}
func TestIsLegitimateBotNoDNS(t *testing.T) {
resolver := &fakeResolver{}
setupBotTest(t, resolver)
err := FileInit("testdata", "test_data_bots.json", "bots")
require.NoError(t, err)
tests := []struct {
name string
ip string
ua string
path string
expected bool
}{
{"exact IP", "10.1.2.3", "", "/", true},
{"exact IPv6", "2001:db8::42", "", "/", true},
{"exact IP with port", "10.1.2.3:54321", "", "/", true},
{"IPv4-mapped IPv6", "::ffff:10.1.2.3", "", "/", true},
{"range hit", "192.0.2.77", "anything", "/any/path", true},
{"range miss", "192.0.3.77", "anything", "/any/path", false},
{"UA+path+range", "69.162.124.230", "UptimeRobot/2.0", "/health/db", true},
{"UA+path+IP", "216.144.250.150", "uptimerobot", "/status", true},
{"UA+path+v6 range with port", "[2607:ff68:107::25]:443", "UptimeRobot/2.0", "/health", true},
{"UA case-insensitive", "69.162.124.230", "UPTIMEROBOT", "/health", true},
{"path case-insensitive", "69.162.124.230", "uptimerobot", "/HEALTH", true},
{"path mismatch", "69.162.124.230", "uptimerobot", "/index.html", false},
{"UA mismatch", "69.162.124.230", "definitely-not-it", "/health", false},
{"empty UA against UA filter", "69.162.124.230", "", "/health", false},
{"garbage IP", "not-an-ip", "uptimerobot", "/health", false},
{"empty IP", "", "uptimerobot", "/health", false},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.expected, IsLegitimateBot(tc.ip, tc.ua, tc.path))
})
}
// none of the above may have triggered a DNS lookup: either they matched
// on IP/range, or they failed a precondition / address parsing.
// "range miss" and "path/UA mismatch" don't reach the googlebot entry's
// rdns because its UA precondition doesn't match those UAs.
assert.Equal(t, int32(0), resolver.ptrCalls.Load(), "no DNS lookup expected")
}
func TestIsLegitimateBotNoFiles(t *testing.T) {
resolver := &fakeResolver{}
setupBotTest(t, resolver)
assert.False(t, IsLegitimateBot("10.1.2.3", "googlebot", "/"))
assert.Equal(t, int32(0), resolver.ptrCalls.Load())
}
func TestIsLegitimateBotFCrDNS(t *testing.T) {
tests := []struct {
name string
resolver *fakeResolver
ip string
ua string
expected bool
}{
{
name: "happy path, trailing dot and mixed case PTR",
resolver: &fakeResolver{
ptr: map[string][]string{"66.249.66.1": {"Crawl-66-249-66-1.GoogleBot.com."}},
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
},
ip: "66.249.66.1",
ua: "Mozilla/5.0 (compatible; Googlebot/2.1)",
expected: true,
},
{
name: "spoofed PTR, forward lookup does not confirm",
resolver: &fakeResolver{
ptr: map[string][]string{"203.0.113.5": {"crawl.googlebot.com."}},
fwd: map[string][]net.IPAddr{"crawl.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
},
ip: "203.0.113.5",
ua: "googlebot",
expected: false,
},
{
name: "anchored pattern rejects lookalike domain",
resolver: &fakeResolver{
ptr: map[string][]string{"203.0.113.6": {"evilgooglebot.com."}},
fwd: map[string][]net.IPAddr{"evilgooglebot.com": {{IP: net.ParseIP("203.0.113.6")}}},
},
ip: "203.0.113.6",
ua: "googlebot",
expected: false,
},
{
name: "no PTR record",
resolver: &fakeResolver{
ptr: map[string][]string{},
},
ip: "203.0.113.7",
ua: "googlebot",
expected: false,
},
{
name: "forward confirms via second domain suffix",
resolver: &fakeResolver{
ptr: map[string][]string{"66.249.66.2": {"rate-limited-proxy.google.com."}},
fwd: map[string][]net.IPAddr{"rate-limited-proxy.google.com": {{IP: net.ParseIP("66.249.66.2")}}},
},
ip: "66.249.66.2",
ua: "googlebot",
expected: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
setupBotTest(t, tc.resolver)
err := FileInit("testdata", "test_data_bots.json", "bots")
require.NoError(t, err)
assert.Equal(t, tc.expected, IsLegitimateBot(tc.ip, tc.ua, "/"))
})
}
}
func TestIsLegitimateBotDNSCache(t *testing.T) {
resolver := &fakeResolver{
ptr: map[string][]string{"66.249.66.1": {"crawl-66-249-66-1.googlebot.com."}},
fwd: map[string][]net.IPAddr{"crawl-66-249-66-1.googlebot.com": {{IP: net.ParseIP("66.249.66.1")}}},
}
setupBotTest(t, resolver)
err := FileInit("testdata", "test_data_bots.json", "bots")
require.NoError(t, err)
assert.True(t, IsLegitimateBot("66.249.66.1", "googlebot", "/"))
assert.True(t, IsLegitimateBot("66.249.66.1", "googlebot", "/other"))
assert.Equal(t, int32(1), resolver.ptrCalls.Load(), "second call must be served from cache")
// negative results are cached too
assert.False(t, IsLegitimateBot("203.0.113.9", "googlebot", "/"))
assert.False(t, IsLegitimateBot("203.0.113.9", "googlebot", "/"))
assert.Equal(t, int32(2), resolver.ptrCalls.Load(), "failed lookup must be cached as negative")
}
// bot data files are loaded per-file via FileInit(dir, name, "bots"), driven
// by the data reference on each appsec-rule (see LoadCollection).
func TestFileInitBots(t *testing.T) {
setupBotTest(t, nil)
botsDir := t.TempDir()
write := func(name, content string) {
require.NoError(t, os.WriteFile(filepath.Join(botsDir, name), []byte(content), 0o644))
}
write("google.json", `{"name":"googlebot","user_agent":"googlebot","rdns":["googlebot.com"]}`)
write("partners.json", `{"name":"partner","ranges":["192.0.2.0/24"]}`)
require.NoError(t, FileInit(botsDir, "google.json", "bots"))
require.NoError(t, FileInit(botsDir, "partners.json", "bots"))
assert.Len(t, dataFileBots, 2)
assert.Contains(t, dataFileBots, "google.json")
assert.Contains(t, dataFileBots, "partners.json")
// any-file semantics: an entry from the second file matches
assert.True(t, IsLegitimateBot("192.0.2.10", "", "/"))
}
func TestFileInitBotsInvalidFile(t *testing.T) {
setupBotTest(t, nil)
botsDir := t.TempDir()
// first line is valid, second has no identity verification: the
// malformed line makes the whole load fail.
require.NoError(t, os.WriteFile(filepath.Join(botsDir, "bad.json"),
[]byte(`{"name":"ok","ips":["10.0.0.1"]}`+"\n"+`{"name":"ua-only","user_agent":"x"}`), 0o644))
require.Error(t, FileInit(botsDir, "bad.json", "bots"))
}
func TestParseBotAddr(t *testing.T) {
tests := []struct {
in string
expected string
ok bool
}{
{"1.2.3.4", "1.2.3.4", true},
{"1.2.3.4:8080", "1.2.3.4", true},
{"2001:db8::1", "2001:db8::1", true},
{"[2001:db8::1]:443", "2001:db8::1", true},
{"::ffff:1.2.3.4", "1.2.3.4", true},
{"fe80::1%eth0", "fe80::1", true},
{"garbage", "", false},
{"", "", false},
{"1.2.3.4:port:extra", "", false},
}
for _, tc := range tests {
t.Run(tc.in, func(t *testing.T) {
addr, ok := parseBotAddr(tc.in)
assert.Equal(t, tc.ok, ok)
if tc.ok {
assert.Equal(t, tc.expected, addr.String())
}
})
}
}
+7
View File
@@ -531,6 +531,13 @@ var exprFuncs = []exprCustomFunc{
new(func(string) *net.IPNet),
},
},
{
name: "IPToCountry",
function: IPToCountry,
signature: []any{
new(func(string) string),
},
},
{
name: "JA4H",
function: JA4H,
+30
View File
@@ -21,6 +21,36 @@ func GeoIPEnrich(params ...any) (any, error) {
return city, nil
}
// IPToCountryString resolves an IP address string to its ISO-3166 alpha-2
// country code, returning an empty string on any nil/parse/lookup failure.
// Intended for internal Go callers; expr rules use IPToCountry.
func IPToCountryString(ip string) string {
if geoIPCityReader == nil || ip == "" {
return ""
}
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
return ""
}
city, err := geoIPCityReader.City(parsedIP)
if err != nil || city == nil {
return ""
}
return city.Country.IsoCode
}
// IPToCountry wraps IPToCountryString for expr. Lets rules write
// `IPToCountry(req.RemoteAddr)` instead of
// `GeoIPEnrich(req.RemoteAddr)?.Country.IsoCode` when they only need the
// country and want nil-safety.
func IPToCountry(params ...any) (any, error) {
ip, _ := params[0].(string)
return IPToCountryString(ip), nil
}
func GeoIPASNEnrich(params ...any) (any, error) {
if geoIPASNReader == nil {
return nil, nil
+10
View File
@@ -122,6 +122,7 @@ func Init(databaseClient *database.Client) error {
dataFileRegex = make(map[string][]*regexp.Regexp)
dataFileRe2 = make(map[string][]*re2.Regexp)
dataFileMap = make(map[string]*fileMapEntry)
dataFileBots = make(map[string][]*botEntry)
dbClient = databaseClient
XMLCacheInit()
@@ -131,12 +132,15 @@ func Init(databaseClient *database.Client) error {
// ResetDataFiles clears all datafile-related global variables.
// This should be called during HUP reload to ensure clean state.
// The DNS cache (pkg/dnscache) is deliberately kept: DNS facts don't change
// with the configuration, and a reload shouldn't trigger a re-lookup storm.
func ResetDataFiles() {
dataFile = make(map[string][]string)
dataFileRegex = make(map[string][]*regexp.Regexp)
dataFileRe2 = make(map[string][]*re2.Regexp)
dataFileRegexCache = make(map[string]gcache.Cache)
dataFileMap = make(map[string]*fileMapEntry)
dataFileBots = make(map[string][]*botEntry)
}
func RegexpCacheInit(filename string, cacheCfg enrichment.DataProvider) error {
@@ -240,6 +244,10 @@ func FileInit(directory string, filename string, fileType string) error {
if err := fileMapInit(filename, scanner.Text()); err != nil {
return err
}
case "bots":
if err := botFileInit(filename, scanner.Text()); err != nil {
return err
}
}
}
@@ -322,6 +330,8 @@ func existsInFileMaps(filename string, ftype string) (bool, error) {
_, ok = dataFile[filename]
case "map":
_, ok = dataFileMap[filename]
case "bots":
_, ok = dataFileBots[filename]
default:
err = fmt.Errorf("unknown data type '%s' for : '%s'", ftype, filename)
}

Some files were not shown because too many files have changed in this diff Show More