From c2239474131ee10a90ba8aacd30429ba7290cc83 Mon Sep 17 00:00:00 2001 From: Luke Okraszewski Date: Fri, 20 Feb 2026 15:31:03 +0000 Subject: [PATCH] [ci] add differential benchmark workflows (#63100) This commit does the following: - Move existing smoke benchmark workflows to seperate workflow. - Add differential benchmark workflows - Split benchmarks into heavy and micro categories - Add make target arguments for benchmark time and count - Move smoketests to run only on PRs - Differential benchmarks run on merge queue Benchmarks above 1ms/op are considered heavy and should make use of iteration based benchtime to prevent the CI job from taking too much time. Benchmarks are skipped based on env vars, this is an alternative to splitting benchmark targets into seperate files with build tags. The smoke tests should be much faster and as such are set to run on PRs to ensure the targets are not broken in the change. The current parameters for differential benchmarks attempt to balance CI runner time and confidence for benchstat. Differential benchmarks determine the base commit to run the tests against, this happens within the same job to remove the runner to runner variance. --- .github/workflows/benchmark-code-nonroot.yaml | 80 ++++++++++++++----- .github/workflows/benchmark-code-root.yaml | 65 ++++++++++----- .../benchmark-code-smoke-nonroot.yaml | 75 +++++++++++++++++ .../workflows/benchmark-code-smoke-root.yaml | 75 +++++++++++++++++ Makefile | 12 ++- lib/auth/auth_with_roles_test.go | 11 +++ .../userloginstate/generator_bench_test.go | 1 - lib/cache/cache_test.go | 4 + lib/cache/node_test.go | 3 + lib/client/client_store_test.go | 3 + lib/inventory/store_test.go | 3 + lib/services/local/perf_test.go | 3 + lib/services/role_test.go | 3 + lib/srv/db/benchmark_test.go | 3 + lib/srv/regular/sshserver_unix_test.go | 26 +++++- lib/utils/concurrentqueue/queue_test.go | 3 + lib/utils/fanoutbuffer/buffer_test.go | 3 + lib/utils/sortcache/sortcache_test.go | 3 + tool/tbot/main_test.go | 3 + tool/tctl/common/tctl_test.go | 3 + tool/teleport-update/main_test.go | 3 + tool/teleport/common/teleport_test.go | 3 + tool/tsh/common/tsh_test.go | 3 + 23 files changed, 343 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/benchmark-code-smoke-nonroot.yaml create mode 100644 .github/workflows/benchmark-code-smoke-root.yaml diff --git a/.github/workflows/benchmark-code-nonroot.yaml b/.github/workflows/benchmark-code-nonroot.yaml index 7bdbbdbd282..03b7a612aba 100644 --- a/.github/workflows/benchmark-code-nonroot.yaml +++ b/.github/workflows/benchmark-code-nonroot.yaml @@ -1,9 +1,7 @@ -name: Benchmarks (Go) -run-name: Benchmarks (Go) - ${{ github.run_id }} - @${{ github.actor }} +name: Benchmarks Diff (Go) +run-name: Benchmarks Diff (Go) - ${{ github.run_id }} - @${{ github.actor }} on: - pull_request: - merge_group: jobs: @@ -16,7 +14,6 @@ jobs: changed: ${{ steps.changes.outputs.changed }} steps: - name: Checkout - if: ${{ github.event_name == 'merge_group' }} uses: actions/checkout@v6 - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: changes @@ -33,39 +30,80 @@ jobs: - 'build.assets/Dockerfile*' - 'Makefile' - test: - name: Benchmarks (Go) + benchmark: + name: Benchmarks (Go) - ${{ matrix.name }} needs: changes - if: ${{ !startsWith(github.head_ref, 'dependabot/') && needs.changes.outputs.changed == 'true' }} + if: ${{ needs.changes.outputs.changed == 'true' }} runs-on: ubuntu-22.04-32core - permissions: contents: read + strategy: + # The benchmarks are split by heavy and micro categories. + # Heavy benchmarks expect to take >1ms per op, these use fixed iteration count. + # Micro benchmarks use standard 1s benchtime. + # Both times run for a multiple count to determine run-to-run variance. + matrix: + include: + - name: heavy + BENCH_ARGS: "" + BENCH_TIME: "2x" + BENCH_COUNT: 6 + - name: micro + BENCH_ARGS: "-short" + BENCH_TIME: "1s" + BENCH_COUNT: 6 + fail-fast: false container: image: ghcr.io/gravitational/teleport-buildbox:teleport19 env: TELEPORT_XAUTH_TEST: yes WEBASSETS_SKIP_BUILD: 1 + BENCH_TIME: ${{ matrix.BENCH_TIME }} + BENCH_COUNT: ${{ matrix.BENCH_COUNT }} + GIT_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + GIT_HEAD_SHA: ${{ github.sha }} steps: - - name: Checkout Teleport + - name: Checkout uses: actions/checkout@v6 + with: + fetch-depth: 2 - name: Prepare workspace - id: prepare uses: ./.github/actions/prepare-workspace - # Run benchmarks once to make sure they don't break - # Must be run separate since gotestsum is not compatible with benchmark output - - name: Run Benchmarks Once - timeout-minutes: 20 - shell: bash # Overriding default shell which is `sh -e` - run: make test-go-bench | sed -u -E "s/^(FAIL\s+github)/::error title=Benchmark Failed::\1/" + - name: Run head benchmarks + timeout-minutes: 30 + run: | + make test-go-bench "BENCH_ARGS=${{ matrix.BENCH_ARGS }}" BENCH_OUTPUT=test-logs/bench-${GIT_HEAD_SHA}.txt - - name: Construct Summary + - name: Checkout base commit + uses: actions/checkout@v6 + with: + clean: false # Preserve previous results + ref: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + + - name: Run base benchmarks + timeout-minutes: 30 + run: | + make test-go-bench "BENCH_ARGS=${{ matrix.BENCH_ARGS }}" BENCH_OUTPUT=test-logs/bench-${GIT_BASE_SHA}.txt + + - name: Generate benchstat output shell: bash run: | - echo '```' >> $GITHUB_STEP_SUMMARY - go run golang.org/x/perf/cmd/benchstat@latest test-logs/bench.txt \ - | sed -E -e 's/^pkg:\s+(github.*)/\n```\n## \1\n\n```/' >> "$GITHUB_STEP_SUMMARY" + make benchstat BENCH_FILES="test-logs/bench-${GIT_BASE_SHA}.txt test-logs/bench-${GIT_HEAD_SHA}.txt" + + - name: Publish benchstat summary + shell: bash + run: | + { + echo "## Go benchmark diff" + echo "" + echo "**Base:** \`${GIT_BASE_SHA}\`" + echo "**Head:** \`${GIT_HEAD_SHA}\`" + echo "" + echo '```' + cat test-logs/benchstat.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/benchmark-code-root.yaml b/.github/workflows/benchmark-code-root.yaml index 18fd890e8da..b0af125a411 100644 --- a/.github/workflows/benchmark-code-root.yaml +++ b/.github/workflows/benchmark-code-root.yaml @@ -1,9 +1,7 @@ -name: Benchmarks (Root) (Go) -run-name: Benchmarks (Root) (Go) - ${{ github.run_id }} - @${{ github.actor }} +name: Benchmarks Diff (Root) (Go) +run-name: Benchmarks Diff (Root) (Go) - ${{ github.run_id }} - @${{ github.actor }} on: - pull_request: - merge_group: jobs: @@ -16,7 +14,6 @@ jobs: changed: ${{ steps.changes.outputs.changed }} steps: - name: Checkout - if: ${{ github.event_name == 'merge_group' }} uses: actions/checkout@v6 - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: changes @@ -33,12 +30,11 @@ jobs: - 'build.assets/Dockerfile*' - 'Makefile' - test: - name: Benchmarks (Root) + benchmark: + name: Benchmarks (Root) (Go) needs: changes - if: ${{ !startsWith(github.head_ref, 'dependabot/') && needs.changes.outputs.changed == 'true' }} + if: ${{ needs.changes.outputs.changed == 'true' }} runs-on: ubuntu-22.04-32core - permissions: contents: read @@ -46,26 +42,53 @@ jobs: image: ghcr.io/gravitational/teleport-buildbox:teleport19 options: --cap-add=SYS_ADMIN --privileged env: - WEBASSETS_SKIP_BUILD: 1 TELEPORT_XAUTH_TEST: yes + WEBASSETS_SKIP_BUILD: 1 + BENCH_TIME: "2s" + BENCH_COUNT: 10 + GIT_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + GIT_HEAD_SHA: ${{ github.sha }} steps: - - name: Checkout Teleport + - name: Checkout uses: actions/checkout@v6 + with: + fetch-depth: 2 - name: Prepare workspace uses: ./.github/actions/prepare-workspace - # Run benchmarks once to make sure they don't break - # Must be run separate since gotestsum is not compatible with benchmark output - - name: Run Benchmarks Once - timeout-minutes: 5 - shell: bash # Overriding default shell which is `sh -e` - run: make test-go-bench-root | sed -u -E "s/^(FAIL\s+github)/::error title=Benchmark Failed::\1/" + - name: Run head benchmarks + timeout-minutes: 10 + run: | + make test-go-bench-root BENCH_OUTPUT=test-logs/bench-${GIT_HEAD_SHA}.txt - - name: Construct Summary + - name: Checkout base commit + uses: actions/checkout@v6 + with: + clean: false # Preserve previous results + ref: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + + - name: Run base benchmarks + timeout-minutes: 10 + run: | + make test-go-bench-root BENCH_OUTPUT=test-logs/bench-${GIT_BASE_SHA}.txt + + - name: Generate benchstat output shell: bash run: | - echo '```' >> $GITHUB_STEP_SUMMARY - go run golang.org/x/perf/cmd/benchstat@latest test-logs/bench.txt \ - | sed -E -e 's/^pkg:\s+(github.*)/\n```\n## \1\n\n```/' >> "$GITHUB_STEP_SUMMARY" + make benchstat BENCH_FILES="test-logs/bench-${GIT_BASE_SHA}.txt test-logs/bench-${GIT_HEAD_SHA}.txt" + + - name: Publish benchstat summary + shell: bash + run: | + { + echo "## Go benchmark diff" + echo "" + echo "**Base:** \`${GIT_BASE_SHA}\`" + echo "**Head:** \`${GIT_HEAD_SHA}\`" + echo "" + echo '```' + cat test-logs/benchstat.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/benchmark-code-smoke-nonroot.yaml b/.github/workflows/benchmark-code-smoke-nonroot.yaml new file mode 100644 index 00000000000..3c0b56732f1 --- /dev/null +++ b/.github/workflows/benchmark-code-smoke-nonroot.yaml @@ -0,0 +1,75 @@ +# Smoketest that runs bechmarks a single time to ensure they don't break. +name: Benchmarks Smoketest (Go) +run-name: Benchmarks Smoketest (Go) - ${{ github.run_id }} - @${{ github.actor }} + +on: + pull_request: + +jobs: + changes: + name: Check for relevant changes + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + changed: ${{ steps.changes.outputs.changed }} + steps: + - name: Checkout + if: ${{ github.event_name == 'merge_group' }} + uses: actions/checkout@v6 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: changes + with: + base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} + ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }} + filters: | + changed: + - '.github/workflows/benchmark-code-smoke-nonroot.yaml' + - '**.go' + - 'go.mod' + - 'go.sum' + - 'build.assets/Makefile' + - 'build.assets/Dockerfile*' + - 'Makefile' + + test: + name: Benchmarks (Go) + needs: changes + if: ${{ !startsWith(github.head_ref, 'dependabot/') && needs.changes.outputs.changed == 'true' }} + runs-on: ubuntu-22.04-32core + + permissions: + contents: read + + container: + image: ghcr.io/gravitational/teleport-buildbox:teleport19 + env: + TELEPORT_XAUTH_TEST: yes + WEBASSETS_SKIP_BUILD: 1 + + steps: + - name: Checkout Teleport + uses: actions/checkout@v6 + + - name: Prepare workspace + id: prepare + uses: ./.github/actions/prepare-workspace + + # Run benchmarks once to make sure they don't break + # Must be run separate since gotestsum is not compatible with benchmark output + - name: Run Benchmarks Once + timeout-minutes: 20 + shell: bash # Overriding default shell which is `sh -e` + run: make test-go-bench | sed -u -E "s/^(FAIL\s+github)/::error title=Benchmark Failed::\1/" + + - name: Generate benchstat output + shell: bash + run: | + make benchstat BENCH_FILES="test-logs/bench.txt" + + - name: Construct Summary + shell: bash + run: | + echo '```' >> $GITHUB_STEP_SUMMARY + cat test-logs/benchstat.txt \ + | sed -E -e 's/^pkg:\s+(github.*)/\n```\n## \1\n\n```/' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/benchmark-code-smoke-root.yaml b/.github/workflows/benchmark-code-smoke-root.yaml new file mode 100644 index 00000000000..47c96625f71 --- /dev/null +++ b/.github/workflows/benchmark-code-smoke-root.yaml @@ -0,0 +1,75 @@ +# Smoketest that runs bechmarks a single time to ensure they don't break. +name: Benchmarks Smoketest (Root) (Go) +run-name: Benchmarks Smoketest (Root) (Go) - ${{ github.run_id }} - @${{ github.actor }} + +on: + pull_request: + +jobs: + changes: + name: Check for relevant changes + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + changed: ${{ steps.changes.outputs.changed }} + steps: + - name: Checkout + if: ${{ github.event_name == 'merge_group' }} + uses: actions/checkout@v6 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: changes + with: + base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} + ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }} + filters: | + changed: + - '.github/workflows/benchmark-code-smoke-root.yaml' + - '**.go' + - 'go.mod' + - 'go.sum' + - 'build.assets/Makefile' + - 'build.assets/Dockerfile*' + - 'Makefile' + + test: + name: Benchmarks (Root) + needs: changes + if: ${{ !startsWith(github.head_ref, 'dependabot/') && needs.changes.outputs.changed == 'true' }} + runs-on: ubuntu-22.04-32core + + permissions: + contents: read + + container: + image: ghcr.io/gravitational/teleport-buildbox:teleport19 + options: --cap-add=SYS_ADMIN --privileged + env: + WEBASSETS_SKIP_BUILD: 1 + TELEPORT_XAUTH_TEST: yes + + steps: + - name: Checkout Teleport + uses: actions/checkout@v6 + + - name: Prepare workspace + uses: ./.github/actions/prepare-workspace + + # Run benchmarks once to make sure they don't break + # Must be run separate since gotestsum is not compatible with benchmark output + - name: Run Benchmarks Once + timeout-minutes: 5 + shell: bash # Overriding default shell which is `sh -e` + run: make test-go-bench-root | sed -u -E "s/^(FAIL\s+github)/::error title=Benchmark Failed::\1/" + + - name: Generate benchstat output + shell: bash + run: | + make benchstat BENCH_FILES="test-logs/bench.txt" + + - name: Construct Summary + shell: bash + run: | + echo '```' >> $GITHUB_STEP_SUMMARY + cat test-logs/benchstat.txt \ + | sed -E -e 's/^pkg:\s+(github.*)/\n```\n## \1\n\n```/' >> "$GITHUB_STEP_SUMMARY" diff --git a/Makefile b/Makefile index 9c3b5de0821..3ac9db884ca 100644 --- a/Makefile +++ b/Makefile @@ -999,7 +999,7 @@ ifneq ("$(TOUCHID_TAG)", "") | $(GOTESTSUM) --junitfile $(TEST_LOG_DIR)/unit-tests-touchid.xml --jsonfile $(TEST_LOG_DIR)/unit-tests-touchid.json --raw-command -- cat endif -# Runs benchmarks once to make sure they pass. +# By default, the parameters run each benchmark only once. # This is intended to run in CI during unit testing to make sure benchmarks don't break. # To limit noise and improve speed this will only run on packages that have benchmarks. # Race detection is not enabled because it significantly slows down benchmarks. @@ -1007,26 +1007,32 @@ endif .PHONY: test-go-bench test-go-bench: BENCHMARK_SKIP_PATTERN = "^BenchmarkRoot" test-go-bench: BENCH_OUTPUT ?= $(TEST_LOG_DIR)/bench.txt +test-go-bench: BENCH_ARGS ?= +test-go-bench: BENCH_TIME ?= "1x" +test-go-bench: BENCH_COUNT ?= 1 test-go-bench: $(BENCHFIND) | $(TEST_LOG_DIR) @PKGS=$$($(BENCHFIND) --tags=$(BUILD_TAGS)) ; \ if [ -z "$$PKGS" ]; then \ echo "No benchmark packages found"; \ exit 1; \ fi ; \ - go test -run ^$$ -bench . -skip $(BENCHMARK_SKIP_PATTERN) -benchtime 1x $$PKGS \ + go test -run ^$$ -bench . -skip $(BENCHMARK_SKIP_PATTERN) -count $(BENCH_COUNT) -benchtime $(BENCH_TIME) $(BENCH_ARGS) $$PKGS \ | tee $(BENCH_OUTPUT) .PHONY: test-go-bench-root test-go-bench-root: BENCHMARK_PATTERN = "^BenchmarkRoot" test-go-bench-root: BENCHMARK_SKIP_PATTERN = "" test-go-bench-root: BENCH_OUTPUT ?= $(TEST_LOG_DIR)/bench.txt +test-go-bench-root: BENCH_ARGS ?= +test-go-bench-root: BENCH_TIME ?= "1x" +test-go-bench-root: BENCH_COUNT ?= 1 test-go-bench-root: $(BENCHFIND) | $(TEST_LOG_DIR) @PKGS=$$($(BENCHFIND) --tags=$(BUILD_TAGS)) ; \ if [ -z "$$PKGS" ]; then \ echo "No benchmark packages found"; \ exit 1; \ fi ; \ - go test -run ^$$ -bench $(BENCHMARK_PATTERN) -skip $(BENCHMARK_SKIP_PATTERN) -benchtime 1x $$PKGS \ + go test -run ^$$ -bench $(BENCHMARK_PATTERN) -skip $(BENCHMARK_SKIP_PATTERN) -count $(BENCH_COUNT) -benchtime $(BENCH_TIME) $(BENCH_ARGS) $$PKGS \ | tee $(BENCH_OUTPUT) # Make sure untagged vnetdaemon code build/tests. diff --git a/lib/auth/auth_with_roles_test.go b/lib/auth/auth_with_roles_test.go index f599b692978..8f408f20be8 100644 --- a/lib/auth/auth_with_roles_test.go +++ b/lib/auth/auth_with_roles_test.go @@ -2205,6 +2205,10 @@ func TestSessionRecordingConfigRBAC(t *testing.T) { // PASS // ok github.com/gravitational/teleport/lib/auth 11.679s func BenchmarkListNodes(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } + const nodeCount = 50_000 const roleCount = 32 @@ -6781,6 +6785,9 @@ func TestUnifiedResources_IdentityCenter(t *testing.T) { } func BenchmarkListUnifiedResourcesFilter(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } const nodeCount = 150_000 const roleCount = 32 const dbCount = 150_000 @@ -6983,6 +6990,10 @@ func BenchmarkListUnifiedResourcesFilter(b *testing.B) { // PASS // ok github.com/gravitational/teleport/lib/auth 2.878s func BenchmarkListUnifiedResources(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } + const nodeCount = 150_000 const roleCount = 32 const dbCount = 150_000 diff --git a/lib/auth/userloginstate/generator_bench_test.go b/lib/auth/userloginstate/generator_bench_test.go index 7df5635aa0f..ef3b6d55c48 100644 --- a/lib/auth/userloginstate/generator_bench_test.go +++ b/lib/auth/userloginstate/generator_bench_test.go @@ -63,7 +63,6 @@ func BenchmarkGenerate(b *testing.B) { } b.ReportAllocs() - b.ResetTimer() for b.Loop() { _, err := svc.generate(b.Context(), user, backendSvc, false /* pure */) diff --git a/lib/cache/cache_test.go b/lib/cache/cache_test.go index 76c4c59b2b7..5efa1d5027d 100644 --- a/lib/cache/cache_test.go +++ b/lib/cache/cache_test.go @@ -1012,6 +1012,10 @@ cpu: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz BenchmarkListResourcesWithSort-8 1 2351035036 ns/op */ func BenchmarkListResourcesWithSort(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } + p, err := newPack(b, ForAuth) require.NoError(b, err) defer p.Close() diff --git a/lib/cache/node_test.go b/lib/cache/node_test.go index c4fc2387e3e..2cdb1e8bcb5 100644 --- a/lib/cache/node_test.go +++ b/lib/cache/node_test.go @@ -120,6 +120,9 @@ func TestNodes(t *testing.T) { } func BenchmarkGetMaxNodes(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } benchGetNodes(b, 1_000_000) } diff --git a/lib/client/client_store_test.go b/lib/client/client_store_test.go index c70770cda61..60799f3346f 100644 --- a/lib/client/client_store_test.go +++ b/lib/client/client_store_test.go @@ -437,6 +437,9 @@ func TestProxySSHConfig(t *testing.T) { // fast to avoid adding latency to all kubectl calls. It should tolerate being // called many times in parallel. func BenchmarkLoadKeysToKubeFromStore(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } key, err := cryptosuites.GenerateKeyWithAlgorithm(cryptosuites.ECDSAP256) require.NoError(b, err) diff --git a/lib/inventory/store_test.go b/lib/inventory/store_test.go index 23f49e80301..93911c8a514 100644 --- a/lib/inventory/store_test.go +++ b/lib/inventory/store_test.go @@ -36,6 +36,9 @@ cpu: Intel(R) Xeon(R) CPU @ 2.80GHz BenchmarkStore-4 3 480249642 ns/op */ func BenchmarkStore(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } const insertions = 100_000 const uniqueServers = 10_000 const readMod = 100 diff --git a/lib/services/local/perf_test.go b/lib/services/local/perf_test.go index 645f23a208b..c7abf175286 100644 --- a/lib/services/local/perf_test.go +++ b/lib/services/local/perf_test.go @@ -36,6 +36,9 @@ import ( // BenchmarkGetNodes verifies the performance of the GetNodes operation // on local (sqlite) databases (as used by the cache system). func BenchmarkGetNodes(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } ctx := context.Background() type testCase struct { diff --git a/lib/services/role_test.go b/lib/services/role_test.go index 6e9dc09b8c7..132539d8016 100644 --- a/lib/services/role_test.go +++ b/lib/services/role_test.go @@ -7436,6 +7436,9 @@ func TestCheckAccessToUserGroups(t *testing.T) { // go tool pprof --pdf cpu.prof > cpu.pdf // go tool pprof --pdf mem.prof > mem.pdf func BenchmarkCheckConditionalAccessToServer(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } servers := make([]*types.ServerV2, 0, 4000) // Create 4,000 servers with random IDs. diff --git a/lib/srv/db/benchmark_test.go b/lib/srv/db/benchmark_test.go index b54fb79ebee..c054b83475e 100644 --- a/lib/srv/db/benchmark_test.go +++ b/lib/srv/db/benchmark_test.go @@ -43,6 +43,9 @@ BenchmarkPostgresReadLargeTable/size=8000-10 3 215046472 ns/op // BenchmarkPostgresReadLargeTable is a benchmark for read-heavy usage of Postgres. // Depending on the message size we may get different performance, due to the way the respective engine is written. func BenchmarkPostgresReadLargeTable(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } ctx := context.Background() testCtx := setupTestContext(ctx, b, withSelfHostedPostgres("postgres", func(db *types.DatabaseV3) { db.SetStaticLabels(map[string]string{"foo": "bar"}) diff --git a/lib/srv/regular/sshserver_unix_test.go b/lib/srv/regular/sshserver_unix_test.go index a13551c921b..f3788ff3141 100644 --- a/lib/srv/regular/sshserver_unix_test.go +++ b/lib/srv/regular/sshserver_unix_test.go @@ -21,14 +21,19 @@ package regular import ( "context" + "io" + "log/slog" "os/user" + "runtime" "sync" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/gravitational/teleport/api/constants" tracessh "github.com/gravitational/teleport/api/observability/tracing/ssh" + "github.com/gravitational/teleport/lib/srv" "github.com/gravitational/teleport/lib/utils/host" "github.com/gravitational/teleport/lib/utils/testutils" ) @@ -56,9 +61,26 @@ func BenchmarkRootExecCommand(b *testing.B) { for _, test := range cases { b.Run(test.name, func(b *testing.B) { - var opts []ServerOption + if test.createUser && runtime.GOOS != constants.LinuxOS { + b.Skip("Skip benchmark with user creation on non-linux OS") + } + + opts := []ServerOption{ + // TODO(okraport): Disable child logs to reduce noise in benchmark results. + // Re-enable once we have a better way to benchmark with logging enabled. + func(s *Server) error { + s.childLogConfig = &srv.ChildLogConfig{ + ExecLogConfig: srv.ExecLogConfig{ + Level: &slog.LevelVar{}, + }, + Writer: io.Discard, + } + return nil + }, + } + if test.createUser { - opts = []ServerOption{SetCreateHostUser(true)} + opts = append(opts, SetCreateHostUser(true)) } f := newFixtureWithoutDiskBasedLogging(b, opts...) diff --git a/lib/utils/concurrentqueue/queue_test.go b/lib/utils/concurrentqueue/queue_test.go index e8a62df3c93..610c3562038 100644 --- a/lib/utils/concurrentqueue/queue_test.go +++ b/lib/utils/concurrentqueue/queue_test.go @@ -172,6 +172,9 @@ cpu: Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz BenchmarkQueue-16 156 7342841 ns/op */ func BenchmarkQueue(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } const workers = 16 const iters = 4096 workfn := func(v int) int { diff --git a/lib/utils/fanoutbuffer/buffer_test.go b/lib/utils/fanoutbuffer/buffer_test.go index fba32731a82..f746f70eece 100644 --- a/lib/utils/fanoutbuffer/buffer_test.go +++ b/lib/utils/fanoutbuffer/buffer_test.go @@ -43,6 +43,9 @@ BenchmarkBuffer/100000-events-10000-cursors-4 1 12335906365 ns/op // levels of concurrency. Note that these scenarios are very contrived and may not reflect real-world // performance (e.g. benchmarks append to the buffer with a fixed chunk size). func BenchmarkBuffer(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } bbs := []struct { events, cursors int }{ diff --git a/lib/utils/sortcache/sortcache_test.go b/lib/utils/sortcache/sortcache_test.go index dd36d21339e..8815a4ea11d 100644 --- a/lib/utils/sortcache/sortcache_test.go +++ b/lib/utils/sortcache/sortcache_test.go @@ -285,6 +285,9 @@ func TestNextKey(t *testing.T) { // cpu: Intel(R) Xeon(R) CPU @ 2.80GHz // BenchmarkSortCache-4 12 250820820 ns/op func BenchmarkSortCache(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } const ( concurrency = 100 resourcesPerKind = 50_000 diff --git a/tool/tbot/main_test.go b/tool/tbot/main_test.go index a0c699ba140..a2bb87fedfd 100644 --- a/tool/tbot/main_test.go +++ b/tool/tbot/main_test.go @@ -43,6 +43,9 @@ func TestMain(m *testing.M) { } func BenchmarkInit(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } executable, err := os.Executable() require.NoError(b, err) diff --git a/tool/tctl/common/tctl_test.go b/tool/tctl/common/tctl_test.go index 00c450f9338..b060f418522 100644 --- a/tool/tctl/common/tctl_test.go +++ b/tool/tctl/common/tctl_test.go @@ -57,6 +57,9 @@ func TestMain(m *testing.M) { } func BenchmarkInit(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } executable, err := os.Executable() require.NoError(b, err) diff --git a/tool/teleport-update/main_test.go b/tool/teleport-update/main_test.go index 444d0373aef..6f348d4a2af 100644 --- a/tool/teleport-update/main_test.go +++ b/tool/teleport-update/main_test.go @@ -40,6 +40,9 @@ func TestMain(m *testing.M) { } func BenchmarkInit(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } executable, err := os.Executable() require.NoError(b, err) diff --git a/tool/teleport/common/teleport_test.go b/tool/teleport/common/teleport_test.go index 74815bb54b9..68e819d277e 100644 --- a/tool/teleport/common/teleport_test.go +++ b/tool/teleport/common/teleport_test.go @@ -52,6 +52,9 @@ func TestMain(m *testing.M) { } func BenchmarkInit(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } executable, err := os.Executable() require.NoError(b, err) diff --git a/tool/tsh/common/tsh_test.go b/tool/tsh/common/tsh_test.go index b9fa5f0d9d6..6e6fdeeacb7 100644 --- a/tool/tsh/common/tsh_test.go +++ b/tool/tsh/common/tsh_test.go @@ -146,6 +146,9 @@ func TestMain(m *testing.M) { } func BenchmarkInit(b *testing.B) { + if testing.Short() { + b.Skip("skipping heavy benchmark") + } executable, err := os.Executable() require.NoError(b, err)