ci(buildkite): build tests once, split test jobs by timing (#4278)

.buildkite/cache.yml

@@ -5,10 +5,11 @@
 # (fallback_limit), so a Cargo.lock change restores the newest entry for the
 # same toolchain and cargo rebuilds only the dependencies that changed.
 #
-# Only main saves (.buildkite/scripts/cache-save.sh); branch and PR builds
-# restore main's entries. The cluster's cache registry policy also scopes
-# entries by branch, so a branch build can never write into what main
-# restores.
+# main saves every cache. Branch and PR builds restore main's entries and
+# save only the ones with no exact entry to restore, e.g. after changing
+# Cargo.lock (cache-restore.sh, cache-save.sh). The cluster's cache registry
+# policy scopes entries by branch, so a branch build can never write into
+# what main restores.
 #
 # Bump `v1` to invalidate a cache; an entry is written once per address.
 
@@ -33,8 +34,9 @@ caches:
   - name: target
     cache_key:
       - target
-      # v2: dev/test builds without debuginfo (pipeline env).
-      - v2
+      # v3: check and clippy build different feature sets, and the tests
+      # build in build-tests.
+      - v3
       - agent: step
       - agent: os
       - agent: arch
@@ -87,3 +89,4 @@ caches:
       - checksum: .buildkite/scripts/illumos.sh
     target_paths:
       - ~/.cache/anyvm
+

.buildkite/hooks/post-command

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-# Gives this job's nextest JUnit report (.config/nextest.toml's ci profile) a
-# name unique to the job, so the test-engine step can collect every shard's
-# without them overwriting each other. post-command runs whether or not the
-# command passed, and before the step's artifact_paths are uploaded.
-set -euo pipefail
-
-if [ -f test-results/junit.xml ]; then
-  mv test-results/junit.xml "test-results/junit-${BUILDKITE_JOB_ID}.xml"
-fi

.buildkite/pipeline.yml

@@ -1,20 +1,24 @@
 # atuin CI. Replaces the GitHub Actions workflows rust.yml (except
 # Windows, which stays in .github/workflows/windows.yml until Buildkite's
 # hosted Windows agents are generally available), nix.yml, codespell.yml,
-# shellcheck.yml, vale.yml, installer.yml and nightly.yml, with the same
-# jobs and commands.
+# shellcheck.yml, vale.yml, installer.yml and nightly.yml.
 #
-# Each step sources .buildkite/scripts/toolchain.sh (rustup + pinned tools,
-# and the `section` log-group helper). Caching is Buildkite Cache
+# Each Rust step sources .buildkite/scripts/toolchain.sh (rustup + pinned
+# tools, and the `section` log-group helper). Caching is Buildkite Cache
 # (.buildkite/cache.yml): the cargo registry, plus a per-step target/ pruned
 # to third-party dependency builds (prune-target.sh), keyed on the toolchain
-# and Cargo.lock. Saves run only after the step's commands have succeeded,
-# and only on main (cache-save.sh): branch and PR builds restore main's.
+# and Cargo.lock. Saves run only after the step's commands have succeeded.
+# main saves every cache; branch and PR builds restore main's and save only
+# what had no exact entry to restore (cache-restore.sh, cache-save.sh).
+#
+# Tests build once per OS into a nextest archive (build-tests.sh), which a
+# step of parallel jobs then runs. Buildkite Test Engine splits them across
+# the jobs by how long each test took on main (run-tests.sh,
+# nextest-bktec.py). Raise a test step's parallelism as the suite grows.
 #
 # Queues: light jobs use the cluster's default queue (linux-small, 2 vCPU).
 # Compile-bound jobs use linux-large (8 vCPU) and macOS jobs macos-large
-# (12 vCPU), both default queues in every Buildkite cluster. Test steps are
-# split across jobs with `parallelism` + nextest --partition.
+# (12 vCPU), both default queues in every Buildkite cluster.
 #
 # Not ported yet: the `paths-ignore: ui/**` filters on rust.yml and nix.yml
 # (if_changed needs a diff base set for main builds first).
@@ -23,6 +27,30 @@
 # everything plus the nightly tests, which also keeps main's cache entries
 # from expiring.
 
+# YAML anchors for what the steps below repeat; Buildkite ignores this key.
+common:
+  - &main-only build.branch == pipeline.default_branch
+  - &restore ".buildkite/scripts/cache-restore.sh --name cargo_registry --name target --name rustup --name apt_debs"
+  # No rustup cache on macOS: the image ships several toolchains, so the
+  # entry is 1.7 GB (20s to restore) against ~4s to install 1.98.0.
+  - &restore-macos ".buildkite/scripts/cache-restore.sh --name cargo_registry --name target"
+  - &toolchain source .buildkite/scripts/toolchain.sh
+  - &save-toolchain .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
+  - &prune 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
+  - &save-target .buildkite/scripts/cache-save.sh --name target
+  - &linux-large
+    queue: linux-large
+  # Test Engine's client (bktec), for the steps that run tests. It
+  # authenticates with the job's Buildkite OIDC token, which the suite's OIDC
+  # policy accepts from this pipeline, so there's no token to keep secret.
+  - &test-engine
+    BUILDKITE_TEST_ENGINE_SUITE_SLUG: atuin
+    BUILDKITE_TEST_ENGINE_TEST_RUNNER: custom
+    BUILDKITE_TEST_ENGINE_TEST_CMD: "python3 .buildkite/scripts/nextest-bktec.py run {{testExamples}}"
+    BUILDKITE_TEST_ENGINE_RESULT_PATH: test-results/test-engine.json
+  - &macos-large
+    queue: macos-large
+
 env:
   CARGO_TERM_COLOR: always
   # No debuginfo in dev/test builds: a big share of their compile and link
@@ -31,7 +59,7 @@ env:
   CARGO_PROFILE_DEV_DEBUG: "0"
   CARGO_PROFILE_TEST_DEBUG: "0"
   # .config/nextest.toml's ci profile: writes test-results/junit.xml, which
-  # the test-engine step uploads to Buildkite Test Engine.
+  # nextest-bktec.py reports to Test Engine.
   NEXTEST_PROFILE: ci
 
 steps:
@@ -40,12 +68,9 @@ steps:
     # main only: PRs are type-checked in the release profile by `check`
     # (cargo check --release), and nearly all of this job is LLVM codegen
     # for the atuin binary (~2 min on any machine).
-    if: build.branch == pipeline.default_branch
-    notify:
-      - github_check:
-          name: "buildkite/build"
-    agents:
-      queue: linux-large
+    if: *main-only
+    notify: [github_check: {name: buildkite/build}]
+    agents: *linux-large
     timeout_in_minutes: 45
     env:
       # Only proves the release profile compiles (release.yml ships the
@@ -53,111 +78,124 @@ steps:
       CARGO_PROFILE_RELEASE_LTO: "off"
       CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "256"
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
+      - *restore
+      - *toolchain
+      - *save-toolchain
       - 'section ":hammer: cargo build --release" cargo build --workspace --locked --release'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
+      - *prune
+      - *save-target
 
-  - label: ":test_tube: Unit tests (Linux) %n"
-    key: unit-test
-    notify:
-      - github_check:
-          name: "buildkite/unit-test"
-    agents:
-      queue: linux-large
-    # Split with nextest --partition; raise this as the suite grows.
-    parallelism: 2
+  # Unit, integration and e2e tests, compiled once and run below.
+  - label: ":package: Build tests (Linux)"
+    key: build-tests
+    notify: [github_check: {name: buildkite/build-tests}]
+    agents: *linux-large
     timeout_in_minutes: 45
     env:
       CARGO_TOOLS: nextest
-    # JUnit report, renamed per job by .buildkite/hooks/post-command; the
-    # test-engine step uploads them.
-    artifact_paths: "test-results/junit-*.xml"
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
-      - 'section ":test_tube: cargo nextest (lib + bins)" cargo nextest run $$(nextest_partition) --lib --bins || { .buildkite/scripts/proptest-regressions.sh; exit 1; }'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
+      - *restore
+      - *toolchain
+      - *save-toolchain
+      - .buildkite/scripts/build-tests.sh linux --lib --bins --tests
+      - *prune
+      - *save-target
 
-  - label: ":test_tube: Unit tests (macOS) %n"
-    key: unit-test-macos
-    notify:
-      - github_check:
-          name: "buildkite/unit-test-macos"
-    agents:
-      queue: macos-large
-    # Split with nextest --partition; raise this as the suite grows.
+  - label: ":test_tube: Tests (Linux) %n"
+    key: tests
+    depends_on: build-tests
+    notify: [github_check: {name: buildkite/tests}]
+    agents: *linux-large
+    # ~60s of test time plus ~15s of setup per job (Postgres, packages, the
+    # archive): two jobs finish in under a minute.
     parallelism: 2
+    timeout_in_minutes: 30
+    env:
+      <<: *test-engine
+      RUST_TOOLCHAIN: none
+      CARGO_TOOLS: nextest bktec
+      APT_PACKAGES: "zsh fish xz-utils"
+      ATUIN_DB_URI: postgres://atuin:[email protected]:5432/atuin
+      ATUIN_E2E_REQUIRE_SHELLS: "1"
+      # Test databases live in tempdirs; tmpfs keeps concurrent SQLite WAL
+      # fsyncs from stalling the disk.
+      TEST_TMPDIR: /dev/shm
+    command:
+      - .buildkite/scripts/start-postgres.sh
+      - .buildkite/scripts/cache-restore.sh --name apt_debs
+      - *toolchain
+      - .buildkite/scripts/cache-save.sh --name apt_debs
+      - 'section ":shell: Install ble.sh" .buildkite/scripts/install-blesh.sh'
+      - .buildkite/scripts/run-tests.sh linux
+
+  # Unit and e2e tests; the integration tests need Postgres, which only the
+  # Linux agents run.
+  - label: ":package: Build tests (macOS)"
+    key: build-tests-macos
+    notify: [github_check: {name: buildkite/build-tests-macos}]
+    agents: *macos-large
     timeout_in_minutes: 45
     env:
       CARGO_TOOLS: nextest
-    # JUnit report, renamed per job by .buildkite/hooks/post-command; the
-    # test-engine step uploads them.
-    artifact_paths: "test-results/junit-*.xml"
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target
-      - source .buildkite/scripts/toolchain.sh
-      # No rustup cache on macOS: the image ships several toolchains, so the
-      # entry is 1.7 GB (20s to restore) against ~4s to install 1.98.0.
-      - 'section ":test_tube: cargo nextest (lib + bins)" cargo nextest run $$(nextest_partition) --lib --bins || { .buildkite/scripts/proptest-regressions.sh; exit 1; }'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
+      - *restore-macos
+      - *toolchain
+      - .buildkite/scripts/build-tests.sh macos --lib --bins --test "e2e_*"
+      - *prune
+      - *save-target
 
-  - label: ":mag: Check (Linux): all features, no features, release"
+  - label: ":test_tube: Tests (macOS) %n"
+    key: tests-macos
+    depends_on: build-tests-macos
+    notify: [github_check: {name: buildkite/tests-macos}]
+    agents: *macos-large
+    # ~30s of test time plus ~8s of setup: one job finishes in under a
+    # minute. Kept as a parallel step so raising this is all it takes.
+    parallelism: 1
+    timeout_in_minutes: 30
+    env:
+      <<: *test-engine
+      RUST_TOOLCHAIN: none
+      CARGO_TOOLS: nextest bktec
+      BREW_PACKAGES: "bash fish"
+      ATUIN_E2E_REQUIRE_SHELLS: "1"
+    command:
+      - *toolchain
+      # Test with Homebrew's bash, not macOS's /bin/bash 3.2.
+      # ($$ escapes pipeline-upload interpolation.)
+      - export ATUIN_E2E_BASH="$$(brew --prefix bash)/bin/bash"
+      - 'section ":shell: Install ble.sh" .buildkite/scripts/install-blesh.sh'
+      - .buildkite/scripts/run-tests.sh macos
+
+  # All features are type-checked by clippy below.
+  - label: ":mag: Check (Linux)"
     key: check
-    notify:
-      - github_check:
-          name: "buildkite/check"
+    notify: [github_check: {name: buildkite/check}]
+    agents: *linux-large
     timeout_in_minutes: 45
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
+      - *restore
+      - *toolchain
+      - *save-toolchain
       - 'section ":package: cargo fetch" cargo fetch --locked'
       # Registry contents are the same for every step; this one saves them.
       - .buildkite/scripts/cache-save.sh --name cargo_registry
-      # --all-features enables vendored-tls, which compiles OpenSSL from
-      # source; type-checking can use the system libssl instead.
-      - 'section ":mag: check (all features)" env OPENSSL_NO_VENDOR=1 cargo check --all-features --workspace'
       - 'section ":mag: check (no features)" cargo check --no-default-features --workspace'
-      - 'section ":mag: check (release profile)" cargo check --workspace --locked --release'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
-
-  # The rest of rust.yml's check matrix, in parallel with the step above.
-  - label: ":mag: Check (Linux): sync, server, client only"
-    key: check-subsets
-    notify:
-      - github_check:
-          name: "buildkite/check-subsets"
-    timeout_in_minutes: 45
-    command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
       - 'section ":mag: check (sync)" cargo check --no-default-features --features sync --workspace'
       - 'section ":mag: check (server)" cargo check -p atuin-server'
       - 'section ":mag: check (client only)" cargo check --no-default-features --features client --workspace'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
+      - 'section ":mag: check (release profile)" cargo check --workspace --locked --release'
+      - *prune
+      - *save-target
 
   - label: ":mag: Check (macOS)"
     key: check-macos
-    notify:
-      - github_check:
-          name: "buildkite/check-macos"
-    agents:
-      queue: macos-large
+    notify: [github_check: {name: buildkite/check-macos}]
+    agents: *macos-large
     timeout_in_minutes: 45
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target
-      - source .buildkite/scripts/toolchain.sh
-      # No rustup cache on macOS: the image ships several toolchains, so the
-      # entry is 1.7 GB (20s to restore) against ~4s to install 1.98.0.
+      - *restore-macos
+      - *toolchain
       - 'section ":package: cargo fetch" cargo fetch --locked'
       # The registry key includes the OS, so macOS saves its own.
       - .buildkite/scripts/cache-save.sh --name cargo_registry
@@ -169,166 +207,79 @@ steps:
       - 'section ":mag: check (server)" cargo check -p atuin-server'
       - 'section ":mag: check (client only)" cargo check --no-default-features --features client --workspace'
       - 'section ":mag: check (release profile)" cargo check --workspace --locked --release'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
+      - *prune
+      - *save-target
 
   - label: ":crab: MSRV (1.95.0)"
     key: msrv
-    notify:
-      - github_check:
-          name: "buildkite/msrv"
+    notify: [github_check: {name: buildkite/msrv}]
     timeout_in_minutes: 45
     env:
       # rust-toolchain.toml pins the dev toolchain; override it explicitly.
       RUST_TOOLCHAIN: "1.95.0"
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
+      - *restore
+      - *toolchain
+      - *save-toolchain
       - 'section ":crab: check on MSRV (all features)" cargo check --locked --workspace --all-features'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
-
-  - label: ":postgres: Integration tests %n"
-    key: integration-test
-    notify:
-      - github_check:
-          name: "buildkite/integration-test"
-    agents:
-      queue: linux-large
-    # Split with nextest --partition; raise this as the suite grows.
-    parallelism: 2
-    timeout_in_minutes: 45
-    env:
-      CARGO_TOOLS: nextest
-      ATUIN_DB_URI: postgres://atuin:[email protected]:5432/atuin
-    # JUnit report, renamed per job by .buildkite/hooks/post-command; the
-    # test-engine step uploads them.
-    artifact_paths: "test-results/junit-*.xml"
-    command:
-      - .buildkite/scripts/start-postgres.sh
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
-      # e2e tests have their own step. Test databases live in tempdirs;
-      # tmpfs keeps concurrent SQLite WAL fsyncs from stalling the disk.
-      - 'section ":test_tube: cargo nextest (integration)" env TMPDIR=/dev/shm cargo nextest run $$(nextest_partition) --test "*" -E "not binary(/e2e_.*/)" || { .buildkite/scripts/proptest-regressions.sh; exit 1; }'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
-
-  - label: ":terminal: e2e (Linux) %n"
-    key: e2e
-    notify:
-      - github_check:
-          name: "buildkite/e2e"
-    # Split with nextest --partition; raise this as the suite grows.
-    parallelism: 2
-    agents:
-      queue: linux-large
-    timeout_in_minutes: 45
-    env:
-      CARGO_TOOLS: nextest
-      APT_PACKAGES: "zsh fish xz-utils"
-      ATUIN_E2E_REQUIRE_SHELLS: "1"
-    # JUnit report, renamed per job by .buildkite/hooks/post-command; the
-    # test-engine step uploads them.
-    artifact_paths: "test-results/junit-*.xml"
-    command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
-      - 'section ":shell: Install ble.sh" .buildkite/scripts/install-blesh.sh'
-      - 'section ":terminal: cargo nextest (e2e)" cargo nextest run $$(nextest_partition) -p atuin --test "e2e_*"'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
-
-  - label: ":terminal: e2e (macOS)"
-    key: e2e-macos
-    notify:
-      - github_check:
-          name: "buildkite/e2e-macos"
-    agents:
-      queue: macos-large
-    timeout_in_minutes: 45
-    env:
-      CARGO_TOOLS: nextest
-      BREW_PACKAGES: "bash fish"
-      ATUIN_E2E_REQUIRE_SHELLS: "1"
-    # JUnit report, renamed per job by .buildkite/hooks/post-command; the
-    # test-engine step uploads them.
-    artifact_paths: "test-results/junit-*.xml"
-    command:
-      - buildkite-agent cache restore --name cargo_registry --name target
-      - source .buildkite/scripts/toolchain.sh
-      # No rustup cache on macOS: the image ships several toolchains, so the
-      # entry is 1.7 GB (20s to restore) against ~4s to install 1.98.0.
-      # Test with Homebrew's bash, not macOS's /bin/bash 3.2.
-      # ($$ escapes pipeline-upload interpolation.)
-      - export ATUIN_E2E_BASH="$$(brew --prefix bash)/bin/bash"
-      - 'section ":shell: Install ble.sh" .buildkite/scripts/install-blesh.sh'
-      - 'section ":terminal: cargo nextest (e2e)" cargo nextest run -p atuin --test "e2e_*"'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
+      - *prune
+      - *save-target
 
   - label: ":paperclip: Clippy"
     key: clippy
-    notify:
-      - github_check:
-          name: "buildkite/clippy"
+    notify: [github_check: {name: buildkite/clippy}]
     timeout_in_minutes: 45
     env:
       RUST_COMPONENTS: clippy
+      # --all-features enables vendored-tls, which compiles OpenSSL from
+      # source; linting can use the system libssl instead.
+      OPENSSL_NO_VENDOR: "1"
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
-      # One run over the union of what `cargo clippy` (lib + bins) and
-      # `cargo clippy --tests` lint, so the two share their work.
-      - 'section ":paperclip: clippy (lib, bins, tests)" cargo clippy --lib --bins --tests -- -D warnings'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
+      - *restore
+      - *toolchain
+      - *save-toolchain
+      # All features, and the union of what `cargo clippy` (lib + bins) and
+      # `cargo clippy --tests` lint, in one run. This also type-checks
+      # everything `cargo check --all-features` did.
+      - 'section ":paperclip: clippy (all features; lib, bins, tests)" cargo clippy --all-features --lib --bins --tests -- -D warnings'
+      - *prune
+      - *save-target
 
   - label: ":books: Docs"
     key: docs
-    notify:
-      - github_check:
-          name: "buildkite/docs"
+    notify: [github_check: {name: buildkite/docs}]
     timeout_in_minutes: 45
     env:
       RUSTDOCFLAGS: "-D warnings"
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
+      - *restore
+      - *toolchain
+      - *save-toolchain
       - 'section ":books: cargo doc" cargo doc --document-private-items --no-deps --workspace'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
+      - *prune
+      - *save-target
 
   - label: ":art: Format"
     key: format
-    notify:
-      - github_check:
-          name: "buildkite/format"
+    notify: [github_check: {name: buildkite/format}]
     timeout_in_minutes: 15
     env:
       RUST_TOOLCHAIN: nightly
       RUST_COMPONENTS: rustfmt
     command:
-      - source .buildkite/scripts/toolchain.sh
+      - *toolchain
       - 'section ":art: cargo fmt --check" cargo fmt -- --check'
 
   - label: ":no_entry: cargo-deny"
     key: deny
-    notify:
-      - github_check:
-          name: "buildkite/deny"
+    notify: [github_check: {name: buildkite/deny}]
     timeout_in_minutes: 15
     env:
       CARGO_TOOLS: deny
     command:
-      - buildkite-agent cache restore --name cargo_registry --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
+      - .buildkite/scripts/cache-restore.sh --name cargo_registry --name rustup --name apt_debs
+      - *toolchain
+      - *save-toolchain
       # Same arguments cargo-deny-action passes by default.
       - 'section ":no_entry: cargo deny check" cargo deny --log-level warn --all-features check'
       # Library crates must never (re)add eyre or anyhow; each has its own
@@ -343,113 +294,66 @@ steps:
     # main only: this only guards crates.io publishing (every crate builds
     # from its packaged tarball), which isn't part of the PR flow. On main
     # it still flags publishing breakage well before a release.
-    if: build.branch == pipeline.default_branch
-    notify:
-      - github_check:
-          name: "buildkite/package"
-    agents:
-      queue: linux-large
+    if: *main-only
+    notify: [github_check: {name: buildkite/package}]
+    agents: *linux-large
     timeout_in_minutes: 45
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
+      - *restore
+      - *toolchain
+      - *save-toolchain
       # Make sure the packaged crates build from their tarballs, not from
       # anything left over from the workspace build. The restored target/
       # holds only third-party builds (prune-target.sh), which verification
       # reuses: it builds into the shared target/debug.
       - rm -rf ~/.cargo/registry/{cache,src}/*/atuin* target/package
       - 'section ":package: cargo package" cargo package --workspace'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
-
-  # Uploads every test job's JUnit report to Buildkite Test Engine. A job of
-  # its own, with no checkout, so the token never reaches a job that runs
-  # the repository's code: any test, build script or dependency there could
-  # read it, and this pipeline's logs are public. This can't stop a PR from
-  # editing this file; the secret's access policy restricting it to main
-  # can, and fork PRs only build after a maintainer approves (pipeline
-  # settings).
-  - label: ":test_tube: Upload test results"
-    key: test-engine
-    # Main only: the token's access policy only grants it to main builds, so
-    # anywhere else it would just fail to fetch it. The fork check keeps a
-    # fork's branch that happens to be called main out too.
-    if: build.branch == pipeline.default_branch && build.pull_request.repository.fork != true
-    depends_on:
-      - unit-test
-      - unit-test-macos
-      - integration-test
-      - e2e
-      - e2e-macos
-      - nightly
-    # Report failing runs too, and don't wait on skipped (scheduled-only) steps.
-    allow_dependency_failure: true
-    # A Test Engine outage or rejected upload mustn't fail an otherwise green
-    # build.
-    soft_fail: true
-    # No checkout: no repository files, so no repository hooks or scripts,
-    # run on the one machine that holds the token. It only needs the reports.
-    checkout:
-      skip: true
-    notify:
-      - github_check:
-          name: "buildkite/test-engine"
-    timeout_in_minutes: 10
-    secrets:
-      - ATUIN_TEST_ENGINE_TOKEN
-    plugins:
-      - test-collector#v1.12.0:
-          files: "test-results/junit-*.xml"
-          format: junit
-          api-token-env-name: ATUIN_TEST_ENGINE_TOKEN
-    command:
-      # No reports (e.g. every test job failed before running tests) is not an error.
-      - buildkite-agent artifact download "test-results/junit-*.xml" . || echo "No test reports to upload."
+      - *prune
+      - *save-target
 
   # rust.yml's "Test in VM (illumos)": builds the client and server on a real
   # OmniOS VM (KVM-accelerated; hosted agents expose /dev/kvm). main only,
   # like build and nix-build: it was ~12 min per PR on GitHub Actions.
   - label: ":sunrise: Build on illumos (OmniOS VM)"
     key: illumos
-    if: build.branch == pipeline.default_branch
-    notify:
-      - github_check:
-          name: "buildkite/illumos"
-    agents:
-      queue: linux-large
+    if: *main-only
+    notify: [github_check: {name: buildkite/illumos}]
+    agents: *linux-large
     timeout_in_minutes: 45
     command:
-      - buildkite-agent cache restore --name anyvm --name apt_debs
+      - .buildkite/scripts/cache-restore.sh --name anyvm --name apt_debs
       - .buildkite/scripts/illumos.sh
       - .buildkite/scripts/cache-save.sh --name anyvm --name apt_debs
 
   - label: ":moon: Nightly tests"
     key: nightly
     if: build.source == "schedule"
-    notify:
-      - github_check:
-          name: "buildkite/nightly"
+    notify: [github_check: {name: buildkite/nightly}]
     timeout_in_minutes: 60
     env:
-      CARGO_TOOLS: nextest
-    # JUnit report, renamed per job by .buildkite/hooks/post-command; the
-    # test-engine step uploads them.
-    artifact_paths: "test-results/junit-*.xml"
+      <<: *test-engine
+      CARGO_TOOLS: nextest bktec
+      NEXTEST_COMMAND: cargo nextest run -p atuin-daemon --test nightly --run-ignored all
+      BUILDKITE_TEST_ENGINE_SELECTOR_FILE: test-results/selectors.txt
     command:
-      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
-      - source .buildkite/scripts/toolchain.sh
-      - .buildkite/scripts/cache-save.sh --name rustup --name apt_debs
-      - 'section ":moon: cargo nextest (nightly)" cargo nextest run -p atuin-daemon --test nightly --run-ignored all || { .buildkite/scripts/proptest-regressions.sh; exit 1; }'
-      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
-      - .buildkite/scripts/cache-save.sh --name target
+      - *restore
+      - *toolchain
+      - *save-toolchain
+      # Run through bktec, a one-job split, so the results reach Test Engine
+      # like the other tests'.
+      - mkdir -p test-results
+      - 'echo "--- :hammer: Build and list the nightly tests"'
+      - cargo nextest list -p atuin-daemon --test nightly --run-ignored all --message-format json >test-results/list.json
+      - python3 .buildkite/scripts/nextest-bktec.py selectors test-results/list.json test-results/selectors.txt
+      - test_engine_uploads_on_main
+      - 'section ":moon: cargo nextest (nightly)" bktec run --parallelism 1 --parallel-job 0 || { .buildkite/scripts/proptest-regressions.sh; exit 1; }'
+      - *prune
+      - *save-target
 
   # nix.yml: flake check, build (and run the result), formatting.
   - label: ":nix: Nix flake check"
     key: nix-check
-    notify:
-      - github_check:
-          name: "buildkite/nix-check"
+    notify: [github_check: {name: buildkite/nix-check}]
     timeout_in_minutes: 30
     command:
       - source .buildkite/scripts/lib.sh
@@ -458,15 +362,12 @@ steps:
 
   - label: ":nix: Nix build"
     key: nix-build
-    notify:
-      - github_check:
-          name: "buildkite/nix-build"
     # main only: buildRustPackage builds atuin and every dependency in one
     # derivation, so every change rebuilds from scratch (~4 min). The flake
     # check and nixfmt steps still run on PRs.
-    if: build.branch == pipeline.default_branch
-    agents:
-      queue: linux-large
+    if: *main-only
+    notify: [github_check: {name: buildkite/nix-build}]
+    agents: *linux-large
     timeout_in_minutes: 45
     command:
       - source .buildkite/scripts/lib.sh
@@ -476,9 +377,7 @@ steps:
 
   - label: ":nix: nixfmt"
     key: nixfmt
-    notify:
-      - github_check:
-          name: "buildkite/nixfmt"
+    notify: [github_check: {name: buildkite/nixfmt}]
     timeout_in_minutes: 15
     command:
       - source .buildkite/scripts/lib.sh
@@ -487,9 +386,7 @@ steps:
 
   - label: ":pencil2: Codespell"
     key: codespell
-    notify:
-      - github_check:
-          name: "buildkite/codespell"
+    notify: [github_check: {name: buildkite/codespell}]
     timeout_in_minutes: 10
     command:
       - source .buildkite/scripts/lib.sh
@@ -498,9 +395,7 @@ steps:
 
   - label: ":shell: Shellcheck"
     key: shellcheck
-    notify:
-      - github_check:
-          name: "buildkite/shellcheck"
+    notify: [github_check: {name: buildkite/shellcheck}]
     timeout_in_minutes: 10
     env:
       SHELLCHECK_OPTS: "-e SC2148"
@@ -519,9 +414,7 @@ steps:
       - ".vale.ini"
       - ".vale/styles/config/**"
       - ".buildkite/scripts/vale.sh"
-    notify:
-      - github_check:
-          name: "buildkite/vale"
+    notify: [github_check: {name: buildkite/vale}]
     timeout_in_minutes: 10
     command:
       - source .buildkite/scripts/lib.sh
@@ -532,10 +425,8 @@ steps:
   # main's install.sh), so it runs on main builds only, as before.
   - label: ":inbox_tray: Installer (Linux)"
     key: install
-    if: build.branch == pipeline.default_branch
-    notify:
-      - github_check:
-          name: "buildkite/install"
+    if: *main-only
+    notify: [github_check: {name: buildkite/install}]
     timeout_in_minutes: 10
     command:
       - source .buildkite/scripts/lib.sh
@@ -545,10 +436,8 @@ steps:
 
   - label: ":inbox_tray: Installer (macOS)"
     key: install-macos
-    if: build.branch == pipeline.default_branch
-    notify:
-      - github_check:
-          name: "buildkite/install-macos"
+    if: *main-only
+    notify: [github_check: {name: buildkite/install-macos}]
     agents:
       queue: macos-medium
     timeout_in_minutes: 10

.buildkite/scripts/build-tests.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Compiles the tests once into a nextest archive, lists them as bktec
+# selectors (nextest-bktec.py), and uploads both for run-tests.sh. Runs after
+# toolchain.sh and the target cache restore.
+#
+# Usage: build-tests.sh <os> <cargo target selection...>
+set -euo pipefail
+
+# shellcheck source=lib.sh disable=SC1091
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+os=$1
+shift
+mkdir -p test-archive
+
+section ":package: cargo nextest archive" \
+  cargo nextest archive --archive-file "test-archive/tests-${os}.tar.zst" "$@"
+
+# Listing reuses the build above; nothing recompiles.
+echo "--- :straight_ruler: List the tests"
+cargo nextest list "$@" --message-format json >"test-archive/list-${os}.json"
+python3 .buildkite/scripts/nextest-bktec.py selectors \
+  "test-archive/list-${os}.json" "test-archive/selectors-${os}.txt"
+# The test jobs need the build's checkout path: the test binaries have it
+# compiled in (run-tests.sh).
+echo "$PWD" >"test-archive/checkout-${os}.txt"
+
+section ":arrow_up: Upload the archive and test list" \
+  buildkite-agent artifact upload "test-archive/tests-${os}.tar.zst;test-archive/selectors-${os}.txt;test-archive/checkout-${os}.txt"

.buildkite/scripts/cache-restore.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# `buildkite-agent cache restore "$@"`, noting which caches didn't restore
+# main's (or this branch's) exact entry: a miss, or a fallback to an older
+# entry, as when a branch changes Cargo.lock, the toolchain or a cache
+# version. cache-save.sh saves those on branches.
+set -euo pipefail
+
+log=$(mktemp)
+status=0
+buildkite-agent cache restore "$@" 2>&1 | tee "$log" || status=$?
+
+# The agent logs one outcome per cache:
+#   Cache restored cache_id=target ... fallback_used=false   (exact entry)
+#   Cache restored cache_id=target ... fallback_used=true    (older entry)
+#   Cache not restored (not found) cache_id=target ...
+# (with colour codes, stripped first). If this format ever changes, nothing
+# is recorded and branches just don't save, as before.
+esc=$(printf '\033')
+sed -E "s/${esc}\[[0-9;]*m//g" "$log" | sed -n -E \
+  -e 's/.*Cache restored cache_id=([^ ]+) .*fallback_used=true.*/\1/p' \
+  -e 's/.*Cache not restored .*cache_id=([^ ]+).*/\1/p' \
+  >>"${TMPDIR:-/tmp}/cache-inexact"
+rm -f "$log"
+exit "$status"

.buildkite/scripts/cache-save.sh

@@ -1,13 +1,39 @@
 #!/usr/bin/env bash
-# `buildkite-agent cache save "$@"`, on the default branch only (main,
-# including its scheduled builds). Branch and PR builds restore main's
-# entries (exact or fallback, per the cache registry policy) and never save:
-# a branch's first build would otherwise spend ~20s uploading copies of
-# entries identical to main's, and branches never write caches at all.
+# `buildkite-agent cache save --name <cache>...`. main (including its
+# scheduled builds) saves every cache. Branch and PR builds restore main's
+# entries, so they save only the caches cache-restore.sh found no exact
+# entry for: a branch that changes Cargo.lock, the toolchain or a cache
+# version is warm from its second build, and every other branch skips
+# uploading copies of main's entries. The cache registry policy scopes a
+# branch's entries to that branch, so main never restores them.
 set -euo pipefail
 
-if [ "${BUILDKITE_BRANCH:-}" != "${BUILDKITE_PIPELINE_DEFAULT_BRANCH:-main}" ]; then
-  echo "Not saving caches ($*) on ${BUILDKITE_BRANCH:-?}: only ${BUILDKITE_PIPELINE_DEFAULT_BRANCH:-main} saves."
-  exit 0
+if [ "${BUILDKITE_BRANCH:-}" = "${BUILDKITE_PIPELINE_DEFAULT_BRANCH:-main}" ]; then
+  exec buildkite-agent cache save "$@"
+fi
+
+inexact="${TMPDIR:-/tmp}/cache-inexact"
+args=()
+skipped=()
+saving=0
+while [ $# -gt 0 ]; do
+  if [ "$1" = --name ] && [ $# -ge 2 ]; then
+    if [ -f "$inexact" ] && grep -qxF "$2" "$inexact"; then
+      args+=(--name "$2")
+      saving=$((saving + 1))
+    else
+      skipped+=("$2")
+    fi
+    shift 2
+  else
+    args+=("$1")
+    shift
+  fi
+done
+
+if [ ${#skipped[@]} -gt 0 ]; then
+  echo "Not saving ${skipped[*]} on ${BUILDKITE_BRANCH:-?}: restored an exact entry."
+fi
+if [ "$saving" -gt 0 ]; then
+  buildkite-agent cache save "${args[@]}"
 fi
-buildkite-agent cache save "$@"

.buildkite/scripts/lib.sh

@@ -70,12 +70,12 @@ apt_install() {
     -o APT::Sandbox::User=root $missing
 }
 
-# nextest's `--partition count:K/N` for a step split with Buildkite's
-# `parallelism: N`, and nothing for a single job, so how many ways a test
-# step is split is just that one number in the pipeline.
-nextest_partition() {
-  if [ -n "${BUILDKITE_PARALLEL_JOB_COUNT:-}" ]; then
-    echo "--partition count:$((BUILDKITE_PARALLEL_JOB + 1))/${BUILDKITE_PARALLEL_JOB_COUNT}"
+# bktec uploads test results to Test Engine, for splitting later builds'
+# tests by timing, from main builds only: those are the timings to plan
+# with, and the suite's OIDC policy only lets main builds upload anyway.
+test_engine_uploads_on_main() {
+  if [ "${BUILDKITE_BRANCH:-}" = "${BUILDKITE_PIPELINE_DEFAULT_BRANCH:-}" ]; then
+    export BUILDKITE_TEST_ENGINE_UPLOAD_RESULTS=true
   fi
 }
 

.buildkite/scripts/nextest-bktec.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Glue between Buildkite Test Engine's client (bktec) and cargo-nextest.
+
+bktec asks Test Engine how to split the tests across a step's parallel jobs,
+by how long each took before, then runs this job's share. It has no nextest
+support, so it uses its custom runner: tests are "selectors", one per test,
+written `<binary id>@<test name>`.
+
+  selectors <list.json> <selectors.txt>
+      Write the selectors for every test `cargo nextest list
+      --message-format json` would run.
+  run <selector>...
+      bktec's test command. Runs $NEXTEST_COMMAND on just these tests, then
+      converts nextest's JUnit report into Test Engine JSON at
+      $BUILDKITE_TEST_ENGINE_RESULT_PATH, which bktec reads and (when
+      configured) uploads. Each result is tagged with its selector
+      (test.selector.primary), which is how Test Engine matches timings to
+      the selectors of later plans. Exits with nextest's status.
+"""
+
+import json
+import os
+import shlex
+import subprocess
+import sys
+import uuid
+import xml.etree.ElementTree as ET
+
+# .config/nextest.toml's ci profile (NEXTEST_PROFILE=ci) writes it here.
+JUNIT = "test-results/junit.xml"
+# Keeps each -E well under Linux's 128 KiB limit on a single argument.
+TESTS_PER_FILTER = 500
+
+
+def selectors(list_path, out_path):
+    with open(list_path) as f:
+        listing = json.load(f)
+    lines = [
+        f"{binary_id}@{name}"
+        for binary_id, suite in sorted(listing["rust-suites"].items())
+        for name, case in sorted(suite["testcases"].items())
+        if case["filter-match"]["status"] == "matches"
+    ]
+    if not lines:
+        sys.exit("nextest-bktec: no tests to run")
+    with open(out_path, "w") as f:
+        f.write("\n".join(lines) + "\n")
+    print(f"{len(lines)} tests")
+
+
+def run(chosen):
+    by_binary = {}
+    for selector in chosen:
+        binary_id, _, name = selector.partition("@")
+        by_binary.setdefault(binary_id, []).append(name)
+
+    # nextest runs the union of its -E filters.
+    args = shlex.split(os.environ["NEXTEST_COMMAND"])
+    for binary_id, names in sorted(by_binary.items()):
+        for i in range(0, len(names), TESTS_PER_FILTER):
+            tests = " | ".join(f"test(={n})" for n in names[i : i + TESTS_PER_FILTER])
+            args += ["-E", f"binary_id(={binary_id}) & ({tests})"]
+
+    if os.path.exists(JUNIT):
+        os.remove(JUNIT)
+    status = subprocess.call(args)
+    write_results(set(chosen))
+    sys.exit(status)
+
+
+def write_results(chosen):
+    results = []
+    if os.path.exists(JUNIT):
+        for case in ET.parse(JUNIT).getroot().iter("testcase"):
+            binary_id, name = case.get("classname"), case.get("name")
+            selector = f"{binary_id}@{name}"
+            failure = case.find("failure")
+            if failure is None:
+                failure = case.find("error")
+            if failure is not None:
+                result = "failed"
+            elif case.find("skipped") is not None:
+                result = "skipped"
+            else:
+                result = "passed"
+            entry = {
+                "id": str(uuid.uuid4()),
+                "scope": binary_id,
+                "name": name,
+                "result": result,
+                "history": {"start_at": 0, "duration": float(case.get("time") or 0)},
+                "tags": {"test.selector.primary": selector},
+            }
+            if failure is not None:
+                entry["failure_reason"] = (failure.get("message") or failure.get("type") or "")[:1000]
+            if selector not in chosen:
+                print(f"nextest-bktec: ran {selector}, which wasn't assigned to this job")
+            results.append(entry)
+    path = os.environ.get("BUILDKITE_TEST_ENGINE_RESULT_PATH")
+    if path:
+        os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+        with open(path, "w") as f:
+            json.dump(results, f)
+
+
+if __name__ == "__main__":
+    mode, rest = sys.argv[1], sys.argv[2:]
+    if mode == "selectors":
+        selectors(*rest)
+    elif mode == "run":
+        run(rest)
+    else:
+        sys.exit(f"nextest-bktec: unknown mode {mode}")

.buildkite/scripts/run-tests.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Runs this parallel job's share of the tests build-tests.sh archived for
+# <os>. bktec asks Test Engine for the split (by each test's past timings)
+# and runs its share through nextest-bktec.py. The archive is extracted to
+# target/, next to the sources.
+#
+# Usage: run-tests.sh <os>
+set -euo pipefail
+
+# shellcheck source=lib.sh disable=SC1091
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+os=$1
+archive=test-archive/tests-${os}.tar.zst
+selectors=test-archive/selectors-${os}.txt
+
+section ":arrow_down: Download the tests" \
+  buildkite-agent artifact download "test-archive/*-${os}.*" .
+
+# Compile-time paths in the tests and the binaries they run
+# (env!("CARGO_BIN_EXE_atuin"), env!("CARGO_MANIFEST_DIR"), rstest's
+# #[files], rust-embed's folders) point into the build job's checkout, which
+# is at a different path on every agent. Move this checkout there, so those
+# paths are real ones: a symlink the other way isn't enough, since rust-embed
+# rejects files whose canonical path is outside its folder. The old path
+# becomes a symlink, for the agent.
+built=$(cat "test-archive/checkout-${os}.txt")
+if [ "$built" != "$PWD" ]; then
+  if [ -e "$built" ]; then
+    echo "+++ :rotating_light: $built already exists on this agent"
+    exit 1
+  fi
+  here=$PWD
+  mkdir -p "$(dirname "$built")"
+  mv "$here" "$built"
+  ln -s "$built" "$here"
+  cd "$built"
+fi
+
+# TEST_TMPDIR: where the tests' tempdirs go, without moving the downloads
+# above there too.
+export TMPDIR="${TEST_TMPDIR:-${TMPDIR:-/tmp}}"
+
+echo "+++ :test_tube: Tests (job $((BUILDKITE_PARALLEL_JOB + 1)) of ${BUILDKITE_PARALLEL_JOB_COUNT})"
+test_engine_uploads_on_main
+export BUILDKITE_TEST_ENGINE_SELECTOR_FILE=$selectors
+export NEXTEST_COMMAND="cargo-nextest nextest run --archive-file $archive --workspace-remap $PWD --extract-to $PWD --extract-overwrite --no-tests=pass"
+bktec run || { .buildkite/scripts/proptest-regressions.sh; exit 1; }

.buildkite/scripts/toolchain.sh

@@ -2,9 +2,10 @@
 # toolchain and CI tools. Shared helpers (`section` etc.) come from lib.sh.
 #
 # Per-step knobs, set as step env:
-#   RUST_TOOLCHAIN   toolchain to install (default: rust-toolchain.toml)
+#   RUST_TOOLCHAIN   toolchain to install (default: rust-toolchain.toml), or
+#                    "none" for steps that only run prebuilt tests
 #   RUST_COMPONENTS  extra rustup components, e.g. "clippy"
-#   CARGO_TOOLS      prebuilt tools to install: "nextest", "deny"
+#   CARGO_TOOLS      prebuilt tools to install: "nextest", "deny", "bktec"
 #   APT_PACKAGES     extra apt packages on Linux (libssl-dev + pkg-config always)
 #   BREW_PACKAGES    Homebrew packages on macOS
 #
@@ -20,6 +21,10 @@ NEXTEST_LINUX_SHA256=682c21b777c333e96fd532e114d3a5a894e0729ab88d94c0a9f20f84196
 NEXTEST_MACOS_SHA256=39785160b3c2f6ed9a765049cf4fa79f3b39aa02eb7598a5a0e2a1a0b9ffb9a8
 CARGO_DENY_VERSION=0.20.2
 CARGO_DENY_SHA256=9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f
+# Buildkite Test Engine's client, which splits test steps (nextest-bktec.py).
+BKTEC_VERSION=3.1.0
+BKTEC_LINUX_SHA256=05d1007c40afce8a58d73128c9725d7a1da05a084d34496ef6550e6c7d9ad03f
+BKTEC_MACOS_SHA256=b4cc70f051862144bf656e565e8b6dec6af2b98f3ebfe4c24a5feb5aa317b8cf
 
 echo "--- :rust: Toolchain"
 export CARGO_HOME="$HOME/.cargo" RUSTUP_HOME="$HOME/.rustup"
@@ -31,6 +36,7 @@ case "$(uname -s)" in
     # shellcheck disable=SC2086 # word-split the package list
     apt_install libssl-dev pkg-config ${APT_PACKAGES:-}
     nextest_asset=x86_64-unknown-linux-gnu nextest_sha256=$NEXTEST_LINUX_SHA256
+    bktec_asset=linux_amd64 bktec_sha256=$BKTEC_LINUX_SHA256
     ;;
   Darwin)
     if [ -n "${BREW_PACKAGES:-}" ]; then
@@ -38,6 +44,7 @@ case "$(uname -s)" in
       HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 brew install $BREW_PACKAGES
     fi
     nextest_asset=universal-apple-darwin nextest_sha256=$NEXTEST_MACOS_SHA256
+    bktec_asset=darwin_arm64 bktec_sha256=$BKTEC_MACOS_SHA256
     ;;
   *)
     echo "Unsupported OS: $(uname -s)"
@@ -45,23 +52,25 @@ case "$(uname -s)" in
     ;;
 esac
 
-if ! command -v rustup >/dev/null; then
-  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs |
-    sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
+if [ "${RUST_TOOLCHAIN:-}" != none ]; then
+  if ! command -v rustup >/dev/null; then
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs |
+      sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
+  fi
+  if [ -n "${RUST_TOOLCHAIN:-}" ]; then
+    rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal
+    export RUSTUP_TOOLCHAIN="$RUST_TOOLCHAIN"
+  else
+    # The channel pinned in rust-toolchain.toml. Named explicitly: a bare
+    # `rustup toolchain install` needs rustup 1.28+, and hosted images may
+    # ship an older one.
+    channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml)
+    rustup toolchain install "$channel" --profile minimal
+  fi
+  for component in ${RUST_COMPONENTS:-}; do
+    rustup component add "$component"
+  done
 fi
-if [ -n "${RUST_TOOLCHAIN:-}" ]; then
-  rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal
-  export RUSTUP_TOOLCHAIN="$RUST_TOOLCHAIN"
-else
-  # The channel pinned in rust-toolchain.toml. Named explicitly: a bare
-  # `rustup toolchain install` needs rustup 1.28+, and hosted images may
-  # ship an older one.
-  channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml)
-  rustup toolchain install "$channel" --profile minimal
-fi
-for component in ${RUST_COMPONENTS:-}; do
-  rustup component add "$component"
-done
 
 # rustup may come preinstalled from elsewhere (a system package, the image),
 # in which case nothing has created $CARGO_HOME/bin yet.
@@ -79,6 +88,11 @@ for tool in ${CARGO_TOOLS:-}; do
       tar -xzf /tmp/cargo-deny.tgz -C /tmp
       mv "/tmp/cargo-deny-${CARGO_DENY_VERSION}-x86_64-unknown-linux-musl/cargo-deny" "$CARGO_HOME/bin/"
       ;;
+    bktec)
+      fetch_verified "https://github.com/buildkite/test-engine-client/releases/download/v${BKTEC_VERSION}/bktec_${BKTEC_VERSION}_${bktec_asset}" \
+        "$bktec_sha256" "$CARGO_HOME/bin/bktec"
+      chmod +x "$CARGO_HOME/bin/bktec"
+      ;;
     *)
       echo "Unknown CARGO_TOOLS entry: $tool"
       return 1
@@ -86,5 +100,7 @@ for tool in ${CARGO_TOOLS:-}; do
   esac
 done
 
-rustc --version
-cargo --version
+if [ "${RUST_TOOLCHAIN:-}" != none ]; then
+  rustc --version
+  cargo --version
+fi

.config/nextest.toml

@@ -18,10 +18,8 @@ filter = 'binary_id(=atuin-daemon::scale) and test(/hundred_k/)'
 threads-required = 4
 
 # CI profile (Buildkite sets NEXTEST_PROFILE=ci). Inherits the default
-# profile's settings and overrides; adds a JUnit report for Buildkite Test
-# Engine.
+# profile's settings and overrides; adds a JUnit report, which
+# .buildkite/scripts/nextest-bktec.py reports to Buildkite Test Engine.
 [profile.ci.junit]
-# Relative to target/nextest/ci/. Kept outside target/ so prune-target.sh
-# (which deletes target/nextest/) runs before the test-collector plugin's
-# pre-exit upload without taking the report with it.
+# Relative to target/nextest/ci/, so test-results/ in the workspace root.
 path = "../../../test-results/junit.xml"