ci: gate each deploy on its own checks and skip unchanged subtrees

.buildkite/cache.yml

@@ -51,7 +51,7 @@ caches:
   # to download. Per step, since steps pin different toolchains (MSRV) and
   # components (clippy). Linux only: macOS images ship several toolchains,
   # so there the entry is 1.7 GB and slower to restore than installing. Not
-  # used for the nightly rustfmt step, which should pick up new nightlies.
+  # used for the rustfmt step, whose pinned nightly isn't part of the key.
   - name: rustup
     cache_key:
       - rustup

.buildkite/pipeline.yml

@@ -7,22 +7,21 @@
 # tools, and the `section` log-group helper). Caching is Buildkite Cache
 # (.buildkite/cache.yml): the cargo registry, plus a per-step target/ pruned
 # to third-party dependency builds (prune-target.sh), keyed on the toolchain
-# and Cargo.lock. Saves run only after the step's commands have succeeded.
-# main saves every cache; branch and PR builds restore main's and save only
-# what had no exact entry to restore (cache-restore.sh, cache-save.sh).
+# and Cargo.lock. Saves run only after the commands before them have
+# succeeded. main saves every cache; branch and PR builds restore main's and
+# save only what had no exact entry to restore (cache-restore.sh,
+# cache-save.sh).
 #
 # Tests build once per OS into a nextest archive (build-tests.sh), which a
 # step of parallel jobs then runs. Buildkite Test Engine splits them across
 # the jobs by how long each test took on main (run-tests.sh,
-# nextest-bktec.py). Raise a test step's parallelism as the suite grows.
+# nextest-bktec.py). Raise the Linux test step's parallelism as the suite
+# grows.
 #
 # Queues: light jobs use the cluster's default queue (linux-small, 2 vCPU).
 # Compile-bound jobs use linux-large (8 vCPU) and macOS jobs macos-large
 # (12 vCPU), both default queues in every Buildkite cluster.
 #
-# Not ported yet: the `paths-ignore: ui/**` filters on rust.yml and nix.yml
-# (if_changed needs a diff base set for main builds first).
-#
 # Scheduled builds (the nightly schedule, set in the pipeline settings) run
 # everything plus the nightly tests, which also keeps main's cache entries
 # from expiring.
@@ -30,6 +29,17 @@
 # YAML anchors for what the steps below repeat; Buildkite ignores this key.
 common:
   - &main-only build.branch == pipeline.default_branch
+  - &non-pr build.pull_request.id == null
+  - &infra-retry
+    automatic:
+      - exit_status: -1
+        limit: 1
+      - signal_reason: agent_stop
+        limit: 1
+      - signal_reason: stack_error
+        limit: 1
+      - signal_reason: cancel
+        limit: 1
   - &restore ".buildkite/scripts/cache-restore.sh --name cargo_registry --name target --name rustup --name apt_debs"
   # No rustup cache on macOS: the image ships several toolchains, so the
   # entry is 1.7 GB (20s to restore) against ~4s to install 1.98.0.
@@ -63,27 +73,6 @@ env:
   NEXTEST_PROFILE: ci
 
 steps:
-  - label: ":rust: Build (release)"
-    key: build
-    # main only: PRs are type-checked in the release profile by `check`
-    # (cargo check --release), and nearly all of this job is LLVM codegen
-    # for the atuin binary (~2 min on any machine).
-    if: *main-only
-    agents: *linux-large
-    timeout_in_minutes: 45
-    env:
-      # Only proves the release profile compiles (release.yml ships the
-      # `dist` profile), so skip LTO and spread codegen across more units.
-      CARGO_PROFILE_RELEASE_LTO: "off"
-      CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "256"
-    command:
-      - *restore
-      - *toolchain
-      - *save-toolchain
-      - 'section ":hammer: cargo build --release" cargo build --workspace --locked --release'
-      - *prune
-      - *save-target
-
   # Unit, integration and e2e tests, compiled once and run below.
   - label: ":package: Build tests (Linux)"
     key: build-tests
@@ -127,35 +116,24 @@ steps:
 
   # Unit and e2e tests; the integration tests need Postgres, which only the
   # Linux agents run.
-  - label: ":package: Build tests (macOS)"
-    key: build-tests-macos
+  - label: ":test_tube: Tests (macOS) %n"
+    key: tests-macos
     agents: *macos-large
+    # ~30s of test time, so one job builds and runs them. Raising this needs
+    # the build split back out into its own step.
+    parallelism: 1
     timeout_in_minutes: 45
     env:
-      CARGO_TOOLS: nextest
+      <<: *test-engine
+      CARGO_TOOLS: nextest bktec
+      BREW_PACKAGES: "bash fish"
+      ATUIN_E2E_REQUIRE_SHELLS: "1"
     command:
       - *restore-macos
       - *toolchain
       - .buildkite/scripts/build-tests.sh macos --lib --bins --test "e2e_*"
       - *prune
       - *save-target
-
-  - label: ":test_tube: Tests (macOS) %n"
-    key: tests-macos
-    depends_on: build-tests-macos
-    agents: *macos-large
-    # ~30s of test time plus ~8s of setup: one job finishes in under a
-    # minute. Kept as a parallel step so raising this is all it takes.
-    parallelism: 1
-    timeout_in_minutes: 30
-    env:
-      <<: *test-engine
-      RUST_TOOLCHAIN: none
-      CARGO_TOOLS: nextest bktec
-      BREW_PACKAGES: "bash fish"
-      ATUIN_E2E_REQUIRE_SHELLS: "1"
-    command:
-      - *toolchain
       # Test with Homebrew's bash, not macOS's /bin/bash 3.2.
       # ($$ escapes pipeline-upload interpolation.)
       - export ATUIN_E2E_BASH="$$(brew --prefix bash)/bin/bash"
@@ -251,9 +229,10 @@ steps:
 
   - label: ":art: Format"
     key: format
-    timeout_in_minutes: 15
+    timeout_in_minutes: 5
+    retry: *infra-retry
     env:
-      RUST_TOOLCHAIN: nightly
+      RUST_TOOLCHAIN: nightly-2026-10-09
       RUST_COMPONENTS: rustfmt
     command:
       - *toolchain
@@ -262,6 +241,7 @@ steps:
   - label: ":no_entry: cargo-deny"
     key: deny
     timeout_in_minutes: 15
+    retry: *infra-retry
     env:
       CARGO_TOOLS: deny
     command:
@@ -279,10 +259,10 @@ steps:
 
   - label: ":package: Package"
     key: package
-    # main only: this only guards crates.io publishing (every crate builds
+    # Not on PRs: this only guards crates.io publishing (every crate builds
     # from its packaged tarball), which isn't part of the PR flow. On main
     # it still flags publishing breakage well before a release.
-    if: *main-only
+    if: *non-pr
     agents: *linux-large
     timeout_in_minutes: 45
     command:
@@ -299,11 +279,11 @@ steps:
       - *save-target
 
   # rust.yml's "Test in VM (illumos)": builds the client and server on a real
-  # OmniOS VM (KVM-accelerated; hosted agents expose /dev/kvm). main only,
-  # like build and nix-build: it was ~12 min per PR on GitHub Actions.
+  # OmniOS VM (KVM-accelerated; hosted agents expose /dev/kvm). Not on PRs,
+  # like nix-build: it was ~12 min per PR on GitHub Actions.
   - label: ":sunrise: Build on illumos (OmniOS VM)"
     key: illumos
-    if: *main-only
+    if: *non-pr
     agents: *linux-large
     timeout_in_minutes: 45
     command:
@@ -339,6 +319,7 @@ steps:
   - label: ":nix: Nix flake check"
     key: nix-check
     timeout_in_minutes: 30
+    retry: *infra-retry
     command:
       - source .buildkite/scripts/lib.sh
       - 'section ":nix: Install Nix" install_nix'
@@ -360,9 +341,21 @@ steps:
       - 'section ":nix: nix build" nix run --inputs-from . nixpkgs#cachix -- watch-exec atuin -- nix build --print-build-logs'
       - 'section ":nix: Run the built binary" env -u CACHIX_AUTH_TOKEN ./result/bin/atuin --version'
 
+  - label: ":nix: Nix build"
+    key: nix-build-no-push
+    if: build.pull_request.id == null && build.branch != pipeline.default_branch
+    agents: *linux-large
+    timeout_in_minutes: 45
+    command:
+      - source .buildkite/scripts/lib.sh
+      - 'section ":nix: Install Nix" install_nix'
+      - 'section ":nix: nix build" nix build --print-build-logs'
+      - 'section ":nix: Run the built binary" ./result/bin/atuin --version'
+
   - label: ":nix: nixfmt"
     key: nixfmt
     timeout_in_minutes: 15
+    retry: *infra-retry
     command:
       - source .buildkite/scripts/lib.sh
       - 'section ":nix: Install Nix" install_nix'
@@ -371,6 +364,7 @@ steps:
   - label: ":pencil2: Codespell"
     key: codespell
     timeout_in_minutes: 10
+    retry: *infra-retry
     command:
       - source .buildkite/scripts/lib.sh
       - 'section ":pencil2: Install codespell" install_codespell'
@@ -379,6 +373,7 @@ steps:
   - label: ":shell: Shellcheck"
     key: shellcheck
     timeout_in_minutes: 10
+    retry: *infra-retry
     env:
       SHELLCHECK_OPTS: "-e SC2148"
     command:
@@ -387,8 +382,6 @@ steps:
       - 'section ":shell: shellcheck" .buildkite/scripts/shellcheck.sh'
 
   # vale.yml ran only on pull requests touching the docs or the Vale config.
-  # On main builds there's no diff against main, so if_changed skips it there
-  # too.
   - label: ":memo: Vale (docs prose)"
     key: vale
     if_changed:
@@ -397,16 +390,17 @@ steps:
       - ".vale/styles/config/**"
       - ".buildkite/scripts/vale.sh"
     timeout_in_minutes: 10
+    retry: *infra-retry
     command:
       - source .buildkite/scripts/lib.sh
       - 'section ":memo: Install Vale" install_vale'
       - 'section ":memo: vale" .buildkite/scripts/vale.sh'
 
   # installer.yml: the published installer (setup.atuin.sh, served from
-  # main's install.sh), so it runs on main builds only, as before.
+  # main's install.sh), so it runs on scheduled builds only.
   - label: ":inbox_tray: Installer (Linux)"
     key: install
-    if: *main-only
+    if: build.source == "schedule"
     timeout_in_minutes: 10
     command:
       - source .buildkite/scripts/lib.sh
@@ -416,7 +410,7 @@ steps:
 
   - label: ":inbox_tray: Installer (macOS)"
     key: install-macos
-    if: *main-only
+    if: build.source == "schedule"
     agents:
       queue: macos-medium
     timeout_in_minutes: 10

.buildkite/scripts/illumos.sh

@@ -22,7 +22,12 @@ section ":package: Fetch anyvm ${ANYVM_VERSION}" \
   fetch_verified "https://dl.tty0.dev/anyvm-org/anyvm/releases/download/v${ANYVM_VERSION}/anyvm.py" \
   "$ANYVM_SHA256" /tmp/anyvm.py
 
-# Run inside the VM: the same steps as the old job's prepare + run.
+profile=dev
+if [ "${BUILDKITE_SOURCE:-}" = schedule ]; then
+  profile=release
+fi
+
+# Run inside the VM.
 guest_script="
 set -e
 cd /work
@@ -32,10 +37,9 @@ pkg install pkg-config openssl || [ \$? -eq 4 ]
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs |
   sh -s -- -y --profile minimal --default-toolchain ${RUST_VERSION}
 . \"\$HOME/.cargo/env\"
-echo '--- :hammer: cargo build -p atuin-client --release'
-cargo build -p atuin-client --locked --release
-echo '--- :hammer: cargo build -p atuin-server --release'
-cargo build -p atuin-server --locked --release
+export CARGO_PROFILE_DEV_DEBUG=0 CARGO_INCREMENTAL=0
+echo '--- :hammer: cargo build (${profile})'
+cargo build --locked --profile ${profile} -p atuin-client -p atuin-server
 "
 
 # The downloaded image lives in the `anyvm` cache; --snapshot runs from it

.buildkite/scripts/lib.sh

@@ -24,7 +24,7 @@ verify_sha256() { # <sha256> <file>
 }
 
 fetch_verified() { # <url> <sha256> <file>
-  curl -fsSL -o "$3" "$1"
+  curl -fsSL --retry 3 --retry-all-errors -o "$3" "$1"
   verify_sha256 "$2" "$3"
 }