atuin/.buildkite/pipeline.yml

# atuin CI on Buildkite: PROTOTYPE, running alongside GitHub Actions to
# compare. Ports rust.yml (Linux + macOS; Windows and the illumos VM job
# stay on GitHub Actions for now), nix.yml, codespell.yml, shellcheck.yml,
# vale.yml, installer.yml and nightly.yml, with the same jobs and commands.
#
# Each step sources .buildkite/scripts/toolchain.sh (rustup + pinned tools,
# and the `section` log-group helper). Caching is Buildkite Cache
# (.buildkite/cache.yml): the cargo registry, plus a per-step target/ pruned
# to third-party dependency builds (prune-target.sh), keyed on the toolchain
# and Cargo.lock. Saves run only after the step's commands have succeeded.
#
# Queues: light jobs use the cluster's default queue (linux-small, 2 vCPU).
# Compile-bound jobs use linux-large (8 vCPU) and macOS jobs macos-large
# (12 vCPU), both default queues in every Buildkite cluster. Test steps are
# split across jobs with `parallelism` + nextest --partition.
#
# Not ported yet: the `paths-ignore: ui/**` filters on rust.yml and nix.yml
# (if_changed needs a diff base set for main builds first).
#
# Scheduled builds (the nightly schedule, set in the pipeline settings) run
# everything plus the nightly tests, which also keeps main's cache entries
# from expiring.

env:
  CARGO_TERM_COLOR: always
  # No debuginfo in dev/test builds: a big share of their compile and link
  # time, unused in CI. debug_assertions stay on; panics still report
  # file:line, backtraces just lose line numbers.
  CARGO_PROFILE_DEV_DEBUG: "0"
  CARGO_PROFILE_TEST_DEBUG: "0"
  # .config/nextest.toml's ci profile: writes test-results/junit.xml, which
  # the test-collector plugin uploads to Buildkite Test Engine.
  NEXTEST_PROFILE: ci

steps:
  - label: ":rust: Build (release)"
    key: build
    # main only: PRs are type-checked in the release profile by `check`
    # (cargo check --release), and nearly all of this job is LLVM codegen
    # for the atuin binary (~2 min on any machine).
    if: build.branch == pipeline.default_branch
    notify:
      - github_check:
          name: "buildkite/build"
    agents:
      queue: linux-large
    timeout_in_minutes: 45
    env:
      # Only proves the release profile compiles (release.yml ships the
      # `dist` profile), so skip LTO and spread codegen across more units.
      CARGO_PROFILE_RELEASE_LTO: "off"
      CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "256"
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      - 'section ":hammer: cargo build --release" cargo build --workspace --locked --release'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":test_tube: Unit tests (Linux) %n"
    key: unit-test
    notify:
      - github_check:
          name: "buildkite/unit-test"
    agents:
      queue: linux-large
    # Split with nextest --partition; raise this as the suite grows.
    parallelism: 2
    timeout_in_minutes: 45
    env:
      CARGO_TOOLS: nextest
    secrets:
      - ATUIN_TEST_ENGINE_TOKEN
    plugins: &test_collector
      - test-collector#v1.12.0:
          files: test-results/junit.xml
          format: junit
          api-token-env-name: ATUIN_TEST_ENGINE_TOKEN
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      - 'section ":test_tube: cargo nextest (lib + bins)" cargo nextest run $$(nextest_partition) --lib --bins || { .buildkite/scripts/proptest-regressions.sh; exit 1; }'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":test_tube: Unit tests (macOS) %n"
    key: unit-test-macos
    notify:
      - github_check:
          name: "buildkite/unit-test-macos"
    agents:
      queue: macos-large
    # Split with nextest --partition; raise this as the suite grows.
    parallelism: 2
    timeout_in_minutes: 45
    env:
      CARGO_TOOLS: nextest
    secrets:
      - ATUIN_TEST_ENGINE_TOKEN
    plugins: *test_collector
    command:
      - buildkite-agent cache restore --name cargo_registry --name target
      - source .buildkite/scripts/toolchain.sh
      # No rustup cache on macOS: the image ships several toolchains, so the
      # entry is 1.7 GB (20s to restore) against ~4s to install 1.98.0.
      - 'section ":test_tube: cargo nextest (lib + bins)" cargo nextest run $$(nextest_partition) --lib --bins || { .buildkite/scripts/proptest-regressions.sh; exit 1; }'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":mag: Check (Linux): all features, no features, release"
    key: check
    notify:
      - github_check:
          name: "buildkite/check"
    timeout_in_minutes: 45
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      - 'section ":package: cargo fetch" cargo fetch --locked'
      # Registry contents are the same for every step; this one saves them.
      - buildkite-agent cache save --name cargo_registry
      # --all-features enables vendored-tls, which compiles OpenSSL from
      # source; type-checking can use the system libssl instead.
      - 'section ":mag: check (all features)" env OPENSSL_NO_VENDOR=1 cargo check --all-features --workspace'
      - 'section ":mag: check (no features)" cargo check --no-default-features --workspace'
      - 'section ":mag: check (release profile)" cargo check --workspace --locked --release'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  # The rest of rust.yml's check matrix, in parallel with the step above.
  - label: ":mag: Check (Linux): sync, server, client only"
    key: check-subsets
    notify:
      - github_check:
          name: "buildkite/check-subsets"
    timeout_in_minutes: 45
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      - 'section ":mag: check (sync)" cargo check --no-default-features --features sync --workspace'
      - 'section ":mag: check (server)" cargo check -p atuin-server'
      - 'section ":mag: check (client only)" cargo check --no-default-features --features client --workspace'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":mag: Check (macOS)"
    key: check-macos
    notify:
      - github_check:
          name: "buildkite/check-macos"
    agents:
      queue: macos-large
    timeout_in_minutes: 45
    command:
      - buildkite-agent cache restore --name cargo_registry --name target
      - source .buildkite/scripts/toolchain.sh
      # No rustup cache on macOS: the image ships several toolchains, so the
      # entry is 1.7 GB (20s to restore) against ~4s to install 1.98.0.
      - 'section ":package: cargo fetch" cargo fetch --locked'
      # The registry key includes the OS, so macOS saves its own.
      - buildkite-agent cache save --name cargo_registry
      # No system libssl on macOS: --all-features builds vendored OpenSSL,
      # as it does on GitHub Actions.
      - 'section ":mag: check (all features)" cargo check --all-features --workspace'
      - 'section ":mag: check (no features)" cargo check --no-default-features --workspace'
      - 'section ":mag: check (sync)" cargo check --no-default-features --features sync --workspace'
      - 'section ":mag: check (server)" cargo check -p atuin-server'
      - 'section ":mag: check (client only)" cargo check --no-default-features --features client --workspace'
      - 'section ":mag: check (release profile)" cargo check --workspace --locked --release'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":crab: MSRV (1.95.0)"
    key: msrv
    notify:
      - github_check:
          name: "buildkite/msrv"
    timeout_in_minutes: 45
    env:
      # rust-toolchain.toml pins the dev toolchain; override it explicitly.
      RUST_TOOLCHAIN: "1.95.0"
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      - 'section ":crab: check on MSRV (all features)" cargo check --locked --workspace --all-features'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":postgres: Integration tests %n"
    key: integration-test
    notify:
      - github_check:
          name: "buildkite/integration-test"
    agents:
      queue: linux-large
    # Split with nextest --partition; raise this as the suite grows.
    parallelism: 2
    timeout_in_minutes: 45
    env:
      CARGO_TOOLS: nextest
      ATUIN_DB_URI: postgres://atuin:pass@localhost:5432/atuin
    secrets:
      - ATUIN_TEST_ENGINE_TOKEN
    plugins: *test_collector
    command:
      - .buildkite/scripts/start-postgres.sh
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      # e2e tests have their own step. Test databases live in tempdirs;
      # tmpfs keeps concurrent SQLite WAL fsyncs from stalling the disk.
      - 'section ":test_tube: cargo nextest (integration)" env TMPDIR=/dev/shm cargo nextest run $$(nextest_partition) --test "*" -E "not binary(/e2e_.*/)" || { .buildkite/scripts/proptest-regressions.sh; exit 1; }'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":terminal: e2e (Linux) %n"
    key: e2e
    notify:
      - github_check:
          name: "buildkite/e2e"
    # Split with nextest --partition; raise this as the suite grows.
    parallelism: 2
    agents:
      queue: linux-large
    timeout_in_minutes: 45
    env:
      CARGO_TOOLS: nextest
      APT_PACKAGES: "zsh fish xz-utils"
      ATUIN_E2E_REQUIRE_SHELLS: "1"
    secrets:
      - ATUIN_TEST_ENGINE_TOKEN
    plugins: *test_collector
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      - 'section ":shell: Install ble.sh" .buildkite/scripts/install-blesh.sh'
      - 'section ":terminal: cargo nextest (e2e)" cargo nextest run $$(nextest_partition) -p atuin --test "e2e_*"'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":terminal: e2e (macOS)"
    key: e2e-macos
    notify:
      - github_check:
          name: "buildkite/e2e-macos"
    agents:
      queue: macos-large
    timeout_in_minutes: 45
    env:
      CARGO_TOOLS: nextest
      BREW_PACKAGES: "bash fish"
      ATUIN_E2E_REQUIRE_SHELLS: "1"
    secrets:
      - ATUIN_TEST_ENGINE_TOKEN
    plugins: *test_collector
    command:
      - buildkite-agent cache restore --name cargo_registry --name target
      - source .buildkite/scripts/toolchain.sh
      # No rustup cache on macOS: the image ships several toolchains, so the
      # entry is 1.7 GB (20s to restore) against ~4s to install 1.98.0.
      # Test with Homebrew's bash, not macOS's /bin/bash 3.2.
      # ($$ escapes pipeline-upload interpolation.)
      - export ATUIN_E2E_BASH="$$(brew --prefix bash)/bin/bash"
      - 'section ":shell: Install ble.sh" .buildkite/scripts/install-blesh.sh'
      - 'section ":terminal: cargo nextest (e2e)" cargo nextest run -p atuin --test "e2e_*"'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":paperclip: Clippy"
    key: clippy
    notify:
      - github_check:
          name: "buildkite/clippy"
    timeout_in_minutes: 45
    env:
      RUST_COMPONENTS: clippy
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      # One run over the union of what `cargo clippy` (lib + bins) and
      # `cargo clippy --tests` lint, so the two share their work.
      - 'section ":paperclip: clippy (lib, bins, tests)" cargo clippy --lib --bins --tests -- -D warnings'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":books: Docs"
    key: docs
    notify:
      - github_check:
          name: "buildkite/docs"
    timeout_in_minutes: 45
    env:
      RUSTDOCFLAGS: "-D warnings"
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      - 'section ":books: cargo doc" cargo doc --document-private-items --no-deps --workspace'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  - label: ":art: Format"
    key: format
    notify:
      - github_check:
          name: "buildkite/format"
    timeout_in_minutes: 15
    env:
      RUST_TOOLCHAIN: nightly
      RUST_COMPONENTS: rustfmt
    command:
      - source .buildkite/scripts/toolchain.sh
      - 'section ":art: cargo fmt --check" cargo fmt -- --check'

  - label: ":no_entry: cargo-deny"
    key: deny
    notify:
      - github_check:
          name: "buildkite/deny"
    timeout_in_minutes: 15
    env:
      CARGO_TOOLS: deny
    command:
      - buildkite-agent cache restore --name cargo_registry --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      # Same arguments cargo-deny-action passes by default.
      - 'section ":no_entry: cargo deny check" cargo deny --log-level warn --all-features check'
      # Library crates must never (re)add eyre or anyhow; each has its own
      # crate-scoped deny.toml.
      - 'section ":no_entry: bans: atuin-common" cargo deny --log-level warn --manifest-path crates/atuin-common/Cargo.toml --all-features check bans'
      - 'section ":no_entry: bans: atuin-history" cargo deny --log-level warn --manifest-path crates/atuin-history/Cargo.toml --all-features check bans'
      - 'section ":no_entry: bans: atuin-domain" cargo deny --log-level warn --manifest-path crates/atuin-domain/Cargo.toml --all-features check bans'
      - 'section ":no_entry: bans: atuin-pty-proxy" cargo deny --log-level warn --manifest-path crates/atuin-pty-proxy/Cargo.toml --all-features check bans'

  - label: ":package: Package"
    key: package
    # main only: this only guards crates.io publishing (every crate builds
    # from its packaged tarball), which isn't part of the PR flow. On main
    # it still flags publishing breakage well before a release.
    if: build.branch == pipeline.default_branch
    notify:
      - github_check:
          name: "buildkite/package"
    agents:
      queue: linux-large
    timeout_in_minutes: 45
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      # Make sure the packaged crates build from their tarballs, not from
      # anything left over from the workspace build. The restored target/
      # holds only third-party builds (prune-target.sh), which verification
      # reuses: it builds into the shared target/debug.
      - rm -rf ~/.cargo/registry/{cache,src}/*/atuin* target/package
      - 'section ":package: cargo package" cargo package --workspace'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  # rust.yml's "Test in VM (illumos)": builds the client and server on a real
  # OmniOS VM (KVM-accelerated; hosted agents expose /dev/kvm). main only,
  # like build and nix-build: it was ~12 min per PR on GitHub Actions.
  - label: ":sunrise: Build on illumos (OmniOS VM)"
    key: illumos
    if: build.branch == pipeline.default_branch
    notify:
      - github_check:
          name: "buildkite/illumos"
    agents:
      queue: linux-large
    timeout_in_minutes: 45
    command:
      - buildkite-agent cache restore --name anyvm --name apt_debs
      - .buildkite/scripts/illumos.sh
      - buildkite-agent cache save --name anyvm --name apt_debs

  - label: ":moon: Nightly tests"
    key: nightly
    if: build.source == "schedule"
    notify:
      - github_check:
          name: "buildkite/nightly"
    timeout_in_minutes: 60
    env:
      CARGO_TOOLS: nextest
    secrets:
      - ATUIN_TEST_ENGINE_TOKEN
    plugins: *test_collector
    command:
      - buildkite-agent cache restore --name cargo_registry --name target --name rustup --name apt_debs
      - source .buildkite/scripts/toolchain.sh
      - buildkite-agent cache save --name rustup --name apt_debs
      - 'section ":moon: cargo nextest (nightly)" cargo nextest run -p atuin-daemon --test nightly --run-ignored all || { .buildkite/scripts/proptest-regressions.sh; exit 1; }'
      - 'section ":broom: Prune target/" .buildkite/scripts/prune-target.sh'
      - buildkite-agent cache save --name target

  # nix.yml: flake check, build (and run the result), formatting.
  - label: ":nix: Nix flake check"
    key: nix-check
    notify:
      - github_check:
          name: "buildkite/nix-check"
    timeout_in_minutes: 30
    command:
      - source .buildkite/scripts/lib.sh
      - 'section ":nix: Install Nix" install_nix'
      - 'section ":nix: nix flake check" nix flake check --print-build-logs'

  - label: ":nix: Nix build"
    key: nix-build
    notify:
      - github_check:
          name: "buildkite/nix-build"
    # main only: buildRustPackage builds atuin and every dependency in one
    # derivation, so every change rebuilds from scratch (~4 min). The flake
    # check and nixfmt steps still run on PRs.
    if: build.branch == pipeline.default_branch
    agents:
      queue: linux-large
    timeout_in_minutes: 45
    command:
      - source .buildkite/scripts/lib.sh
      - 'section ":nix: Install Nix" install_nix'
      - 'section ":nix: nix build" nix build --print-build-logs'
      - 'section ":nix: Run the built binary" ./result/bin/atuin --version'

  - label: ":nix: nixfmt"
    key: nixfmt
    notify:
      - github_check:
          name: "buildkite/nixfmt"
    timeout_in_minutes: 15
    command:
      - source .buildkite/scripts/lib.sh
      - 'section ":nix: Install Nix" install_nix'
      - 'section ":nix: nix fmt --ci" nix fmt -- --ci'

  - label: ":pencil2: Codespell"
    key: codespell
    notify:
      - github_check:
          name: "buildkite/codespell"
    timeout_in_minutes: 10
    command:
      - source .buildkite/scripts/lib.sh
      - 'section ":pencil2: Install codespell" install_codespell'
      - 'section ":pencil2: codespell" codespell --count --exclude-file CHANGELOG.md'

  - label: ":shell: Shellcheck"
    key: shellcheck
    notify:
      - github_check:
          name: "buildkite/shellcheck"
    timeout_in_minutes: 10
    env:
      SHELLCHECK_OPTS: "-e SC2148"
    command:
      - source .buildkite/scripts/lib.sh
      - 'section ":shell: Install ShellCheck" install_shellcheck'
      - 'section ":shell: shellcheck" .buildkite/scripts/shellcheck.sh'

  # vale.yml ran only on pull requests touching the docs or the Vale config.
  # On main builds there's no diff against main, so if_changed skips it there
  # too.
  - label: ":memo: Vale (docs prose)"
    key: vale
    if_changed:
      - "docs/docs/**"
      - ".vale.ini"
      - ".vale/styles/config/**"
      - ".buildkite/scripts/vale.sh"
    notify:
      - github_check:
          name: "buildkite/vale"
    timeout_in_minutes: 10
    command:
      - source .buildkite/scripts/lib.sh
      - 'section ":memo: Install Vale" install_vale'
      - 'section ":memo: vale" .buildkite/scripts/vale.sh'

  # installer.yml: the published installer (setup.atuin.sh, served from
  # main's install.sh), so it runs on main builds only, as before.
  - label: ":inbox_tray: Installer (Linux)"
    key: install
    if: build.branch == pipeline.default_branch
    notify:
      - github_check:
          name: "buildkite/install"
    timeout_in_minutes: 10
    command:
      - source .buildkite/scripts/lib.sh
      - 'section ":shell: Install zsh" apt_install zsh'
      - 'section ":inbox_tray: Installer via bash" .buildkite/scripts/test-installer.sh bash'
      - 'section ":inbox_tray: Installer via zsh" .buildkite/scripts/test-installer.sh zsh'

  - label: ":inbox_tray: Installer (macOS)"
    key: install-macos
    if: build.branch == pipeline.default_branch
    notify:
      - github_check:
          name: "buildkite/install-macos"
    agents:
      queue: macos-medium
    timeout_in_minutes: 10
    command:
      - source .buildkite/scripts/lib.sh
      - 'section ":inbox_tray: Installer via bash" .buildkite/scripts/test-installer.sh bash'
      - 'section ":inbox_tray: Installer via zsh" .buildkite/scripts/test-installer.sh zsh'