bootstrap.sh

#!/usr/bin/env bash
# Install required dependencies first (and restart your shell):
#   ./bootstrap.sh install_deps
#
# Usage: ./bootstrap.sh [cmd]"
#   ./bootstrap.sh: Max parallelism. Only use on serious hardware.
#   ./bootstrap.sh gentle: Less parallelism. Gentler on hardware. Slow.
#   ./bootstrap.sh check: Check required toolchains and versions are installed.
#   ./bootstrap.sh clean: Force a complete clean of the repo. Erases untracked files, be careful!

### TOOLCHAIN INSTALLATIONS ############################################################################################
# Expected toolchain versions.
export expected_min_clang_version=20.0.0
export expected_min_cmake_version=3.24
export expected_min_node_version=24.12.0
export expected_min_zig_version=0.15.1
export expected_abs_rust_version=1.89.0
export expected_abs_wasi_version=27.0
export expected_abs_foundry_version=1.4.1
export expected_abs_yarn_version=4.5.2

function ensure {
  command -v $1 &>/dev/null
}

function install_wasi_sdk {
  if cat /opt/wasi-sdk/VERSION 2> /dev/null | grep $expected_abs_wasi_version > /dev/null; then
    return
  fi
  local arch=$(uname -m)
  local os=$(os)
  local triple=$expected_abs_wasi_version-$arch-$os
  curl -LOs https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${expected_abs_wasi_version%%.*}/wasi-sdk-$triple.tar.gz
  tar xzf wasi-sdk-$triple.tar.gz
  rm wasi-sdk-$triple.tar.gz
  echo "Installing wasi sdk at /opt/wasi-sdk..."
  sudo rm -rf /opt/wasi-sdk
  sudo mv wasi-sdk-$triple /opt/wasi-sdk
}

function install_foundry {
  curl -L https://foundry.paradigm.xyz | bash
  ~/.foundry/bin/foundryup -i $expected_abs_foundry_version
}

function install_zig {
  if ! ensure zvm; then
    curl -s https://www.zvm.app/install.sh | bash
    export PATH="$PATH:$HOME/.zvm/bin"
    export PATH="$PATH:$HOME/.zvm/self"
  fi
  zvm i $expected_min_zig_version
}

function install_rustup {
  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $expected_abs_rust_version
}

function install_node {
  if ! ensure nvm; then
    # Files need to exist if you want nvm installer to update them.
    case $SHELL in
      */zsh) touch $HOME/.zshrc ;;
      */bash) touch $HOME/.bashrc ;;
    esac
    curl -s -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh | bash
    . "$HOME/.nvm/nvm.sh" --no-use
  fi
  nvm install --lts
  nvm alias default lts/*
}

function install_node_utils {
  . "$HOME/.nvm/nvm.sh"
  npm i -g corepack solhint
}

function install_llvm {
  wget https://apt.llvm.org/llvm.sh && \
    chmod +x llvm.sh && \
    ./llvm.sh 20 all && \
    rm llvm.sh
}

function install_yq {
  curl -sL https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_$(dpkg --print-architecture) \
    -o $AZTEC_DEV_BIN/yq && chmod +x $AZTEC_DEV_BIN/yq
}

function install_ldid {
  curl -sL https://github.com/ProcursusTeam/ldid/releases/download/v2.1.5-procursus7/ldid_linux_x86_64 \
    -o $AZTEC_DEV_BIN/ldid && chmod +x $AZTEC_DEV_BIN/ldid
}

export -f install_wasi_sdk install_foundry install_zig install_rustup install_node install_node_utils install_llvm \
          install_yq install_ldid ensure

function install_linux_deps {
  if ! ensure apt; then
    echo "Installation requires the apt package manager."
    exit 1
  fi
  mkdir -p "$AZTEC_DEV_BIN"
  spinner "Installing apt dependencies..." "sudo apt install -y jq parallel curl wget zstd redis-tools lsb-release software-properties-common gnupg build-essential cmake ninja-build xxd doxygen"
  spinner "Installing llvm..." install_llvm
  spinner "Installing yq..." install_yq
  spinner "Installing ldid..." install_ldid
  spinner "Installing rustup..." install_rustup
  spinner "Installing wasi-sdk..." install_wasi_sdk
  spinner "Installing foundry..." install_foundry
  spinner "Installing zig..." install_zig
  spinner "Installing node..." install_node
  spinner "Installing node utils..." install_node_utils
}

function install_macos_deps {
  # Check if brew is available.
  if ! ensure brew; then
    echo "Installation requires Homebrew."
    echo "Install it from https://brew.sh"
    exit 1
  fi
  spinner "Installing brew dependencies..." \
    "brew install cmake ninja llvm@20 doxygen coreutils grep gnu-sed parallel yq zstd redis util-linux libusb jq bash"

  # Make clang 20 available.
  local llvm_bin="$(brew --prefix)/Cellar/llvm@20/20.1.8/bin"
  mkdir -p "$AZTEC_DEV_BIN"
  ln -sf "$llvm_bin/clang" "$AZTEC_DEV_BIN/clang-20"
  ln -sf "$llvm_bin/clang++" "$AZTEC_DEV_BIN/clang++-20"
  ln -sf "$llvm_bin/clang-format" "$AZTEC_DEV_BIN/clang-format-20"

  spinner "Installing wasi-sdk..." install_wasi_sdk
  spinner "Installing foundry..." install_foundry
  spinner "Installing rustup..." install_rustup
  spinner "Installing zig..." install_zig
  spinner "Installing node..." install_node
  spinner "Installing node utils..." install_node_utils
}

function install_deps {
  case "$(os)" in
    linux) install_linux_deps ;;
    macos) install_macos_deps ;;
    *)
      echo -e "${bold}${red}Unknown operating system.${reset}"
      echo "We encourage use of our dev container. See build-images/README.md."
      exit 1
      ;;
  esac

  echo
  if [ -t 0 ]; then
    echo "Done! Starting fresh shell..."
    exec $SHELL
  else
    echo "Done! You'll need to start a fresh shell to see PATH updates."
    echo
  fi
}

# Special case for installing dependencies (can run on older bash).
if [ "${1:-}" = "install_deps" ]; then
  set -euo pipefail
  source $(git rev-parse --show-toplevel)/ci3/source_base
  install_deps
  exit 0
fi

### START OF MAIN BOOTSTRAP SCRIPT #####################################################################################
source $(git rev-parse --show-toplevel)/ci3/source_bootstrap

# Enable abbreviated output by default.
export DENOISE=${DENOISE:-1}

# Number of TXE servers to run when testing.
export NUM_TXES=1

# Number of jobs for make. Defaults to number of CPUs.
# TODO: We should dial this back on consumer hardware, maybe to just 1.
export MAKEFLAGS="-j${MAKE_JOBS:-$(get_num_cpus)}"

# We append all test commands to this file as they become available during build.
# The test engine feeds it into parallel.
export test_cmds_file="/tmp/test_cmds"
export bench_cmds_file="/tmp/bench_cmds"

### CLEANUP ON EXIT ####################################################################################################
function cleanup {
  set +e
  if [ -n "${test_engine_pid:-}" ]; then
    echo "Sending SIGTERM to test engine process group..."
    kill -SIGTERM -- -$test_engine_pid &>/dev/null
    wait $test_engine_pid
    test_engine_pid=
  fi
  if [ -n "${make_pid:-}" ]; then
    echo "Sending SIGTERM to make process..."
    kill -SIGTERM $make_pid &>/dev/null
    wait $make_pid
    make_pid=
  fi
  stop_txes
}
trap cleanup EXIT

### TOOLCHAIN CHECKS ###################################################################################################
function check_minimum_version {
  local min_version=$1
  local installed_version=$2
  if [[ "$(printf '%s\n' "$min_version" "$installed_version" | sort -V | head -n1)" != "$min_version" ]]; then
    return 1
  fi
  return 0
}

function toolchain_incompatible {
  if [ "$(os)" == "unknown" ] || [ "$(os)" == "linux" ] && ! ensure apt; then
    echo -e "${bold}${red}ERROR: Toolchain incompatibility.${reset}"
    echo "We encourage use of our dev container. See build-images/README.md."
  else
    echo -e "${bold}${red}ERROR: Toolchain incompatibility.${reset}"
    echo "You can install requirements with: ./bootstrap.sh install_deps"
  fi
  exit 1
}

# Checks for required utilities, toolchains and their versions.
# DO NOT INSTALL THINGS IN HERE.
function check_toolchains {
  # Check for various required utilities.
  for util in jq parallel awk git curl zstd corepack solhint; do
    if ! ensure $util; then
      echo "$util not found."
      toolchain_incompatible
    fi
  done
  if [ "$(os)" == "linux" ] && ! ensure ldid; then
    echo "ldid not found."
    toolchain_incompatible
  fi
  if ! yq --version | grep "version v4" > /dev/null; then
    echo "yq not found."
    toolchain_incompatible
  fi
  # Check cmake version.
  local cmake_installed_version=$(cmake --version | head -n1 | awk '{print $3}')
  if ! check_minimum_version $expected_min_cmake_version $cmake_installed_version; then
    echo "Minimum cmake version $expected_min_cmake_version not found."
    toolchain_incompatible
  fi
  # Check clang version.
  local clang_installed_version=$(clang++-20 --version | head -n1 | awk '{print $4}')
  if ! check_minimum_version $expected_min_clang_version $clang_installed_version; then
    echo "Minimum clang version $expected_min_clang_version not found."
    toolchain_incompatible
  fi
  # Check zig version.
  local zig_installed_version=$(zig version)
  if ! check_minimum_version $expected_min_zig_version $zig_installed_version; then
    echo "Minimum zig version $expected_min_zig_version not found."
    toolchain_incompatible
  fi
  # Check rustup installed.
  if ! ensure rustup; then
    echo "Rustup not installed."
    toolchain_incompatible
  fi
  if ! rustup show | grep $expected_abs_rust_version > /dev/null; then
    # Cargo will download necessary version of rust at runtime but warn to update the build-image.
    echo -e "${bold}${yellow}WARN: Rust ${expected_abs_rust_version} is not installed. Update build-image.${reset}"
  fi
  # Check wasi-sdk version.
  if ! cat /opt/wasi-sdk/VERSION 2> /dev/null | grep $expected_abs_wasi_version > /dev/null; then
    toolchain_incompatible
  fi
  # Check foundry version.
  for tool in forge anvil; do
    if ! $tool --version 2> /dev/null | grep "$expected_abs_foundry_version" > /dev/null; then
      echo "$tool version $expected_abs_foundry_version not found."
      toolchain_incompatible
    fi
  done
  # Check Node.js version.
  local node_installed_version=$(node --version | cut -d 'v' -f 2)
  if ! check_minimum_version $expected_min_node_version $node_installed_version; then
    echo "Minimum node version $expected_min_node_version not found."
    toolchain_incompatible
  fi
  # Check yarn version. This catches oddities like an overriding .yarnrc.yml outside the repo.
  if [ "$expected_abs_yarn_version" != "$(corepack yarn@$expected_abs_yarn_version --version)" ]; then
    echo "Yarn version $expected_abs_yarn_version not found. Check for a rogue .yarnrc.yml in e.g. home directory."
    toolchain_incompatible
  fi
}

### BUILDING AND TESTING ###############################################################################################
# Install pre-commit git hooks.
function install_hooks {
  hooks_dir=$(git rev-parse --git-path hooks)
  rm -f $hooks_dir/*

  cat <<EOF >$hooks_dir/pre-commit
#!/usr/bin/env bash
set -euo pipefail
(cd barretenberg/cpp && ./format.sh staged)
./yarn-project/precommit.sh
./noir/precommit.sh
./noir-projects/precommit.sh
./yarn-project/constants/precommit.sh
./docs/examples/ts/precommit.sh
EOF
  chmod +x $hooks_dir/pre-commit

  cat <<EOF >$hooks_dir/post-merge
#!/usr/bin/env bash
set -euo pipefail
git submodule update --init --recursive
EOF
  chmod +x $hooks_dir/post-merge
}

function pull_submodules {
  echo_header "pull submodules"
  # If it's an old standalone noir clone, nuke it.
  if [ -d "noir/noir-repo/.git" ]; then
    echo "Removing old noir clone..."
    rm -rf noir/noir-repo
  fi
  denoise "git submodule update --init --recursive --depth 1 --jobs 8 && git -C noir/noir-repo fetch --tags &>/dev/null"
}

function start_txes {
  # Until Kev's kzg lib stops using Tokio.
  export TOKIO_WORKER_THREADS=1

  # Starting txe servers with incrementing port numbers.
  # Base port is below the Linux ephemeral range (32768-60999) to avoid conflicts.
  local txe_base_port=14730
  for i in $(seq 0 $((NUM_TXES-1))); do
    port=$((txe_base_port + i))
    existing_pid=$(lsof -ti :$port || true)
    if [ -n "$existing_pid" ]; then
      echo "Killing existing process $existing_pid on port: $port"
      check_port $port
      kill -9 $existing_pid &>/dev/null || true
      while kill -0 $existing_pid &>/dev/null; do sleep 0.1; done
    fi
    dump_fail "LOG_LEVEL=info TXE_PORT=$port retry 'node --no-warnings ./yarn-project/txe/dest/bin/index.js'" &
    txe_pids+="$! "
  done

  echo "Waiting for TXE's to start..."
  for i in $(seq 0 $((NUM_TXES-1))); do
      local j=0
      while ! nc -z 127.0.0.1 $((txe_base_port + i)) &>/dev/null; do
        if [ $j == 60 ]; then
          echo_stderr "TXE $i failed to start on port $((txe_base_port + i)) after 60s."
          check_port $((txe_base_port + i))
          exit 1
        fi
        sleep 1
        j=$((j+1))
      done
  done
}

function stop_txes {
  if [ -n "${txe_pids:-}" ]; then
    echo "Stopping TXE processes..."
    kill -SIGTERM $txe_pids &>/dev/null || true
    wait $txe_pids || true
    txe_pids=
  fi
}

function prep {
  check_toolchains
  pull_submodules
}

function build {
  prep
  echo_header "build"
  make ${1:-fast}
}

function build_and_test {
  prep
  echo_header "build and test"

  # Start the test engine.
  rm -f $test_cmds_file
  touch $test_cmds_file
  # put it in it's own process group, we can terminate on cleanup.
  setsid color_prefix "test-engine" "denoise \"test_engine $test_cmds_file\"" &
  # setsid makes the child the session leader, so its PID is its PGID.
  test_engine_pid=$!
  echo "Started test engine with PGID $test_engine_pid."

  # Start the build.
  make $1 &
  make_pid=$!

  # As soon as one of the above fails, terminate the other (as part of exit cleanup).
  while [ -n "$make_pid" ] || [ -n "$test_engine_pid" ]; do
    # This will return success only if build or test succeeds.
    # Otherwise it's an error, which is an exit, which means we enter cleanup.
    wait -p finished -n $make_pid $test_engine_pid &>/dev/null

    # If make succeeded, start txes and add tests that depend on them.
    if [ "$finished" == "$make_pid" ]; then
      echo "Makefile build complete, starting TXEs and adding dependent tests..."
      make_pid=

      # TODO: Handle this better so they can be run as part of the Makefile dependency tree.
      start_txes
      make noir-projects-txe-tests

      # Signal tests complete, handled by parallel -E STOP.
      echo STOP >> $test_cmds_file
    fi

    if [ "$finished" == "$test_engine_pid" ]; then
      echo "Test engine completed successfully."
      test_engine_pid=
    fi
  done

  stop_txes

  return 0
}

function bench_cmds {
  if [ "$#" -eq 0 ]; then
    set -- yarn-project/end-to-end yarn-project barretenberg/{ts,cpp,sol} noir-projects/noir-protocol-circuits l1-contracts
  fi
  parallel -k --line-buffer './{}/bootstrap.sh bench_cmds' ::: $@
}

function build_bench {
  # TODO bench for arm64.
  if [ $(arch) == arm64 ]; then
    return
  fi
  parallel --line-buffer --tag --halt now,fail=1 'denoise "{}/bootstrap.sh build_bench"' ::: \
    barretenberg/cpp \
    yarn-project/end-to-end
}
export -f build_bench

function bench_merge {
  find . -path "*/bench-out/*.bench.json" -type f -print0 | \
  xargs -0 -I{} bash -c '
    dir=$1; \
    dir=${dir#./}; \
    dir=${dir%/bench-out*}; \
    jq --arg prefix "$dir/" '\''map(.name |= "\($prefix)\(.)")'\'' "$1"
  ' _ {} | jq -s "add // []" > bench-out/bench.json

}

function bench {
  # TODO bench for arm64.
  if [ $(arch) == arm64 ]; then
    return
  fi
  echo_header "bench all"
  build_bench

  bench_cmds > $bench_cmds_file
  denoise "bench_engine $bench_cmds_file"

  rm -rf bench-out
  mkdir -p bench-out
  bench_merge
  cache_upload bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
}

### RELEASING ##########################################################################################################
function versions {
  local noir_version anvil_version node_version cmake_version clang_version zig_version rustc_version wasi_sdk_version
  noir_version=$(git -C noir/noir-repo describe --tags --always HEAD)
  anvil_version=$(anvil --version | head -n1 | sed -E 's/anvil Version: ([0-9.]+).*/\1/')
  node_version=$(node --version | cut -d 'v' -f 2)
  cmake_version=$(cmake --version | head -n1 | cut -d' ' -f3)
  clang_version=$(clang++-20 --version | head -n1 | cut -d' ' -f4)
  zig_version=$(zig version)
  rustc_version=$(rustc --version | cut -d' ' -f2)
  wasi_sdk_version=$(cat /opt/wasi-sdk/VERSION 2> /dev/null | head -n1)
  echo "noir: $noir_version"
  echo "foundry: $anvil_version"
  echo "node: $node_version"
  echo "cmake: $cmake_version"
  echo "clang: $clang_version"
  echo "zig: $zig_version"
  echo "rustc: $rustc_version"
  echo "wasi-sdk: $wasi_sdk_version"
}

function release_github {
  # Add an easy link for comparing to previous release.
  local compare_link=""
  if gh release view "v$CURRENT_VERSION" &>/dev/null; then
    compare_link=$(echo -e "See changes: https://github.com/AztecProtocol/aztec-packages/compare/v${CURRENT_VERSION}...${COMMIT_HASH}")
  fi
  # Legacy releases. TODO: Eventually remove.
  if gh release view "aztec-packages-v$CURRENT_VERSION" &>/dev/null; then
    compare_link=$(echo -e "See changes: https://github.com/AztecProtocol/aztec-packages/compare/aztec-packages-v${CURRENT_VERSION}...${COMMIT_HASH}")
  fi
  # Determine if this is a prerelease (has a prerelease tag like -rc.1, -alpha, etc.)
  local is_prerelease=false
  if [ -n "$(semver prerelease $REF_NAME)" ]; then
    is_prerelease=true
  fi
  # Ensure we have a commit release.
  if ! gh release view "$REF_NAME" &>/dev/null; then
    local prerelease_flag=""
    if $is_prerelease; then
      prerelease_flag="--prerelease"
    fi
    do_or_dryrun gh release create "$REF_NAME" \
      $prerelease_flag \
      --target $COMMIT_HASH \
      --title "$REF_NAME" \
      --notes "$compare_link"
  elif ! $is_prerelease; then
    # Release exists but this is not a prerelease version - ensure it's marked as a full release
    do_or_dryrun gh release edit "$REF_NAME" --prerelease=false
  fi
}

function release {
  # Releases are triggered when REF_NAME is a valid semver (but can have a leading v).
  # We ensure there is a github release for our REF_NAME, if not on latest (in which case release-please creates it).
  # We derive a dist tag from our prerelease portion of our REF_NAME semver. It is latest if no prerelease.
  echo_header "release all"
  set -x

  # Ensure we have a github release for our REF_NAME.
  # This is in case were are not going through release-please.
  release_github

  projects=(
    barretenberg/cpp
    barretenberg/ts
    barretenberg/rust
    noir
    l1-contracts
    noir-projects/aztec-nr
    yarn-project
    boxes
    aztec-up
    playground
    release-image
  )
  if [ $(arch) == arm64 ]; then
    projects=(
      release-image
    )
  fi

  for project in "${projects[@]}"; do
    $project/bootstrap.sh release
  done
}

function release_dryrun {
  DRY_RUN=1 release
}

### SELF TESTING #######################################################################################################
function test_bootstrap_linux {
  local name=linux-bootstrap-test-ubuntu
  docker volume rm $name-volume &>/dev/null || true
  trap "docker volume rm $name-volume &>/dev/null" EXIT
  docker run --rm -ti --name $name \
    --cpus=32 \
    --ulimit nofile=1048576:1048576 \
    -v $root:/aztec-packages:ro \
    --mount type=volume,src=$name-volume,dst=/root/aztec-packages \
    -w /root \
    ubuntu:24.04 bash -c "
set -euo pipefail
ulimit -n 65536
apt update && apt install -y git sudo
git config --global --add safe.directory /aztec-packages/.git
git clone --branch=$(git branch --show-current) /aztec-packages
cd aztec-packages
./bootstrap.sh install_deps </dev/null
bash -l -i -c 'ulimit -n 65536 && NO_CACHE=1 ./bootstrap.sh gentle'
  "
}

function test_bootstrap_macos {
  local version="${1:-26}"
  local name="bootstrap-test-$version"
  CPU_CORES=32 RAM_SIZE=32g /mnt/user-data/macos/launch_vm.sh $version "" $name
  trap "docker stop macos-$name" EXIT
  local ip=$(docker inspect -f '{{ .NetworkSettings.Networks.bridge.IPAddress }}' macos-$name)
  echo "Waiting for Mac VM ($name) to be accessible at $ip..."
  while ! nc -z $ip 22 &>/dev/null; do sleep 0.5; done
  /mnt/user-data/macos/ssh.sh $name bash -c 'cat > /tmp/mac_bootstrap.sh' <<REMOTE_EOF
set -euo pipefail
ulimit -n 65536
git clone --depth=1 --branch=$(git branch --show-current) https://github.com/aztecprotocol/aztec-packages
cd aztec-packages
./bootstrap.sh install_deps </dev/null
zsh -l -i -c "ulimit -n 65536 && NO_CACHE=1 ./bootstrap.sh gentle"
REMOTE_EOF
  /mnt/user-data/macos/ssh.sh $name -t zsh -l /tmp/mac_bootstrap.sh
}

### COMMAND HANDLER ####################################################################################################
# Handle our command line arguments.
# All the commands that start with ci-* are intended to be callable from
# a fresh repo. They are ideal for calling into from github actions on a new runner
# Current flow: ci3.yml -> .github/ci3.sh -> ci.sh -> this script on a fresh EC2 runner.
case "$cmd" in
  "clean")
    echo "WARNING: This will erase *all* untracked files, including hooks and submodules."
    echo -n "Continue? [y/n] "
    read user_input
    if [[ ! "$user_input" =~ ^[yY](es)?$ ]]; then
      echo "Exiting without cleaning"
      exit 1
    fi

    # Remove hooks and submodules.
    rm -rf .git/hooks/*
    rm -rf .git/modules/*
    for submodule in $(git config --file .gitmodules --get-regexp path | awk '{print $2}'); do
      rm -rf $submodule
    done

    # Remove all untracked files, directories, nested repos, and .gitignore files.
    git clean -ffdx
  ;;
  "check")
    check_toolchains
    echo "Toolchains look good! 🎉"
  ;;
  "")
    install_hooks
    build
  ;;
  "gentle")
    install_hooks
    prep
    if [ -t 1 ]; then
      export DUMP_FAIL=1
    fi
    CMAKE_BUILD_PARALLEL_LEVEL=$(nproc --ignore=2) CARGO_BUILD_JOBS=$(nproc --ignore=2) MEMSUSPEND=1g make -j1 "$@"
  ;;

  ######################################
  # VARIANTS ON NORMAL PULL-REQUEST CI #
  ######################################
  "ci-fast")
    export CI=1
    export USE_TEST_CACHE=1
    export CI_FULL=0
    build_and_test fast
    ;;
  "ci-full")
    export CI=1
    export USE_TEST_CACHE=1
    export CI_FULL=1
    build_and_test full
    bench
    ;;
  "ci-full-no-test-cache")
    export CI=1
    export USE_TEST_CACHE=0
    export CI_FULL=1
    build_and_test full
    bench
    ;;
  "ci-grind-test")
    export CI=1
    export USE_TEST_CACHE=0

    full_cmd="${1:?full_cmd required}"
    timeout="${2:-}"
    jobs_pct="${3:-200}"
    memsuspend_pct="${4:-50}"
    commit="${5:-}"

    grind_test "$full_cmd" "$timeout" "$jobs_pct" "$memsuspend_pct" "$commit"
    ;;

  ##########################################
  # NETWORK DEPLOYMENTS WITH BENCHES/TESTS #
  ##########################################
  "ci-network-deploy")
    # Args: <env_file> <namespace> [docker_image] [test_set]
    export CI=1
    env_file="${1:?env_file is required}"
    namespace="${2:?namespace is required}"
    docker_image="${3:-}"
    test_set="${4:-}"
    build
    # If no docker image provided, build and push to aztecdev
    if [ -z "$docker_image" ]; then
      release-image/bootstrap.sh push_pr
      docker_image="aztecprotocol/aztecdev:$(git rev-parse HEAD)"
    fi
    # Set up environment and deploy using spartan
    export NAMESPACE="$namespace"
    export AZTEC_DOCKER_IMAGE="$docker_image"
    deploy_exit_code=0
    spartan/bootstrap.sh network_deploy "${env_file}" "$test_set" || deploy_exit_code=$?
    # Merge and upload deploy benchmarks (deploy_network.sh writes to spartan/bench-out/)
    rm -rf bench-out
    mkdir -p bench-out
    bench_merge
    cache_upload deploy-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
    exit $deploy_exit_code
    ;;
  "ci-network-tests")
    # Args: <env_file> <namespace>
    export CI=1
    env_file="${1:?env_file is required}"
    namespace="${2:?namespace is required}"
    build
    # Set up environment for tests
    export NAMESPACE="$namespace"
    spartan/bootstrap.sh network_tests "${env_file}"
    ;;
  "ci-network-kind-tests")
    export CI=1
    [ "${SKIP_BUILD:-0}" -eq 0 ] && build
    # Set the docker image to the locally built image and load it into KIND
    export AZTEC_DOCKER_IMAGE="aztecprotocol/aztec:$(git rev-parse HEAD)"
    spartan/bootstrap.sh kind
    kind load docker-image "$AZTEC_DOCKER_IMAGE"
    # Just one test for now
    spartan/bootstrap.sh test-kind-upgrade-rollup
    ;;
  "ci-network-bench")
    # Args: <env_file> <namespace> [docker_image]
    # Deploys network and runs benchmarks. Cleanup should be done separately.
    export CI=1
    env_file="${1:?env_file is required}"
    namespace="${2:?namespace is required}"
    docker_image="${3:-}"
    build
    # If no docker image provided, build and push to aztecdev
    if [ -z "$docker_image" ]; then
      release-image/bootstrap.sh push_pr
      docker_image="aztecprotocol/aztecdev:$(git rev-parse HEAD)"
    fi
    # Set up environment and deploy using spartan
    export NAMESPACE="$namespace"
    export AZTEC_DOCKER_IMAGE="$docker_image"
    spartan/bootstrap.sh network_deploy "${env_file}"
    # Run benchmarks
    spartan/bootstrap.sh network_bench "${env_file}"
    rm -rf bench-out
    mkdir -p bench-out
    bench_merge
    cache_upload spartan-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
    ;;
  "ci-network-proving-bench")
    # Args: <env_file> <namespace> [docker_image]
    # Deploys network and runs proving benchmarks. Cleanup should be done separately.
    export CI=1
    env_file="${1:?env_file is required}"
    namespace="${2:?namespace is required}"
    docker_image="${3:-}"
    build
    # If no docker image provided, build and push to aztecdev
    if [ -z "$docker_image" ]; then
      release-image/bootstrap.sh push_pr
      docker_image="aztecprotocol/aztecdev:$(git rev-parse HEAD)"
    fi
    # Set up environment and deploy using spartan
    export NAMESPACE="$namespace"
    export AZTEC_DOCKER_IMAGE="$docker_image"
    spartan/bootstrap.sh network_deploy "${env_file}"
    # Run proving benchmarks
    spartan/bootstrap.sh proving_bench "${env_file}"
    rm -rf bench-out
    mkdir -p bench-out
    bench_merge
    cache_upload spartan-proving-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
    ;;
  "ci-network-teardown")
    # Args: <env_file> <namespace>
    # Tears down a deployed network.
    export CI=1
    env_file="${1:?env_file is required}"
    namespace="${2:?namespace is required}"
    # Set up environment for teardown
    export NAMESPACE="$namespace"
    denoise "spartan/bootstrap.sh network_teardown ${env_file}"
    ;;

  ############
  # RELEASES #
  ############
  "ci-release")
    export CI=1
    export USE_TEST_CACHE=1
    if ! semver check $REF_NAME; then
      exit 1
    fi
    build release
    release
    ;;

  ##########################
  # MERGE TRAIN CI SUBSETS #
  ##########################
  "ci-docs")
    export CI=1
    export USE_TEST_CACHE=1
    BOOTSTRAP_TO=yarn-project ./bootstrap.sh
    docs/bootstrap.sh ci
    ;;
  "ci-barretenberg-debug")
    export CI=1
    export NATIVE_PRESET=debug
    export AVM=0
    export AVM_TRANSPILER=0
    barretenberg/cpp/bootstrap.sh build
    ;;
  "ci-barretenberg")
    export CI=1
    export USE_TEST_CACHE=1
    export AVM=0
    export AVM_TRANSPILER=0
    barretenberg/cpp/bootstrap.sh ci
    ;;
  "ci-barretenberg-full")
    export CI=1
    export USE_TEST_CACHE=1
    export AVM=0
    export AVM_TRANSPILER=0
    pull_submodules
    noir/bootstrap.sh build_native  # Build nargo for acir_tests
    barretenberg/bootstrap.sh ci
    barretenberg/cpp/bootstrap.sh build_bench
    ;;

  #######################
  # AVM QA ONE OFF JOBS #
  #######################
  "ci-avm-inputs-collection")
    # Nightly job: Run e2e tests with AVM circuit inputs dumping, upload to cache
    export CI=1
    # Use tree hash for tarball name - consistent across all environments
    export AVM_INPUTS_HASH=$(git rev-parse HEAD^{tree})
    build
    yarn-project/end-to-end/bootstrap.sh test_and_collect_avm_inputs
    ;;
  "ci-avm-check-circuit")
    # Nightly job: Download cached AVM inputs and run check-circuit on each
    export CI=1
    # Use tree hash for tarball name - consistent across all environments
    export AVM_INPUTS_HASH=$(git rev-parse HEAD^{tree})
    build
    yarn-project/end-to-end/bootstrap.sh avm_check_circuit
    ;;
  ##########################################
  # ROLLUP UPGRADE DEPLOYMENT              #
  ##########################################
  "ci-deploy-rollup-upgrade")
    # Env vars: NETWORK, GCP_PROJECT_ID (for GCP secrets)
    # Args: <registry_address> [KEY=VALUE...]
    export CI=1
    build
    exec spartan/scripts/deploy_rollup_upgrade.sh "$@"
    ;;

  #################
  # SOCKET FIX    #
  #################
  "ci-socket-fix")
    export CI=1
    build
    scripts/socket-fix-ci.sh "$@"
    ;;

  ##############################################
  # Default handler, calls our above functions #
  ##############################################
  *)
    default_cmd_handler "$@"
    ;;
esac