From 0610c7df8a2d230bd9a205d000d9d44842c3478e Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:43:01 -0600 Subject: [PATCH] ci: Run the smoke test VM with gcloud instead of Vagrant smoke-test-vm.sh creates the VM with gcloud, copies the repository to it over SSH, runs test/smoke-test.sh and test/executor-smoke-test.sh, and asks GCP to delete the VM without waiting. GCP deletes each VM an hour after it starts, and each job deletes the VMs and disks older jobs left behind. The boot disk is 20 GB. If the VM stops answering SSH while the repository is copied to it, the script retries the copy on the same VM. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5 Co-authored-by: Amp --- .buildkite/pipeline.yaml | 7 ++- .buildkite/smoke-test-vm.sh | 121 ++++++++++++++++++++++++++++++++++++ .buildkite/vagrant-run.sh | 39 ------------ .gitignore | 1 - AGENTS.md | 8 ++- test/Vagrantfile | 74 ---------------------- test/servers.yaml | 17 ----- 7 files changed, 130 insertions(+), 137 deletions(-) create mode 100755 .buildkite/smoke-test-vm.sh delete mode 100755 .buildkite/vagrant-run.sh delete mode 100644 test/Vagrantfile delete mode 100644 test/servers.yaml diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 2a2595ad6..f5f2e72f5 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,8 +1,7 @@ $schema: "http://json.schemastore.org/buildkite" env: - # GCP VM image for the Vagrant smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images` + # GCP VM image for the smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images` IMAGE: "e2e-db8a40720-2024-08-26t08-44-10z" - VAGRANT_RUN_ENV: "CI" steps: - label: ":docker:" command: .buildkite/validate-docker-compose.sh @@ -20,9 +19,11 @@ steps: # so it finds no changes in a commit which is already on main, unless it is the newest - &smoke_test label: ":rice: docker-compose-test" - command: .buildkite/vagrant-run.sh docker-test + command: .buildkite/smoke-test-vm.sh artifact_paths: ./*.log agents: { queue: "vagrant" } + # smoke-test-vm.sh deletes VMs and disks older than an hour, so no job may run that long + timeout_in_minutes: 30 if: build.branch == "main" # Builds on other branches skip the smoke test if they only change files it does not read. diff --git a/.buildkite/smoke-test-vm.sh b/.buildkite/smoke-test-vm.sh new file mode 100755 index 000000000..d5b746029 --- /dev/null +++ b/.buildkite/smoke-test-vm.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +# Runs test/smoke-test.sh and test/executor-smoke-test.sh on a new GCP VM booted from $IMAGE, +# then deletes the VM + +cd "$(dirname "${BASH_SOURCE[0]}")/.." +set -euo pipefail + +project="sourcegraph-ci" +zone="us-central1-f" + +# GCP creates a disk, then a VM, both with the same name. Naming them after the job +# lets the cleanup find them if the job is cancelled before the VM exists +instance_name_prefix="docker-test-job-" +instance_name="${instance_name_prefix}${BUILDKITE_JOB_ID}" + +# The builder account key, and an SSH key for this job only +temporary_directory="$(mktemp -d)" + +# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses, +# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM +request_deletion() { + { + gcloud compute instances list --project="$project" --filter="$1" --format="value(selfLink)" && + gcloud compute disks list --project="$project" --filter="($1) AND -users:*" --format="value(selfLink)" + } | tee /dev/stderr | + xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \ + --request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)" +} + +cleanup() { + request_deletion "name=${instance_name}" || + echo "Could not delete this job's VM. GCP deletes it an hour after it started" + rm -rf "$temporary_directory" +} +trap cleanup EXIT + +# The VM's guest agent authorizes this key for the buildkite user. +# The keepalive ends the connection to a VM which stopped answering, after a minute +ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$temporary_directory/ssh-key" +ssh_options=(-i "$temporary_directory/ssh-key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR + -o ConnectTimeout=5 -o ServerAliveInterval=15 -o ServerAliveCountMax=4) + +echo --- ":lock: builder account key" +gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project="$project" >"$temporary_directory/builder-key.json" +DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project="$project")" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$temporary_directory/builder-key.json" + +echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup" +# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job. +# GCP deletes the VMs itself after an hour, but not a disk whose VM was never created +request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks" + +echo "--- :gcloud: starting VM $instance_name" +# `beta`, because --max-run-duration is only there in the agents' gcloud 417. It is in every track from gcloud 483 +ip_address="$(gcloud beta compute instances create "$instance_name" \ + --project="$project" --zone="$zone" \ + --image="$IMAGE" --image-project="$project" \ + --machine-type=custom-16-20480 \ + --boot-disk-size=20GB --boot-disk-type=pd-standard \ + --max-run-duration=1h --instance-termination-action=DELETE \ + --network=default --no-address \ + --service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \ + --labels=team=dev-infra,type=smoke-test \ + --metadata="ssh-keys=buildkite:$(cat "$temporary_directory/ssh-key.pub")" \ + --format="value(networkInterfaces[0].networkIP)")" +remote="buildkite@${ip_address}" + +echo --- ":ssh: waiting for the VM to accept SSH" +deadline=$((SECONDS + 300)) +until ssh "${ssh_options[@]}" "$remote" true; do + if ((SECONDS > deadline)); then + echo "The VM did not accept SSH within 5 minutes" + exit 1 + fi + sleep 2 +done + +copy_repository() { + tar --create --gzip --exclude=.git --exclude=node_modules . | + ssh "${ssh_options[@]}" "$remote" "sudo mkdir -p /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker" +} + +echo --- ":package: copying the repository to the VM" +# A VM can accept SSH, then stop answering for a while. ssh exits with 255 when the connection fails, +# so retry the copy on the same VM. The tests have not started yet, so nothing runs twice +exit_code=0 +copy_repository || exit_code=$? +for retry in {1..10}; do + if [ "$exit_code" != 255 ]; then + break + fi + echo "The VM does not answer SSH, retry $retry of 10 in 15 seconds" + sleep 15 + exit_code=0 + copy_repository || exit_code=$? +done + +if [ "$exit_code" = 0 ]; then + echo --- ":docker: running the smoke tests" + # Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list + remote_script=' +set -euo pipefail +read -r DOCKER_PASSWORD +export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD +/deploy-sourcegraph-docker/test/smoke-test.sh +# The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails +command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; } +/deploy-sourcegraph-docker/test/executor-smoke-test.sh +' + # shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM + ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD" || exit_code=$? + + if [ "$exit_code" != 0 ]; then + # Copy back the container logs the smoke test wrote, for the step's artifacts + ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true + fi +fi + +# Not a plain `exit`, after which shellcheck reports that cleanup is never invoked (SC2329) +[ "$exit_code" = 0 ] || exit "$exit_code" diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh deleted file mode 100755 index 3be84b10e..000000000 --- a/.buildkite/vagrant-run.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -cd "$(dirname "${BASH_SOURCE[0]}")/.." -set -euo pipefail - -box="$1" -exit_code=0 - -pushd "test" - -cleanup() { - vagrant destroy -f "$box" -} - -echo --- ":vagrant: installing plugins" -vagrant --version -vagrant plugin install vagrant-google --plugin-version '2.7.0' -vagrant plugin install vagrant-env -vagrant plugin install vagrant-scp - -trap cleanup EXIT - -echo --- ":lock: builder account key" -KEY_PATH="/tmp/e2e-builder.json" -if [ ! -f ${KEY_PATH} ]; then - gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}" -fi -export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}" -export DOCKER_USERNAME=sourcegraph -DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" -export DOCKER_PASSWORD - -echo --- ":vagrant: starting box $box" -vagrant up "$box" --provider=google || exit_code=$? - -if [ "$exit_code" != 0 ]; then - vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../ - exit $exit_code -fi diff --git a/.gitignore b/.gitignore index 9cf3cdc72..3ecd4b8fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .env .envrc -.vagrant *.log node_modules/ diff --git a/AGENTS.md b/AGENTS.md index e14e992d0..61ec38f75 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,20 +17,22 @@ CI runs these on every branch. `.tool-versions` pins Node, the GitHub CLI, and s - **Compose files:** `.buildkite/validate-docker-compose.sh` runs `docker compose config` on every Compose file, in each combination the docs say to run. Add a line to it for every new Compose file or combination. - **Formatting:** `npm ci && npm run prettier-check`; `npm run prettier` rewrites the files. Prettier formats the JavaScript and JSON files; the repository has no other file type its glob matches. `.prettierignore` excludes Markdown and `.github/`, and no formatter runs on YAML, so match the surrounding YAML style by hand. - **Shell scripts:** `git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck`. -- **Smoke test:** the `docker-compose-test` step runs `.buildkite/vagrant-run.sh docker-test`. See below. +- **Smoke test:** the `docker-compose-test` step runs `.buildkite/smoke-test-vm.sh`. See below. Use `docker compose` (v2) in scripts and docs, not `docker-compose`. ## Smoke test -`.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. +`.buildkite/smoke-test-vm.sh` boots a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. - `test/smoke-test.sh` starts the base file, `jaeger/`, `executors/`, and `test/docker-compose.override.yaml` with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend and Jaeger respond. It counts the services itself, so a new long-running service in those files needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count. Add a new optional Compose file to its `compose_files`. - `test/executor-smoke-test.sh` checks that the executor registers with the frontend, and serves exactly the queues which `test/docker-compose.override.yaml` gives it. The script names them again, so change both files together. - A service which is not ready until another service is needs a `start_period` on its health check, or `--wait` fails. -- A test failure is not retried, and the step uploads every container's logs as artifacts. Only the image pull is retried: `test/smoke-test.sh` tries `docker compose pull` three times. +- A test failure is not retried, and the step uploads every container's logs as artifacts. Only connections are retried: the script waits up to 5 minutes for the VM to accept SSH, retries the SSH connection which copies the repository to the VM, and `test/smoke-test.sh` tries `docker compose pull` three times. - A branch which only changes files listed in the step's `if_changed` `exclude` skips the step. Builds on `main` always run it. When the smoke test starts reading a file on that list, remove it from the list. +- GCP deletes each VM an hour after it starts, and the script deletes the VMs and disks which older jobs left behind. Keep the step's `timeout_in_minutes` under an hour. - The VM image is built from `gcp/images/e2e` in sourcegraph/infrastructure. +- The CI agents' queue is named `vagrant`. Nothing on `main` uses Vagrant. ## Conventions diff --git a/test/Vagrantfile b/test/Vagrantfile deleted file mode 100644 index 09bd66030..000000000 --- a/test/Vagrantfile +++ /dev/null @@ -1,74 +0,0 @@ -# -*- mode: ruby -*- -# # vi: set ft=ruby : -# Specify minimum Vagrant version and Vagrant API version -Vagrant.require_version '>= 1.6.0' -VAGRANTFILE_API_VERSION = '2'.freeze -# Require YAML module -require 'yaml' - -# Read YAML file with box details -servers = YAML.load_file('servers.yaml') - -# Create boxes -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # Iterate through entries in YAML file - servers.each do |server| - config.env.enable - config.vm.define server['name'] do |srv| - srv.vm.box = server['box'] - srv.vm.synced_folder '../', '/deploy-sourcegraph-docker' - srv.vm.boot_timeout = 600 - - # Setup for CI or Local - case ENV['VAGRANT_RUN_ENV'] - when 'CI' - project_id = server['project_id'] - external_ip = server['external_ip'] - use_private_ip = server['use_private_ip'] - username = server['username'] - ssh_key_path = server['ssh_key_path'] - else - project_id = ENV['VAGRANT_GCP_PROJECT_ID'] || 'sourcegraph-server' - external_ip = nil - use_private_ip = false - username = ENV['VAGRANT_SSH_USER'] || ENV['USER'] - ssh_key_path = ENV['VAGRANT_SSH_KEY'] || '~/.ssh/id_rsa' - end - - srv.vm.provider :google do |g, o| - g.machine_type = server['machine_type'] - g.image = ENV['IMAGE'] - g.image_project_id = 'sourcegraph-ci' - g.google_project_id = project_id - g.name = "#{server['name']}-"+"#{Time.now.to_i}"+"#{rand(100000)}" - g.network = server['network'] - g.external_ip = external_ip - g.use_private_ip = use_private_ip - g.service_account = server['service_account'] - g.labels = { - "team" => "dev-infra", - "type" => "vagrant" - } - g.scopes = [ - "compute-rw", - "storage-ro" - ] - o.ssh.username = username - o.ssh.private_key_path = ssh_key_path - end - - srv.vm.provision 'shell', inline: <<-SHELL - #!/usr/bin/env bash - cat << EOF >> /root/.profile -export GIT_BRANCH=#{ENV['BUILDKITE_BRANCH']} -export DOCKER_USERNAME=#{ENV['DOCKER_USERNAME']} -export DOCKER_PASSWORD=#{ENV['DOCKER_PASSWORD']} -EOF - SHELL - - server['shell_commands'].each do |sh| - srv.vm.provision 'shell', inline: sh - end - end - end -end diff --git a/test/servers.yaml b/test/servers.yaml deleted file mode 100644 index f68c942c9..000000000 --- a/test/servers.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: docker-test - box: google/gce - machine_type: "custom-16-20480" - project_id: sourcegraph-ci - external_ip: false - use_private_ip: true - network: default - username: buildkite - ssh_key_path: "~/.ssh/id_rsa" - service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com - shell_commands: - - "/deploy-sourcegraph-docker/test/smoke-test.sh" - # The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails - - "command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; }" - - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh" -