From 696957b352847eaa9290f71f92c998745a60515a Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:44:20 -0600 Subject: [PATCH] ci: Run the smoke test VM with gcloud and ssh instead of Vagrant Vagrant came in 2020 so one Vagrantfile could run the smoke test in a local VirtualBox VM or on GCE. Only the GCE provider is left, so Vagrant now only adds Ruby, the vagrant-google plugin, a pinned Vagrant on the agents, and an rsync of the repo. smoke-test-vm.sh replaces vagrant-run.sh, the Vagrantfile, and servers.yaml. It creates the VM with the same settings, authorizes a key made for the job, copies the repo over ssh, and runs the steps servers.yaml listed as root, sending the Docker Hub password over stdin. The boot disk grows from vagrant-google's default 10 GB to 20 GB, which the Ubuntu 26.04 e2e image will need for Docker 29's containerd image store. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1c770-8d7e-76ad-b47f-2a2ff80873cc --- .buildkite/pipeline.yaml | 7 ++- .buildkite/smoke-test-vm.sh | 100 ++++++++++++++++++++++++++++++++++++ .buildkite/vagrant-run.sh | 61 ---------------------- .gitignore | 1 - test/Vagrantfile | 65 ----------------------- test/servers.yaml | 17 ------ 6 files changed, 103 insertions(+), 148 deletions(-) create mode 100755 .buildkite/smoke-test-vm.sh delete mode 100755 .buildkite/vagrant-run.sh delete mode 100644 test/Vagrantfile delete mode 100644 test/servers.yaml diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 826b9f888..1091d0f8f 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,8 +1,7 @@ $schema: "http://json.schemastore.org/buildkite" env: - # GCP VM image for the Vagrant smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images` + # GCP VM image for the smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images` IMAGE: "e2e-db8a40720-2024-08-26t08-44-10z" - VAGRANT_RUN_ENV: "CI" steps: - label: ":docker:" command: .buildkite/validate-docker-compose.sh @@ -21,10 +20,10 @@ steps: # The branch is named here, because the Buildkite pipeline's default branch is still "master" - &smoke_test label: ":rice: docker-compose-test" - command: .buildkite/vagrant-run.sh docker-test + command: .buildkite/smoke-test-vm.sh artifact_paths: ./*.log agents: { queue: "vagrant" } - # vagrant-run.sh deletes VMs and disks older than an hour, so no job may run that long + # smoke-test-vm.sh deletes VMs and disks older than an hour, so no job may run that long timeout_in_minutes: 30 if: build.branch == "main" diff --git a/.buildkite/smoke-test-vm.sh b/.buildkite/smoke-test-vm.sh new file mode 100755 index 000000000..7c9b1198c --- /dev/null +++ b/.buildkite/smoke-test-vm.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# Runs test/smoke-test.sh on a new GCP VM booted from $IMAGE, then deletes the VM + +cd "$(dirname "${BASH_SOURCE[0]}")/.." +set -euo pipefail + +KEY_PATH="$(mktemp)" +project="sourcegraph-ci" +zone="us-central1-f" + +# GCP creates a disk, then a VM, both with this name. Naming them after the job +# lets the cleanup find them if the job is cancelled before the VM exists +instance_name_prefix="docker-test-job-" +instance_name="${instance_name_prefix}${BUILDKITE_JOB_ID}" + +# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses, +# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM +request_deletion() { + { + gcloud compute instances list --project="$project" --filter="$1" --format="value(selfLink)" && + gcloud compute disks list --project="$project" --filter="($1) AND -users:*" --format="value(selfLink)" + } | tee /dev/stderr | + xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \ + --request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)" +} + +# Fall back to a blocking delete, so a failed deletion request cannot leak a VM +cleanup() { + deletion_status=0 + request_deletion "name=${instance_name}" || + gcloud compute instances delete "$instance_name" --project="$project" --zone="$zone" --quiet || + deletion_status=$? + rm -rf "$KEY_PATH" "$ssh_directory" + return "$deletion_status" +} + +# A key for this job only, which the VM's guest agent authorizes for the buildkite user +ssh_directory="$(mktemp -d)" +ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$ssh_directory/key" +ssh_options=(-i "$ssh_directory/key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5) + +trap cleanup EXIT + +echo --- ":lock: builder account key" +gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci >"${KEY_PATH}" +DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH" + +echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup" +# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job +request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks" + +echo --- ":gcloud: starting VM $instance_name" +ip_address="$(gcloud compute instances create "$instance_name" \ + --project="$project" --zone="$zone" \ + --image="$IMAGE" --image-project="$project" \ + --machine-type=custom-16-20480 \ + --boot-disk-size=20GB --boot-disk-type=pd-standard \ + --network=default --no-address \ + --service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \ + --labels=team=dev-infra,type=smoke-test \ + --metadata="ssh-keys=buildkite:$(cat "$ssh_directory/key.pub")" \ + --format="value(networkInterfaces[0].networkIP)")" +remote="buildkite@${ip_address}" + +echo --- ":ssh: waiting for the VM to accept SSH" +deadline=$((SECONDS + 600)) +until ssh "${ssh_options[@]}" "$remote" true; do + if ((SECONDS > deadline)); then + echo "The VM did not accept SSH within 10 minutes" + exit 1 + fi + sleep 2 +done + +echo --- ":package: copying the repository to the VM" +tar --create --gzip --exclude=.git --exclude=node_modules . | + ssh "${ssh_options[@]}" "$remote" "sudo mkdir /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker" + +echo --- ":docker: running the smoke tests" +# Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list +remote_script=' +set -euo pipefail +read -r DOCKER_PASSWORD +export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD +/deploy-sourcegraph-docker/test/smoke-test.sh +# The VM image has no jq. Skip /etc/apt/sources.list.d: the image lacks the key for its Chrome repository, so a full update fails +apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq +/deploy-sourcegraph-docker/test/executor-smoke-test.sh +' +exit_code=0 +# shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM +ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD" || exit_code=$? + +if [ "$exit_code" != 0 ]; then + # Copy back the container logs the smoke test wrote, for the step's artifacts + ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true + exit "$exit_code" +fi diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh deleted file mode 100755 index c4afd4569..000000000 --- a/.buildkite/vagrant-run.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -cd "$(dirname "${BASH_SOURCE[0]}")/.." -set -euo pipefail - -box="$1" -exit_code=0 -KEY_PATH="$(mktemp)" - -pushd "test" - -# Vagrant creates a disk, then a VM, both with this name, and only knows about them once GCP has finished -# creating the VM. Naming them after the job lets the cleanup find them if the job is cancelled before that -instance_name_prefix="${box}-job-" -export VAGRANT_INSTANCE_NAME="${instance_name_prefix}${BUILDKITE_JOB_ID}" - -# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses, -# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM -request_deletion() ( - export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH" - { - gcloud compute instances list --project=sourcegraph-ci --filter="$1" --format="value(selfLink)" && - gcloud compute disks list --project=sourcegraph-ci --filter="($1) AND -users:*" --format="value(selfLink)" - } | tee /dev/stderr | - xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \ - --request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)" -) - -# Fall back to a blocking destroy, so a failed deletion request cannot leak a VM -cleanup() { - deletion_status=0 - request_deletion "name=${VAGRANT_INSTANCE_NAME}" || vagrant destroy -f "$box" || deletion_status=$? - rm -f "$KEY_PATH" - return "$deletion_status" -} - -echo --- ":vagrant: installing plugins" -vagrant --version -# Agents on the same host share ~/.vagrant.d, so only the first job on a new host installs -[[ "$(vagrant plugin list)" == *"vagrant-google (2.7.0"* ]] || vagrant plugin install vagrant-google --plugin-version '2.7.0' - -trap cleanup EXIT - -echo --- ":lock: builder account key" -gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}" -export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}" -DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" -export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD - -echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup" -# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job -request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks" - -echo --- ":vagrant: starting box $box" -vagrant up "$box" --provider=google || exit_code=$? - -if [ "$exit_code" != 0 ]; then - vagrant plugin install vagrant-scp - vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../ - exit $exit_code -fi diff --git a/.gitignore b/.gitignore index 9cf3cdc72..3ecd4b8fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .env .envrc -.vagrant *.log node_modules/ diff --git a/test/Vagrantfile b/test/Vagrantfile deleted file mode 100644 index ebd5446ca..000000000 --- a/test/Vagrantfile +++ /dev/null @@ -1,65 +0,0 @@ -# -*- mode: ruby -*- -# # vi: set ft=ruby : -# Specify minimum Vagrant version and Vagrant API version -Vagrant.require_version '>= 1.6.0' -VAGRANTFILE_API_VERSION = '2'.freeze -# Require YAML module -require 'yaml' - -# Read YAML file with box details -servers = YAML.load_file('servers.yaml') - -# Create boxes -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # Iterate through entries in YAML file - servers.each do |server| - config.vm.define server['name'] do |srv| - srv.vm.box = server['box'] - srv.vm.synced_folder '../', '/deploy-sourcegraph-docker' - srv.vm.boot_timeout = 600 - - # Setup for CI or Local - case ENV['VAGRANT_RUN_ENV'] - when 'CI' - project_id = server['project_id'] - external_ip = server['external_ip'] - use_private_ip = server['use_private_ip'] - username = server['username'] - ssh_key_path = server['ssh_key_path'] - else - project_id = ENV['VAGRANT_GCP_PROJECT_ID'] || 'sourcegraph-server' - external_ip = nil - use_private_ip = false - username = ENV['VAGRANT_SSH_USER'] || ENV['USER'] - ssh_key_path = ENV['VAGRANT_SSH_KEY'] || '~/.ssh/id_rsa' - end - - srv.vm.provider :google do |g, o| - g.machine_type = server['machine_type'] - g.image = ENV['IMAGE'] - g.image_project_id = 'sourcegraph-ci' - g.google_project_id = project_id - # .buildkite/vagrant-run.sh names the VM after its job, so it can delete the VM by name - g.name = ENV['VAGRANT_INSTANCE_NAME'] || "#{server['name']}-"+"#{Time.now.to_i}"+"#{rand(100000)}" - g.network = server['network'] - g.external_ip = external_ip - g.use_private_ip = use_private_ip - g.service_account = server['service_account'] - g.labels = { - "team" => "dev-infra", - "type" => "vagrant" - } - g.scopes = [ - "compute-rw", - "storage-ro" - ] - o.ssh.username = username - o.ssh.private_key_path = ssh_key_path - end - - server['shell_commands'].each do |sh| - srv.vm.provision 'shell', inline: sh, env: ENV.slice('DOCKER_USERNAME', 'DOCKER_PASSWORD') - end - end - end -end diff --git a/test/servers.yaml b/test/servers.yaml deleted file mode 100644 index 88a9cc366..000000000 --- a/test/servers.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: docker-test - box: google/gce - machine_type: "custom-16-20480" - project_id: sourcegraph-ci - external_ip: false - use_private_ip: true - network: default - username: buildkite - ssh_key_path: "~/.ssh/id_rsa" - service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com - shell_commands: - - "/deploy-sourcegraph-docker/test/smoke-test.sh" - # The VM image has no jq. Skip /etc/apt/sources.list.d: the image lacks the key for its Chrome repository, so a full update fails - - "apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq" - - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh" -