ci: Run the smoke test VM with gcloud instead of Vagrant

.buildkite/pipeline.yaml

@@ -1,8 +1,7 @@
 $schema: "http://json.schemastore.org/buildkite"
 env:
-  # GCP VM image for the Vagrant smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images`
+  # GCP VM image for the smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images`
   IMAGE: "e2e-db8a40720-2024-08-26t08-44-10z"
-  VAGRANT_RUN_ENV: "CI"
 steps:
   - label: ":docker:"
     command: .buildkite/validate-docker-compose.sh
@@ -20,9 +19,11 @@ steps:
   # so it finds no changes in a commit which is already on main, unless it is the newest
   - &smoke_test
     label: ":rice: docker-compose-test"
-    command: .buildkite/vagrant-run.sh docker-test
+    command: .buildkite/smoke-test-vm.sh
     artifact_paths: ./*.log
     agents: { queue: "vagrant" }
+    # smoke-test-vm.sh deletes VMs and disks older than an hour, so no job may run that long
+    timeout_in_minutes: 30
     if: build.branch == "main"
 
   # Builds on other branches skip the smoke test if they only change files it does not read.

.buildkite/smoke-test-vm.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+# Runs test/smoke-test.sh and test/executor-smoke-test.sh on a new GCP VM booted from $IMAGE,
+# then deletes the VM
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+set -euo pipefail
+
+project="sourcegraph-ci"
+zone="us-central1-f"
+
+# GCP creates a disk, then a VM, both with the same name. Naming them after the job
+# lets the cleanup find them if the job is cancelled before the VM exists
+instance_name_prefix="docker-test-job-"
+instance_name="${instance_name_prefix}${BUILDKITE_JOB_ID}"
+
+# The builder account key, and an SSH key for this job only
+temporary_directory="$(mktemp -d)"
+
+# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses,
+# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM
+request_deletion() {
+	{
+		gcloud compute instances list --project="$project" --filter="$1" --format="value(selfLink)" &&
+			gcloud compute disks list --project="$project" --filter="($1) AND -users:*" --format="value(selfLink)"
+	} | tee /dev/stderr |
+		xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \
+			--request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)"
+}
+
+cleanup() {
+	request_deletion "name=${instance_name}" ||
+		echo "Could not delete this job's VM. GCP deletes it an hour after it started"
+	rm -rf "$temporary_directory"
+}
+trap cleanup EXIT
+
+# The VM's guest agent authorizes this key for the buildkite user.
+# The keepalive ends the connection to a VM which stopped answering, after a minute
+ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$temporary_directory/ssh-key"
+ssh_options=(-i "$temporary_directory/ssh-key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR
+	-o ConnectTimeout=5 -o ServerAliveInterval=15 -o ServerAliveCountMax=4)
+
+echo --- ":lock: builder account key"
+gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project="$project" >"$temporary_directory/builder-key.json"
+DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project="$project")"
+export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$temporary_directory/builder-key.json"
+
+echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
+# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job.
+# GCP deletes the VMs itself after an hour, but not a disk whose VM was never created
+request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks"
+
+echo "--- :gcloud: starting VM $instance_name"
+# `beta`, because --max-run-duration is only there in the agents' gcloud 417. It is in every track from gcloud 483
+ip_address="$(gcloud beta compute instances create "$instance_name" \
+	--project="$project" --zone="$zone" \
+	--image="$IMAGE" --image-project="$project" \
+	--machine-type=custom-16-20480 \
+	--boot-disk-size=20GB --boot-disk-type=pd-standard \
+	--max-run-duration=1h --instance-termination-action=DELETE \
+	--network=default --no-address \
+	--service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \
+	--labels=team=dev-infra,type=smoke-test \
+	--metadata="ssh-keys=buildkite:$(cat "$temporary_directory/ssh-key.pub")" \
+	--format="value(networkInterfaces[0].networkIP)")"
+remote="buildkite@${ip_address}"
+
+echo --- ":ssh: waiting for the VM to accept SSH"
+deadline=$((SECONDS + 300))
+until ssh "${ssh_options[@]}" "$remote" true; do
+	if ((SECONDS > deadline)); then
+		echo "The VM did not accept SSH within 5 minutes"
+		exit 1
+	fi
+	sleep 2
+done
+
+copy_repository() {
+	tar --create --gzip --exclude=.git --exclude=node_modules . |
+		ssh "${ssh_options[@]}" "$remote" "sudo mkdir -p /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker"
+}
+
+echo --- ":package: copying the repository to the VM"
+# A VM can accept SSH, then stop answering for a while. ssh exits with 255 when the connection fails,
+# so retry the copy on the same VM. The tests have not started yet, so nothing runs twice
+exit_code=0
+copy_repository || exit_code=$?
+for retry in {1..10}; do
+	if [ "$exit_code" != 255 ]; then
+		break
+	fi
+	echo "The VM does not answer SSH, retry $retry of 10 in 15 seconds"
+	sleep 15
+	exit_code=0
+	copy_repository || exit_code=$?
+done
+
+if [ "$exit_code" = 0 ]; then
+	echo --- ":docker: running the smoke tests"
+	# Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list
+	remote_script='
+set -euo pipefail
+read -r DOCKER_PASSWORD
+export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD
+/deploy-sourcegraph-docker/test/smoke-test.sh
+# The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails
+command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; }
+/deploy-sourcegraph-docker/test/executor-smoke-test.sh
+'
+	# shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM
+	ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD" || exit_code=$?
+
+	if [ "$exit_code" != 0 ]; then
+		# Copy back the container logs the smoke test wrote, for the step's artifacts
+		ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true
+	fi
+fi
+
+# Not a plain `exit`, after which shellcheck reports that cleanup is never invoked (SC2329)
+[ "$exit_code" = 0 ] || exit "$exit_code"

.gitignore

@@ -1,5 +1,4 @@
 .env
 .envrc
-.vagrant
 *.log
 node_modules/

AGENTS.md

@@ -17,20 +17,22 @@ CI runs these on every branch. `.tool-versions` pins Node, the GitHub CLI, and s
 - **Compose files:** `.buildkite/validate-docker-compose.sh` runs `docker compose config` on every Compose file, in each combination the docs say to run. Add a line to it for every new Compose file or combination.
 - **Formatting:** `npm ci && npm run prettier-check`; `npm run prettier` rewrites the files. Prettier formats the JavaScript and JSON files; the repository has no other file type its glob matches. `.prettierignore` excludes Markdown and `.github/`, and no formatter runs on YAML, so match the surrounding YAML style by hand.
 - **Shell scripts:** `git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck`.
-- **Smoke test:** the `docker-compose-test` step runs `.buildkite/vagrant-run.sh docker-test`. See below.
+- **Smoke test:** the `docker-compose-test` step runs `.buildkite/smoke-test-vm.sh`. See below.
 
 Use `docker compose` (v2) in scripts and docs, not `docker-compose`.
 
 ## Smoke test
 
-`.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks.
+`.buildkite/smoke-test-vm.sh` boots a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks.
 
 - `test/smoke-test.sh` starts the base file, `jaeger/`, `executors/`, and `test/docker-compose.override.yaml` with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend and Jaeger respond. It counts the services itself, so a new long-running service in those files needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count. Add a new optional Compose file to its `compose_files`.
 - `test/executor-smoke-test.sh` checks that the executor registers with the frontend, and serves exactly the queues which `test/docker-compose.override.yaml` gives it. The script names them again, so change both files together.
 - A service which is not ready until another service is needs a `start_period` on its health check, or `--wait` fails.
-- A test failure is not retried, and the step uploads every container's logs as artifacts. Only the image pull is retried: `test/smoke-test.sh` tries `docker compose pull` three times.
+- A test failure is not retried, and the step uploads every container's logs as artifacts. Only connections are retried: the script waits up to 5 minutes for the VM to accept SSH, retries the SSH connection which copies the repository to the VM, and `test/smoke-test.sh` tries `docker compose pull` three times.
 - A branch which only changes files listed in the step's `if_changed` `exclude` skips the step. Builds on `main` always run it. When the smoke test starts reading a file on that list, remove it from the list.
+- GCP deletes each VM an hour after it starts, and the script deletes the VMs and disks which older jobs left behind. Keep the step's `timeout_in_minutes` under an hour.
 - The VM image is built from `gcp/images/e2e` in sourcegraph/infrastructure.
+- The CI agents' queue is named `vagrant`. Nothing on `main` uses Vagrant.
 
 ## Conventions