ci: Speed up the smoke test

.buildkite/pipeline.yaml

@@ -12,10 +12,24 @@ steps:
     command: .buildkite/prettier-check.sh
     agents: { queue: "vagrant" }
 
-  - label: ":rice: docker-compose-test"
+  # Builds on main always run the smoke test. if_changed compares against origin/main,
+  # so it finds no changes in a commit which is already on main, unless it is the newest.
+  # The branch is named here, because the Buildkite pipeline's default branch is still "master"
+  - &smoke_test
+    label: ":rice: docker-compose-test"
     command: .buildkite/vagrant-run.sh docker-test
     artifact_paths: ./*.log
     agents: { queue: "vagrant" }
+    # vagrant-run.sh deletes VMs and disks older than an hour, so no job may run that long
+    timeout_in_minutes: 30
+    if: build.branch == "main"
+
+  # Builds on other branches skip the smoke test if they only change docs
+  - <<: *smoke_test
+    if: build.branch != "main"
+    if_changed:
+      include: "**"
+      exclude: "**.md"
 
   # Please keep in mind that the release manifest uses specific branch names when creating releases.
   # Therefore, if you update them, or if you decide to change how we detect what kind of build we're dealing

.buildkite/vagrant-run.sh

@@ -5,34 +5,56 @@ set -euo pipefail
 
 box="$1"
 exit_code=0
+KEY_PATH="/tmp/e2e-builder.json"
 
 pushd "test"
 
+# Vagrant creates a disk, then a VM, both with this name, and only knows about them once GCP has finished
+# creating the VM. Naming them after the job lets the cleanup find them if the job is cancelled before that
+instance_name_prefix="${box}-job-"
+export VAGRANT_INSTANCE_NAME="${instance_name_prefix}${BUILDKITE_JOB_ID}"
+
+# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses,
+# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM
+request_deletion() (
+	export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH"
+	{
+		gcloud compute instances list --project=sourcegraph-ci --filter="$1" --format="value(selfLink)" &&
+			gcloud compute disks list --project=sourcegraph-ci --filter="($1) AND -users:*" --format="value(selfLink)"
+	} | tee /dev/stderr |
+		xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \
+			--request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)"
+)
+
+# Fall back to a blocking destroy, so a failed deletion request cannot leak a VM
 cleanup() {
-	vagrant destroy -f "$box"
+	request_deletion "name=${VAGRANT_INSTANCE_NAME}" || vagrant destroy -f "$box"
 }
 
 echo --- ":vagrant: installing plugins"
 vagrant --version
-vagrant plugin install vagrant-google --plugin-version '2.7.0'
-vagrant plugin install vagrant-env
-vagrant plugin install vagrant-scp
+# Agents on the same host share ~/.vagrant.d, so only the first job on a new host installs
+[[ "$(vagrant plugin list)" == *"vagrant-google (2.7.0"* ]] || vagrant plugin install vagrant-google --plugin-version '2.7.0'
 
 trap cleanup EXIT
 
 echo --- ":lock: builder account key"
-KEY_PATH="/tmp/e2e-builder.json"
 if [ ! -f ${KEY_PATH} ]; then
   gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}"
 fi
 export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}"
 export DOCKER_USERNAME=sourcegraph
 export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
 
+echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
+# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job
+request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks"
+
 echo --- ":vagrant: starting box $box"
 vagrant up "$box" --provider=google || exit_code=$?
 
 if [ "$exit_code" != 0 ]; then
+	vagrant plugin install vagrant-scp
 	vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../
 	exit $exit_code
 fi

test/Vagrantfile

@@ -13,7 +13,6 @@ servers = YAML.load_file('servers.yaml')
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # Iterate through entries in YAML file
   servers.each do |server|
-    config.env.enable
     config.vm.define server['name'] do |srv|
       srv.vm.box = server['box']
       srv.vm.synced_folder '../', '/deploy-sourcegraph-docker'
@@ -40,7 +39,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         g.image = ENV['IMAGE']
         g.image_project_id = 'sourcegraph-ci'
         g.google_project_id = project_id
-        g.name = "#{server['name']}-"+"#{Time.now.to_i}"+"#{rand(100000)}"
+        # .buildkite/vagrant-run.sh names the VM after its job, so it can delete the VM by name
+        g.name = ENV['VAGRANT_INSTANCE_NAME'] || "#{server['name']}-"+"#{Time.now.to_i}"+"#{rand(100000)}"
         g.network = server['network']
         g.external_ip = external_ip
         g.use_private_ip = use_private_ip

test/smoke-test.sh

@@ -15,10 +15,24 @@ expect_containers="25"
 
 deploy_sourcegraph() {
 	cd $(dirname "${BASH_SOURCE[0]}")/..
+	docker-compose version
+	# `up` pulls images one at a time, `pull` pulls them in parallel
+	docker-compose --file docker-compose/docker-compose.yaml pull --quiet
 	docker-compose --file docker-compose/docker-compose.yaml up -d -t 600
+}
 
-	echo "Giving containers 90s to start..."
-	sleep 90
+wait_until_healthy() {
+	echo "Waiting up to 5 minutes for the frontend to respond, and container health checks to pass..."
+	for _ in {1..60}; do
+		if curl --fail --silent --output /dev/null http://localhost:80/healthz &&
+			[[ -z "$(docker ps --quiet --filter health=starting --filter health=unhealthy)" ]]; then
+			return
+		fi
+		sleep 5
+	done
+	docker ps --format '{{.Names}}: {{.Status}}'
+	echo "TEST FAILURE: containers were not healthy after 5 minutes"
+	exit 1
 }
 
 test_count() {
@@ -46,6 +60,13 @@ test_containers() {
 		sleep 1
 	done
 
+	echo "TEST: No container has restarted"
+	restarted_containers=$(docker ps --quiet | xargs docker inspect --format '{{.Name}} restarted {{.RestartCount}} times' | grep -v ' restarted 0 times' || true)
+	if [[ -n "$restarted_containers" ]]; then
+		echo "TEST FAILURE: $restarted_containers"
+		exit 1
+	fi
+
 	echo "TEST: Checking frontend is accessible"
 	curl -f http://localhost:80
 	curl -f http://localhost:80/healthz
@@ -70,5 +91,6 @@ trap catch_errors EXIT
 
 configure_docker
 deploy_sourcegraph
+wait_until_healthy
 test_count
 test_containers