From 93a03e2ca634d20a9335c4f52250d6625d2aa64d Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 15:15:05 -0600
Subject: [PATCH 01/25] ci: Speed up the smoke test

Skip waiting on the VM deletion, poll container health instead of sleeping
90 seconds, pull images in parallel, install fewer Vagrant plugins, and
skip the smoke test on docs-only builds.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c731-7931-7091-9b5e-664cc2379c79
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/pipeline.yaml  |  4 ++++
 .buildkite/vagrant-run.sh | 20 +++++++++++++++-----
 test/Vagrantfile          |  1 -
 test/smoke-test.sh        | 19 +++++++++++++++++--
 4 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index efa4e4125..afabeac5c 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -16,6 +16,10 @@ steps:
     command: .buildkite/vagrant-run.sh docker-test
     artifact_paths: ./*.log
     agents: { queue: "vagrant" }
+    # Skip the smoke test on builds which only change docs
+    if_changed:
+      include: "**"
+      exclude: "**.md"
 
   # Please keep in mind that the release manifest uses specific branch names when creating releases.
   # Therefore, if you update them, or if you decide to change how we detect what kind of build we're dealing
diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh
index a54aeb97d..1403e0dd8 100755
--- a/.buildkite/vagrant-run.sh
+++ b/.buildkite/vagrant-run.sh
@@ -5,23 +5,32 @@ set -euo pipefail
 
 box="$1"
 exit_code=0
+KEY_PATH="/tmp/e2e-builder.json"
 
 pushd "test"
 
+# Ask GCP to delete the VM, without waiting the 1-2 minutes the deletion takes
+request_instance_deletion() (
+	export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH"
+	instance_name="$(cat ".vagrant/machines/${box}/google/id" 2>/dev/null)" &&
+		instance_url="$(gcloud compute instances list --project=sourcegraph-ci --filter="name=${instance_name}" --format="value(selfLink)")" &&
+		curl --fail --silent --show-error --output /dev/null --request DELETE \
+			--header "Authorization: Bearer $(gcloud auth print-access-token)" "$instance_url"
+)
+
+# Fall back to a blocking destroy, so a failed deletion request cannot leak a VM
 cleanup() {
-	vagrant destroy -f "$box"
+	request_instance_deletion || vagrant destroy -f "$box"
 }
 
 echo --- ":vagrant: installing plugins"
 vagrant --version
-vagrant plugin install vagrant-google --plugin-version '2.7.0'
-vagrant plugin install vagrant-env
-vagrant plugin install vagrant-scp
+# Agents on the same host share ~/.vagrant.d, so only the first job on a new host installs
+[[ "$(vagrant plugin list)" == *"vagrant-google (2.7.0"* ]] || vagrant plugin install vagrant-google --plugin-version '2.7.0'
 
 trap cleanup EXIT
 
 echo --- ":lock: builder account key"
-KEY_PATH="/tmp/e2e-builder.json"
 if [ ! -f ${KEY_PATH} ]; then
   gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}"
 fi
@@ -33,6 +42,7 @@ echo --- ":vagrant: starting box $box"
 vagrant up "$box" --provider=google || exit_code=$?
 
 if [ "$exit_code" != 0 ]; then
+	vagrant plugin install vagrant-scp
 	vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../
 	exit $exit_code
 fi
diff --git a/test/Vagrantfile b/test/Vagrantfile
index 09bd66030..ef3901e09 100644
--- a/test/Vagrantfile
+++ b/test/Vagrantfile
@@ -13,7 +13,6 @@ servers = YAML.load_file('servers.yaml')
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # Iterate through entries in YAML file
   servers.each do |server|
-    config.env.enable
     config.vm.define server['name'] do |srv|
       srv.vm.box = server['box']
       srv.vm.synced_folder '../', '/deploy-sourcegraph-docker'
diff --git a/test/smoke-test.sh b/test/smoke-test.sh
index 5aac87dca..6eb98584e 100755
--- a/test/smoke-test.sh
+++ b/test/smoke-test.sh
@@ -15,10 +15,24 @@ expect_containers="25"
 
 deploy_sourcegraph() {
 	cd $(dirname "${BASH_SOURCE[0]}")/..
+	docker-compose version
+	# `up` pulls images one at a time, `pull` pulls them in parallel
+	docker-compose --file docker-compose/docker-compose.yaml pull --quiet
 	docker-compose --file docker-compose/docker-compose.yaml up -d -t 600
+}
 
-	echo "Giving containers 90s to start..."
-	sleep 90
+wait_until_healthy() {
+	echo "Waiting up to 5 minutes for the frontend to respond, and container health checks to pass..."
+	for _ in {1..60}; do
+		if curl --fail --silent --output /dev/null http://localhost:80/healthz &&
+			[[ -z "$(docker ps --quiet --filter health=starting --filter health=unhealthy)" ]]; then
+			return
+		fi
+		sleep 5
+	done
+	docker ps --format '{{.Names}}: {{.Status}}'
+	echo "TEST FAILURE: containers were not healthy after 5 minutes"
+	exit 1
 }
 
 test_count() {
@@ -70,5 +84,6 @@ trap catch_errors EXIT
 
 configure_docker
 deploy_sourcegraph
+wait_until_healthy
 test_count
 test_containers

From b7eea59a00fb1b9ae7992c362077eb4bae3928e6 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 15:24:25 -0600
Subject: [PATCH 02/25] ci: Boot the smoke test VM from a pd-balanced disk

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c731-7931-7091-9b5e-664cc2379c79
Co-authored-by: Amp <amp@ampcode.com>
---
 test/Vagrantfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/Vagrantfile b/test/Vagrantfile
index ef3901e09..4806a15ef 100644
--- a/test/Vagrantfile
+++ b/test/Vagrantfile
@@ -36,6 +36,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
       srv.vm.provider :google do |g, o|
         g.machine_type = server['machine_type']
+        # The default, pd-standard, is slow to boot from and to pull images onto
+        g.disk_type = 'pd-balanced'
         g.image = ENV['IMAGE']
         g.image_project_id = 'sourcegraph-ci'
         g.google_project_id = project_id

From 00f7714aa2a266d20d5f790a754a41bacb066562 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 15:29:49 -0600
Subject: [PATCH 03/25] Revert "ci: Boot the smoke test VM from a pd-balanced
 disk"

Measured on build 16813: image pulls took 90s, against 93s on pd-standard
in build 16811, so the disk is not what limits them.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c731-7931-7091-9b5e-664cc2379c79
Co-authored-by: Amp <amp@ampcode.com>
---
 test/Vagrantfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/Vagrantfile b/test/Vagrantfile
index 4806a15ef..ef3901e09 100644
--- a/test/Vagrantfile
+++ b/test/Vagrantfile
@@ -36,8 +36,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
       srv.vm.provider :google do |g, o|
         g.machine_type = server['machine_type']
-        # The default, pd-standard, is slow to boot from and to pull images onto
-        g.disk_type = 'pd-balanced'
         g.image = ENV['IMAGE']
         g.image_project_id = 'sourcegraph-ci'
         g.google_project_id = project_id

From 3d7a993b54ba7088f00dc02eb7950bed14458cf6 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 15:53:29 -0600
Subject: [PATCH 04/25] ci: Always run the smoke test on main, and fail it on
 container restarts

if_changed compares a build against origin/main. For a commit which is
already on main, and is not the newest, that comparison is empty, so the
smoke test was skipped. Only apply if_changed on other branches.

The health check wait is shorter than the sleep it replaced, so a
crash-looping container was more likely to be seen as running. Fail the
test if any container has restarted.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c731-7931-7091-9b5e-664cc2379c79
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/pipeline.yaml | 11 +++++++++--
 test/smoke-test.sh       |  7 +++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index afabeac5c..94570eee9 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -12,11 +12,18 @@ steps:
     command: .buildkite/prettier-check.sh
     agents: { queue: "vagrant" }
 
-  - label: ":rice: docker-compose-test"
+  # Builds on main always run the smoke test. if_changed compares against origin/main,
+  # so it finds no changes in a commit which is already on main, unless it is the newest
+  - &smoke_test
+    label: ":rice: docker-compose-test"
     command: .buildkite/vagrant-run.sh docker-test
     artifact_paths: ./*.log
     agents: { queue: "vagrant" }
-    # Skip the smoke test on builds which only change docs
+    if: build.branch == pipeline.default_branch
+
+  # Builds on other branches skip the smoke test if they only change docs
+  - <<: *smoke_test
+    if: build.branch != pipeline.default_branch
     if_changed:
       include: "**"
       exclude: "**.md"
diff --git a/test/smoke-test.sh b/test/smoke-test.sh
index 6eb98584e..1c558b21e 100755
--- a/test/smoke-test.sh
+++ b/test/smoke-test.sh
@@ -60,6 +60,13 @@ test_containers() {
 		sleep 1
 	done
 
+	echo "TEST: No container has restarted"
+	restarted_containers=$(docker ps --quiet | xargs docker inspect --format '{{.Name}} restarted {{.RestartCount}} times' | grep -v ' restarted 0 times' || true)
+	if [[ -n "$restarted_containers" ]]; then
+		echo "TEST FAILURE: $restarted_containers"
+		exit 1
+	fi
+
 	echo "TEST: Checking frontend is accessible"
 	curl -f http://localhost:80
 	curl -f http://localhost:80/healthz

From fa0fa89e4555c2a5ffd16a4360de5e7798e61f8b Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 15:59:38 -0600
Subject: [PATCH 05/25] ci: Name the main branch in the smoke test conditions

The Buildkite pipeline's default branch is still "master", so
pipeline.default_branch never matched a build on main.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c731-7931-7091-9b5e-664cc2379c79
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/pipeline.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index 94570eee9..c9a6acd17 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -13,17 +13,18 @@ steps:
     agents: { queue: "vagrant" }
 
   # Builds on main always run the smoke test. if_changed compares against origin/main,
-  # so it finds no changes in a commit which is already on main, unless it is the newest
+  # so it finds no changes in a commit which is already on main, unless it is the newest.
+  # The branch is named here, because the Buildkite pipeline's default branch is still "master"
   - &smoke_test
     label: ":rice: docker-compose-test"
     command: .buildkite/vagrant-run.sh docker-test
     artifact_paths: ./*.log
     agents: { queue: "vagrant" }
-    if: build.branch == pipeline.default_branch
+    if: build.branch == "main"
 
   # Builds on other branches skip the smoke test if they only change docs
   - <<: *smoke_test
-    if: build.branch != pipeline.default_branch
+    if: build.branch != "main"
     if_changed:
       include: "**"
       exclude: "**.md"

From e509a4210cc99193a6411287c59b602b2f86f25f Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:19:06 -0600
Subject: [PATCH 06/25] ci: Delete the smoke test VM by its Buildkite job
 label, and sweep stale VMs

A build cancelled while GCP was still creating the VM leaked it, because
Vagrant only records the VM once the creation has finished, so the cleanup
found nothing to delete. The VM now carries its Buildkite job ID as a label,
and the cleanup deletes by that label.

As a backstop for jobs which die before their cleanup runs, every job deletes
labelled VMs older than an hour. The step times out after 30 minutes, so no
running job owns a VM that old.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c731-7931-7091-9b5e-664cc2379c79
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/pipeline.yaml  |  2 ++
 .buildkite/vagrant-run.sh | 16 ++++++++++------
 test/Vagrantfile          |  6 ++++--
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index c9a6acd17..eff33a8b5 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -20,6 +20,8 @@ steps:
     command: .buildkite/vagrant-run.sh docker-test
     artifact_paths: ./*.log
     agents: { queue: "vagrant" }
+    # vagrant-run.sh deletes VMs older than an hour, so no job may run that long
+    timeout_in_minutes: 30
     if: build.branch == "main"
 
   # Builds on other branches skip the smoke test if they only change docs
diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh
index 1403e0dd8..920b753e1 100755
--- a/.buildkite/vagrant-run.sh
+++ b/.buildkite/vagrant-run.sh
@@ -9,18 +9,18 @@ KEY_PATH="/tmp/e2e-builder.json"
 
 pushd "test"
 
-# Ask GCP to delete the VM, without waiting the 1-2 minutes the deletion takes
+# Ask GCP to delete the VMs matching a filter, without waiting the 1-2 minutes a deletion takes
 request_instance_deletion() (
 	export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH"
-	instance_name="$(cat ".vagrant/machines/${box}/google/id" 2>/dev/null)" &&
-		instance_url="$(gcloud compute instances list --project=sourcegraph-ci --filter="name=${instance_name}" --format="value(selfLink)")" &&
-		curl --fail --silent --show-error --output /dev/null --request DELETE \
-			--header "Authorization: Bearer $(gcloud auth print-access-token)" "$instance_url"
+	gcloud compute instances list --project=sourcegraph-ci --filter="$1" --format="value(selfLink)" | tee /dev/stderr |
+		xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \
+			--request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)"
 )
 
+# Find this job's VM by its label, because Vagrant only knows about the VM once GCP has finished creating it.
 # Fall back to a blocking destroy, so a failed deletion request cannot leak a VM
 cleanup() {
-	request_instance_deletion || vagrant destroy -f "$box"
+	request_instance_deletion "labels.buildkite-job-id=${BUILDKITE_JOB_ID}" || vagrant destroy -f "$box"
 }
 
 echo --- ":vagrant: installing plugins"
@@ -38,6 +38,10 @@ export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}"
 export DOCKER_USERNAME=sourcegraph
 export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
 
+echo --- ":wastebasket: deleting VMs left behind by jobs which died before their cleanup"
+# The smoke test step times out after 30 minutes, so a VM older than an hour has no job
+request_instance_deletion "labels.buildkite-job-id:* AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs"
+
 echo --- ":vagrant: starting box $box"
 vagrant up "$box" --provider=google || exit_code=$?
 
diff --git a/test/Vagrantfile b/test/Vagrantfile
index ef3901e09..bf738b5cf 100644
--- a/test/Vagrantfile
+++ b/test/Vagrantfile
@@ -46,8 +46,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         g.service_account = server['service_account']
         g.labels = {
           "team" => "dev-infra",
-          "type" => "vagrant"
-        }
+          "type" => "vagrant",
+          # .buildkite/vagrant-run.sh deletes VMs by this label
+          "buildkite-job-id" => ENV['BUILDKITE_JOB_ID']
+        }.compact
         g.scopes = [
             "compute-rw",
             "storage-ro"

From 38a3df3eb8dd176438463acfd1c3b16267cb6857 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:25:57 -0600
Subject: [PATCH 07/25] ci: Name the smoke test VM after its job, and delete
 orphaned disks too

Vagrant creates the boot disk before the VM, and the disk carries no labels.
A build cancelled between the two left the disk behind, and the label lookup
from the previous commit could not find it. sourcegraph-ci holds three such
disks, from September.

The VM and its disk are now named docker-test-job-<Buildkite job ID>. The
cleanup and the stale sweep delete VMs by that name, and disks by that name
which no VM uses. This replaces the label.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c731-7931-7091-9b5e-664cc2379c79
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/pipeline.yaml  |  2 +-
 .buildkite/vagrant-run.sh | 24 ++++++++++++++++--------
 test/Vagrantfile          |  9 ++++-----
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index eff33a8b5..65b37cc6e 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -20,7 +20,7 @@ steps:
     command: .buildkite/vagrant-run.sh docker-test
     artifact_paths: ./*.log
     agents: { queue: "vagrant" }
-    # vagrant-run.sh deletes VMs older than an hour, so no job may run that long
+    # vagrant-run.sh deletes VMs and disks older than an hour, so no job may run that long
     timeout_in_minutes: 30
     if: build.branch == "main"
 
diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh
index 920b753e1..d6fd33380 100755
--- a/.buildkite/vagrant-run.sh
+++ b/.buildkite/vagrant-run.sh
@@ -9,18 +9,26 @@ KEY_PATH="/tmp/e2e-builder.json"
 
 pushd "test"
 
-# Ask GCP to delete the VMs matching a filter, without waiting the 1-2 minutes a deletion takes
-request_instance_deletion() (
+# Vagrant creates a disk, then a VM, both with this name, and only knows about them once GCP has finished
+# creating the VM. Naming them after the job lets the cleanup find them if the job is cancelled before that
+instance_name_prefix="${box}-job-"
+export VAGRANT_INSTANCE_NAME="${instance_name_prefix}${BUILDKITE_JOB_ID}"
+
+# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses,
+# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM
+request_deletion() (
 	export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH"
-	gcloud compute instances list --project=sourcegraph-ci --filter="$1" --format="value(selfLink)" | tee /dev/stderr |
+	{
+		gcloud compute instances list --project=sourcegraph-ci --filter="$1" --format="value(selfLink)" &&
+			gcloud compute disks list --project=sourcegraph-ci --filter="($1) AND -users:*" --format="value(selfLink)"
+	} | tee /dev/stderr |
 		xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \
 			--request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)"
 )
 
-# Find this job's VM by its label, because Vagrant only knows about the VM once GCP has finished creating it.
 # Fall back to a blocking destroy, so a failed deletion request cannot leak a VM
 cleanup() {
-	request_instance_deletion "labels.buildkite-job-id=${BUILDKITE_JOB_ID}" || vagrant destroy -f "$box"
+	request_deletion "name=${VAGRANT_INSTANCE_NAME}" || vagrant destroy -f "$box"
 }
 
 echo --- ":vagrant: installing plugins"
@@ -38,9 +46,9 @@ export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}"
 export DOCKER_USERNAME=sourcegraph
 export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
 
-echo --- ":wastebasket: deleting VMs left behind by jobs which died before their cleanup"
-# The smoke test step times out after 30 minutes, so a VM older than an hour has no job
-request_instance_deletion "labels.buildkite-job-id:* AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs"
+echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
+# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job
+request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks"
 
 echo --- ":vagrant: starting box $box"
 vagrant up "$box" --provider=google || exit_code=$?
diff --git a/test/Vagrantfile b/test/Vagrantfile
index bf738b5cf..671f25254 100644
--- a/test/Vagrantfile
+++ b/test/Vagrantfile
@@ -39,17 +39,16 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         g.image = ENV['IMAGE']
         g.image_project_id = 'sourcegraph-ci'
         g.google_project_id = project_id
-        g.name = "#{server['name']}-"+"#{Time.now.to_i}"+"#{rand(100000)}"
+        # .buildkite/vagrant-run.sh names the VM after its job, so it can delete the VM by name
+        g.name = ENV['VAGRANT_INSTANCE_NAME'] || "#{server['name']}-"+"#{Time.now.to_i}"+"#{rand(100000)}"
         g.network = server['network']
         g.external_ip = external_ip
         g.use_private_ip = use_private_ip
         g.service_account = server['service_account']
         g.labels = {
           "team" => "dev-infra",
-          "type" => "vagrant",
-          # .buildkite/vagrant-run.sh deletes VMs by this label
-          "buildkite-job-id" => ENV['BUILDKITE_JOB_ID']
-        }.compact
+          "type" => "vagrant"
+        }
         g.scopes = [
             "compute-rw",
             "storage-ro"

From 55fd45c2ad23439addf4d831ed39be6245c6c5dd Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:16:45 -0600
Subject: [PATCH 08/25] ci: Validate every Compose file, and stop the db-only
 file drifting

- db-only-migrate.docker-compose.yaml now extends the pgsql and codeintel-db
  services in docker-compose.yaml, instead of copying them
- CI runs `docker compose config` on every Compose file, combined the way
  its docs say to run it
- executor.docker-compose.yaml declares the sourcegraph network, so the
  standalone command in its README works on Compose v2
- dev/run.sh uses `docker compose`, like the README

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c724-4acc-77b0-a43d-b94f22933b8a
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/validate-docker-compose.sh         | 16 ++++--
 .../db-only-migrate.docker-compose.yaml       | 54 ++++---------------
 docker-compose/dev/run.sh                     |  2 +-
 .../executors/executor.docker-compose.yaml    |  3 ++
 4 files changed, 27 insertions(+), 48 deletions(-)

diff --git a/.buildkite/validate-docker-compose.sh b/.buildkite/validate-docker-compose.sh
index 3c7bac418..ae74f69ba 100755
--- a/.buildkite/validate-docker-compose.sh
+++ b/.buildkite/validate-docker-compose.sh
@@ -1,7 +1,17 @@
 #!/usr/bin/env bash
 
-cd "$(dirname "${BASH_SOURCE[0]}")/.."
+cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose"
 set -euxo pipefail
 
-cd docker-compose 
-docker-compose -f docker-compose.yaml config -q
+docker compose version
+
+# Every Compose file, combined the way its docs say to run it
+docker compose --file docker-compose.yaml config --quiet
+docker compose --file db-only-migrate.docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file dev/docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file executors/executor.docker-compose.yaml config --quiet
+docker compose --file executors/executor.docker-compose.yaml config --quiet
+for override in examples/*/docker-compose.override.yaml; do
+	docker compose --file docker-compose.yaml --file "$override" config --quiet
+done
diff --git a/docker-compose/db-only-migrate.docker-compose.yaml b/docker-compose/db-only-migrate.docker-compose.yaml
index 152de093d..6302de6e2 100644
--- a/docker-compose/db-only-migrate.docker-compose.yaml
+++ b/docker-compose/db-only-migrate.docker-compose.yaml
@@ -1,54 +1,20 @@
-# This file contains the pgsql service definition (the primary Sourcegraph database deployment), 
-# as well as the codeintel-db service definition (a separate postgres deployment). Both run postgres.
-# This file is used for database-only operations such as backup and restore.
+# This file starts only the pgsql service (the primary Sourcegraph database),
+# and the codeintel-db service (a separate Postgres deployment).
+# It is used for database-only operations such as backup and restore.
 # See https://sourcegraph.com/docs/self-hosted/deploy/docker-compose/operations
 #
-# 🚨This file MUST be kept in sync with the pgsql definition in docker-compose/docker-compose.yaml
+# Both services are defined in docker-compose.yaml, so this file cannot drift from it
 version: '2.4'
 services:
-  # Description: PostgreSQL database for various data.
-  #
-  # Disk: 128GB / persistent SSD
-  # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP
-  # Ports exposed to the public internet: none
-  #
   pgsql:
-    container_name: pgsql
-    image: 'index.docker.io/sourcegraph/postgresql-16:8.1.4690@sha256:1bd9b7314bdb83d12162c20e1c8da580ca27c43a4f391404a30339faeefa033b'
-    cpus: 4
-    mem_limit: '2g'
-    security_opt:
-      - no-new-privileges:true
-    healthcheck:
-      test: '/liveness.sh'
-      interval: 10s
-      timeout: 1s
-      retries: 3
-      start_period: 15s
-    volumes:
-      - 'pgsql:/data/'
-    networks:
-      - sourcegraph
-    restart: always
+    extends:
+      file: docker-compose.yaml
+      service: pgsql
 
   codeintel-db:
-    container_name: codeintel-db
-    image: 'index.docker.io/sourcegraph/postgresql-16:8.1.4690@sha256:1bd9b7314bdb83d12162c20e1c8da580ca27c43a4f391404a30339faeefa033b'
-    cpus: 4
-    mem_limit: '2g'
-    security_opt:
-      - no-new-privileges:true
-    healthcheck:
-      test: '/liveness.sh'
-      interval: 10s
-      timeout: 1s
-      retries: 3
-      start_period: 15s
-    volumes:
-      - 'codeintel-db:/data/'
-    networks:
-      - sourcegraph
-    restart: always
+    extends:
+      file: docker-compose.yaml
+      service: codeintel-db
 
 volumes:
   pgsql:
diff --git a/docker-compose/dev/run.sh b/docker-compose/dev/run.sh
index f5c3881b4..e7b85ff19 100755
--- a/docker-compose/dev/run.sh
+++ b/docker-compose/dev/run.sh
@@ -3,7 +3,7 @@
 cd "$(dirname "${BASH_SOURCE[0]}")/../.."
 set -euxo pipefail
 
-docker-compose \
+docker compose \
     -f docker-compose/docker-compose.yaml \
     -f docker-compose/jaeger/docker-compose.yaml \
     -f docker-compose/dev/docker-compose.yaml \
diff --git a/docker-compose/executors/executor.docker-compose.yaml b/docker-compose/executors/executor.docker-compose.yaml
index cf5dadeba..87e43dcc3 100644
--- a/docker-compose/executors/executor.docker-compose.yaml
+++ b/docker-compose/executors/executor.docker-compose.yaml
@@ -31,3 +31,6 @@ services:
     networks:
       - sourcegraph
     restart: always
+
+networks:
+  sourcegraph:

From db14865e8ecab1dd99613ea340db5da3168c5483 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:16:45 -0600
Subject: [PATCH 09/25] ci: Count services instead of hard-coding 25, and run
 the executor test

- The smoke test compares running containers against the services in
  docker-compose.yaml, so adding or removing a service needs no test edit
- Remove the loop which claimed to watch containers for 5 minutes, but
  checked them twice, one second apart
- Run test/executor-smoke-test.sh after the smoke test

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c724-4acc-77b0-a43d-b94f22933b8a
Co-authored-by: Amp <amp@ampcode.com>
---
 test/servers.yaml  |  3 +++
 test/smoke-test.sh | 29 ++++++++---------------------
 2 files changed, 11 insertions(+), 21 deletions(-)

diff --git a/test/servers.yaml b/test/servers.yaml
index 36780dbe7..930331f38 100644
--- a/test/servers.yaml
+++ b/test/servers.yaml
@@ -11,4 +11,7 @@
   service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com
   shell_commands:
    - "/deploy-sourcegraph-docker/test/smoke-test.sh"
+   # The VM image has no jq
+   - "apt-get update --quiet && apt-get install --quiet --yes jq"
+   - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh"
 
diff --git a/test/smoke-test.sh b/test/smoke-test.sh
index 1c558b21e..e733e4333 100755
--- a/test/smoke-test.sh
+++ b/test/smoke-test.sh
@@ -3,7 +3,7 @@ set -euxfo pipefail
 
 configure_docker() {
   set +x
-  if [ -n "${DOCKER_USERNAME}" ] && [ -n "${DOCKER_PASSWORD}" ]; then
+  if [ -n "${DOCKER_USERNAME:-}" ] && [ -n "${DOCKER_PASSWORD:-}" ]; then
     docker login -u "${DOCKER_USERNAME}" --password-stdin <<<"$DOCKER_PASSWORD"
   fi
   set -x
@@ -11,10 +11,8 @@ configure_docker() {
   gcloud auth configure-docker us-central1-docker.pkg.dev,us-docker.pkg.dev
 }
 
-expect_containers="25"
-
 deploy_sourcegraph() {
-	cd $(dirname "${BASH_SOURCE[0]}")/..
+	cd "$(dirname "${BASH_SOURCE[0]}")/.."
 	docker-compose version
 	# `up` pulls images one at a time, `pull` pulls them in parallel
 	docker-compose --file docker-compose/docker-compose.yaml pull --quiet
@@ -36,30 +34,19 @@ wait_until_healthy() {
 }
 
 test_count() {
-	echo "TEST: Number of expected containers created"
-	containers_count=$(docker ps --format '{{.Names}}' | wc -l)
+	echo "TEST: Every service has a running container"
+	# migrator exits when it is done, every other service keeps running
+	expect_containers=$(docker-compose --file docker-compose/docker-compose.yaml config --services | grep --count --invert-match '^migrator$')
+	containers_count=$(docker ps --filter status=running --format '{{.Names}}' | wc -l)
 	if [[ "$containers_count" -ne "$expect_containers" ]]; then
-		docker ps --format '{{.Names}}'
+		docker ps --all --format '{{.Names}}: {{.Status}}'
 		echo
-		echo "TEST FAILURE: expected $expect_containers containers, found $containers_count"
+		echo "TEST FAILURE: expected $expect_containers running containers, found $containers_count"
 		exit 1
 	fi
 }
 
 test_containers() {
-	echo "TEST: Checking every 10s that containers are running for 5 minutes..."
-	for i in {0..1}; do
-		containers=$(docker ps --format '{{.Names}}' | xargs -I{} -n1 sh -c "printf '{}: ' && docker inspect --format '{{.State.Status}}' {}")
-		containers_running=$(echo "$containers" | grep -c "running")
-		if [[ "$containers_running" -ne "$expect_containers" ]]; then
-			containers_failing=$(docker ps --format '{{.Names}}:{{.Status}}' | grep -v Up | cut -f 1 -d :)
-			echo "TEST FAILURE: expected $expect_containers containers running, found $containers_running. The following containers are failing: $containers_failing"
-			exit 1
-		fi
-		echo "Containers running OK.. waiting 10s"
-		sleep 1
-	done
-
 	echo "TEST: No container has restarted"
 	restarted_containers=$(docker ps --quiet | xargs docker inspect --format '{{.Name}} restarted {{.RestartCount}} times' | grep -v ' restarted 0 times' || true)
 	if [[ -n "$restarted_containers" ]]; then

From 4ea4ebb891204fadc6e15c5f5b6022d2e743f48a Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:16:45 -0600
Subject: [PATCH 10/25] ci: Stop sharing the builder key file, and writing
 credentials to the VM profile

- Each job fetches the builder key into its own temporary file, and removes
  it when the job ends
- The Docker Hub credentials reach the smoke test as provisioner environment
  variables, instead of being appended to /root/.profile

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c724-4acc-77b0-a43d-b94f22933b8a
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/vagrant-run.sh | 15 ++++++++-------
 test/Vagrantfile          | 11 +----------
 2 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh
index d6fd33380..c4afd4569 100755
--- a/.buildkite/vagrant-run.sh
+++ b/.buildkite/vagrant-run.sh
@@ -5,7 +5,7 @@ set -euo pipefail
 
 box="$1"
 exit_code=0
-KEY_PATH="/tmp/e2e-builder.json"
+KEY_PATH="$(mktemp)"
 
 pushd "test"
 
@@ -28,7 +28,10 @@ request_deletion() (
 
 # Fall back to a blocking destroy, so a failed deletion request cannot leak a VM
 cleanup() {
-	request_deletion "name=${VAGRANT_INSTANCE_NAME}" || vagrant destroy -f "$box"
+	deletion_status=0
+	request_deletion "name=${VAGRANT_INSTANCE_NAME}" || vagrant destroy -f "$box" || deletion_status=$?
+	rm -f "$KEY_PATH"
+	return "$deletion_status"
 }
 
 echo --- ":vagrant: installing plugins"
@@ -39,12 +42,10 @@ vagrant --version
 trap cleanup EXIT
 
 echo --- ":lock: builder account key"
-if [ ! -f ${KEY_PATH} ]; then
-  gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}"
-fi
+gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}"
 export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}"
-export DOCKER_USERNAME=sourcegraph
-export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
+DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
+export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD
 
 echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
 # The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job
diff --git a/test/Vagrantfile b/test/Vagrantfile
index 671f25254..ebd5446ca 100644
--- a/test/Vagrantfile
+++ b/test/Vagrantfile
@@ -57,17 +57,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         o.ssh.private_key_path = ssh_key_path
       end
 
-      srv.vm.provision 'shell', inline: <<-SHELL
-        #!/usr/bin/env bash
-        cat << EOF >> /root/.profile
-export GIT_BRANCH=#{ENV['BUILDKITE_BRANCH']}
-export DOCKER_USERNAME=#{ENV['DOCKER_USERNAME']}
-export DOCKER_PASSWORD=#{ENV['DOCKER_PASSWORD']}
-EOF
-        SHELL
-
       server['shell_commands'].each do |sh|
-        srv.vm.provision 'shell', inline: sh
+        srv.vm.provision 'shell', inline: sh, env: ENV.slice('DOCKER_USERNAME', 'DOCKER_PASSWORD')
       end
     end
   end

From dcc9c21642150dce0de103870d66083e4cda6951 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:16:45 -0600
Subject: [PATCH 11/25] release: Make the batch-change tag step safe to retry

Same block as release.yaml: a retried finalize finds its own tag, and only
a tag on a different commit is an error.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c724-4acc-77b0-a43d-b94f22933b8a
Co-authored-by: Amp <amp@ampcode.com>
---
 batch-change/release.yaml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/batch-change/release.yaml b/batch-change/release.yaml
index 2534f0a07..66cbb53ac 100644
--- a/batch-change/release.yaml
+++ b/batch-change/release.yaml
@@ -138,7 +138,16 @@ promoteToPublic:
           git checkout "${promote_branch}"
           git switch -c "${release_branch}"
 
-          git tag {{version}}
+          # A retried finalize finds its own tag; only a tag elsewhere is an error
+          if tagged=$(git rev-parse -q --verify "refs/tags/{{version}}^{commit}"); then
+            if [ "${tagged}" != "$(git rev-parse HEAD)" ]; then
+              echo "Tag {{version}} already exists at ${tagged}, not at ${promote_branch} ($(git rev-parse HEAD))" >&2
+              exit 1
+            fi
+            echo "Tag {{version}} already points at ${promote_branch}, not recreating it"
+          else
+            git tag {{version}}
+          fi
           git push origin ${release_branch} --tags
 
           # Web URL to the tag

From 18827bd4cf731ed6d790344ceaa5a86129f63fd9 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:16:45 -0600
Subject: [PATCH 12/25] ci: Run shellcheck on every shell script

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c724-4acc-77b0-a43d-b94f22933b8a
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/pipeline.yaml | 4 ++++
 .tool-versions           | 1 +
 2 files changed, 5 insertions(+)

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index 65b37cc6e..826b9f888 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -12,6 +12,10 @@ steps:
     command: .buildkite/prettier-check.sh
     agents: { queue: "vagrant" }
 
+  - label: ":bash:"
+    command: git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck
+    agents: { queue: "vagrant" }
+
   # Builds on main always run the smoke test. if_changed compares against origin/main,
   # so it finds no changes in a commit which is already on main, unless it is the newest.
   # The branch is named here, because the Buildkite pipeline's default branch is still "master"
diff --git a/.tool-versions b/.tool-versions
index 790e537c9..2e2b59f27 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1,3 +1,4 @@
 nodejs                   16.7.0
 yarn                     1.22.4
 github-cli               2.46.0
+shellcheck               0.11.0

From 47d36f15e3afb4ae37e804fa3436be188225143f Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:24:32 -0600
Subject: [PATCH 13/25] ci: Install jq from the Ubuntu repositories only

The VM image lacks the signing key for its Chrome apt repository, so a
full `apt-get update` fails.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c724-4acc-77b0-a43d-b94f22933b8a
Co-authored-by: Amp <amp@ampcode.com>
---
 test/servers.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/servers.yaml b/test/servers.yaml
index 930331f38..88a9cc366 100644
--- a/test/servers.yaml
+++ b/test/servers.yaml
@@ -11,7 +11,7 @@
   service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com
   shell_commands:
    - "/deploy-sourcegraph-docker/test/smoke-test.sh"
-   # The VM image has no jq
-   - "apt-get update --quiet && apt-get install --quiet --yes jq"
+   # The VM image has no jq. Skip /etc/apt/sources.list.d: the image lacks the key for its Chrome repository, so a full update fails
+   - "apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq"
    - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh"
 

From ad8e2f2fc49f611f7ef81e98498fcbaf8befc699 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:44:20 -0600
Subject: [PATCH 14/25] ci: Run the smoke test VM with gcloud and ssh instead
 of Vagrant

Vagrant came in 2020 so one Vagrantfile could run the smoke test in a local
VirtualBox VM or on GCE. Only the GCE provider is left, so Vagrant now only
adds Ruby, the vagrant-google plugin, a pinned Vagrant on the agents, and an
rsync of the repo.

smoke-test-vm.sh replaces vagrant-run.sh, the Vagrantfile, and servers.yaml.
It creates the VM with the same settings, authorizes a key made for the job,
copies the repo over ssh, and runs the steps servers.yaml listed as root, sending the
Docker Hub password over stdin. The boot disk grows from vagrant-google's
default 10 GB to 20 GB, which the Ubuntu 26.04 e2e image will need for
Docker 29's containerd image store.

Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a1c770-8d7e-76ad-b47f-2a2ff80873cc
---
 .buildkite/pipeline.yaml    |   7 ++-
 .buildkite/smoke-test-vm.sh | 100 ++++++++++++++++++++++++++++++++++++
 .buildkite/vagrant-run.sh   |  61 ----------------------
 .gitignore                  |   1 -
 test/Vagrantfile            |  65 -----------------------
 test/servers.yaml           |  17 ------
 6 files changed, 103 insertions(+), 148 deletions(-)
 create mode 100755 .buildkite/smoke-test-vm.sh
 delete mode 100755 .buildkite/vagrant-run.sh
 delete mode 100644 test/Vagrantfile
 delete mode 100644 test/servers.yaml

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index 826b9f888..1091d0f8f 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -1,8 +1,7 @@
 $schema: "http://json.schemastore.org/buildkite"
 env:
-  # GCP VM image for the Vagrant smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images`
+  # GCP VM image for the smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images`
   IMAGE: "e2e-db8a40720-2024-08-26t08-44-10z"
-  VAGRANT_RUN_ENV: "CI"
 steps:
   - label: ":docker:"
     command: .buildkite/validate-docker-compose.sh
@@ -21,10 +20,10 @@ steps:
   # The branch is named here, because the Buildkite pipeline's default branch is still "master"
   - &smoke_test
     label: ":rice: docker-compose-test"
-    command: .buildkite/vagrant-run.sh docker-test
+    command: .buildkite/smoke-test-vm.sh
     artifact_paths: ./*.log
     agents: { queue: "vagrant" }
-    # vagrant-run.sh deletes VMs and disks older than an hour, so no job may run that long
+    # smoke-test-vm.sh deletes VMs and disks older than an hour, so no job may run that long
     timeout_in_minutes: 30
     if: build.branch == "main"
 
diff --git a/.buildkite/smoke-test-vm.sh b/.buildkite/smoke-test-vm.sh
new file mode 100755
index 000000000..7c9b1198c
--- /dev/null
+++ b/.buildkite/smoke-test-vm.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+# Runs test/smoke-test.sh on a new GCP VM booted from $IMAGE, then deletes the VM
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+set -euo pipefail
+
+KEY_PATH="$(mktemp)"
+project="sourcegraph-ci"
+zone="us-central1-f"
+
+# GCP creates a disk, then a VM, both with this name. Naming them after the job
+# lets the cleanup find them if the job is cancelled before the VM exists
+instance_name_prefix="docker-test-job-"
+instance_name="${instance_name_prefix}${BUILDKITE_JOB_ID}"
+
+# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses,
+# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM
+request_deletion() {
+	{
+		gcloud compute instances list --project="$project" --filter="$1" --format="value(selfLink)" &&
+			gcloud compute disks list --project="$project" --filter="($1) AND -users:*" --format="value(selfLink)"
+	} | tee /dev/stderr |
+		xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \
+			--request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)"
+}
+
+# Fall back to a blocking delete, so a failed deletion request cannot leak a VM
+cleanup() {
+	deletion_status=0
+	request_deletion "name=${instance_name}" ||
+		gcloud compute instances delete "$instance_name" --project="$project" --zone="$zone" --quiet ||
+		deletion_status=$?
+	rm -rf "$KEY_PATH" "$ssh_directory"
+	return "$deletion_status"
+}
+
+# A key for this job only, which the VM's guest agent authorizes for the buildkite user
+ssh_directory="$(mktemp -d)"
+ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$ssh_directory/key"
+ssh_options=(-i "$ssh_directory/key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5)
+
+trap cleanup EXIT
+
+echo --- ":lock: builder account key"
+gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci >"${KEY_PATH}"
+DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
+export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH"
+
+echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
+# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job
+request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks"
+
+echo --- ":gcloud: starting VM $instance_name"
+ip_address="$(gcloud compute instances create "$instance_name" \
+	--project="$project" --zone="$zone" \
+	--image="$IMAGE" --image-project="$project" \
+	--machine-type=custom-16-20480 \
+	--boot-disk-size=20GB --boot-disk-type=pd-standard \
+	--network=default --no-address \
+	--service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \
+	--labels=team=dev-infra,type=smoke-test \
+	--metadata="ssh-keys=buildkite:$(cat "$ssh_directory/key.pub")" \
+	--format="value(networkInterfaces[0].networkIP)")"
+remote="buildkite@${ip_address}"
+
+echo --- ":ssh: waiting for the VM to accept SSH"
+deadline=$((SECONDS + 600))
+until ssh "${ssh_options[@]}" "$remote" true; do
+	if ((SECONDS > deadline)); then
+		echo "The VM did not accept SSH within 10 minutes"
+		exit 1
+	fi
+	sleep 2
+done
+
+echo --- ":package: copying the repository to the VM"
+tar --create --gzip --exclude=.git --exclude=node_modules . |
+	ssh "${ssh_options[@]}" "$remote" "sudo mkdir /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker"
+
+echo --- ":docker: running the smoke tests"
+# Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list
+remote_script='
+set -euo pipefail
+read -r DOCKER_PASSWORD
+export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD
+/deploy-sourcegraph-docker/test/smoke-test.sh
+# The VM image has no jq. Skip /etc/apt/sources.list.d: the image lacks the key for its Chrome repository, so a full update fails
+apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq
+/deploy-sourcegraph-docker/test/executor-smoke-test.sh
+'
+exit_code=0
+# shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM
+ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD" || exit_code=$?
+
+if [ "$exit_code" != 0 ]; then
+	# Copy back the container logs the smoke test wrote, for the step's artifacts
+	ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true
+	exit "$exit_code"
+fi
diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh
deleted file mode 100755
index c4afd4569..000000000
--- a/.buildkite/vagrant-run.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env bash
-
-cd "$(dirname "${BASH_SOURCE[0]}")/.."
-set -euo pipefail
-
-box="$1"
-exit_code=0
-KEY_PATH="$(mktemp)"
-
-pushd "test"
-
-# Vagrant creates a disk, then a VM, both with this name, and only knows about them once GCP has finished
-# creating the VM. Naming them after the job lets the cleanup find them if the job is cancelled before that
-instance_name_prefix="${box}-job-"
-export VAGRANT_INSTANCE_NAME="${instance_name_prefix}${BUILDKITE_JOB_ID}"
-
-# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses,
-# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM
-request_deletion() (
-	export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH"
-	{
-		gcloud compute instances list --project=sourcegraph-ci --filter="$1" --format="value(selfLink)" &&
-			gcloud compute disks list --project=sourcegraph-ci --filter="($1) AND -users:*" --format="value(selfLink)"
-	} | tee /dev/stderr |
-		xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \
-			--request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)"
-)
-
-# Fall back to a blocking destroy, so a failed deletion request cannot leak a VM
-cleanup() {
-	deletion_status=0
-	request_deletion "name=${VAGRANT_INSTANCE_NAME}" || vagrant destroy -f "$box" || deletion_status=$?
-	rm -f "$KEY_PATH"
-	return "$deletion_status"
-}
-
-echo --- ":vagrant: installing plugins"
-vagrant --version
-# Agents on the same host share ~/.vagrant.d, so only the first job on a new host installs
-[[ "$(vagrant plugin list)" == *"vagrant-google (2.7.0"* ]] || vagrant plugin install vagrant-google --plugin-version '2.7.0'
-
-trap cleanup EXIT
-
-echo --- ":lock: builder account key"
-gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}"
-export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}"
-DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
-export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD
-
-echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
-# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job
-request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks"
-
-echo --- ":vagrant: starting box $box"
-vagrant up "$box" --provider=google || exit_code=$?
-
-if [ "$exit_code" != 0 ]; then
-	vagrant plugin install vagrant-scp
-	vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../
-	exit $exit_code
-fi
diff --git a/.gitignore b/.gitignore
index 9cf3cdc72..3ecd4b8fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,4 @@
 .env
 .envrc
-.vagrant
 *.log
 node_modules/
diff --git a/test/Vagrantfile b/test/Vagrantfile
deleted file mode 100644
index ebd5446ca..000000000
--- a/test/Vagrantfile
+++ /dev/null
@@ -1,65 +0,0 @@
-# -*- mode: ruby -*-
-# # vi: set ft=ruby :
-# Specify minimum Vagrant version and Vagrant API version
-Vagrant.require_version '>= 1.6.0'
-VAGRANTFILE_API_VERSION = '2'.freeze
-# Require YAML module
-require 'yaml'
-
-# Read YAML file with box details
-servers = YAML.load_file('servers.yaml')
-
-# Create boxes
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  # Iterate through entries in YAML file
-  servers.each do |server|
-    config.vm.define server['name'] do |srv|
-      srv.vm.box = server['box']
-      srv.vm.synced_folder '../', '/deploy-sourcegraph-docker'
-      srv.vm.boot_timeout = 600
-
-      # Setup for CI or Local
-      case ENV['VAGRANT_RUN_ENV']
-      when 'CI'
-        project_id = server['project_id']
-        external_ip = server['external_ip']
-        use_private_ip = server['use_private_ip']
-        username = server['username']
-        ssh_key_path = server['ssh_key_path']
-      else
-        project_id = ENV['VAGRANT_GCP_PROJECT_ID'] || 'sourcegraph-server'
-        external_ip = nil
-        use_private_ip = false
-        username = ENV['VAGRANT_SSH_USER'] || ENV['USER']
-        ssh_key_path = ENV['VAGRANT_SSH_KEY'] || '~/.ssh/id_rsa'
-      end
-
-      srv.vm.provider :google do |g, o|
-        g.machine_type = server['machine_type']
-        g.image = ENV['IMAGE']
-        g.image_project_id = 'sourcegraph-ci'
-        g.google_project_id = project_id
-        # .buildkite/vagrant-run.sh names the VM after its job, so it can delete the VM by name
-        g.name = ENV['VAGRANT_INSTANCE_NAME'] || "#{server['name']}-"+"#{Time.now.to_i}"+"#{rand(100000)}"
-        g.network = server['network']
-        g.external_ip = external_ip
-        g.use_private_ip = use_private_ip
-        g.service_account = server['service_account']
-        g.labels = {
-          "team" => "dev-infra",
-          "type" => "vagrant"
-        }
-        g.scopes = [
-            "compute-rw",
-            "storage-ro"
-        ]
-        o.ssh.username = username
-        o.ssh.private_key_path = ssh_key_path
-      end
-
-      server['shell_commands'].each do |sh|
-        srv.vm.provision 'shell', inline: sh, env: ENV.slice('DOCKER_USERNAME', 'DOCKER_PASSWORD')
-      end
-    end
-  end
-end
diff --git a/test/servers.yaml b/test/servers.yaml
deleted file mode 100644
index 88a9cc366..000000000
--- a/test/servers.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
----
-- name: docker-test
-  box: google/gce
-  machine_type: "custom-16-20480"
-  project_id: sourcegraph-ci
-  external_ip: false
-  use_private_ip: true
-  network: default
-  username: buildkite
-  ssh_key_path: "~/.ssh/id_rsa"
-  service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com
-  shell_commands:
-   - "/deploy-sourcegraph-docker/test/smoke-test.sh"
-   # The VM image has no jq. Skip /etc/apt/sources.list.d: the image lacks the key for its Chrome repository, so a full update fails
-   - "apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq"
-   - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh"
-

From 9c516f691b98e4ff3b59e8e941a48596c8937d5c Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 16:46:48 -0600
Subject: [PATCH 15/25] ci: Deploy Jaeger and the executor in the smoke test

The smoke test started only docker-compose.yaml, then started a second,
separate executor with `docker compose run`. Jaeger never ran.

Start all three Compose files with one `up`, so the container count,
health check wait, and restart check cover every service. The executor
exits without a password and a queue name, so a test override file sets
them, and executor-smoke-test.sh reads the password from the container.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c731-7931-7091-9b5e-664cc2379c79
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/validate-docker-compose.sh |  2 ++
 test/docker-compose.override.yaml     |  8 +++++++
 test/executor-smoke-test.sh           | 31 +++++++++------------------
 test/smoke-test.sh                    | 17 ++++++++++++---
 4 files changed, 34 insertions(+), 24 deletions(-)
 create mode 100644 test/docker-compose.override.yaml

diff --git a/.buildkite/validate-docker-compose.sh b/.buildkite/validate-docker-compose.sh
index ae74f69ba..8a716a3d7 100755
--- a/.buildkite/validate-docker-compose.sh
+++ b/.buildkite/validate-docker-compose.sh
@@ -12,6 +12,8 @@ docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml conf
 docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file dev/docker-compose.yaml config --quiet
 docker compose --file docker-compose.yaml --file executors/executor.docker-compose.yaml config --quiet
 docker compose --file executors/executor.docker-compose.yaml config --quiet
+# The combination the smoke test deploys
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file executors/executor.docker-compose.yaml --file ../test/docker-compose.override.yaml config --quiet
 for override in examples/*/docker-compose.override.yaml; do
 	docker compose --file docker-compose.yaml --file "$override" config --quiet
 done
diff --git a/test/docker-compose.override.yaml b/test/docker-compose.override.yaml
new file mode 100644
index 000000000..2cc86799c
--- /dev/null
+++ b/test/docker-compose.override.yaml
@@ -0,0 +1,8 @@
+version: '2.4'
+services:
+  # The executor exits unless both are set.
+  # executor-smoke-test.sh sets this password as `executors.accessToken` in the site config
+  executor:
+    environment:
+      - EXECUTOR_FRONTEND_PASSWORD=executor-smoke-test-token
+      - EXECUTOR_QUEUE_NAME=batches
diff --git a/test/executor-smoke-test.sh b/test/executor-smoke-test.sh
index f92e1b474..052c829d1 100755
--- a/test/executor-smoke-test.sh
+++ b/test/executor-smoke-test.sh
@@ -3,18 +3,18 @@ set -euo pipefail
 
 # Executor smoke test for deploy-sourcegraph-docker
 #
-# Requires: a running Sourcegraph docker-compose stack (from smoke-test.sh or manual start).
-# This script creates a site admin account, configures executor access, starts an executor
-# container, and verifies it registers with the Sourcegraph instance.
+# Requires: a running Sourcegraph docker-compose stack with an executor container
+# (from smoke-test.sh or manual start).
+# This script creates a site admin account, configures executor access, and verifies
+# the executor registers with the Sourcegraph instance.
 #
-# Dependencies: curl, jq, docker compose
+# Dependencies: curl, jq, docker
 
 FRONTEND_URL="${FRONTEND_URL:-http://localhost:80}"
-EXECUTOR_TOKEN="executor-smoke-test-token"
+EXECUTOR_TOKEN=$(docker exec executor printenv EXECUTOR_FRONTEND_PASSWORD)
 ADMIN_EMAIL="smoke-test@sourcegraph.com"
 ADMIN_USERNAME="smoke-admin"
 ADMIN_PASSWORD="smoke-test-password-123!"
-COMPOSE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose" && pwd)"
 
 echo "=== Executor Smoke Test ==="
 
@@ -74,19 +74,8 @@ curl -s -o /dev/null -X POST "${FRONTEND_URL}/.api/graphql" \
     --data-raw "{\"query\":\"mutation UpdateSiteConfig(\$input: String!) { updateSiteConfiguration(lastID: $LAST_ID, input: \$input) }\", \"variables\":{\"input\":$NEW_CONFIG}}"
 echo "  Site config updated"
 
-# Step 5: Start executor container
-echo "Step 5: Starting executor container..."
-EXECUTOR_CONTAINER=$(docker compose \
-    -f "${COMPOSE_DIR}/docker-compose.yaml" \
-    -f "${COMPOSE_DIR}/executors/executor.docker-compose.yaml" \
-    run -d \
-    -e "EXECUTOR_FRONTEND_PASSWORD=${EXECUTOR_TOKEN}" \
-    -e EXECUTOR_QUEUE_NAME=batches \
-    executor 2>&1 | tail -1)
-echo "  Executor container started: ${EXECUTOR_CONTAINER:0:12}"
-
-# Step 6: Wait for executor to register
-echo "Step 6: Waiting for executor to register..."
+# Step 5: Wait for executor to register
+echo "Step 5: Waiting for executor to register..."
 MAX_ATTEMPTS=24
 for i in $(seq 1 $MAX_ATTEMPTS); do
     COUNT=$(curl -s -X POST "${FRONTEND_URL}/.api/graphql" \
@@ -110,8 +99,8 @@ echo "=== EXECUTOR SMOKE TEST FAILED ==="
 echo "Executor did not register within $((MAX_ATTEMPTS * 5))s"
 echo ""
 echo "Container status:"
-docker inspect "$EXECUTOR_CONTAINER" --format '{{.State.Status}} (exit code: {{.State.ExitCode}})' 2>/dev/null || echo "  Container not found"
+docker inspect executor --format '{{.State.Status}} (exit code: {{.State.ExitCode}})' 2>/dev/null || echo "  Container not found"
 echo ""
 echo "Container logs:"
-docker logs "$EXECUTOR_CONTAINER" 2>&1 | tail -20 || echo "  No logs available"
+docker logs executor 2>&1 | tail -20 || echo "  No logs available"
 exit 1
diff --git a/test/smoke-test.sh b/test/smoke-test.sh
index e733e4333..3b39ed1c8 100755
--- a/test/smoke-test.sh
+++ b/test/smoke-test.sh
@@ -11,12 +11,20 @@ configure_docker() {
   gcloud auth configure-docker us-central1-docker.pkg.dev,us-docker.pkg.dev
 }
 
+# Every service a customer can deploy, started and checked by the same commands
+compose_files=(
+	--file docker-compose/docker-compose.yaml
+	--file docker-compose/jaeger/docker-compose.yaml
+	--file docker-compose/executors/executor.docker-compose.yaml
+	--file test/docker-compose.override.yaml
+)
+
 deploy_sourcegraph() {
 	cd "$(dirname "${BASH_SOURCE[0]}")/.."
 	docker-compose version
 	# `up` pulls images one at a time, `pull` pulls them in parallel
-	docker-compose --file docker-compose/docker-compose.yaml pull --quiet
-	docker-compose --file docker-compose/docker-compose.yaml up -d -t 600
+	docker-compose "${compose_files[@]}" pull --quiet
+	docker-compose "${compose_files[@]}" up -d -t 600
 }
 
 wait_until_healthy() {
@@ -36,7 +44,7 @@ wait_until_healthy() {
 test_count() {
 	echo "TEST: Every service has a running container"
 	# migrator exits when it is done, every other service keeps running
-	expect_containers=$(docker-compose --file docker-compose/docker-compose.yaml config --services | grep --count --invert-match '^migrator$')
+	expect_containers=$(docker-compose "${compose_files[@]}" config --services | grep --count --invert-match '^migrator$')
 	containers_count=$(docker ps --filter status=running --format '{{.Names}}' | wc -l)
 	if [[ "$containers_count" -ne "$expect_containers" ]]; then
 		docker ps --all --format '{{.Names}}: {{.Status}}'
@@ -58,6 +66,9 @@ test_containers() {
 	curl -f http://localhost:80
 	curl -f http://localhost:80/healthz
 
+	echo "TEST: Checking Jaeger is accessible"
+	curl --fail --silent --show-error --location --output /dev/null http://localhost:16686/-/debug/jaeger/
+
 	echo "ALL TESTS PASSED"
 }
 

From 05cd457eb3249be4c92fff8b5e12c233fe7b45c1 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 17:23:27 -0600
Subject: [PATCH 16/25] ci: Retry the smoke test on a new VM, and have GCP
 delete each VM after an hour

A VM which stops answering SSH did not fail the tests, so the script deletes it
and runs the tests once more on a new VM. A VM which still answers ran the tests,
and they failed, so the script copies its logs and does not retry.

Each VM is created with --max-run-duration=1h and
--instance-termination-action=DELETE, so GCP deletes the VM and its boot disk
even if no job is left to do it. The hourly sweep stays for disks whose VM
was never created.

The script works with both the current Ubuntu 20.04 image and the Ubuntu 26.04
image from sourcegraph/infrastructure#7627, which ships jq.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5
Co-authored-by: Amp <amp@ampcode.com>
---
 .buildkite/pipeline.yaml    |   3 +-
 .buildkite/smoke-test-vm.sh | 144 +++++++++++++++++++++---------------
 2 files changed, 87 insertions(+), 60 deletions(-)

diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index 1091d0f8f..9729ebb08 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -16,8 +16,7 @@ steps:
     agents: { queue: "vagrant" }
 
   # Builds on main always run the smoke test. if_changed compares against origin/main,
-  # so it finds no changes in a commit which is already on main, unless it is the newest.
-  # The branch is named here, because the Buildkite pipeline's default branch is still "master"
+  # so it finds no changes in a commit which is already on main, unless it is the newest
   - &smoke_test
     label: ":rice: docker-compose-test"
     command: .buildkite/smoke-test-vm.sh
diff --git a/.buildkite/smoke-test-vm.sh b/.buildkite/smoke-test-vm.sh
index 7c9b1198c..0f0884e97 100755
--- a/.buildkite/smoke-test-vm.sh
+++ b/.buildkite/smoke-test-vm.sh
@@ -1,18 +1,21 @@
 #!/usr/bin/env bash
 
-# Runs test/smoke-test.sh on a new GCP VM booted from $IMAGE, then deletes the VM
+# Runs test/smoke-test.sh and test/executor-smoke-test.sh on a new GCP VM booted from $IMAGE,
+# then deletes the VM
 
 cd "$(dirname "${BASH_SOURCE[0]}")/.."
 set -euo pipefail
 
-KEY_PATH="$(mktemp)"
 project="sourcegraph-ci"
 zone="us-central1-f"
 
-# GCP creates a disk, then a VM, both with this name. Naming them after the job
+# GCP creates a disk, then a VM, both with the same name. Naming them after the job
 # lets the cleanup find them if the job is cancelled before the VM exists
 instance_name_prefix="docker-test-job-"
-instance_name="${instance_name_prefix}${BUILDKITE_JOB_ID}"
+job_instance_name_prefix="${instance_name_prefix}${BUILDKITE_JOB_ID}-"
+
+# The builder account key, and an SSH key for this job only
+temporary_directory="$(mktemp -d)"
 
 # Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses,
 # without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM
@@ -25,76 +28,101 @@ request_deletion() {
 			--request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)"
 }
 
-# Fall back to a blocking delete, so a failed deletion request cannot leak a VM
 cleanup() {
-	deletion_status=0
-	request_deletion "name=${instance_name}" ||
-		gcloud compute instances delete "$instance_name" --project="$project" --zone="$zone" --quiet ||
-		deletion_status=$?
-	rm -rf "$KEY_PATH" "$ssh_directory"
-	return "$deletion_status"
+	request_deletion "name~^${job_instance_name_prefix}" ||
+		echo "Could not delete this job's VMs. GCP deletes each VM an hour after it started"
+	rm -rf "$temporary_directory"
 }
-
-# A key for this job only, which the VM's guest agent authorizes for the buildkite user
-ssh_directory="$(mktemp -d)"
-ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$ssh_directory/key"
-ssh_options=(-i "$ssh_directory/key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5)
-
 trap cleanup EXIT
 
+# The VM's guest agent authorizes this key for the buildkite user.
+# The keepalive ends the connection to a VM which stopped answering, after a minute
+ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$temporary_directory/ssh-key"
+ssh_options=(-i "$temporary_directory/ssh-key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR
+	-o ConnectTimeout=5 -o ServerAliveInterval=15 -o ServerAliveCountMax=4)
+
 echo --- ":lock: builder account key"
-gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci >"${KEY_PATH}"
-DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
-export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH"
+gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project="$project" >"$temporary_directory/builder-key.json"
+DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project="$project")"
+export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$temporary_directory/builder-key.json"
 
 echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
-# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job
+# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job.
+# GCP deletes the VMs itself after an hour, but not a disk whose VM was never created
 request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks"
 
-echo --- ":gcloud: starting VM $instance_name"
-ip_address="$(gcloud compute instances create "$instance_name" \
-	--project="$project" --zone="$zone" \
-	--image="$IMAGE" --image-project="$project" \
-	--machine-type=custom-16-20480 \
-	--boot-disk-size=20GB --boot-disk-type=pd-standard \
-	--network=default --no-address \
-	--service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \
-	--labels=team=dev-infra,type=smoke-test \
-	--metadata="ssh-keys=buildkite:$(cat "$ssh_directory/key.pub")" \
-	--format="value(networkInterfaces[0].networkIP)")"
-remote="buildkite@${ip_address}"
-
-echo --- ":ssh: waiting for the VM to accept SSH"
-deadline=$((SECONDS + 600))
-until ssh "${ssh_options[@]}" "$remote" true; do
-	if ((SECONDS > deadline)); then
-		echo "The VM did not accept SSH within 10 minutes"
-		exit 1
-	fi
-	sleep 2
-done
+# The caller checks the exit status of the two functions below, which stops `set -e`
+# from applying inside them, so they return on the first failure themselves
+
+# Sets $remote once the VM exists
+start_vm() {
+	echo "--- :gcloud: starting VM $instance_name"
+	# `beta`, because --max-run-duration is only there in the agents' gcloud 417. It is in every track from gcloud 483
+	ip_address="$(gcloud beta compute instances create "$instance_name" \
+		--project="$project" --zone="$zone" \
+		--image="$IMAGE" --image-project="$project" \
+		--machine-type=custom-16-20480 \
+		--boot-disk-size=20GB --boot-disk-type=pd-standard \
+		--max-run-duration=1h --instance-termination-action=DELETE \
+		--network=default --no-address \
+		--service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \
+		--labels=team=dev-infra,type=smoke-test \
+		--metadata="ssh-keys=buildkite:$(cat "$temporary_directory/ssh-key.pub")" \
+		--format="value(networkInterfaces[0].networkIP)")" || return
+	remote="buildkite@${ip_address}"
+
+	echo --- ":ssh: waiting for the VM to accept SSH"
+	deadline=$((SECONDS + 300))
+	until ssh "${ssh_options[@]}" "$remote" true; do
+		if ((SECONDS > deadline)); then
+			echo "The VM did not accept SSH within 5 minutes"
+			return 1
+		fi
+		sleep 2
+	done
+}
 
-echo --- ":package: copying the repository to the VM"
-tar --create --gzip --exclude=.git --exclude=node_modules . |
-	ssh "${ssh_options[@]}" "$remote" "sudo mkdir /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker"
+run_smoke_tests() {
+	echo --- ":package: copying the repository to the VM"
+	tar --create --gzip --exclude=.git --exclude=node_modules . |
+		ssh "${ssh_options[@]}" "$remote" "sudo mkdir /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker" ||
+		return
 
-echo --- ":docker: running the smoke tests"
-# Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list
-remote_script='
+	echo --- ":docker: running the smoke tests"
+	# Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list
+	remote_script='
 set -euo pipefail
 read -r DOCKER_PASSWORD
 export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD
 /deploy-sourcegraph-docker/test/smoke-test.sh
-# The VM image has no jq. Skip /etc/apt/sources.list.d: the image lacks the key for its Chrome repository, so a full update fails
-apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq
+# The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails
+command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; }
 /deploy-sourcegraph-docker/test/executor-smoke-test.sh
 '
-exit_code=0
-# shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM
-ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD" || exit_code=$?
+	# shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM
+	ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD"
+}
+
+for attempt in 1 2; do
+	instance_name="${job_instance_name_prefix}${attempt}"
+	remote=""
+	exit_code=0
+	{ start_vm && run_smoke_tests; } || exit_code=$?
+	if [ "$exit_code" = 0 ]; then
+		break
+	fi
 
-if [ "$exit_code" != 0 ]; then
+	# A VM which still answers SSH ran the tests, and they failed.
 	# Copy back the container logs the smoke test wrote, for the step's artifacts
-	ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true
-	exit "$exit_code"
-fi
+	if [ -n "$remote" ] && ssh -o ConnectTimeout=15 "${ssh_options[@]}" "$remote" true; then
+		ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true
+		break
+	fi
+
+	# A VM which does not answer SSH did not fail the tests, so run them again on a new VM
+	echo "VM $instance_name does not answer SSH, in attempt $attempt of 2"
+	request_deletion "name=${instance_name}" || true
+done
+
+# Not a plain `exit`, after which shellcheck reports that cleanup is never invoked (SC2329)
+[ "$exit_code" = 0 ] || exit "$exit_code"

From ec0c7485bdb0a4128597ec3a8199b78cd19a0e52 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 17:23:27 -0600
Subject: [PATCH 17/25] ci: Save every container's logs when the smoke test
 fails, and use the Compose plugin if the VM has it

The old trap only saved the logs of running containers whose status was not Up,
so it saved nothing for an exited or unhealthy container.

The Ubuntu 26.04 VM image has the Compose plugin instead of the standalone
docker-compose 1.29.2 binary.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5
Co-authored-by: Amp <amp@ampcode.com>
---
 test/smoke-test.sh | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/test/smoke-test.sh b/test/smoke-test.sh
index 3b39ed1c8..6b284f81a 100755
--- a/test/smoke-test.sh
+++ b/test/smoke-test.sh
@@ -21,10 +21,16 @@ compose_files=(
 
 deploy_sourcegraph() {
 	cd "$(dirname "${BASH_SOURCE[0]}")/.."
-	docker-compose version
+	# The Ubuntu 20.04 VM image only has the standalone docker-compose 1.29.2
+	if docker compose version; then
+		compose=(docker compose)
+	else
+		compose=(docker-compose)
+		docker-compose version
+	fi
 	# `up` pulls images one at a time, `pull` pulls them in parallel
-	docker-compose "${compose_files[@]}" pull --quiet
-	docker-compose "${compose_files[@]}" up -d -t 600
+	"${compose[@]}" "${compose_files[@]}" pull --quiet
+	"${compose[@]}" "${compose_files[@]}" up -d -t 600
 }
 
 wait_until_healthy() {
@@ -44,7 +50,7 @@ wait_until_healthy() {
 test_count() {
 	echo "TEST: Every service has a running container"
 	# migrator exits when it is done, every other service keeps running
-	expect_containers=$(docker-compose "${compose_files[@]}" config --services | grep --count --invert-match '^migrator$')
+	expect_containers=$("${compose[@]}" "${compose_files[@]}" config --services | grep --count --invert-match '^migrator$')
 	containers_count=$(docker ps --filter status=running --format '{{.Names}}' | wc -l)
 	if [[ "$containers_count" -ne "$expect_containers" ]]; then
 		docker ps --all --format '{{.Names}}: {{.Status}}'
@@ -72,20 +78,18 @@ test_containers() {
 	echo "ALL TESTS PASSED"
 }
 
-catch_errors() {
-	count=$(docker ps --format '{{.Names}}:{{.Status}}' | grep -c -v Up) || true
-	if [[ $count -ne 0 ]]; then
-		containers_failing=$(docker ps --format '{{.Names}}:{{.Status}}' | grep -v Up | cut -f 1 -d :)
-		echo
-		for cf in $containers_failing; do
-			echo "$cf is failing. Review the log files uploaded as artefacts to see errors."
-			docker logs -t "$cf" >"$cf".log 2>&1
+# On failure, save every container's logs, which the CI step uploads as artifacts
+save_logs_on_failure() {
+	exit_code=$?
+	if [[ $exit_code -ne 0 ]]; then
+		docker ps --all --format '{{.Names}}: {{.Status}}'
+		for container in $(docker ps --all --format '{{.Names}}'); do
+			docker logs --timestamps "$container" >"$container.log" 2>&1 || true
 		done
-		exit 1
 	fi
 }
 
-trap catch_errors EXIT
+trap save_logs_on_failure EXIT
 
 configure_docker
 deploy_sourcegraph

From 66694b0a0ab6ef8987bb9af806aad9b306446412 Mon Sep 17 00:00:00 2001
From: Travis Lyons <travis.lyons@sourcegraph.com>
Date: Thu, 25 Jun 2026 17:47:07 +0000
Subject: [PATCH 18/25] docs: add missing AGENTS.md (and README where missing)

---
 AGENTS.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 AGENTS.md

diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 000000000..171cdc667
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,74 @@
+# AGENTS.md
+
+Guidance for coding agents working in this repository.
+
+## What this repo is
+
+This is the **deployment reference for Sourcegraph with Docker Compose** (`github.com/sourcegraph/deploy-sourcegraph-docker`). It is primarily configuration (Docker Compose YAML, service configs, dashboards) plus small Go and shell tooling for testing and releases — not an application codebase.
+
+The `main` branch tracks development. Versioned branches (e.g. `3.19`) correspond to released Sourcegraph versions.
+
+## Where the main code/config lives
+
+- `docker-compose/` — the deployment reference; `docker-compose/docker-compose.yaml` is the primary entrypoint, with `dev/`, `examples/`, `executors/`, and `jaeger/` variants.
+- Service config directories: `caddy/`, `nginx/`, `pgsql/`, `codeintel-db/`, `codeinsights-db/`, `grafana/`, `prometheus/`, `otel-collector/`.
+- `test/` — smoke and upgrade tests (`smoke-test.sh`, `upgrade-test.go`, `Vagrantfile`, `servers.yaml`).
+- `tools/` — Go-pinned CLIs (`update-docker-tags`) and `migrate.sh`.
+- `.buildkite/` — CI scripts and pipeline.
+
+## Toolchain versions
+
+Defined in `.tool-versions` (asdf/mise): Go `1.19.8`, Node `16.7.0`, Yarn `1.22.4`, shellcheck `0.7.1`, github-cli `2.46.0`, checkov.
+
+## Setup, build, and checks
+
+These mirror what CI (`.buildkite/pipeline.yaml`) actually runs.
+
+Install JS dev dependencies (for prettier):
+
+```bash
+yarn
+```
+
+Validate the Docker Compose config:
+
+```bash
+cd docker-compose
+docker-compose -f docker-compose.yaml config -q
+```
+
+Or via the CI wrapper from the repo root:
+
+```bash
+.buildkite/validate-docker-compose.sh
+```
+
+Format / check formatting (prettier, config in `prettier.config.js`):
+
+```bash
+yarn run prettier        # write
+yarn run prettier-check  # check only
+```
+
+Run the Terraform/IaC security scan (Checkov):
+
+```bash
+.buildkite/ci-checkov.sh
+```
+
+## Testing
+
+Smoke and upgrade tests run on GCP via Vagrant (see `TESTING.md`):
+
+```bash
+.buildkite/vagrant-run.sh docker-test   # TEST_TYPE=docker-compose-test
+```
+
+This starts a GCP instance, uploads the repo, and runs the docker-compose smoke test. Expected output: `docker-test: ALL TESTS PASSED`.
+
+## Conventions
+
+- Keep changes minimal and config-focused; this is a deployment reference consumed by customers.
+- All `*.{js,json,ts,tsx,graphql,md,scss}` files are formatted with prettier — run `yarn run prettier` before committing.
+- Shell scripts should pass shellcheck.
+- Releases are driven by `release.yaml` and the `sg release` steps in `.buildkite/pipeline.yaml`; release branches use `internal/release-*` and `promote/release-*` naming.

From 69a9f5b7e1f23cb811d2c94831e162699cd726a4 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 15:21:09 -0600
Subject: [PATCH 19/25] docs: Correct AGENTS.md for the current repository

The file described the repository as of June. Since then the Go tooling,
Checkov, shellcheck, TESTING.md, tools/, and nginx/ were removed.

- Drop statements about files and tools that no longer exist
- Prettier skips Markdown and YAML; say so
- The smoke test reads CI secrets and creates a GCP VM; point agents at
  the pull request's Buildkite build instead of running it
- Add the rules an agent cannot infer: keep db-only-migrate in sync,
  update expect_containers, leave image tags to the release process,
  backport labels, and the branches the release tooling owns
---
 AGENTS.md | 86 ++++++++++++++-----------------------------------------
 1 file changed, 22 insertions(+), 64 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 171cdc667..a3a18fec3 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -1,74 +1,32 @@
 # AGENTS.md
 
-Guidance for coding agents working in this repository.
+This repository is the deployment reference for [Sourcegraph with Docker Compose](https://sourcegraph.com/docs/self-hosted/deploy/docker-compose). Customers merge each release tag into their own copy of this repository, run `docker-compose/docker-compose.yaml`, and keep their changes in a `docker-compose.override.yaml` file. The repository holds configuration and a few shell scripts, no application code.
 
-## What this repo is
+## Layout
 
-This is the **deployment reference for Sourcegraph with Docker Compose** (`github.com/sourcegraph/deploy-sourcegraph-docker`). It is primarily configuration (Docker Compose YAML, service configs, dashboards) plus small Go and shell tooling for testing and releases — not an application codebase.
+- `docker-compose/docker-compose.yaml`: the base file every customer runs. It mounts the service configuration in `caddy/`, `pgsql/`, `codeintel-db/`, `codeinsights-db/`, `grafana/`, `prometheus/`, and `otel-collector/`.
+- `docker-compose/db-only-migrate.docker-compose.yaml`: a copy of the `pgsql` and `codeintel-db` services for backup and restore. Make every change to those two services in both files.
+- `docker-compose/examples/`: example override files. `docker-compose/executors/` and `docker-compose/jaeger/` are optional extra compose files. `docker-compose/dev/` is an overlay for developing this repository.
+- `test/` and `.buildkite/`: the smoke test and the CI pipeline.
+- `release.yaml`: the release steps that `sg release` runs.
 
-The `main` branch tracks development. Versioned branches (e.g. `3.19`) correspond to released Sourcegraph versions.
+## Checks
 
-## Where the main code/config lives
+- `.tool-versions` pins Node, Yarn, and the GitHub CLI; CI installs them with mise.
+- Validate the base file with `docker compose -f docker-compose/docker-compose.yaml config -q`. CI runs the same check in `.buildkite/validate-docker-compose.sh` with the standalone `docker-compose` binary, so the base file must stay valid for that binary. CI validates no other compose file.
+- Check formatting with `yarn && yarn run prettier-check`; `yarn run prettier` rewrites the files. Prettier covers only JavaScript and JSON files. `.prettierignore` excludes Markdown and `.github/`, and no formatter runs on YAML, so match the surrounding YAML style by hand.
+- The `docker-compose-test` step in `.buildkite/pipeline.yaml` starts the base file on a GCP VM and runs `test/smoke-test.sh`. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run `.buildkite/vagrant-run.sh docker-test` yourself only when a person asks.
+- `test/smoke-test.sh` compares the number of running containers with `expect_containers`. Update that number in the same pull request that adds or removes a service.
+- CI does not run `test/executor-smoke-test.sh`. Run it against a running deployment when you change `docker-compose/executors/`.
 
-- `docker-compose/` — the deployment reference; `docker-compose/docker-compose.yaml` is the primary entrypoint, with `dev/`, `examples/`, `executors/`, and `jaeger/` variants.
-- Service config directories: `caddy/`, `nginx/`, `pgsql/`, `codeintel-db/`, `codeinsights-db/`, `grafana/`, `prometheus/`, `otel-collector/`.
-- `test/` — smoke and upgrade tests (`smoke-test.sh`, `upgrade-test.go`, `Vagrantfile`, `servers.yaml`).
-- `tools/` — Go-pinned CLIs (`update-docker-tags`) and `migrate.sh`.
-- `.buildkite/` — CI scripts and pipeline.
-
-## Toolchain versions
-
-Defined in `.tool-versions` (asdf/mise): Go `1.19.8`, Node `16.7.0`, Yarn `1.22.4`, shellcheck `0.7.1`, github-cli `2.46.0`, checkov.
-
-## Setup, build, and checks
-
-These mirror what CI (`.buildkite/pipeline.yaml`) actually runs.
-
-Install JS dev dependencies (for prettier):
-
-```bash
-yarn
-```
-
-Validate the Docker Compose config:
-
-```bash
-cd docker-compose
-docker-compose -f docker-compose.yaml config -q
-```
-
-Or via the CI wrapper from the repo root:
-
-```bash
-.buildkite/validate-docker-compose.sh
-```
-
-Format / check formatting (prettier, config in `prettier.config.js`):
-
-```bash
-yarn run prettier        # write
-yarn run prettier-check  # check only
-```
-
-Run the Terraform/IaC security scan (Checkov):
-
-```bash
-.buildkite/ci-checkov.sh
-```
-
-## Testing
-
-Smoke and upgrade tests run on GCP via Vagrant (see `TESTING.md`):
-
-```bash
-.buildkite/vagrant-run.sh docker-test   # TEST_TYPE=docker-compose-test
-```
+## Conventions
 
-This starts a GCP instance, uploads the repo, and runs the docker-compose smoke test. Expected output: `docker-test: ALL TESTS PASSED`.
+- Customer override files refer to the base file's service names, volume names, published ports, and environment variable names. Describe every rename or removal of one in the pull request description.
+- Every `image:` has the form `index.docker.io/sourcegraph/<name>:<version>@sha256:<digest>`. The release process sets the versions and digests (`sg ops update-images` in `release.yaml`) and brings them to `main` in "Update main: build v…" pull requests. Leave them unchanged; Renovate is turned off for these images for the same reason.
+- `.github/PULL_REQUEST_TEMPLATE.md` requires a "Test plan" section in every pull request description.
 
-## Conventions
+## Branches and releases
 
-- Keep changes minimal and config-focused; this is a deployment reference consumed by customers.
-- All `*.{js,json,ts,tsx,graphql,md,scss}` files are formatted with prettier — run `yarn run prettier` before committing.
-- Shell scripts should pass shellcheck.
-- Releases are driven by `release.yaml` and the `sg release` steps in `.buildkite/pipeline.yaml`; release branches use `internal/release-*` and `promote/release-*` naming.
+- `main` tracks development. Each release is a tag such as `v8.1.0`.
+- To copy a pull request to a release branch such as `8.1.x`, add the label `backport 8.1.x` to it. `.github/workflows/backport.yml` opens the backport pull request once the original merges.
+- The release tooling owns the branches named `internal/release-*`, `promote/release-*`, and `release-v*`, and the `v*` tags. Do not create, push to, or delete them: CI runs the release steps on a branch whose name starts with `internal/release-` or `promote/release-`, and each `release-v*` branch holds the commit of a tag that customers deploy.

From c852c57b124b02d089ae1c1d6f562d0a23b8e479 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 15:50:55 -0600
Subject: [PATCH 20/25] docs: Cover both clones and forks in the AGENTS.md
 upgrade sentence

Customers who clone this repository check out the release tag; only
customers who fork it merge the tag into their own branch.
---
 AGENTS.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/AGENTS.md b/AGENTS.md
index a3a18fec3..0284a673b 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -1,6 +1,6 @@
 # AGENTS.md
 
-This repository is the deployment reference for [Sourcegraph with Docker Compose](https://sourcegraph.com/docs/self-hosted/deploy/docker-compose). Customers merge each release tag into their own copy of this repository, run `docker-compose/docker-compose.yaml`, and keep their changes in a `docker-compose.override.yaml` file. The repository holds configuration and a few shell scripts, no application code.
+This repository is the deployment reference for [Sourcegraph with Docker Compose](https://sourcegraph.com/docs/self-hosted/deploy/docker-compose). Customers run `docker-compose/docker-compose.yaml` at a release tag: they check the tag out in a clone of this repository, or merge it into their own fork. We recommend they keep their changes in a `docker-compose.override.yaml` file. The repository holds configuration and a few shell scripts, no application code.
 
 ## Layout
 

From 555b08de5133359a25bdb3b8dd807592d5694668 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 17:23:54 -0600
Subject: [PATCH 21/25] docs: Bring AGENTS.md up to date with the CI changes

The db-only file now extends the base file, CI validates every compose file,
runs shellcheck and the executor test, counts the services itself, and runs
the smoke test VM with gcloud instead of Vagrant.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5
Co-authored-by: Amp <amp@ampcode.com>
---
 AGENTS.md | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 0284a673b..c08789561 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -5,19 +5,24 @@ This repository is the deployment reference for [Sourcegraph with Docker Compose
 ## Layout
 
 - `docker-compose/docker-compose.yaml`: the base file every customer runs. It mounts the service configuration in `caddy/`, `pgsql/`, `codeintel-db/`, `codeinsights-db/`, `grafana/`, `prometheus/`, and `otel-collector/`.
-- `docker-compose/db-only-migrate.docker-compose.yaml`: a copy of the `pgsql` and `codeintel-db` services for backup and restore. Make every change to those two services in both files.
+- `docker-compose/db-only-migrate.docker-compose.yaml`: starts only the `pgsql` and `codeintel-db` services, for backup and restore. It `extends` both from the base file, so change them there.
 - `docker-compose/examples/`: example override files. `docker-compose/executors/` and `docker-compose/jaeger/` are optional extra compose files. `docker-compose/dev/` is an overlay for developing this repository.
-- `test/` and `.buildkite/`: the smoke test and the CI pipeline.
+- `test/` and `.buildkite/`: the smoke tests and the CI pipeline.
 - `release.yaml`: the release steps that `sg release` runs.
 
 ## Checks
 
-- `.tool-versions` pins Node, Yarn, and the GitHub CLI; CI installs them with mise.
-- Validate the base file with `docker compose -f docker-compose/docker-compose.yaml config -q`. CI runs the same check in `.buildkite/validate-docker-compose.sh` with the standalone `docker-compose` binary, so the base file must stay valid for that binary. CI validates no other compose file.
+- `.tool-versions` pins Node, Yarn, the GitHub CLI, and shellcheck; CI installs them with mise.
+- `.buildkite/validate-docker-compose.sh` runs `docker compose config` on every compose file, in each combination the docs say to run. Add a line to it for every new compose file or combination.
 - Check formatting with `yarn && yarn run prettier-check`; `yarn run prettier` rewrites the files. Prettier covers only JavaScript and JSON files. `.prettierignore` excludes Markdown and `.github/`, and no formatter runs on YAML, so match the surrounding YAML style by hand.
-- The `docker-compose-test` step in `.buildkite/pipeline.yaml` starts the base file on a GCP VM and runs `test/smoke-test.sh`. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run `.buildkite/vagrant-run.sh docker-test` yourself only when a person asks.
-- `test/smoke-test.sh` compares the number of running containers with `expect_containers`. Update that number in the same pull request that adds or removes a service.
-- CI does not run `test/executor-smoke-test.sh`. Run it against a running deployment when you change `docker-compose/executors/`.
+- CI runs shellcheck on every shell script: `git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck`.
+- The `docker-compose-test` step runs `.buildkite/smoke-test-vm.sh`, which boots a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. The step reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. Branches which only change Markdown files skip the step.
+  - `test/smoke-test.sh` starts the base file, `jaeger/`, `executors/`, and `test/docker-compose.override.yaml`, then checks that every service has a running container, no container restarted, the health checks pass, and the frontend and Jaeger respond. It counts the services itself, so a new service in those files needs no change to the test. Add a new optional compose file to its `compose_files`.
+  - `test/executor-smoke-test.sh` checks that the executor registers with the frontend.
+- If the VM stops answering SSH, the script runs the tests once more on a new VM. A test failure is not retried, and the step uploads every container's logs as artifacts.
+- GCP deletes each VM an hour after it starts, and the script deletes the VMs and disks which older jobs left behind. Keep the step's `timeout_in_minutes` under an hour.
+- The VM image is built from `gcp/images/e2e` in sourcegraph/infrastructure. The release branches name their image too, so leave old images in place.
+- The CI agents' queue is still named `vagrant`. Release branches up to `8.1.x` run the smoke test with Vagrant (`.buildkite/vagrant-run.sh`), so a CI change on `main` does not backport cleanly.
 
 ## Conventions
 

From b13253fc1fcecc460135529eaeb331bae9555505 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 17:30:06 -0600
Subject: [PATCH 22/25] ci: Use the Compose plugin in the smoke test without a
 docker-compose fallback

The Ubuntu 20.04 VM image has Docker Compose v2.18.1 as a plugin (build 16848), so the fallback to the standalone docker-compose never ran.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5
Co-authored-by: Amp <amp@ampcode.com>
---
 test/smoke-test.sh | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/test/smoke-test.sh b/test/smoke-test.sh
index 6b284f81a..010a05409 100755
--- a/test/smoke-test.sh
+++ b/test/smoke-test.sh
@@ -21,16 +21,10 @@ compose_files=(
 
 deploy_sourcegraph() {
 	cd "$(dirname "${BASH_SOURCE[0]}")/.."
-	# The Ubuntu 20.04 VM image only has the standalone docker-compose 1.29.2
-	if docker compose version; then
-		compose=(docker compose)
-	else
-		compose=(docker-compose)
-		docker-compose version
-	fi
+	docker compose version
 	# `up` pulls images one at a time, `pull` pulls them in parallel
-	"${compose[@]}" "${compose_files[@]}" pull --quiet
-	"${compose[@]}" "${compose_files[@]}" up -d -t 600
+	docker compose "${compose_files[@]}" pull --quiet
+	docker compose "${compose_files[@]}" up -d -t 600
 }
 
 wait_until_healthy() {
@@ -50,7 +44,7 @@ wait_until_healthy() {
 test_count() {
 	echo "TEST: Every service has a running container"
 	# migrator exits when it is done, every other service keeps running
-	expect_containers=$("${compose[@]}" "${compose_files[@]}" config --services | grep --count --invert-match '^migrator$')
+	expect_containers=$(docker compose "${compose_files[@]}" config --services | grep --count --invert-match '^migrator$')
 	containers_count=$(docker ps --filter status=running --format '{{.Names}}' | wc -l)
 	if [[ "$containers_count" -ne "$expect_containers" ]]; then
 		docker ps --all --format '{{.Names}}: {{.Status}}'

From 869abd2363cfc31abdf7fdd1f5757cf93f26ce03 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 17:33:04 -0600
Subject: [PATCH 23/25] ci: Wait for the containers with docker compose up
 --wait

Replaces the polling loop. The Ubuntu 20.04 VM image's Compose v2.18.1 has --wait and --wait-timeout, so this does not need the new image.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5
Co-authored-by: Amp <amp@ampcode.com>
---
 test/smoke-test.sh | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/test/smoke-test.sh b/test/smoke-test.sh
index 010a05409..5ef9b93ae 100755
--- a/test/smoke-test.sh
+++ b/test/smoke-test.sh
@@ -24,21 +24,8 @@ deploy_sourcegraph() {
 	docker compose version
 	# `up` pulls images one at a time, `pull` pulls them in parallel
 	docker compose "${compose_files[@]}" pull --quiet
-	docker compose "${compose_files[@]}" up -d -t 600
-}
-
-wait_until_healthy() {
-	echo "Waiting up to 5 minutes for the frontend to respond, and container health checks to pass..."
-	for _ in {1..60}; do
-		if curl --fail --silent --output /dev/null http://localhost:80/healthz &&
-			[[ -z "$(docker ps --quiet --filter health=starting --filter health=unhealthy)" ]]; then
-			return
-		fi
-		sleep 5
-	done
-	docker ps --format '{{.Names}}: {{.Status}}'
-	echo "TEST FAILURE: containers were not healthy after 5 minutes"
-	exit 1
+	# --wait returns when every container is running, and healthy if it has a health check
+	docker compose "${compose_files[@]}" up -d -t 600 --wait --wait-timeout 300
 }
 
 test_count() {
@@ -87,6 +74,5 @@ trap save_logs_on_failure EXIT
 
 configure_docker
 deploy_sourcegraph
-wait_until_healthy
 test_count
 test_containers

From 44081db197bf23213618eebd8b91cc2596ddcac8 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 17:38:23 -0600
Subject: [PATCH 24/25] Revert "ci: Wait for the containers with docker compose
 up --wait"

This reverts commit 869abd2363cfc31abdf7fdd1f5757cf93f26ce03.

Build 16853: `up --wait` failed with "container searcher-0 is unhealthy" while `docker ps` showed it healthy in the same second. searcher has no health check start period, so it reports unhealthy until the frontend is up, and --wait fails on the first unhealthy report. The polling loop tolerates that.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5
Co-authored-by: Amp <amp@ampcode.com>
---
 test/smoke-test.sh | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/test/smoke-test.sh b/test/smoke-test.sh
index 5ef9b93ae..010a05409 100755
--- a/test/smoke-test.sh
+++ b/test/smoke-test.sh
@@ -24,8 +24,21 @@ deploy_sourcegraph() {
 	docker compose version
 	# `up` pulls images one at a time, `pull` pulls them in parallel
 	docker compose "${compose_files[@]}" pull --quiet
-	# --wait returns when every container is running, and healthy if it has a health check
-	docker compose "${compose_files[@]}" up -d -t 600 --wait --wait-timeout 300
+	docker compose "${compose_files[@]}" up -d -t 600
+}
+
+wait_until_healthy() {
+	echo "Waiting up to 5 minutes for the frontend to respond, and container health checks to pass..."
+	for _ in {1..60}; do
+		if curl --fail --silent --output /dev/null http://localhost:80/healthz &&
+			[[ -z "$(docker ps --quiet --filter health=starting --filter health=unhealthy)" ]]; then
+			return
+		fi
+		sleep 5
+	done
+	docker ps --format '{{.Names}}: {{.Status}}'
+	echo "TEST FAILURE: containers were not healthy after 5 minutes"
+	exit 1
 }
 
 test_count() {
@@ -74,5 +87,6 @@ trap save_logs_on_failure EXIT
 
 configure_docker
 deploy_sourcegraph
+wait_until_healthy
 test_count
 test_containers

From 81aa31f8838182aa588d3bcb382a954ba7e8eae5 Mon Sep 17 00:00:00 2001
From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com>
Date: Tue, 10 Nov 2026 17:43:19 -0600
Subject: [PATCH 25/25] ci: Retry the image pull in the smoke test

Build 16854 failed when Docker Hub's token endpoint timed out during the pull.

Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5
Co-authored-by: Amp <amp@ampcode.com>
---
 test/smoke-test.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/smoke-test.sh b/test/smoke-test.sh
index 010a05409..2c8255cdf 100755
--- a/test/smoke-test.sh
+++ b/test/smoke-test.sh
@@ -22,8 +22,12 @@ compose_files=(
 deploy_sourcegraph() {
 	cd "$(dirname "${BASH_SOURCE[0]}")/.."
 	docker compose version
-	# `up` pulls images one at a time, `pull` pulls them in parallel
-	docker compose "${compose_files[@]}" pull --quiet
+	# `up` pulls images one at a time, `pull` pulls them in parallel.
+	# Docker Hub times out now and then; `up` pulls whatever is still missing after three attempts
+	for _ in 1 2 3; do
+		docker compose "${compose_files[@]}" pull --quiet && break
+		sleep 10
+	done
 	docker compose "${compose_files[@]}" up -d -t 600
 }