ci: Smoke test on a gcloud VM with a 1 hour TTL, a retry, Jaeger, and the executor

.buildkite/pipeline.yaml

@@ -1,8 +1,7 @@
 $schema: "http://json.schemastore.org/buildkite"
 env:
-  # GCP VM image for the Vagrant smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images`
+  # GCP VM image for the smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images`
   IMAGE: "e2e-db8a40720-2024-08-26t08-44-10z"
-  VAGRANT_RUN_ENV: "CI"
 steps:
   - label: ":docker:"
     command: .buildkite/validate-docker-compose.sh
@@ -12,10 +11,27 @@ steps:
     command: .buildkite/prettier-check.sh
     agents: { queue: "vagrant" }
 
-  - label: ":rice: docker-compose-test"
-    command: .buildkite/vagrant-run.sh docker-test
+  - label: ":bash:"
+    command: git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck
+    agents: { queue: "vagrant" }
+
+  # Builds on main always run the smoke test. if_changed compares against origin/main,
+  # so it finds no changes in a commit which is already on main, unless it is the newest
+  - &smoke_test
+    label: ":rice: docker-compose-test"
+    command: .buildkite/smoke-test-vm.sh
     artifact_paths: ./*.log
     agents: { queue: "vagrant" }
+    # smoke-test-vm.sh deletes VMs and disks older than an hour, so no job may run that long
+    timeout_in_minutes: 30
+    if: build.branch == "main"
+
+  # Builds on other branches skip the smoke test if they only change docs
+  - <<: *smoke_test
+    if: build.branch != "main"
+    if_changed:
+      include: "**"
+      exclude: "**.md"
 
   # Please keep in mind that the release manifest uses specific branch names when creating releases.
   # Therefore, if you update them, or if you decide to change how we detect what kind of build we're dealing

.buildkite/smoke-test-vm.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+# Runs test/smoke-test.sh and test/executor-smoke-test.sh on a new GCP VM booted from $IMAGE,
+# then deletes the VM
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+set -euo pipefail
+
+project="sourcegraph-ci"
+zone="us-central1-f"
+
+# GCP creates a disk, then a VM, both with the same name. Naming them after the job
+# lets the cleanup find them if the job is cancelled before the VM exists
+instance_name_prefix="docker-test-job-"
+job_instance_name_prefix="${instance_name_prefix}${BUILDKITE_JOB_ID}-"
+
+# The builder account key, and an SSH key for this job only
+temporary_directory="$(mktemp -d)"
+
+# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses,
+# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM
+request_deletion() {
+	{
+		gcloud compute instances list --project="$project" --filter="$1" --format="value(selfLink)" &&
+			gcloud compute disks list --project="$project" --filter="($1) AND -users:*" --format="value(selfLink)"
+	} | tee /dev/stderr |
+		xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \
+			--request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)"
+}
+
+cleanup() {
+	request_deletion "name~^${job_instance_name_prefix}" ||
+		echo "Could not delete this job's VMs. GCP deletes each VM an hour after it started"
+	rm -rf "$temporary_directory"
+}
+trap cleanup EXIT
+
+# The VM's guest agent authorizes this key for the buildkite user.
+# The keepalive ends the connection to a VM which stopped answering, after a minute
+ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$temporary_directory/ssh-key"
+ssh_options=(-i "$temporary_directory/ssh-key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR
+	-o ConnectTimeout=5 -o ServerAliveInterval=15 -o ServerAliveCountMax=4)
+
+echo --- ":lock: builder account key"
+gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project="$project" >"$temporary_directory/builder-key.json"
+DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project="$project")"
+export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$temporary_directory/builder-key.json"
+
+echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
+# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job.
+# GCP deletes the VMs itself after an hour, but not a disk whose VM was never created
+request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks"
+
+# The caller checks the exit status of the two functions below, which stops `set -e`
+# from applying inside them, so they return on the first failure themselves
+
+# Sets $remote once the VM exists
+start_vm() {
+	echo "--- :gcloud: starting VM $instance_name"
+	# `beta`, because --max-run-duration is only there in the agents' gcloud 417. It is in every track from gcloud 483
+	ip_address="$(gcloud beta compute instances create "$instance_name" \
+		--project="$project" --zone="$zone" \
+		--image="$IMAGE" --image-project="$project" \
+		--machine-type=custom-16-20480 \
+		--boot-disk-size=20GB --boot-disk-type=pd-standard \
+		--max-run-duration=1h --instance-termination-action=DELETE \
+		--network=default --no-address \
+		--service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \
+		--labels=team=dev-infra,type=smoke-test \
+		--metadata="ssh-keys=buildkite:$(cat "$temporary_directory/ssh-key.pub")" \
+		--format="value(networkInterfaces[0].networkIP)")" || return
+	remote="buildkite@${ip_address}"
+
+	echo --- ":ssh: waiting for the VM to accept SSH"
+	deadline=$((SECONDS + 300))
+	until ssh "${ssh_options[@]}" "$remote" true; do
+		if ((SECONDS > deadline)); then
+			echo "The VM did not accept SSH within 5 minutes"
+			return 1
+		fi
+		sleep 2
+	done
+}
+
+run_smoke_tests() {
+	echo --- ":package: copying the repository to the VM"
+	tar --create --gzip --exclude=.git --exclude=node_modules . |
+		ssh "${ssh_options[@]}" "$remote" "sudo mkdir /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker" ||
+		return
+
+	echo --- ":docker: running the smoke tests"
+	# Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list
+	remote_script='
+set -euo pipefail
+read -r DOCKER_PASSWORD
+export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD
+/deploy-sourcegraph-docker/test/smoke-test.sh
+# The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails
+command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; }
+/deploy-sourcegraph-docker/test/executor-smoke-test.sh
+'
+	# shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM
+	ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD"
+}
+
+for attempt in 1 2; do
+	instance_name="${job_instance_name_prefix}${attempt}"
+	remote=""
+	exit_code=0
+	{ start_vm && run_smoke_tests; } || exit_code=$?
+	if [ "$exit_code" = 0 ]; then
+		break
+	fi
+
+	# A VM which still answers SSH ran the tests, and they failed.
+	# Copy back the container logs the smoke test wrote, for the step's artifacts
+	if [ -n "$remote" ] && ssh -o ConnectTimeout=15 "${ssh_options[@]}" "$remote" true; then
+		ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true
+		break
+	fi
+
+	# A VM which does not answer SSH did not fail the tests, so run them again on a new VM
+	echo "VM $instance_name does not answer SSH, in attempt $attempt of 2"
+	request_deletion "name=${instance_name}" || true
+done
+
+# Not a plain `exit`, after which shellcheck reports that cleanup is never invoked (SC2329)
+[ "$exit_code" = 0 ] || exit "$exit_code"

.buildkite/validate-docker-compose.sh

@@ -1,7 +1,19 @@
 #!/usr/bin/env bash
 
-cd "$(dirname "${BASH_SOURCE[0]}")/.."
+cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose"
 set -euxo pipefail
 
-cd docker-compose 
-docker-compose -f docker-compose.yaml config -q
+docker compose version
+
+# Every Compose file, combined the way its docs say to run it
+docker compose --file docker-compose.yaml config --quiet
+docker compose --file db-only-migrate.docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file dev/docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file executors/executor.docker-compose.yaml config --quiet
+docker compose --file executors/executor.docker-compose.yaml config --quiet
+# The combination the smoke test deploys
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file executors/executor.docker-compose.yaml --file ../test/docker-compose.override.yaml config --quiet
+for override in examples/*/docker-compose.override.yaml; do
+	docker compose --file docker-compose.yaml --file "$override" config --quiet
+done

.gitignore

@@ -1,5 +1,4 @@
 .env
 .envrc
-.vagrant
 *.log
 node_modules/

.tool-versions

@@ -1,3 +1,4 @@
 nodejs                   16.20.2
 yarn                     1.22.4
 github-cli               2.103.3
+shellcheck               0.11.0

AGENTS.md

@@ -0,0 +1,37 @@
+# AGENTS.md
+
+This repository is the deployment reference for [Sourcegraph with Docker Compose](https://sourcegraph.com/docs/self-hosted/deploy/docker-compose). Customers run `docker-compose/docker-compose.yaml` at a release tag: they check the tag out in a clone of this repository, or merge it into their own fork. We recommend they keep their changes in a `docker-compose.override.yaml` file. The repository holds configuration and a few shell scripts, no application code.
+
+## Layout
+
+- `docker-compose/docker-compose.yaml`: the base file every customer runs. It mounts the service configuration in `caddy/`, `pgsql/`, `codeintel-db/`, `codeinsights-db/`, `grafana/`, `prometheus/`, and `otel-collector/`.
+- `docker-compose/db-only-migrate.docker-compose.yaml`: starts only the `pgsql` and `codeintel-db` services, for backup and restore. It `extends` both from the base file, so change them there.
+- `docker-compose/examples/`: example override files. `docker-compose/executors/` and `docker-compose/jaeger/` are optional extra compose files. `docker-compose/dev/` is an overlay for developing this repository.
+- `test/` and `.buildkite/`: the smoke tests and the CI pipeline.
+- `release.yaml`: the release steps that `sg release` runs.
+
+## Checks
+
+- `.tool-versions` pins Node, Yarn, the GitHub CLI, and shellcheck; CI installs them with mise.
+- `.buildkite/validate-docker-compose.sh` runs `docker compose config` on every compose file, in each combination the docs say to run. Add a line to it for every new compose file or combination.
+- Check formatting with `yarn && yarn run prettier-check`; `yarn run prettier` rewrites the files. Prettier covers only JavaScript and JSON files. `.prettierignore` excludes Markdown and `.github/`, and no formatter runs on YAML, so match the surrounding YAML style by hand.
+- CI runs shellcheck on every shell script: `git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck`.
+- The `docker-compose-test` step runs `.buildkite/smoke-test-vm.sh`, which boots a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. The step reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. Branches which only change Markdown files skip the step.
+  - `test/smoke-test.sh` starts the base file, `jaeger/`, `executors/`, and `test/docker-compose.override.yaml`, then checks that every service has a running container, no container restarted, the health checks pass, and the frontend and Jaeger respond. It counts the services itself, so a new service in those files needs no change to the test. Add a new optional compose file to its `compose_files`.
+  - `test/executor-smoke-test.sh` checks that the executor registers with the frontend.
+- If the VM stops answering SSH, the script runs the tests once more on a new VM. A test failure is not retried, and the step uploads every container's logs as artifacts.
+- GCP deletes each VM an hour after it starts, and the script deletes the VMs and disks which older jobs left behind. Keep the step's `timeout_in_minutes` under an hour.
+- The VM image is built from `gcp/images/e2e` in sourcegraph/infrastructure. The release branches name their image too, so leave old images in place.
+- The CI agents' queue is still named `vagrant`. Release branches up to `8.1.x` run the smoke test with Vagrant (`.buildkite/vagrant-run.sh`), so a CI change on `main` does not backport cleanly.
+
+## Conventions
+
+- Customer override files refer to the base file's service names, volume names, published ports, and environment variable names. Describe every rename or removal of one in the pull request description.
+- Every `image:` has the form `index.docker.io/sourcegraph/<name>:<version>@sha256:<digest>`. The release process sets the versions and digests (`sg ops update-images` in `release.yaml`) and brings them to `main` in "Update main: build v…" pull requests. Leave them unchanged; Renovate is turned off for these images for the same reason.
+- `.github/PULL_REQUEST_TEMPLATE.md` requires a "Test plan" section in every pull request description.
+
+## Branches and releases
+
+- `main` tracks development. Each release is a tag such as `v8.1.0`.
+- To copy a pull request to a release branch such as `8.1.x`, add the label `backport 8.1.x` to it. `.github/workflows/backport.yml` opens the backport pull request once the original merges.
+- The release tooling owns the branches named `internal/release-*`, `promote/release-*`, and `release-v*`, and the `v*` tags. Do not create, push to, or delete them: CI runs the release steps on a branch whose name starts with `internal/release-` or `promote/release-`, and each `release-v*` branch holds the commit of a tag that customers deploy.

batch-change/release.yaml

@@ -138,7 +138,16 @@ promoteToPublic:
           git checkout "${promote_branch}"
           git switch -c "${release_branch}"
 
-          git tag {{version}}
+          # A retried finalize finds its own tag; only a tag elsewhere is an error
+          if tagged=$(git rev-parse -q --verify "refs/tags/{{version}}^{commit}"); then
+            if [ "${tagged}" != "$(git rev-parse HEAD)" ]; then
+              echo "Tag {{version}} already exists at ${tagged}, not at ${promote_branch} ($(git rev-parse HEAD))" >&2
+              exit 1
+            fi
+            echo "Tag {{version}} already points at ${promote_branch}, not recreating it"
+          else
+            git tag {{version}}
+          fi
           git push origin ${release_branch} --tags
 
           # Web URL to the tag

docker-compose/db-only-migrate.docker-compose.yaml

@@ -1,54 +1,20 @@
-# This file contains the pgsql service definition (the primary Sourcegraph database deployment), 
-# as well as the codeintel-db service definition (a separate postgres deployment). Both run postgres.
-# This file is used for database-only operations such as backup and restore.
+# This file starts only the pgsql service (the primary Sourcegraph database),
+# and the codeintel-db service (a separate Postgres deployment).
+# It is used for database-only operations such as backup and restore.
 # See https://sourcegraph.com/docs/self-hosted/deploy/docker-compose/operations
 #
-# 🚨This file MUST be kept in sync with the pgsql definition in docker-compose/docker-compose.yaml
+# Both services are defined in docker-compose.yaml, so this file cannot drift from it
 version: '2.4'
 services:
-  # Description: PostgreSQL database for various data.
-  #
-  # Disk: 128GB / persistent SSD
-  # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP
-  # Ports exposed to the public internet: none
-  #
   pgsql:
-    container_name: pgsql
-    image: 'index.docker.io/sourcegraph/postgresql-16:8.1.4690@sha256:1bd9b7314bdb83d12162c20e1c8da580ca27c43a4f391404a30339faeefa033b'
-    cpus: 4
-    mem_limit: '2g'
-    security_opt:
-      - no-new-privileges:true
-    healthcheck:
-      test: '/liveness.sh'
-      interval: 10s
-      timeout: 1s
-      retries: 3
-      start_period: 15s
-    volumes:
-      - 'pgsql:/data/'
-    networks:
-      - sourcegraph
-    restart: always
+    extends:
+      file: docker-compose.yaml
+      service: pgsql
 
   codeintel-db:
-    container_name: codeintel-db
-    image: 'index.docker.io/sourcegraph/postgresql-16:8.1.4690@sha256:1bd9b7314bdb83d12162c20e1c8da580ca27c43a4f391404a30339faeefa033b'
-    cpus: 4
-    mem_limit: '2g'
-    security_opt:
-      - no-new-privileges:true
-    healthcheck:
-      test: '/liveness.sh'
-      interval: 10s
-      timeout: 1s
-      retries: 3
-      start_period: 15s
-    volumes:
-      - 'codeintel-db:/data/'
-    networks:
-      - sourcegraph
-    restart: always
+    extends:
+      file: docker-compose.yaml
+      service: codeintel-db
 
 volumes:
   pgsql:

docker-compose/dev/run.sh

@@ -3,7 +3,7 @@
 cd "$(dirname "${BASH_SOURCE[0]}")/../.."
 set -euxo pipefail
 
-docker-compose \
+docker compose \
     -f docker-compose/docker-compose.yaml \
     -f docker-compose/jaeger/docker-compose.yaml \
     -f docker-compose/dev/docker-compose.yaml \

docker-compose/executors/executor.docker-compose.yaml

@@ -31,3 +31,6 @@ services:
     networks:
       - sourcegraph
     restart: always
+
+networks:
+  sourcegraph: