ci: Close the gaps in what CI checks

.buildkite/pipeline.yaml

@@ -12,6 +12,10 @@ steps:
     command: .buildkite/prettier-check.sh
     agents: { queue: "vagrant" }
 
+  - label: ":bash:"
+    command: git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck
+    agents: { queue: "vagrant" }
+
   # Builds on main always run the smoke test. if_changed compares against origin/main,
   # so it finds no changes in a commit which is already on main, unless it is the newest.
   # The branch is named here, because the Buildkite pipeline's default branch is still "master"

.buildkite/vagrant-run.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 
 box="$1"
 exit_code=0
-KEY_PATH="/tmp/e2e-builder.json"
+KEY_PATH="$(mktemp)"
 
 pushd "test"
 
@@ -28,7 +28,10 @@ request_deletion() (
 
 # Fall back to a blocking destroy, so a failed deletion request cannot leak a VM
 cleanup() {
-	request_deletion "name=${VAGRANT_INSTANCE_NAME}" || vagrant destroy -f "$box"
+	deletion_status=0
+	request_deletion "name=${VAGRANT_INSTANCE_NAME}" || vagrant destroy -f "$box" || deletion_status=$?
+	rm -f "$KEY_PATH"
+	return "$deletion_status"
 }
 
 echo --- ":vagrant: installing plugins"
@@ -39,12 +42,10 @@ vagrant --version
 trap cleanup EXIT
 
 echo --- ":lock: builder account key"
-if [ ! -f ${KEY_PATH} ]; then
-  gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}"
-fi
+gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}"
 export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}"
-export DOCKER_USERNAME=sourcegraph
-export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
+DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
+export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD
 
 echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
 # The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job

.buildkite/validate-docker-compose.sh

@@ -1,7 +1,17 @@
 #!/usr/bin/env bash
 
-cd "$(dirname "${BASH_SOURCE[0]}")/.."
+cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose"
 set -euxo pipefail
 
-cd docker-compose 
-docker-compose -f docker-compose.yaml config -q
+docker compose version
+
+# Every Compose file, combined the way its docs say to run it
+docker compose --file docker-compose.yaml config --quiet
+docker compose --file db-only-migrate.docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file dev/docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file executors/executor.docker-compose.yaml config --quiet
+docker compose --file executors/executor.docker-compose.yaml config --quiet
+for override in examples/*/docker-compose.override.yaml; do
+	docker compose --file docker-compose.yaml --file "$override" config --quiet
+done

.tool-versions

@@ -1,3 +1,4 @@
 nodejs                   16.7.0
 yarn                     1.22.4
 github-cli               2.46.0
+shellcheck               0.11.0

batch-change/release.yaml

@@ -138,7 +138,16 @@ promoteToPublic:
           git checkout "${promote_branch}"
           git switch -c "${release_branch}"
 
-          git tag {{version}}
+          # A retried finalize finds its own tag; only a tag elsewhere is an error
+          if tagged=$(git rev-parse -q --verify "refs/tags/{{version}}^{commit}"); then
+            if [ "${tagged}" != "$(git rev-parse HEAD)" ]; then
+              echo "Tag {{version}} already exists at ${tagged}, not at ${promote_branch} ($(git rev-parse HEAD))" >&2
+              exit 1
+            fi
+            echo "Tag {{version}} already points at ${promote_branch}, not recreating it"
+          else
+            git tag {{version}}
+          fi
           git push origin ${release_branch} --tags
 
           # Web URL to the tag

docker-compose/db-only-migrate.docker-compose.yaml

@@ -1,54 +1,20 @@
-# This file contains the pgsql service definition (the primary Sourcegraph database deployment), 
-# as well as the codeintel-db service definition (a separate postgres deployment). Both run postgres.
-# This file is used for database-only operations such as backup and restore.
+# This file starts only the pgsql service (the primary Sourcegraph database),
+# and the codeintel-db service (a separate Postgres deployment).
+# It is used for database-only operations such as backup and restore.
 # See https://sourcegraph.com/docs/self-hosted/deploy/docker-compose/operations
 #
-# 🚨This file MUST be kept in sync with the pgsql definition in docker-compose/docker-compose.yaml
+# Both services are defined in docker-compose.yaml, so this file cannot drift from it
 version: '2.4'
 services:
-  # Description: PostgreSQL database for various data.
-  #
-  # Disk: 128GB / persistent SSD
-  # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP
-  # Ports exposed to the public internet: none
-  #
   pgsql:
-    container_name: pgsql
-    image: 'index.docker.io/sourcegraph/postgresql-16:8.1.2687@sha256:cc810cd21f1df7776aa270f49bdd50a4ec3375623ffae1f0238c20dec054bf03'
-    cpus: 4
-    mem_limit: '2g'
-    security_opt:
-      - no-new-privileges:true
-    healthcheck:
-      test: '/liveness.sh'
-      interval: 10s
-      timeout: 1s
-      retries: 3
-      start_period: 15s
-    volumes:
-      - 'pgsql:/data/'
-    networks:
-      - sourcegraph
-    restart: always
+    extends:
+      file: docker-compose.yaml
+      service: pgsql
 
   codeintel-db:
-    container_name: codeintel-db
-    image: 'index.docker.io/sourcegraph/postgresql-16:8.1.2687@sha256:cc810cd21f1df7776aa270f49bdd50a4ec3375623ffae1f0238c20dec054bf03'
-    cpus: 4
-    mem_limit: '2g'
-    security_opt:
-      - no-new-privileges:true
-    healthcheck:
-      test: '/liveness.sh'
-      interval: 10s
-      timeout: 1s
-      retries: 3
-      start_period: 15s
-    volumes:
-      - 'codeintel-db:/data/'
-    networks:
-      - sourcegraph
-    restart: always
+    extends:
+      file: docker-compose.yaml
+      service: codeintel-db
 
 volumes:
   pgsql:

docker-compose/dev/run.sh

@@ -3,7 +3,7 @@
 cd "$(dirname "${BASH_SOURCE[0]}")/../.."
 set -euxo pipefail
 
-docker-compose \
+docker compose \
     -f docker-compose/docker-compose.yaml \
     -f docker-compose/jaeger/docker-compose.yaml \
     -f docker-compose/dev/docker-compose.yaml \

docker-compose/executors/executor.docker-compose.yaml

@@ -31,3 +31,6 @@ services:
     networks:
       - sourcegraph
     restart: always
+
+networks:
+  sourcegraph:

test/Vagrantfile

@@ -57,17 +57,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         o.ssh.private_key_path = ssh_key_path
       end
 
-      srv.vm.provision 'shell', inline: <<-SHELL
-        #!/usr/bin/env bash
-        cat << EOF >> /root/.profile
-export GIT_BRANCH=#{ENV['BUILDKITE_BRANCH']}
-export DOCKER_USERNAME=#{ENV['DOCKER_USERNAME']}
-export DOCKER_PASSWORD=#{ENV['DOCKER_PASSWORD']}
-EOF
-        SHELL
-
       server['shell_commands'].each do |sh|
-        srv.vm.provision 'shell', inline: sh
+        srv.vm.provision 'shell', inline: sh, env: ENV.slice('DOCKER_USERNAME', 'DOCKER_PASSWORD')
       end
     end
   end

test/servers.yaml

@@ -11,4 +11,7 @@
   service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com
   shell_commands:
    - "/deploy-sourcegraph-docker/test/smoke-test.sh"
+   # The VM image has no jq. Skip /etc/apt/sources.list.d: the image lacks the key for its Chrome repository, so a full update fails
+   - "apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq"
+   - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh"
 

test/smoke-test.sh

@@ -3,18 +3,16 @@ set -euxfo pipefail
 
 configure_docker() {
   set +x
-  if [ -n "${DOCKER_USERNAME}" ] && [ -n "${DOCKER_PASSWORD}" ]; then
+  if [ -n "${DOCKER_USERNAME:-}" ] && [ -n "${DOCKER_PASSWORD:-}" ]; then
     docker login -u "${DOCKER_USERNAME}" --password-stdin <<<"$DOCKER_PASSWORD"
   fi
   set -x
   gcloud auth configure-docker
   gcloud auth configure-docker us-central1-docker.pkg.dev,us-docker.pkg.dev
 }
 
-expect_containers="25"
-
 deploy_sourcegraph() {
-	cd $(dirname "${BASH_SOURCE[0]}")/..
+	cd "$(dirname "${BASH_SOURCE[0]}")/.."
 	docker-compose version
 	# `up` pulls images one at a time, `pull` pulls them in parallel
 	docker-compose --file docker-compose/docker-compose.yaml pull --quiet
@@ -36,30 +34,19 @@ wait_until_healthy() {
 }
 
 test_count() {
-	echo "TEST: Number of expected containers created"
-	containers_count=$(docker ps --format '{{.Names}}' | wc -l)
+	echo "TEST: Every service has a running container"
+	# migrator exits when it is done, every other service keeps running
+	expect_containers=$(docker-compose --file docker-compose/docker-compose.yaml config --services | grep --count --invert-match '^migrator$')
+	containers_count=$(docker ps --filter status=running --format '{{.Names}}' | wc -l)
 	if [[ "$containers_count" -ne "$expect_containers" ]]; then
-		docker ps --format '{{.Names}}'
+		docker ps --all --format '{{.Names}}: {{.Status}}'
 		echo
-		echo "TEST FAILURE: expected $expect_containers containers, found $containers_count"
+		echo "TEST FAILURE: expected $expect_containers running containers, found $containers_count"
 		exit 1
 	fi
 }
 
 test_containers() {
-	echo "TEST: Checking every 10s that containers are running for 5 minutes..."
-	for i in {0..1}; do
-		containers=$(docker ps --format '{{.Names}}' | xargs -I{} -n1 sh -c "printf '{}: ' && docker inspect --format '{{.State.Status}}' {}")
-		containers_running=$(echo "$containers" | grep -c "running")
-		if [[ "$containers_running" -ne "$expect_containers" ]]; then
-			containers_failing=$(docker ps --format '{{.Names}}:{{.Status}}' | grep -v Up | cut -f 1 -d :)
-			echo "TEST FAILURE: expected $expect_containers containers running, found $containers_running. The following containers are failing: $containers_failing"
-			exit 1
-		fi
-		echo "Containers running OK.. waiting 10s"
-		sleep 1
-	done
-
 	echo "TEST: No container has restarted"
 	restarted_containers=$(docker ps --quiet | xargs docker inspect --format '{{.Name}} restarted {{.RestartCount}} times' | grep -v ' restarted 0 times' || true)
 	if [[ -n "$restarted_containers" ]]; then