ci: Check every Compose file and shell script, and test Jaeger and the executor

.buildkite/pipeline.yaml

@@ -12,6 +12,10 @@ steps:
     command: .buildkite/prettier-check.sh
     agents: { queue: "vagrant" }
 
+  - label: ":bash:"
+    command: git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck
+    agents: { queue: "vagrant" }
+
   # Builds on main always run the smoke test. if_changed compares against origin/main,
   # so it finds no changes in a commit which is already on main, unless it is the newest
   - &smoke_test

.buildkite/vagrant-run.sh

@@ -27,7 +27,8 @@ if [ ! -f ${KEY_PATH} ]; then
 fi
 export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}"
 export DOCKER_USERNAME=sourcegraph
-export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
+DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)"
+export DOCKER_PASSWORD
 
 echo --- ":vagrant: starting box $box"
 vagrant up "$box" --provider=google || exit_code=$?

.buildkite/validate-docker-compose.sh

@@ -1,7 +1,19 @@
 #!/usr/bin/env bash
 
-cd "$(dirname "${BASH_SOURCE[0]}")/.."
+cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose"
 set -euxo pipefail
 
-cd docker-compose 
-docker-compose -f docker-compose.yaml config -q
+docker compose version
+
+# Every Compose file, combined the way its docs say to run it
+docker compose --file docker-compose.yaml config --quiet
+docker compose --file db-only-migrate.docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file dev/docker-compose.yaml config --quiet
+docker compose --file docker-compose.yaml --file executors/executor.docker-compose.yaml config --quiet
+docker compose --file executors/executor.docker-compose.yaml config --quiet
+# The combination the smoke test deploys
+docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file executors/executor.docker-compose.yaml --file ../test/docker-compose.override.yaml config --quiet
+for override in examples/*/docker-compose.override.yaml; do
+	docker compose --file docker-compose.yaml --file "$override" config --quiet
+done

.tool-versions

@@ -1,2 +1,3 @@
-nodejs                   26.13.0
 github-cli               2.104.0
+nodejs                   26.13.0
+shellcheck               0.11.0

AGENTS.md

@@ -12,18 +12,21 @@ This repository is the deployment reference for [Sourcegraph with Docker Compose
 
 ## Checks
 
-CI runs these on every branch. `.tool-versions` pins Node and the GitHub CLI; CI installs them with mise. `renovate.json` turns Renovate off so that these tools are upgraded together with the `sourcegraph/sourcegraph` monorepo, not on their own.
+CI runs these on every branch. `.tool-versions` pins Node, the GitHub CLI, and shellcheck; CI installs them with mise. `renovate.json` turns Renovate off so that these tools are upgraded together with the `sourcegraph/sourcegraph` monorepo, not on their own.
 
-- **Compose files:** `.buildkite/validate-docker-compose.sh` runs `docker-compose config` on the base file, with the standalone `docker-compose` binary, so the base file must stay valid for that binary. CI validates no other Compose file.
+- **Compose files:** `.buildkite/validate-docker-compose.sh` runs `docker compose config` on every Compose file, in each combination the docs say to run. Add a line to it for every new Compose file or combination.
 - **Formatting:** `npm ci && npm run prettier-check`; `npm run prettier` rewrites the files. Prettier formats the JavaScript and JSON files; the repository has no other file type its glob matches. `.prettierignore` excludes Markdown and `.github/`, and no formatter runs on YAML, so match the surrounding YAML style by hand.
+- **Shell scripts:** `git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck`.
 - **Smoke test:** the `docker-compose-test` step runs `.buildkite/vagrant-run.sh docker-test`. See below.
 
+Use `docker compose` (v2) in scripts and docs, not `docker-compose`.
+
 ## Smoke test
 
-`.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs `test/smoke-test.sh` on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks.
+`.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks.
 
-- `test/smoke-test.sh` starts the base file with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend responds. It counts the services itself, so a new long-running service needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count.
-- CI does not run `test/executor-smoke-test.sh`. Run it against a running deployment when you change `docker-compose/executors/`.
+- `test/smoke-test.sh` starts the base file, `jaeger/`, `executors/`, and `test/docker-compose.override.yaml` with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend and Jaeger respond. It counts the services itself, so a new long-running service in those files needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count. Add a new optional Compose file to its `compose_files`.
+- `test/executor-smoke-test.sh` checks that the executor registers with the frontend, and serves exactly the queues which `test/docker-compose.override.yaml` gives it. The script names them again, so change both files together.
 - A service which is not ready until another service is needs a `start_period` on its health check, or `--wait` fails.
 - A test failure is not retried, and the step uploads every container's logs as artifacts. Only the image pull is retried: `test/smoke-test.sh` tries `docker compose pull` three times.
 - A branch which only changes files listed in the step's `if_changed` `exclude` skips the step. Builds on `main` always run it. When the smoke test starts reading a file on that list, remove it from the list.

docker-compose/dev/README.md

@@ -3,7 +3,7 @@
 For use developing Sourcegraph's docker-compose deployment only.
 For deployments, refer to the official [Sourcegraph with Docker Compose](https://sourcegraph.com/docs/self-hosted/deploy/docker-compose) documentation.
 
-This folder includes a helper script, [run.sh](./run.sh), that will run the `docker-compose` command (with any provided arguments) while automatically specifying the appropriate set of overlays.
+This folder includes a helper script, [run.sh](./run.sh), that will run the `docker compose` command (with any provided arguments) while automatically specifying the appropriate set of overlays.
 
 Example usage:
 

docker-compose/dev/run.sh

@@ -3,7 +3,7 @@
 cd "$(dirname "${BASH_SOURCE[0]}")/../.."
 set -euxo pipefail
 
-docker-compose \
+docker compose \
     -f docker-compose/docker-compose.yaml \
     -f docker-compose/jaeger/docker-compose.yaml \
     -f docker-compose/dev/docker-compose.yaml \

docker-compose/executors/executor.docker-compose.yaml

@@ -30,3 +30,6 @@ services:
     networks:
       - sourcegraph
     restart: always
+
+networks:
+  sourcegraph:

test/docker-compose.override.yaml

@@ -0,0 +1,9 @@
+services:
+  # The executor exits unless both are set.
+  # executor-smoke-test.sh sets this password as `executors.accessToken` in the site config
+  executor:
+    environment:
+      - EXECUTOR_FRONTEND_PASSWORD=executor-smoke-test-token
+      # Every queue; executor-smoke-test.sh checks for these names.
+      # The base file leaves EXECUTOR_QUEUE_NAME empty, which the executor requires with this
+      - EXECUTOR_QUEUE_NAMES=batches,batches-hooks,codeintel

test/executor-smoke-test.sh

@@ -3,18 +3,20 @@ set -euo pipefail
 
 # Executor smoke test for deploy-sourcegraph-docker
 #
-# Requires: a running Sourcegraph docker-compose stack (from smoke-test.sh or manual start).
-# This script creates a site admin account, configures executor access, starts an executor
-# container, and verifies it registers with the Sourcegraph instance.
+# Requires: a running Sourcegraph docker-compose stack with an executor container
+# (from smoke-test.sh or manual start).
+# This script creates a site admin account, configures executor access, and verifies
+# the executor registers with the Sourcegraph instance, with every queue.
 #
-# Dependencies: curl, jq, docker compose
+# Dependencies: curl, jq, docker
 
 FRONTEND_URL="${FRONTEND_URL:-http://localhost:80}"
-EXECUTOR_TOKEN="executor-smoke-test-token"
+EXECUTOR_TOKEN=$(docker exec executor printenv EXECUTOR_FRONTEND_PASSWORD)
 ADMIN_EMAIL="smoke-test@sourcegraph.com"
 ADMIN_USERNAME="smoke-admin"
 ADMIN_PASSWORD="smoke-test-password-123!"
-COMPOSE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose" && pwd)"
+# The queues test/docker-compose.override.yaml gives the executor, sorted
+EXPECTED_QUEUE_NAMES="batches,batches-hooks,codeintel"
 
 echo "=== Executor Smoke Test ==="
 
@@ -74,29 +76,27 @@ curl -s -o /dev/null -X POST "${FRONTEND_URL}/.api/graphql" \
     --data-raw "{\"query\":\"mutation UpdateSiteConfig(\$input: String!) { updateSiteConfiguration(lastID: $LAST_ID, input: \$input) }\", \"variables\":{\"input\":$NEW_CONFIG}}"
 echo "  Site config updated"
 
-# Step 5: Start executor container
-echo "Step 5: Starting executor container..."
-EXECUTOR_CONTAINER=$(docker compose \
-    -f "${COMPOSE_DIR}/docker-compose.yaml" \
-    -f "${COMPOSE_DIR}/executors/executor.docker-compose.yaml" \
-    run -d \
-    -e "EXECUTOR_FRONTEND_PASSWORD=${EXECUTOR_TOKEN}" \
-    -e EXECUTOR_QUEUE_NAME=batches \
-    executor 2>&1 | tail -1)
-echo "  Executor container started: ${EXECUTOR_CONTAINER:0:12}"
-
-# Step 6: Wait for executor to register
-echo "Step 6: Waiting for executor to register..."
+# Step 5: Wait for executor to register
+echo "Step 5: Waiting for executor to register..."
 MAX_ATTEMPTS=24
 for i in $(seq 1 $MAX_ATTEMPTS); do
-    COUNT=$(curl -s -X POST "${FRONTEND_URL}/.api/graphql" \
+    EXECUTORS=$(curl -s -X POST "${FRONTEND_URL}/.api/graphql" \
         -H "Authorization: token $TOKEN" \
         -H 'Content-Type: application/json' \
-        -d '{"query":"{ executors(query: \"\", active: false) { totalCount } }"}' \
-        | jq -r '.data.executors.totalCount // 0')
+        -d '{"query":"{ executors(query: \"\", active: false) { totalCount nodes { queueNames } } }"}')
+    COUNT=$(jq -r '.data.executors.totalCount // 0' <<<"$EXECUTORS")
 
     if [ "$COUNT" -ge 1 ] 2>/dev/null; then
         echo "  Executor registered after $((i * 5))s"
+        # One line per executor: its queue names, sorted
+        QUEUE_NAMES=$(jq -r '.data.executors.nodes[] | .queueNames // [] | sort | join(",")' <<<"$EXECUTORS")
+        if ! grep --quiet --line-regexp --fixed-strings "$EXPECTED_QUEUE_NAMES" <<<"$QUEUE_NAMES"; then
+            echo ""
+            echo "=== EXECUTOR SMOKE TEST FAILED ==="
+            echo "No executor serves the queues \"$EXPECTED_QUEUE_NAMES\". Registered: \"$QUEUE_NAMES\""
+            exit 1
+        fi
+        echo "  Executor serves the queues $EXPECTED_QUEUE_NAMES"
         echo ""
         echo "=== EXECUTOR SMOKE TEST PASSED ==="
         exit 0
@@ -110,8 +110,8 @@ echo "=== EXECUTOR SMOKE TEST FAILED ==="
 echo "Executor did not register within $((MAX_ATTEMPTS * 5))s"
 echo ""
 echo "Container status:"
-docker inspect "$EXECUTOR_CONTAINER" --format '{{.State.Status}} (exit code: {{.State.ExitCode}})' 2>/dev/null || echo "  Container not found"
+docker inspect executor --format '{{.State.Status}} (exit code: {{.State.ExitCode}})' 2>/dev/null || echo "  Container not found"
 echo ""
 echo "Container logs:"
-docker logs "$EXECUTOR_CONTAINER" 2>&1 | tail -20 || echo "  No logs available"
+docker logs executor 2>&1 | tail -20 || echo "  No logs available"
 exit 1

test/servers.yaml

@@ -11,4 +11,7 @@
   service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com
   shell_commands:
    - "/deploy-sourcegraph-docker/test/smoke-test.sh"
+   # The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails
+   - "command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; }"
+   - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh"
 

test/smoke-test.sh

@@ -11,7 +11,13 @@ configure_docker() {
   gcloud auth configure-docker us-central1-docker.pkg.dev,us-docker.pkg.dev
 }
 
-compose_files=(--file docker-compose/docker-compose.yaml)
+# Every service a customer can deploy, started and checked by the same commands
+compose_files=(
+	--file docker-compose/docker-compose.yaml
+	--file docker-compose/jaeger/docker-compose.yaml
+	--file docker-compose/executors/executor.docker-compose.yaml
+	--file test/docker-compose.override.yaml
+)
 
 deploy_sourcegraph() {
 	cd "$(dirname "${BASH_SOURCE[0]}")/.."
@@ -52,6 +58,9 @@ test_containers() {
 	curl -f http://localhost:80
 	curl -f http://localhost:80/healthz
 
+	echo "TEST: Checking Jaeger is accessible"
+	curl --fail --silent --show-error --location --output /dev/null http://localhost:16686/-/debug/jaeger/
+
 	echo "ALL TESTS PASSED"
 }