From 9a0ab7160e5e70763b7625f4a54af00d8f1c723e Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:43:50 -0600 Subject: [PATCH 1/3] ci: Check every Compose file and shell script, and test Jaeger and the executor Validates every Compose file in the combinations the docs give, runs shellcheck on every shell script, and deploys Jaeger and the executor in the smoke test. The VM now runs test/executor-smoke-test.sh after test/smoke-test.sh. Declares the sourcegraph network in executor.docker-compose.yaml, which the standalone command in its README needs, and switches dev/run.sh to docker compose. Fixes the one shellcheck warning, in vagrant-run.sh. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5 Co-authored-by: Amp --- .buildkite/pipeline.yaml | 4 +++ .buildkite/vagrant-run.sh | 3 +- .buildkite/validate-docker-compose.sh | 18 +++++++++-- .tool-versions | 3 +- AGENTS.md | 13 +++++--- docker-compose/dev/README.md | 2 +- docker-compose/dev/run.sh | 2 +- .../executors/executor.docker-compose.yaml | 3 ++ test/docker-compose.override.yaml | 7 +++++ test/executor-smoke-test.sh | 31 ++++++------------- test/servers.yaml | 3 ++ test/smoke-test.sh | 11 ++++++- 12 files changed, 66 insertions(+), 34 deletions(-) create mode 100644 test/docker-compose.override.yaml diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 8b2c1608a..2a2595ad6 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -12,6 +12,10 @@ steps: command: .buildkite/prettier-check.sh agents: { queue: "vagrant" } + - label: ":bash:" + command: git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck + agents: { queue: "vagrant" } + # Builds on main always run the smoke test. if_changed compares against origin/main, # so it finds no changes in a commit which is already on main, unless it is the newest - &smoke_test diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh index a54aeb97d..3be84b10e 100755 --- a/.buildkite/vagrant-run.sh +++ b/.buildkite/vagrant-run.sh @@ -27,7 +27,8 @@ if [ ! -f ${KEY_PATH} ]; then fi export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}" export DOCKER_USERNAME=sourcegraph -export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" +DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" +export DOCKER_PASSWORD echo --- ":vagrant: starting box $box" vagrant up "$box" --provider=google || exit_code=$? diff --git a/.buildkite/validate-docker-compose.sh b/.buildkite/validate-docker-compose.sh index 3c7bac418..8a716a3d7 100755 --- a/.buildkite/validate-docker-compose.sh +++ b/.buildkite/validate-docker-compose.sh @@ -1,7 +1,19 @@ #!/usr/bin/env bash -cd "$(dirname "${BASH_SOURCE[0]}")/.." +cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose" set -euxo pipefail -cd docker-compose -docker-compose -f docker-compose.yaml config -q +docker compose version + +# Every Compose file, combined the way its docs say to run it +docker compose --file docker-compose.yaml config --quiet +docker compose --file db-only-migrate.docker-compose.yaml config --quiet +docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml config --quiet +docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file dev/docker-compose.yaml config --quiet +docker compose --file docker-compose.yaml --file executors/executor.docker-compose.yaml config --quiet +docker compose --file executors/executor.docker-compose.yaml config --quiet +# The combination the smoke test deploys +docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file executors/executor.docker-compose.yaml --file ../test/docker-compose.override.yaml config --quiet +for override in examples/*/docker-compose.override.yaml; do + docker compose --file docker-compose.yaml --file "$override" config --quiet +done diff --git a/.tool-versions b/.tool-versions index e7a0c88e3..45aa0d196 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,3 @@ -nodejs 26.13.0 github-cli 2.104.0 +nodejs 26.13.0 +shellcheck 0.11.0 diff --git a/AGENTS.md b/AGENTS.md index 905246fd7..707a1a426 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,18 +12,21 @@ This repository is the deployment reference for [Sourcegraph with Docker Compose ## Checks -CI runs these on every branch. `.tool-versions` pins Node and the GitHub CLI; CI installs them with mise. `renovate.json` turns Renovate off so that these tools are upgraded together with the `sourcegraph/sourcegraph` monorepo, not on their own. +CI runs these on every branch. `.tool-versions` pins Node, the GitHub CLI, and shellcheck; CI installs them with mise. `renovate.json` turns Renovate off so that these tools are upgraded together with the `sourcegraph/sourcegraph` monorepo, not on their own. -- **Compose files:** `.buildkite/validate-docker-compose.sh` runs `docker-compose config` on the base file, with the standalone `docker-compose` binary, so the base file must stay valid for that binary. CI validates no other Compose file. +- **Compose files:** `.buildkite/validate-docker-compose.sh` runs `docker compose config` on every Compose file, in each combination the docs say to run. Add a line to it for every new Compose file or combination. - **Formatting:** `npm ci && npm run prettier-check`; `npm run prettier` rewrites the files. Prettier formats the JavaScript and JSON files; the repository has no other file type its glob matches. `.prettierignore` excludes Markdown and `.github/`, and no formatter runs on YAML, so match the surrounding YAML style by hand. +- **Shell scripts:** `git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck`. - **Smoke test:** the `docker-compose-test` step runs `.buildkite/vagrant-run.sh docker-test`. See below. +Use `docker compose` (v2) in scripts and docs, not `docker-compose`. + ## Smoke test -`.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs `test/smoke-test.sh` on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. +`.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. -- `test/smoke-test.sh` starts the base file with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend responds. It counts the services itself, so a new long-running service needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count. -- CI does not run `test/executor-smoke-test.sh`. Run it against a running deployment when you change `docker-compose/executors/`. +- `test/smoke-test.sh` starts the base file, `jaeger/`, `executors/`, and `test/docker-compose.override.yaml` with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend and Jaeger respond. It counts the services itself, so a new long-running service in those files needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count. Add a new optional Compose file to its `compose_files`. +- `test/executor-smoke-test.sh` checks that the executor registers with the frontend. - A service which is not ready until another service is needs a `start_period` on its health check, or `--wait` fails. - A test failure is not retried, and the step uploads every container's logs as artifacts. Only the image pull is retried: `test/smoke-test.sh` tries `docker compose pull` three times. - A branch which only changes files listed in the step's `if_changed` `exclude` skips the step. Builds on `main` always run it. When the smoke test starts reading a file on that list, remove it from the list. diff --git a/docker-compose/dev/README.md b/docker-compose/dev/README.md index bf75a6e23..c05d4dfa1 100644 --- a/docker-compose/dev/README.md +++ b/docker-compose/dev/README.md @@ -3,7 +3,7 @@ For use developing Sourcegraph's docker-compose deployment only. For deployments, refer to the official [Sourcegraph with Docker Compose](https://sourcegraph.com/docs/self-hosted/deploy/docker-compose) documentation. -This folder includes a helper script, [run.sh](./run.sh), that will run the `docker-compose` command (with any provided arguments) while automatically specifying the appropriate set of overlays. +This folder includes a helper script, [run.sh](./run.sh), that will run the `docker compose` command (with any provided arguments) while automatically specifying the appropriate set of overlays. Example usage: diff --git a/docker-compose/dev/run.sh b/docker-compose/dev/run.sh index f5c3881b4..e7b85ff19 100755 --- a/docker-compose/dev/run.sh +++ b/docker-compose/dev/run.sh @@ -3,7 +3,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/../.." set -euxo pipefail -docker-compose \ +docker compose \ -f docker-compose/docker-compose.yaml \ -f docker-compose/jaeger/docker-compose.yaml \ -f docker-compose/dev/docker-compose.yaml \ diff --git a/docker-compose/executors/executor.docker-compose.yaml b/docker-compose/executors/executor.docker-compose.yaml index 473bc7072..8f49bc9a8 100644 --- a/docker-compose/executors/executor.docker-compose.yaml +++ b/docker-compose/executors/executor.docker-compose.yaml @@ -30,3 +30,6 @@ services: networks: - sourcegraph restart: always + +networks: + sourcegraph: diff --git a/test/docker-compose.override.yaml b/test/docker-compose.override.yaml new file mode 100644 index 000000000..8f59f0c1c --- /dev/null +++ b/test/docker-compose.override.yaml @@ -0,0 +1,7 @@ +services: + # The executor exits unless both are set. + # executor-smoke-test.sh sets this password as `executors.accessToken` in the site config + executor: + environment: + - EXECUTOR_FRONTEND_PASSWORD=executor-smoke-test-token + - EXECUTOR_QUEUE_NAME=batches diff --git a/test/executor-smoke-test.sh b/test/executor-smoke-test.sh index f92e1b474..052c829d1 100755 --- a/test/executor-smoke-test.sh +++ b/test/executor-smoke-test.sh @@ -3,18 +3,18 @@ set -euo pipefail # Executor smoke test for deploy-sourcegraph-docker # -# Requires: a running Sourcegraph docker-compose stack (from smoke-test.sh or manual start). -# This script creates a site admin account, configures executor access, starts an executor -# container, and verifies it registers with the Sourcegraph instance. +# Requires: a running Sourcegraph docker-compose stack with an executor container +# (from smoke-test.sh or manual start). +# This script creates a site admin account, configures executor access, and verifies +# the executor registers with the Sourcegraph instance. # -# Dependencies: curl, jq, docker compose +# Dependencies: curl, jq, docker FRONTEND_URL="${FRONTEND_URL:-http://localhost:80}" -EXECUTOR_TOKEN="executor-smoke-test-token" +EXECUTOR_TOKEN=$(docker exec executor printenv EXECUTOR_FRONTEND_PASSWORD) ADMIN_EMAIL="smoke-test@sourcegraph.com" ADMIN_USERNAME="smoke-admin" ADMIN_PASSWORD="smoke-test-password-123!" -COMPOSE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose" && pwd)" echo "=== Executor Smoke Test ===" @@ -74,19 +74,8 @@ curl -s -o /dev/null -X POST "${FRONTEND_URL}/.api/graphql" \ --data-raw "{\"query\":\"mutation UpdateSiteConfig(\$input: String!) { updateSiteConfiguration(lastID: $LAST_ID, input: \$input) }\", \"variables\":{\"input\":$NEW_CONFIG}}" echo " Site config updated" -# Step 5: Start executor container -echo "Step 5: Starting executor container..." -EXECUTOR_CONTAINER=$(docker compose \ - -f "${COMPOSE_DIR}/docker-compose.yaml" \ - -f "${COMPOSE_DIR}/executors/executor.docker-compose.yaml" \ - run -d \ - -e "EXECUTOR_FRONTEND_PASSWORD=${EXECUTOR_TOKEN}" \ - -e EXECUTOR_QUEUE_NAME=batches \ - executor 2>&1 | tail -1) -echo " Executor container started: ${EXECUTOR_CONTAINER:0:12}" - -# Step 6: Wait for executor to register -echo "Step 6: Waiting for executor to register..." +# Step 5: Wait for executor to register +echo "Step 5: Waiting for executor to register..." MAX_ATTEMPTS=24 for i in $(seq 1 $MAX_ATTEMPTS); do COUNT=$(curl -s -X POST "${FRONTEND_URL}/.api/graphql" \ @@ -110,8 +99,8 @@ echo "=== EXECUTOR SMOKE TEST FAILED ===" echo "Executor did not register within $((MAX_ATTEMPTS * 5))s" echo "" echo "Container status:" -docker inspect "$EXECUTOR_CONTAINER" --format '{{.State.Status}} (exit code: {{.State.ExitCode}})' 2>/dev/null || echo " Container not found" +docker inspect executor --format '{{.State.Status}} (exit code: {{.State.ExitCode}})' 2>/dev/null || echo " Container not found" echo "" echo "Container logs:" -docker logs "$EXECUTOR_CONTAINER" 2>&1 | tail -20 || echo " No logs available" +docker logs executor 2>&1 | tail -20 || echo " No logs available" exit 1 diff --git a/test/servers.yaml b/test/servers.yaml index 36780dbe7..f68c942c9 100644 --- a/test/servers.yaml +++ b/test/servers.yaml @@ -11,4 +11,7 @@ service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com shell_commands: - "/deploy-sourcegraph-docker/test/smoke-test.sh" + # The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails + - "command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; }" + - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh" diff --git a/test/smoke-test.sh b/test/smoke-test.sh index b94af127e..2e1725859 100755 --- a/test/smoke-test.sh +++ b/test/smoke-test.sh @@ -11,7 +11,13 @@ configure_docker() { gcloud auth configure-docker us-central1-docker.pkg.dev,us-docker.pkg.dev } -compose_files=(--file docker-compose/docker-compose.yaml) +# Every service a customer can deploy, started and checked by the same commands +compose_files=( + --file docker-compose/docker-compose.yaml + --file docker-compose/jaeger/docker-compose.yaml + --file docker-compose/executors/executor.docker-compose.yaml + --file test/docker-compose.override.yaml +) deploy_sourcegraph() { cd "$(dirname "${BASH_SOURCE[0]}")/.." @@ -52,6 +58,9 @@ test_containers() { curl -f http://localhost:80 curl -f http://localhost:80/healthz + echo "TEST: Checking Jaeger is accessible" + curl --fail --silent --show-error --location --output /dev/null http://localhost:16686/-/debug/jaeger/ + echo "ALL TESTS PASSED" } From c126e6313671b15048ec47a5f9985a22125d0b1b Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:43:41 -0600 Subject: [PATCH 2/3] ci: Give the smoke test's executor every queue Sets EXECUTOR_QUEUE_NAMES to batches, batches-hooks, and codeintel, instead of EXECUTOR_QUEUE_NAME to batches. EXECUTOR_QUEUE_NAME takes one queue, and the executor refuses to start when both variables are set, so the override leaves the base file's empty EXECUTOR_QUEUE_NAME alone. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5 --- test/docker-compose.override.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/docker-compose.override.yaml b/test/docker-compose.override.yaml index 8f59f0c1c..dfcdb311b 100644 --- a/test/docker-compose.override.yaml +++ b/test/docker-compose.override.yaml @@ -4,4 +4,5 @@ services: executor: environment: - EXECUTOR_FRONTEND_PASSWORD=executor-smoke-test-token - - EXECUTOR_QUEUE_NAME=batches + # Every queue. The base file leaves EXECUTOR_QUEUE_NAME empty, which the executor requires with this + - EXECUTOR_QUEUE_NAMES=batches,batches-hooks,codeintel From 8c7c2e6bbeb279553afebefd2c40a617ec8c76bb Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:50:19 -0600 Subject: [PATCH 3/3] ci: Check the queue names of the smoke test's executor The executor smoke test passed as soon as any executor registered. It now also fails unless an executor serves exactly the three queues which test/docker-compose.override.yaml sets: batches, batches-hooks, and codeintel. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5 Co-authored-by: Amp --- AGENTS.md | 2 +- test/docker-compose.override.yaml | 3 ++- test/executor-smoke-test.sh | 19 +++++++++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 707a1a426..e14e992d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ Use `docker compose` (v2) in scripts and docs, not `docker-compose`. `.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. - `test/smoke-test.sh` starts the base file, `jaeger/`, `executors/`, and `test/docker-compose.override.yaml` with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend and Jaeger respond. It counts the services itself, so a new long-running service in those files needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count. Add a new optional Compose file to its `compose_files`. -- `test/executor-smoke-test.sh` checks that the executor registers with the frontend. +- `test/executor-smoke-test.sh` checks that the executor registers with the frontend, and serves exactly the queues which `test/docker-compose.override.yaml` gives it. The script names them again, so change both files together. - A service which is not ready until another service is needs a `start_period` on its health check, or `--wait` fails. - A test failure is not retried, and the step uploads every container's logs as artifacts. Only the image pull is retried: `test/smoke-test.sh` tries `docker compose pull` three times. - A branch which only changes files listed in the step's `if_changed` `exclude` skips the step. Builds on `main` always run it. When the smoke test starts reading a file on that list, remove it from the list. diff --git a/test/docker-compose.override.yaml b/test/docker-compose.override.yaml index dfcdb311b..8bf2dda90 100644 --- a/test/docker-compose.override.yaml +++ b/test/docker-compose.override.yaml @@ -4,5 +4,6 @@ services: executor: environment: - EXECUTOR_FRONTEND_PASSWORD=executor-smoke-test-token - # Every queue. The base file leaves EXECUTOR_QUEUE_NAME empty, which the executor requires with this + # Every queue; executor-smoke-test.sh checks for these names. + # The base file leaves EXECUTOR_QUEUE_NAME empty, which the executor requires with this - EXECUTOR_QUEUE_NAMES=batches,batches-hooks,codeintel diff --git a/test/executor-smoke-test.sh b/test/executor-smoke-test.sh index 052c829d1..0ab29cd9f 100755 --- a/test/executor-smoke-test.sh +++ b/test/executor-smoke-test.sh @@ -6,7 +6,7 @@ set -euo pipefail # Requires: a running Sourcegraph docker-compose stack with an executor container # (from smoke-test.sh or manual start). # This script creates a site admin account, configures executor access, and verifies -# the executor registers with the Sourcegraph instance. +# the executor registers with the Sourcegraph instance, with every queue. # # Dependencies: curl, jq, docker @@ -15,6 +15,8 @@ EXECUTOR_TOKEN=$(docker exec executor printenv EXECUTOR_FRONTEND_PASSWORD) ADMIN_EMAIL="smoke-test@sourcegraph.com" ADMIN_USERNAME="smoke-admin" ADMIN_PASSWORD="smoke-test-password-123!" +# The queues test/docker-compose.override.yaml gives the executor, sorted +EXPECTED_QUEUE_NAMES="batches,batches-hooks,codeintel" echo "=== Executor Smoke Test ===" @@ -78,14 +80,23 @@ echo " Site config updated" echo "Step 5: Waiting for executor to register..." MAX_ATTEMPTS=24 for i in $(seq 1 $MAX_ATTEMPTS); do - COUNT=$(curl -s -X POST "${FRONTEND_URL}/.api/graphql" \ + EXECUTORS=$(curl -s -X POST "${FRONTEND_URL}/.api/graphql" \ -H "Authorization: token $TOKEN" \ -H 'Content-Type: application/json' \ - -d '{"query":"{ executors(query: \"\", active: false) { totalCount } }"}' \ - | jq -r '.data.executors.totalCount // 0') + -d '{"query":"{ executors(query: \"\", active: false) { totalCount nodes { queueNames } } }"}') + COUNT=$(jq -r '.data.executors.totalCount // 0' <<<"$EXECUTORS") if [ "$COUNT" -ge 1 ] 2>/dev/null; then echo " Executor registered after $((i * 5))s" + # One line per executor: its queue names, sorted + QUEUE_NAMES=$(jq -r '.data.executors.nodes[] | .queueNames // [] | sort | join(",")' <<<"$EXECUTORS") + if ! grep --quiet --line-regexp --fixed-strings "$EXPECTED_QUEUE_NAMES" <<<"$QUEUE_NAMES"; then + echo "" + echo "=== EXECUTOR SMOKE TEST FAILED ===" + echo "No executor serves the queues \"$EXPECTED_QUEUE_NAMES\". Registered: \"$QUEUE_NAMES\"" + exit 1 + fi + echo " Executor serves the queues $EXPECTED_QUEUE_NAMES" echo "" echo "=== EXECUTOR SMOKE TEST PASSED ===" exit 0