diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 8b2c1608a..2a2595ad6 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -12,6 +12,10 @@ steps: command: .buildkite/prettier-check.sh agents: { queue: "vagrant" } + - label: ":bash:" + command: git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck + agents: { queue: "vagrant" } + # Builds on main always run the smoke test. if_changed compares against origin/main, # so it finds no changes in a commit which is already on main, unless it is the newest - &smoke_test diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh index a54aeb97d..3be84b10e 100755 --- a/.buildkite/vagrant-run.sh +++ b/.buildkite/vagrant-run.sh @@ -27,7 +27,8 @@ if [ ! -f ${KEY_PATH} ]; then fi export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}" export DOCKER_USERNAME=sourcegraph -export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" +DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" +export DOCKER_PASSWORD echo --- ":vagrant: starting box $box" vagrant up "$box" --provider=google || exit_code=$? diff --git a/.buildkite/validate-docker-compose.sh b/.buildkite/validate-docker-compose.sh index 3c7bac418..8a716a3d7 100755 --- a/.buildkite/validate-docker-compose.sh +++ b/.buildkite/validate-docker-compose.sh @@ -1,7 +1,19 @@ #!/usr/bin/env bash -cd "$(dirname "${BASH_SOURCE[0]}")/.." +cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose" set -euxo pipefail -cd docker-compose -docker-compose -f docker-compose.yaml config -q +docker compose version + +# Every Compose file, combined the way its docs say to run it +docker compose --file docker-compose.yaml config --quiet +docker compose --file db-only-migrate.docker-compose.yaml config --quiet +docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml config --quiet +docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file dev/docker-compose.yaml config --quiet +docker compose --file docker-compose.yaml --file executors/executor.docker-compose.yaml config --quiet +docker compose --file executors/executor.docker-compose.yaml config --quiet +# The combination the smoke test deploys +docker compose --file docker-compose.yaml --file jaeger/docker-compose.yaml --file executors/executor.docker-compose.yaml --file ../test/docker-compose.override.yaml config --quiet +for override in examples/*/docker-compose.override.yaml; do + docker compose --file docker-compose.yaml --file "$override" config --quiet +done diff --git a/.tool-versions b/.tool-versions index e7a0c88e3..45aa0d196 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,3 @@ -nodejs 26.13.0 github-cli 2.104.0 +nodejs 26.13.0 +shellcheck 0.11.0 diff --git a/AGENTS.md b/AGENTS.md index 905246fd7..e14e992d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,18 +12,21 @@ This repository is the deployment reference for [Sourcegraph with Docker Compose ## Checks -CI runs these on every branch. `.tool-versions` pins Node and the GitHub CLI; CI installs them with mise. `renovate.json` turns Renovate off so that these tools are upgraded together with the `sourcegraph/sourcegraph` monorepo, not on their own. +CI runs these on every branch. `.tool-versions` pins Node, the GitHub CLI, and shellcheck; CI installs them with mise. `renovate.json` turns Renovate off so that these tools are upgraded together with the `sourcegraph/sourcegraph` monorepo, not on their own. -- **Compose files:** `.buildkite/validate-docker-compose.sh` runs `docker-compose config` on the base file, with the standalone `docker-compose` binary, so the base file must stay valid for that binary. CI validates no other Compose file. +- **Compose files:** `.buildkite/validate-docker-compose.sh` runs `docker compose config` on every Compose file, in each combination the docs say to run. Add a line to it for every new Compose file or combination. - **Formatting:** `npm ci && npm run prettier-check`; `npm run prettier` rewrites the files. Prettier formats the JavaScript and JSON files; the repository has no other file type its glob matches. `.prettierignore` excludes Markdown and `.github/`, and no formatter runs on YAML, so match the surrounding YAML style by hand. +- **Shell scripts:** `git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck`. - **Smoke test:** the `docker-compose-test` step runs `.buildkite/vagrant-run.sh docker-test`. See below. +Use `docker compose` (v2) in scripts and docs, not `docker-compose`. + ## Smoke test -`.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs `test/smoke-test.sh` on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. +`.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. -- `test/smoke-test.sh` starts the base file with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend responds. It counts the services itself, so a new long-running service needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count. -- CI does not run `test/executor-smoke-test.sh`. Run it against a running deployment when you change `docker-compose/executors/`. +- `test/smoke-test.sh` starts the base file, `jaeger/`, `executors/`, and `test/docker-compose.override.yaml` with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend and Jaeger respond. It counts the services itself, so a new long-running service in those files needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count. Add a new optional Compose file to its `compose_files`. +- `test/executor-smoke-test.sh` checks that the executor registers with the frontend, and serves exactly the queues which `test/docker-compose.override.yaml` gives it. The script names them again, so change both files together. - A service which is not ready until another service is needs a `start_period` on its health check, or `--wait` fails. - A test failure is not retried, and the step uploads every container's logs as artifacts. Only the image pull is retried: `test/smoke-test.sh` tries `docker compose pull` three times. - A branch which only changes files listed in the step's `if_changed` `exclude` skips the step. Builds on `main` always run it. When the smoke test starts reading a file on that list, remove it from the list. diff --git a/docker-compose/dev/README.md b/docker-compose/dev/README.md index bf75a6e23..c05d4dfa1 100644 --- a/docker-compose/dev/README.md +++ b/docker-compose/dev/README.md @@ -3,7 +3,7 @@ For use developing Sourcegraph's docker-compose deployment only. For deployments, refer to the official [Sourcegraph with Docker Compose](https://sourcegraph.com/docs/self-hosted/deploy/docker-compose) documentation. -This folder includes a helper script, [run.sh](./run.sh), that will run the `docker-compose` command (with any provided arguments) while automatically specifying the appropriate set of overlays. +This folder includes a helper script, [run.sh](./run.sh), that will run the `docker compose` command (with any provided arguments) while automatically specifying the appropriate set of overlays. Example usage: diff --git a/docker-compose/dev/run.sh b/docker-compose/dev/run.sh index f5c3881b4..e7b85ff19 100755 --- a/docker-compose/dev/run.sh +++ b/docker-compose/dev/run.sh @@ -3,7 +3,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/../.." set -euxo pipefail -docker-compose \ +docker compose \ -f docker-compose/docker-compose.yaml \ -f docker-compose/jaeger/docker-compose.yaml \ -f docker-compose/dev/docker-compose.yaml \ diff --git a/docker-compose/executors/executor.docker-compose.yaml b/docker-compose/executors/executor.docker-compose.yaml index 473bc7072..8f49bc9a8 100644 --- a/docker-compose/executors/executor.docker-compose.yaml +++ b/docker-compose/executors/executor.docker-compose.yaml @@ -30,3 +30,6 @@ services: networks: - sourcegraph restart: always + +networks: + sourcegraph: diff --git a/test/docker-compose.override.yaml b/test/docker-compose.override.yaml new file mode 100644 index 000000000..8bf2dda90 --- /dev/null +++ b/test/docker-compose.override.yaml @@ -0,0 +1,9 @@ +services: + # The executor exits unless both are set. + # executor-smoke-test.sh sets this password as `executors.accessToken` in the site config + executor: + environment: + - EXECUTOR_FRONTEND_PASSWORD=executor-smoke-test-token + # Every queue; executor-smoke-test.sh checks for these names. + # The base file leaves EXECUTOR_QUEUE_NAME empty, which the executor requires with this + - EXECUTOR_QUEUE_NAMES=batches,batches-hooks,codeintel diff --git a/test/executor-smoke-test.sh b/test/executor-smoke-test.sh index f92e1b474..0ab29cd9f 100755 --- a/test/executor-smoke-test.sh +++ b/test/executor-smoke-test.sh @@ -3,18 +3,20 @@ set -euo pipefail # Executor smoke test for deploy-sourcegraph-docker # -# Requires: a running Sourcegraph docker-compose stack (from smoke-test.sh or manual start). -# This script creates a site admin account, configures executor access, starts an executor -# container, and verifies it registers with the Sourcegraph instance. +# Requires: a running Sourcegraph docker-compose stack with an executor container +# (from smoke-test.sh or manual start). +# This script creates a site admin account, configures executor access, and verifies +# the executor registers with the Sourcegraph instance, with every queue. # -# Dependencies: curl, jq, docker compose +# Dependencies: curl, jq, docker FRONTEND_URL="${FRONTEND_URL:-http://localhost:80}" -EXECUTOR_TOKEN="executor-smoke-test-token" +EXECUTOR_TOKEN=$(docker exec executor printenv EXECUTOR_FRONTEND_PASSWORD) ADMIN_EMAIL="smoke-test@sourcegraph.com" ADMIN_USERNAME="smoke-admin" ADMIN_PASSWORD="smoke-test-password-123!" -COMPOSE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../docker-compose" && pwd)" +# The queues test/docker-compose.override.yaml gives the executor, sorted +EXPECTED_QUEUE_NAMES="batches,batches-hooks,codeintel" echo "=== Executor Smoke Test ===" @@ -74,29 +76,27 @@ curl -s -o /dev/null -X POST "${FRONTEND_URL}/.api/graphql" \ --data-raw "{\"query\":\"mutation UpdateSiteConfig(\$input: String!) { updateSiteConfiguration(lastID: $LAST_ID, input: \$input) }\", \"variables\":{\"input\":$NEW_CONFIG}}" echo " Site config updated" -# Step 5: Start executor container -echo "Step 5: Starting executor container..." -EXECUTOR_CONTAINER=$(docker compose \ - -f "${COMPOSE_DIR}/docker-compose.yaml" \ - -f "${COMPOSE_DIR}/executors/executor.docker-compose.yaml" \ - run -d \ - -e "EXECUTOR_FRONTEND_PASSWORD=${EXECUTOR_TOKEN}" \ - -e EXECUTOR_QUEUE_NAME=batches \ - executor 2>&1 | tail -1) -echo " Executor container started: ${EXECUTOR_CONTAINER:0:12}" - -# Step 6: Wait for executor to register -echo "Step 6: Waiting for executor to register..." +# Step 5: Wait for executor to register +echo "Step 5: Waiting for executor to register..." MAX_ATTEMPTS=24 for i in $(seq 1 $MAX_ATTEMPTS); do - COUNT=$(curl -s -X POST "${FRONTEND_URL}/.api/graphql" \ + EXECUTORS=$(curl -s -X POST "${FRONTEND_URL}/.api/graphql" \ -H "Authorization: token $TOKEN" \ -H 'Content-Type: application/json' \ - -d '{"query":"{ executors(query: \"\", active: false) { totalCount } }"}' \ - | jq -r '.data.executors.totalCount // 0') + -d '{"query":"{ executors(query: \"\", active: false) { totalCount nodes { queueNames } } }"}') + COUNT=$(jq -r '.data.executors.totalCount // 0' <<<"$EXECUTORS") if [ "$COUNT" -ge 1 ] 2>/dev/null; then echo " Executor registered after $((i * 5))s" + # One line per executor: its queue names, sorted + QUEUE_NAMES=$(jq -r '.data.executors.nodes[] | .queueNames // [] | sort | join(",")' <<<"$EXECUTORS") + if ! grep --quiet --line-regexp --fixed-strings "$EXPECTED_QUEUE_NAMES" <<<"$QUEUE_NAMES"; then + echo "" + echo "=== EXECUTOR SMOKE TEST FAILED ===" + echo "No executor serves the queues \"$EXPECTED_QUEUE_NAMES\". Registered: \"$QUEUE_NAMES\"" + exit 1 + fi + echo " Executor serves the queues $EXPECTED_QUEUE_NAMES" echo "" echo "=== EXECUTOR SMOKE TEST PASSED ===" exit 0 @@ -110,8 +110,8 @@ echo "=== EXECUTOR SMOKE TEST FAILED ===" echo "Executor did not register within $((MAX_ATTEMPTS * 5))s" echo "" echo "Container status:" -docker inspect "$EXECUTOR_CONTAINER" --format '{{.State.Status}} (exit code: {{.State.ExitCode}})' 2>/dev/null || echo " Container not found" +docker inspect executor --format '{{.State.Status}} (exit code: {{.State.ExitCode}})' 2>/dev/null || echo " Container not found" echo "" echo "Container logs:" -docker logs "$EXECUTOR_CONTAINER" 2>&1 | tail -20 || echo " No logs available" +docker logs executor 2>&1 | tail -20 || echo " No logs available" exit 1 diff --git a/test/servers.yaml b/test/servers.yaml index 36780dbe7..f68c942c9 100644 --- a/test/servers.yaml +++ b/test/servers.yaml @@ -11,4 +11,7 @@ service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com shell_commands: - "/deploy-sourcegraph-docker/test/smoke-test.sh" + # The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails + - "command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; }" + - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh" diff --git a/test/smoke-test.sh b/test/smoke-test.sh index b94af127e..2e1725859 100755 --- a/test/smoke-test.sh +++ b/test/smoke-test.sh @@ -11,7 +11,13 @@ configure_docker() { gcloud auth configure-docker us-central1-docker.pkg.dev,us-docker.pkg.dev } -compose_files=(--file docker-compose/docker-compose.yaml) +# Every service a customer can deploy, started and checked by the same commands +compose_files=( + --file docker-compose/docker-compose.yaml + --file docker-compose/jaeger/docker-compose.yaml + --file docker-compose/executors/executor.docker-compose.yaml + --file test/docker-compose.override.yaml +) deploy_sourcegraph() { cd "$(dirname "${BASH_SOURCE[0]}")/.." @@ -52,6 +58,9 @@ test_containers() { curl -f http://localhost:80 curl -f http://localhost:80/healthz + echo "TEST: Checking Jaeger is accessible" + curl --fail --silent --show-error --location --output /dev/null http://localhost:16686/-/debug/jaeger/ + echo "ALL TESTS PASSED" }