diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 2a2595ad6..f5f2e72f5 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,8 +1,7 @@ $schema: "http://json.schemastore.org/buildkite" env: - # GCP VM image for the Vagrant smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images` + # GCP VM image for the smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images` IMAGE: "e2e-db8a40720-2024-08-26t08-44-10z" - VAGRANT_RUN_ENV: "CI" steps: - label: ":docker:" command: .buildkite/validate-docker-compose.sh @@ -20,9 +19,11 @@ steps: # so it finds no changes in a commit which is already on main, unless it is the newest - &smoke_test label: ":rice: docker-compose-test" - command: .buildkite/vagrant-run.sh docker-test + command: .buildkite/smoke-test-vm.sh artifact_paths: ./*.log agents: { queue: "vagrant" } + # smoke-test-vm.sh deletes VMs and disks older than an hour, so no job may run that long + timeout_in_minutes: 30 if: build.branch == "main" # Builds on other branches skip the smoke test if they only change files it does not read. diff --git a/.buildkite/smoke-test-vm.sh b/.buildkite/smoke-test-vm.sh new file mode 100755 index 000000000..d5b746029 --- /dev/null +++ b/.buildkite/smoke-test-vm.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +# Runs test/smoke-test.sh and test/executor-smoke-test.sh on a new GCP VM booted from $IMAGE, +# then deletes the VM + +cd "$(dirname "${BASH_SOURCE[0]}")/.." +set -euo pipefail + +project="sourcegraph-ci" +zone="us-central1-f" + +# GCP creates a disk, then a VM, both with the same name. Naming them after the job +# lets the cleanup find them if the job is cancelled before the VM exists +instance_name_prefix="docker-test-job-" +instance_name="${instance_name_prefix}${BUILDKITE_JOB_ID}" + +# The builder account key, and an SSH key for this job only +temporary_directory="$(mktemp -d)" + +# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses, +# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM +request_deletion() { + { + gcloud compute instances list --project="$project" --filter="$1" --format="value(selfLink)" && + gcloud compute disks list --project="$project" --filter="($1) AND -users:*" --format="value(selfLink)" + } | tee /dev/stderr | + xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \ + --request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)" +} + +cleanup() { + request_deletion "name=${instance_name}" || + echo "Could not delete this job's VM. GCP deletes it an hour after it started" + rm -rf "$temporary_directory" +} +trap cleanup EXIT + +# The VM's guest agent authorizes this key for the buildkite user. +# The keepalive ends the connection to a VM which stopped answering, after a minute +ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$temporary_directory/ssh-key" +ssh_options=(-i "$temporary_directory/ssh-key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR + -o ConnectTimeout=5 -o ServerAliveInterval=15 -o ServerAliveCountMax=4) + +echo --- ":lock: builder account key" +gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project="$project" >"$temporary_directory/builder-key.json" +DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project="$project")" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$temporary_directory/builder-key.json" + +echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup" +# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job. +# GCP deletes the VMs itself after an hour, but not a disk whose VM was never created +request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks" + +echo "--- :gcloud: starting VM $instance_name" +# `beta`, because --max-run-duration is only there in the agents' gcloud 417. It is in every track from gcloud 483 +ip_address="$(gcloud beta compute instances create "$instance_name" \ + --project="$project" --zone="$zone" \ + --image="$IMAGE" --image-project="$project" \ + --machine-type=custom-16-20480 \ + --boot-disk-size=20GB --boot-disk-type=pd-standard \ + --max-run-duration=1h --instance-termination-action=DELETE \ + --network=default --no-address \ + --service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \ + --labels=team=dev-infra,type=smoke-test \ + --metadata="ssh-keys=buildkite:$(cat "$temporary_directory/ssh-key.pub")" \ + --format="value(networkInterfaces[0].networkIP)")" +remote="buildkite@${ip_address}" + +echo --- ":ssh: waiting for the VM to accept SSH" +deadline=$((SECONDS + 300)) +until ssh "${ssh_options[@]}" "$remote" true; do + if ((SECONDS > deadline)); then + echo "The VM did not accept SSH within 5 minutes" + exit 1 + fi + sleep 2 +done + +copy_repository() { + tar --create --gzip --exclude=.git --exclude=node_modules . | + ssh "${ssh_options[@]}" "$remote" "sudo mkdir -p /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker" +} + +echo --- ":package: copying the repository to the VM" +# A VM can accept SSH, then stop answering for a while. ssh exits with 255 when the connection fails, +# so retry the copy on the same VM. The tests have not started yet, so nothing runs twice +exit_code=0 +copy_repository || exit_code=$? +for retry in {1..10}; do + if [ "$exit_code" != 255 ]; then + break + fi + echo "The VM does not answer SSH, retry $retry of 10 in 15 seconds" + sleep 15 + exit_code=0 + copy_repository || exit_code=$? +done + +if [ "$exit_code" = 0 ]; then + echo --- ":docker: running the smoke tests" + # Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list + remote_script=' +set -euo pipefail +read -r DOCKER_PASSWORD +export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD +/deploy-sourcegraph-docker/test/smoke-test.sh +# The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails +command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; } +/deploy-sourcegraph-docker/test/executor-smoke-test.sh +' + # shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM + ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD" || exit_code=$? + + if [ "$exit_code" != 0 ]; then + # Copy back the container logs the smoke test wrote, for the step's artifacts + ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true + fi +fi + +# Not a plain `exit`, after which shellcheck reports that cleanup is never invoked (SC2329) +[ "$exit_code" = 0 ] || exit "$exit_code" diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh deleted file mode 100755 index 3be84b10e..000000000 --- a/.buildkite/vagrant-run.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -cd "$(dirname "${BASH_SOURCE[0]}")/.." -set -euo pipefail - -box="$1" -exit_code=0 - -pushd "test" - -cleanup() { - vagrant destroy -f "$box" -} - -echo --- ":vagrant: installing plugins" -vagrant --version -vagrant plugin install vagrant-google --plugin-version '2.7.0' -vagrant plugin install vagrant-env -vagrant plugin install vagrant-scp - -trap cleanup EXIT - -echo --- ":lock: builder account key" -KEY_PATH="/tmp/e2e-builder.json" -if [ ! -f ${KEY_PATH} ]; then - gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}" -fi -export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}" -export DOCKER_USERNAME=sourcegraph -DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" -export DOCKER_PASSWORD - -echo --- ":vagrant: starting box $box" -vagrant up "$box" --provider=google || exit_code=$? - -if [ "$exit_code" != 0 ]; then - vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../ - exit $exit_code -fi diff --git a/.gitignore b/.gitignore index 9cf3cdc72..3ecd4b8fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .env .envrc -.vagrant *.log node_modules/ diff --git a/AGENTS.md b/AGENTS.md index e14e992d0..61ec38f75 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,20 +17,22 @@ CI runs these on every branch. `.tool-versions` pins Node, the GitHub CLI, and s - **Compose files:** `.buildkite/validate-docker-compose.sh` runs `docker compose config` on every Compose file, in each combination the docs say to run. Add a line to it for every new Compose file or combination. - **Formatting:** `npm ci && npm run prettier-check`; `npm run prettier` rewrites the files. Prettier formats the JavaScript and JSON files; the repository has no other file type its glob matches. `.prettierignore` excludes Markdown and `.github/`, and no formatter runs on YAML, so match the surrounding YAML style by hand. - **Shell scripts:** `git ls-files -z '*.sh' .buildkite/hooks | xargs -0 shellcheck`. -- **Smoke test:** the `docker-compose-test` step runs `.buildkite/vagrant-run.sh docker-test`. See below. +- **Smoke test:** the `docker-compose-test` step runs `.buildkite/smoke-test-vm.sh`. See below. Use `docker compose` (v2) in scripts and docs, not `docker-compose`. ## Smoke test -`.buildkite/vagrant-run.sh` uses Vagrant to boot a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. +`.buildkite/smoke-test-vm.sh` boots a GCP VM from the image named in `IMAGE` in `.buildkite/pipeline.yaml`, runs the two tests below on it, and deletes it. It reads CI secrets and creates a billed VM, so take its result from the pull request's Buildkite build; run the script yourself only when a person asks. - `test/smoke-test.sh` starts the base file, `jaeger/`, `executors/`, and `test/docker-compose.override.yaml` with `docker compose up --wait`, which fails if any container reports `unhealthy`. It then checks that every service has a running container, no container restarted, and the frontend and Jaeger respond. It counts the services itself, so a new long-running service in those files needs no change to the test. A new service which exits when it is done, like `migrator`, must be left out of the count. Add a new optional Compose file to its `compose_files`. - `test/executor-smoke-test.sh` checks that the executor registers with the frontend, and serves exactly the queues which `test/docker-compose.override.yaml` gives it. The script names them again, so change both files together. - A service which is not ready until another service is needs a `start_period` on its health check, or `--wait` fails. -- A test failure is not retried, and the step uploads every container's logs as artifacts. Only the image pull is retried: `test/smoke-test.sh` tries `docker compose pull` three times. +- A test failure is not retried, and the step uploads every container's logs as artifacts. Only connections are retried: the script waits up to 5 minutes for the VM to accept SSH, retries the SSH connection which copies the repository to the VM, and `test/smoke-test.sh` tries `docker compose pull` three times. - A branch which only changes files listed in the step's `if_changed` `exclude` skips the step. Builds on `main` always run it. When the smoke test starts reading a file on that list, remove it from the list. +- GCP deletes each VM an hour after it starts, and the script deletes the VMs and disks which older jobs left behind. Keep the step's `timeout_in_minutes` under an hour. - The VM image is built from `gcp/images/e2e` in sourcegraph/infrastructure. +- The CI agents' queue is named `vagrant`. Nothing on `main` uses Vagrant. ## Conventions diff --git a/test/Vagrantfile b/test/Vagrantfile deleted file mode 100644 index 09bd66030..000000000 --- a/test/Vagrantfile +++ /dev/null @@ -1,74 +0,0 @@ -# -*- mode: ruby -*- -# # vi: set ft=ruby : -# Specify minimum Vagrant version and Vagrant API version -Vagrant.require_version '>= 1.6.0' -VAGRANTFILE_API_VERSION = '2'.freeze -# Require YAML module -require 'yaml' - -# Read YAML file with box details -servers = YAML.load_file('servers.yaml') - -# Create boxes -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # Iterate through entries in YAML file - servers.each do |server| - config.env.enable - config.vm.define server['name'] do |srv| - srv.vm.box = server['box'] - srv.vm.synced_folder '../', '/deploy-sourcegraph-docker' - srv.vm.boot_timeout = 600 - - # Setup for CI or Local - case ENV['VAGRANT_RUN_ENV'] - when 'CI' - project_id = server['project_id'] - external_ip = server['external_ip'] - use_private_ip = server['use_private_ip'] - username = server['username'] - ssh_key_path = server['ssh_key_path'] - else - project_id = ENV['VAGRANT_GCP_PROJECT_ID'] || 'sourcegraph-server' - external_ip = nil - use_private_ip = false - username = ENV['VAGRANT_SSH_USER'] || ENV['USER'] - ssh_key_path = ENV['VAGRANT_SSH_KEY'] || '~/.ssh/id_rsa' - end - - srv.vm.provider :google do |g, o| - g.machine_type = server['machine_type'] - g.image = ENV['IMAGE'] - g.image_project_id = 'sourcegraph-ci' - g.google_project_id = project_id - g.name = "#{server['name']}-"+"#{Time.now.to_i}"+"#{rand(100000)}" - g.network = server['network'] - g.external_ip = external_ip - g.use_private_ip = use_private_ip - g.service_account = server['service_account'] - g.labels = { - "team" => "dev-infra", - "type" => "vagrant" - } - g.scopes = [ - "compute-rw", - "storage-ro" - ] - o.ssh.username = username - o.ssh.private_key_path = ssh_key_path - end - - srv.vm.provision 'shell', inline: <<-SHELL - #!/usr/bin/env bash - cat << EOF >> /root/.profile -export GIT_BRANCH=#{ENV['BUILDKITE_BRANCH']} -export DOCKER_USERNAME=#{ENV['DOCKER_USERNAME']} -export DOCKER_PASSWORD=#{ENV['DOCKER_PASSWORD']} -EOF - SHELL - - server['shell_commands'].each do |sh| - srv.vm.provision 'shell', inline: sh - end - end - end -end diff --git a/test/servers.yaml b/test/servers.yaml deleted file mode 100644 index f68c942c9..000000000 --- a/test/servers.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: docker-test - box: google/gce - machine_type: "custom-16-20480" - project_id: sourcegraph-ci - external_ip: false - use_private_ip: true - network: default - username: buildkite - ssh_key_path: "~/.ssh/id_rsa" - service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com - shell_commands: - - "/deploy-sourcegraph-docker/test/smoke-test.sh" - # The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails - - "command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; }" - - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh" -