diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 826b9f888..1091d0f8f 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,8 +1,7 @@ $schema: "http://json.schemastore.org/buildkite" env: - # GCP VM image for the Vagrant smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images` + # GCP VM image for the smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images` IMAGE: "e2e-db8a40720-2024-08-26t08-44-10z" - VAGRANT_RUN_ENV: "CI" steps: - label: ":docker:" command: .buildkite/validate-docker-compose.sh @@ -21,10 +20,10 @@ steps: # The branch is named here, because the Buildkite pipeline's default branch is still "master" - &smoke_test label: ":rice: docker-compose-test" - command: .buildkite/vagrant-run.sh docker-test + command: .buildkite/smoke-test-vm.sh artifact_paths: ./*.log agents: { queue: "vagrant" } - # vagrant-run.sh deletes VMs and disks older than an hour, so no job may run that long + # smoke-test-vm.sh deletes VMs and disks older than an hour, so no job may run that long timeout_in_minutes: 30 if: build.branch == "main" diff --git a/.buildkite/smoke-test-vm.sh b/.buildkite/smoke-test-vm.sh new file mode 100755 index 000000000..7c9b1198c --- /dev/null +++ b/.buildkite/smoke-test-vm.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# Runs test/smoke-test.sh on a new GCP VM booted from $IMAGE, then deletes the VM + +cd "$(dirname "${BASH_SOURCE[0]}")/.." +set -euo pipefail + +KEY_PATH="$(mktemp)" +project="sourcegraph-ci" +zone="us-central1-f" + +# GCP creates a disk, then a VM, both with this name. Naming them after the job +# lets the cleanup find them if the job is cancelled before the VM exists +instance_name_prefix="docker-test-job-" +instance_name="${instance_name_prefix}${BUILDKITE_JOB_ID}" + +# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses, +# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM +request_deletion() { + { + gcloud compute instances list --project="$project" --filter="$1" --format="value(selfLink)" && + gcloud compute disks list --project="$project" --filter="($1) AND -users:*" --format="value(selfLink)" + } | tee /dev/stderr | + xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \ + --request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)" +} + +# Fall back to a blocking delete, so a failed deletion request cannot leak a VM +cleanup() { + deletion_status=0 + request_deletion "name=${instance_name}" || + gcloud compute instances delete "$instance_name" --project="$project" --zone="$zone" --quiet || + deletion_status=$? + rm -rf "$KEY_PATH" "$ssh_directory" + return "$deletion_status" +} + +# A key for this job only, which the VM's guest agent authorizes for the buildkite user +ssh_directory="$(mktemp -d)" +ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$ssh_directory/key" +ssh_options=(-i "$ssh_directory/key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5) + +trap cleanup EXIT + +echo --- ":lock: builder account key" +gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci >"${KEY_PATH}" +DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH" + +echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup" +# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job +request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks" + +echo --- ":gcloud: starting VM $instance_name" +ip_address="$(gcloud compute instances create "$instance_name" \ + --project="$project" --zone="$zone" \ + --image="$IMAGE" --image-project="$project" \ + --machine-type=custom-16-20480 \ + --boot-disk-size=20GB --boot-disk-type=pd-standard \ + --network=default --no-address \ + --service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \ + --labels=team=dev-infra,type=smoke-test \ + --metadata="ssh-keys=buildkite:$(cat "$ssh_directory/key.pub")" \ + --format="value(networkInterfaces[0].networkIP)")" +remote="buildkite@${ip_address}" + +echo --- ":ssh: waiting for the VM to accept SSH" +deadline=$((SECONDS + 600)) +until ssh "${ssh_options[@]}" "$remote" true; do + if ((SECONDS > deadline)); then + echo "The VM did not accept SSH within 10 minutes" + exit 1 + fi + sleep 2 +done + +echo --- ":package: copying the repository to the VM" +tar --create --gzip --exclude=.git --exclude=node_modules . | + ssh "${ssh_options[@]}" "$remote" "sudo mkdir /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker" + +echo --- ":docker: running the smoke tests" +# Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list +remote_script=' +set -euo pipefail +read -r DOCKER_PASSWORD +export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD +/deploy-sourcegraph-docker/test/smoke-test.sh +# The VM image has no jq. Skip /etc/apt/sources.list.d: the image lacks the key for its Chrome repository, so a full update fails +apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq +/deploy-sourcegraph-docker/test/executor-smoke-test.sh +' +exit_code=0 +# shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM +ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD" || exit_code=$? + +if [ "$exit_code" != 0 ]; then + # Copy back the container logs the smoke test wrote, for the step's artifacts + ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true + exit "$exit_code" +fi diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh deleted file mode 100755 index c4afd4569..000000000 --- a/.buildkite/vagrant-run.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -cd "$(dirname "${BASH_SOURCE[0]}")/.." -set -euo pipefail - -box="$1" -exit_code=0 -KEY_PATH="$(mktemp)" - -pushd "test" - -# Vagrant creates a disk, then a VM, both with this name, and only knows about them once GCP has finished -# creating the VM. Naming them after the job lets the cleanup find them if the job is cancelled before that -instance_name_prefix="${box}-job-" -export VAGRANT_INSTANCE_NAME="${instance_name_prefix}${BUILDKITE_JOB_ID}" - -# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses, -# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM -request_deletion() ( - export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$KEY_PATH" - { - gcloud compute instances list --project=sourcegraph-ci --filter="$1" --format="value(selfLink)" && - gcloud compute disks list --project=sourcegraph-ci --filter="($1) AND -users:*" --format="value(selfLink)" - } | tee /dev/stderr | - xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \ - --request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)" -) - -# Fall back to a blocking destroy, so a failed deletion request cannot leak a VM -cleanup() { - deletion_status=0 - request_deletion "name=${VAGRANT_INSTANCE_NAME}" || vagrant destroy -f "$box" || deletion_status=$? - rm -f "$KEY_PATH" - return "$deletion_status" -} - -echo --- ":vagrant: installing plugins" -vagrant --version -# Agents on the same host share ~/.vagrant.d, so only the first job on a new host installs -[[ "$(vagrant plugin list)" == *"vagrant-google (2.7.0"* ]] || vagrant plugin install vagrant-google --plugin-version '2.7.0' - -trap cleanup EXIT - -echo --- ":lock: builder account key" -gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}" -export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}" -DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" -export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD - -echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup" -# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job -request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks" - -echo --- ":vagrant: starting box $box" -vagrant up "$box" --provider=google || exit_code=$? - -if [ "$exit_code" != 0 ]; then - vagrant plugin install vagrant-scp - vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../ - exit $exit_code -fi diff --git a/.gitignore b/.gitignore index 9cf3cdc72..3ecd4b8fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .env .envrc -.vagrant *.log node_modules/ diff --git a/test/Vagrantfile b/test/Vagrantfile deleted file mode 100644 index ebd5446ca..000000000 --- a/test/Vagrantfile +++ /dev/null @@ -1,65 +0,0 @@ -# -*- mode: ruby -*- -# # vi: set ft=ruby : -# Specify minimum Vagrant version and Vagrant API version -Vagrant.require_version '>= 1.6.0' -VAGRANTFILE_API_VERSION = '2'.freeze -# Require YAML module -require 'yaml' - -# Read YAML file with box details -servers = YAML.load_file('servers.yaml') - -# Create boxes -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # Iterate through entries in YAML file - servers.each do |server| - config.vm.define server['name'] do |srv| - srv.vm.box = server['box'] - srv.vm.synced_folder '../', '/deploy-sourcegraph-docker' - srv.vm.boot_timeout = 600 - - # Setup for CI or Local - case ENV['VAGRANT_RUN_ENV'] - when 'CI' - project_id = server['project_id'] - external_ip = server['external_ip'] - use_private_ip = server['use_private_ip'] - username = server['username'] - ssh_key_path = server['ssh_key_path'] - else - project_id = ENV['VAGRANT_GCP_PROJECT_ID'] || 'sourcegraph-server' - external_ip = nil - use_private_ip = false - username = ENV['VAGRANT_SSH_USER'] || ENV['USER'] - ssh_key_path = ENV['VAGRANT_SSH_KEY'] || '~/.ssh/id_rsa' - end - - srv.vm.provider :google do |g, o| - g.machine_type = server['machine_type'] - g.image = ENV['IMAGE'] - g.image_project_id = 'sourcegraph-ci' - g.google_project_id = project_id - # .buildkite/vagrant-run.sh names the VM after its job, so it can delete the VM by name - g.name = ENV['VAGRANT_INSTANCE_NAME'] || "#{server['name']}-"+"#{Time.now.to_i}"+"#{rand(100000)}" - g.network = server['network'] - g.external_ip = external_ip - g.use_private_ip = use_private_ip - g.service_account = server['service_account'] - g.labels = { - "team" => "dev-infra", - "type" => "vagrant" - } - g.scopes = [ - "compute-rw", - "storage-ro" - ] - o.ssh.username = username - o.ssh.private_key_path = ssh_key_path - end - - server['shell_commands'].each do |sh| - srv.vm.provision 'shell', inline: sh, env: ENV.slice('DOCKER_USERNAME', 'DOCKER_PASSWORD') - end - end - end -end diff --git a/test/servers.yaml b/test/servers.yaml deleted file mode 100644 index 88a9cc366..000000000 --- a/test/servers.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: docker-test - box: google/gce - machine_type: "custom-16-20480" - project_id: sourcegraph-ci - external_ip: false - use_private_ip: true - network: default - username: buildkite - ssh_key_path: "~/.ssh/id_rsa" - service_account: e2e-builder@sourcegraph-ci.iam.gserviceaccount.com - shell_commands: - - "/deploy-sourcegraph-docker/test/smoke-test.sh" - # The VM image has no jq. Skip /etc/apt/sources.list.d: the image lacks the key for its Chrome repository, so a full update fails - - "apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq" - - "/deploy-sourcegraph-docker/test/executor-smoke-test.sh" -