From 040f145117a9bd0ec7c5ddddce795de97206216b Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:07:46 -0600 Subject: [PATCH 1/3] ci: Retry the smoke test when its VM stops answering SSH vagrant up fails if the test VM drops off the network after its first SSH connection. That is not a test result, so check whether the VM still answers SSH when vagrant up fails, and exit with a status which Buildkite retries once, on a new VM. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c753-39cc-72c6-b71a-0a24497aaf5a Co-authored-by: Amp --- .buildkite/pipeline.yaml | 5 +++++ .buildkite/vagrant-run.sh | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index efa4e4125..75cdc96c4 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -16,6 +16,11 @@ steps: command: .buildkite/vagrant-run.sh docker-test artifact_paths: ./*.log agents: { queue: "vagrant" } + # vagrant-run.sh exits 75 when it loses its SSH connection to the test VM + retry: + automatic: + - exit_status: 75 + limit: 1 # Please keep in mind that the release manifest uses specific branch names when creating releases. # Therefore, if you update them, or if you decide to change how we detect what kind of build we're dealing diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh index a54aeb97d..1d77dce1e 100755 --- a/.buildkite/vagrant-run.sh +++ b/.buildkite/vagrant-run.sh @@ -33,6 +33,11 @@ echo --- ":vagrant: starting box $box" vagrant up "$box" --provider=google || exit_code=$? if [ "$exit_code" != 0 ]; then + # A VM which stopped answering SSH did not fail the test, and has no logs to copy + if ! vagrant ssh "$box" --no-tty --command true -- -o ConnectTimeout=15; then + echo "Lost the SSH connection to $box, so the test has no result. pipeline.yaml retries exit status 75 on a new VM" + exit 75 + fi vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../ exit $exit_code fi From e611a1f531c05e3def20929f6773c8e38f049bf7 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:30:37 -0600 Subject: [PATCH 2/3] ci: Retry on a new VM in vagrant-run.sh instead of in Buildkite Review feedback: keep the retry in the script. When the test VM stops answering SSH, destroy it and run the test once more on a new VM, in the same job. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c753-39cc-72c6-b71a-0a24497aaf5a Co-authored-by: Amp --- .buildkite/pipeline.yaml | 5 ----- .buildkite/vagrant-run.sh | 28 ++++++++++++++++------------ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 75cdc96c4..efa4e4125 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -16,11 +16,6 @@ steps: command: .buildkite/vagrant-run.sh docker-test artifact_paths: ./*.log agents: { queue: "vagrant" } - # vagrant-run.sh exits 75 when it loses its SSH connection to the test VM - retry: - automatic: - - exit_status: 75 - limit: 1 # Please keep in mind that the release manifest uses specific branch names when creating releases. # Therefore, if you update them, or if you decide to change how we detect what kind of build we're dealing diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh index 1d77dce1e..6e7a06d44 100755 --- a/.buildkite/vagrant-run.sh +++ b/.buildkite/vagrant-run.sh @@ -4,7 +4,6 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." set -euo pipefail box="$1" -exit_code=0 pushd "test" @@ -29,15 +28,20 @@ export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}" export DOCKER_USERNAME=sourcegraph export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" -echo --- ":vagrant: starting box $box" -vagrant up "$box" --provider=google || exit_code=$? - -if [ "$exit_code" != 0 ]; then - # A VM which stopped answering SSH did not fail the test, and has no logs to copy - if ! vagrant ssh "$box" --no-tty --command true -- -o ConnectTimeout=15; then - echo "Lost the SSH connection to $box, so the test has no result. pipeline.yaml retries exit status 75 on a new VM" - exit 75 +for attempt in 1 2; do + echo --- ":vagrant: starting box $box, attempt $attempt of 2" + exit_code=0 + vagrant up "$box" --provider=google || exit_code=$? + [ "$exit_code" = 0 ] && break + + # A VM which stopped answering SSH did not fail the test, and has no logs to copy, + # so run the test again on a new VM + if vagrant ssh "$box" --no-tty --command true -- -o ConnectTimeout=15; then + vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../ + break fi - vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../ - exit $exit_code -fi + echo "Lost the SSH connection to $box" + vagrant destroy -f "$box" +done + +exit "$exit_code" From cb84ae366a64ed8beb02840ee53a30c77c9cb3d6 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:42:04 -0600 Subject: [PATCH 3/3] ci: Retry the SSH connection to the same VM instead of making a new VM Review feedback: the job must not create a second VM. Undo the retry of the whole test, and retry only the SSH connection to the VM which vagrant up created. Once it answers, vagrant provision syncs the repo and runs the test on that VM. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c753-39cc-72c6-b71a-0a24497aaf5a Co-authored-by: Amp --- .buildkite/vagrant-run.sh | 41 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh index 6e7a06d44..e7a1458de 100755 --- a/.buildkite/vagrant-run.sh +++ b/.buildkite/vagrant-run.sh @@ -4,6 +4,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." set -euo pipefail box="$1" +exit_code=0 pushd "test" @@ -28,20 +29,28 @@ export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}" export DOCKER_USERNAME=sourcegraph export DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project=sourcegraph-ci)" -for attempt in 1 2; do - echo --- ":vagrant: starting box $box, attempt $attempt of 2" - exit_code=0 - vagrant up "$box" --provider=google || exit_code=$? - [ "$exit_code" = 0 ] && break - - # A VM which stopped answering SSH did not fail the test, and has no logs to copy, - # so run the test again on a new VM - if vagrant ssh "$box" --no-tty --command true -- -o ConnectTimeout=15; then - vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../ - break - fi - echo "Lost the SSH connection to $box" - vagrant destroy -f "$box" -done +ssh_answers() { + vagrant ssh "$box" --no-tty --command true -- -o ConnectTimeout=15 +} -exit "$exit_code" +echo --- ":vagrant: starting box $box" +vagrant up "$box" --provider=google || exit_code=$? + +# vagrant up gives up when the VM stops answering SSH, which is not a test failure. +# Retry the SSH connection to the same VM, then sync the repo and run the test on it +if [ "$exit_code" != 0 ] && ! ssh_answers; then + for attempt in {1..10}; do + echo "$box does not answer SSH, retry $attempt of 10 in 15 seconds" + sleep 15 + if ssh_answers; then + exit_code=0 + vagrant provision "$box" || exit_code=$? + break + fi + done +fi + +if [ "$exit_code" != 0 ]; then + vagrant scp "${box}:/deploy-sourcegraph-docker/*.log" ../ + exit $exit_code +fi