Repository navigation
build: raise the binary ceiling to 50 MB #5159
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-end TUI regressions | |
| # Drives a real tuios binary in a real PTY and asserts on the rendered screen. | |
| # No terminal is needed: the harness allocates the pty itself, so this runs on a | |
| # stock runner with no xvfb or tmux. | |
| # A draft pull request skips the shards. Marking it ready for review starts a | |
| # run, and every push after that runs them. The other workflows still run on a | |
| # draft. A skipped shard shows as "skipping", never as "pass". | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| # No path filter: the shard plan job runs on every pull request, so a | |
| # pull request always shows that this workflow started. A change that | |
| # reaches no test, such as one to the docs, starts no shard. | |
| workflow_dispatch: | |
| inputs: | |
| skip_if_tested: | |
| description: Skip the shards when this commit already passed them as a pull request head | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # A pull request's group is its ref, so a newer push cancels the run in | |
| # progress. Every other run (main, the nightly, a manual run) gets a group of | |
| # its own and is never cancelled or replaced while it waits. | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| # The shard count, the lanes in each shard, and the time each lane may take. | |
| # A shard is one runner. A lane is one go test process, and the lanes of a | |
| # shard run side by side on its runner. shardplan packs the tests into | |
| # E2E_SHARDS times E2E_LANES lanes by the measured times in | |
| # e2e/tui/shardplan/durations.txt. The shard plan job fails when a lane's | |
| # estimate is over E2E_LANE_LIMIT, which leaves a wide margin under the 12 | |
| # minute go test timeout. The shard plan job makes the matrix from E2E_SHARDS. | |
| env: | |
| E2E_SHARDS: "12" | |
| E2E_LANES: "2" | |
| E2E_LANE_LIMIT: 6m | |
| # A partial selection fills lanes to about this much each and leaves the | |
| # rest empty, so a small change does not start every runner. | |
| E2E_LANE_FILL: 3m | |
| jobs: | |
| # ff-merge lands the exact commit that passed as the pull request head, so | |
| # the push to main would run the same suite on the same tree again. This job | |
| # looks for that pull request run. When it finds one, the shards pass at once | |
| # with a note that links it. When it finds none, or the API fails, the shards | |
| # run in full. | |
| tested: | |
| name: e2e-tui already tested | |
| if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.skip_if_tested) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| actions: read | |
| outputs: | |
| url: ${{ steps.find.outputs.url }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| sparse-checkout: scripts | |
| - name: Find a pull request run that passed every shard on this commit | |
| id: find | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| url=$(scripts/e2e-tested.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$E2E_SHARDS" || true) | |
| echo "url=$url" >> "$GITHUB_OUTPUT" | |
| if [ -n "$url" ]; then | |
| echo "This commit passed every shard as a pull request head: $url" | tee -a "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "No pull request run passed every shard on this commit. The shards run in full." | tee -a "$GITHUB_STEP_SUMMARY" | |
| fi | |
| shard-plan: | |
| name: e2e-tui shard plan | |
| runs-on: ubuntu-latest | |
| outputs: | |
| shards: ${{ steps.shards.outputs.shards }} | |
| total: ${{ steps.shards.outputs.total }} | |
| # actions: read lets scripts/e2e-select.sh download the newest test map. | |
| permissions: | |
| contents: read | |
| actions: read | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| # A pull request checks out the merge of its head into the base, so | |
| # HEAD^1 is the base and HEAD^1..HEAD is exactly the change. | |
| fetch-depth: 2 | |
| - uses: ./.github/actions/setup-go | |
| with: | |
| go-version-file: e2e/tui/go.mod | |
| cache-id: e2e-plan | |
| - name: Check that every shard fits its time | |
| working-directory: e2e/tui | |
| env: | |
| TUIOS_E2E: "1" | |
| run: | | |
| set -euo pipefail | |
| go test -list '^Test' ./... | grep '^Test' | go run ./shardplan -shards "$((E2E_SHARDS * E2E_LANES))" -max "$E2E_LANE_LIMIT" | |
| # The shards make the same choice on their own. This copy says on the | |
| # run what was chosen and why. Its notice "mode=all" is how | |
| # scripts/e2e-tested.sh knows that every test ran, so the push to main | |
| # skips the suite only after a pull request run of the whole suite. | |
| - name: Select the tests this change can reach | |
| working-directory: e2e/tui | |
| env: | |
| TUIOS_E2E: "1" | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| GITHUB_OUTPUT="$RUNNER_TEMP/select.out" ../../scripts/e2e-select.sh > "$RUNNER_TEMP/selected.txt" | |
| echo "::notice title=e2e-tui selection::$(grep '^mode=' "$RUNNER_TEMP/select.out")" | |
| # The shards that get a test, by the same plan each shard makes for | |
| # itself below. A shard with no test does not start, so a small change | |
| # takes two or three runners instead of twelve. When the plan finds no | |
| # shard for a selection that holds tests, every shard starts. | |
| - name: Choose the shards that run | |
| id: shards | |
| working-directory: e2e/tui | |
| run: | | |
| set -euo pipefail | |
| mode=$(sed -n 's/^mode=//p' "$RUNNER_TEMP/select.out") | |
| go build -o "$RUNNER_TEMP/shardplan" ./shardplan | |
| lanes=$((E2E_SHARDS * E2E_LANES)) | |
| shards=() | |
| for shard in $(seq 1 "$E2E_SHARDS"); do | |
| case $mode in | |
| all) shards+=("$shard"); continue ;; | |
| none) continue ;; | |
| esac | |
| n=0 | |
| for lane in $(seq 1 "$E2E_LANES"); do | |
| v=$(( (shard - 1) * E2E_LANES + lane )) | |
| c=$("$RUNNER_TEMP/shardplan" -shards "$lanes" -shard "$v" -fill "$E2E_LANE_FILL" < "$RUNNER_TEMP/selected.txt" | grep -c . || true) | |
| n=$((n + c)) | |
| done | |
| [ "$n" -eq 0 ] || shards+=("$shard") | |
| done | |
| if [ "$mode" = some ] && [ "${#shards[@]}" -eq 0 ]; then | |
| echo "::warning title=Shard plan::The selection holds tests but no shard got one, so every shard runs." | |
| mapfile -t shards < <(seq 1 "$E2E_SHARDS") | |
| fi | |
| list=$(IFS=,; echo "${shards[*]:-}") | |
| echo "shards=[$list]" >> "$GITHUB_OUTPUT" | |
| echo "total=$E2E_SHARDS" >> "$GITHUB_OUTPUT" | |
| echo "Shards that run: ${list:-none} of $E2E_SHARDS" | tee -a "$GITHUB_STEP_SUMMARY" | |
| e2e-tui: | |
| runs-on: ubuntu-latest | |
| # actions: read lets scripts/e2e-select.sh download the newest test map. | |
| permissions: | |
| contents: read | |
| actions: read | |
| # The suite spawns multiplexer daemons and shells. The job timeout is a | |
| # backstop; every individual wait inside the suite has its own deadline, so | |
| # a hang is reported as the assertion that timed out rather than as a job | |
| # that was killed. | |
| # | |
| # The suite is split across runners because it grew past a single one. It | |
| # ran about 1000 seconds in one process against a 900 second test timeout, | |
| # and a runner is slower, so three commits in a row were killed rather than | |
| # failed. A kill prints no failing test, which is the worst shape a red | |
| # build can take. | |
| # | |
| # The tests hold no t.Parallel: each one starts a real daemon and a real | |
| # pseudo-terminal, and tests in one process share its environment and | |
| # globals. Each shard runs E2E_LANES go test processes side by side | |
| # instead, each with a list of its own. The tests mostly wait on a pty or | |
| # a daemon: a serial run used about one CPU second in five of its wall | |
| # time, so two lanes on a four-core runner do twice the work in about the | |
| # same time. Four lanes were tried: the runner stayed about two thirds | |
| # idle on average, but the bursts made one to two timing-sensitive tests | |
| # fail in four of seven full runs. Tests that must not overlap share a | |
| # lane through e2e/tui/shardplan/together.txt. | |
| # | |
| # The split was a hash of the test name until seven shards ran between 525 | |
| # and 1054 seconds, and the long one crossed the 1080 second timeout. A | |
| # hash balances the count of tests, and a few tests take a minute or two. | |
| # The split is now by measured time, with twelve shards of two lanes near | |
| # four minutes each. | |
| # | |
| # The shard plan job names the shards that get a test, and only those | |
| # start. Each one still selects and packs its own tests. | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: ${{ fromJSON(needs.shard-plan.outputs.shards) }} | |
| name: e2e-tui (shard ${{ matrix.shard }} of ${{ needs.shard-plan.outputs.total }}) | |
| needs: [tested, shard-plan] | |
| # tested is skipped on a pull request, so the shards must not wait for its | |
| # success. A draft skips the shards. | |
| if: ${{ !cancelled() && needs.shard-plan.result == 'success' && needs.shard-plan.outputs.shards != '[]' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }} | |
| timeout-minutes: 20 | |
| env: | |
| TESTED_URL: ${{ needs.tested.outputs.url }} | |
| steps: | |
| - name: Already passed on the pull request | |
| if: env.TESTED_URL != '' | |
| run: | | |
| echo "::notice title=e2e already passed::This commit passed every shard as a pull request head: $TESTED_URL" | |
| echo "Not run again. This commit passed every shard as a pull request head: $TESTED_URL" >> "$GITHUB_STEP_SUMMARY" | |
| - uses: actions/checkout@v7 | |
| if: env.TESTED_URL == '' | |
| with: | |
| fetch-depth: 2 | |
| - uses: ./.github/actions/setup-go | |
| if: env.TESTED_URL == '' | |
| with: | |
| # The nested e2e module asks for a newer Go than the root module, and | |
| # this job needs both: it builds cmd/tuios from the root and then runs | |
| # the suite from e2e/tui. Installing the higher of the two satisfies | |
| # each, since a module's go directive is a minimum. Reading the root | |
| # go.mod here used to work only because Go quietly fetched the newer | |
| # toolchain when the tests ran, which setup-go now prevents by pinning | |
| # GOTOOLCHAIN to what it installed. | |
| go-version-file: e2e/tui/go.mod | |
| cache-id: e2e | |
| # Every shard computes the same plan from the same test list, diff and | |
| # durations file, so together they run each selected test once. A test | |
| # that is not in the durations file yet counts as the mean. shardplan | |
| # packs E2E_SHARDS times E2E_LANES lanes, and shard N takes lanes | |
| # (N-1)*E2E_LANES+1 to N*E2E_LANES. | |
| # | |
| # On a pull request, scripts/e2e-select.sh keeps only the tests the | |
| # change can reach (see e2e/tui/testselect). Main, the nightly and a | |
| # manual run always select every test. A partial selection is packed | |
| # into as few lanes as it needs, and the shards with none stop here. | |
| # | |
| # Listing needs TUIOS_E2E as well: without it the suite's TestMain | |
| # returns before it can print a name, and the shard would silently be | |
| # empty. | |
| - name: Choose this shard's tests | |
| id: plan | |
| if: env.TESTED_URL == '' | |
| working-directory: e2e/tui | |
| env: | |
| TUIOS_E2E: "1" | |
| GH_TOKEN: ${{ github.token }} | |
| SHARD: ${{ matrix.shard }} | |
| run: | | |
| set -euo pipefail | |
| SHARDS=$E2E_SHARDS | |
| ../../scripts/e2e-select.sh > "$RUNNER_TEMP/selected.txt" 2> "$RUNNER_TEMP/select.log" | |
| cat "$RUNNER_TEMP/select.log" >&2 | |
| lanes=$((SHARDS * E2E_LANES)) | |
| go build -o "$RUNNER_TEMP/shardplan" ./shardplan | |
| for lane in $(seq 1 "$E2E_LANES"); do | |
| v=$(( (SHARD - 1) * E2E_LANES + lane )) | |
| if grep -q '^mode=all$' "$GITHUB_OUTPUT"; then | |
| "$RUNNER_TEMP/shardplan" -shards "$lanes" -shard "$v" < "$RUNNER_TEMP/selected.txt" > "$RUNNER_TEMP/names-$lane.txt" | |
| elif grep -q '^mode=none$' "$GITHUB_OUTPUT"; then | |
| # The change reaches no e2e test, so the list is empty and | |
| # shardplan would refuse it. Every shard stops here. | |
| : > "$RUNNER_TEMP/names-$lane.txt" | |
| else | |
| "$RUNNER_TEMP/shardplan" -shards "$lanes" -shard "$v" -fill "$E2E_LANE_FILL" < "$RUNNER_TEMP/selected.txt" > "$RUNNER_TEMP/names-$lane.txt" | |
| fi | |
| sed -i '/^$/d' "$RUNNER_TEMP/names-$lane.txt" | |
| done | |
| cat "$RUNNER_TEMP"/names-*.txt > "$RUNNER_TEMP/names.txt" | |
| if grep -q '^mode=all$' "$GITHUB_OUTPUT" && [ ! -s "$RUNNER_TEMP/names.txt" ]; then | |
| echo "shard $SHARD of $SHARDS matched no tests, which means the split is broken" >&2 | |
| exit 1 | |
| fi | |
| if [ -s "$RUNNER_TEMP/names.txt" ]; then | |
| echo "run=true" >> "$GITHUB_OUTPUT" | |
| wc -l < "$RUNNER_TEMP/names.txt" | xargs printf 'shard %s of %s runs %s tests\n' "$SHARD" "$SHARDS" | |
| else | |
| echo "run=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice title=No e2e tests in this shard::The change reaches no test this shard would run." | |
| echo "This shard runs no tests: the change reaches none of the tests it would run." >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| # These tests also run their case in fish, and the drag test in zsh, | |
| # and skip a shell that is not installed. Only a shard that holds one | |
| # pays for the packages. | |
| - name: Install fish and zsh for the shards that need them | |
| if: env.TESTED_URL == '' && steps.plan.outputs.run == 'true' | |
| run: | | |
| if grep -qxE 'TestDropPreviewLooks|TestDnDBetweenPanes|TestHerdrFrontAgentStartQuotesForTheShell|TestRailFileDragIntoPane' "$RUNNER_TEMP/names.txt"; then | |
| start=$(date +%s) | |
| sudo apt-get update -qq | |
| sudo apt-get install -y -qq --no-install-recommends fish zsh | |
| echo "Installed fish and zsh in $(( $(date +%s) - start )) s" | |
| fi | |
| # TestRailQuickLookKittyPicture resizes a PDF page, the case the bug was | |
| # reported on, and skips that step without pdftoppm. Only the shard that | |
| # holds the test pays for the package. | |
| - name: Install pdftoppm for the shard that needs it | |
| if: env.TESTED_URL == '' && steps.plan.outputs.run == 'true' | |
| run: | | |
| if grep -qx 'TestRailQuickLookKittyPicture' "$RUNNER_TEMP/names.txt"; then | |
| sudo apt-get update -qq | |
| sudo apt-get install -y -qq --no-install-recommends poppler-utils | |
| pdftoppm -v 2>&1 | head -1 | |
| fi | |
| - name: Build the binary under test | |
| if: env.TESTED_URL == '' && steps.plan.outputs.run == 'true' | |
| run: go build -o "$RUNNER_TEMP/tuios" ./cmd/tuios | |
| - name: Run this shard of the end-to-end suite | |
| if: env.TESTED_URL == '' && steps.plan.outputs.run == 'true' | |
| working-directory: e2e/tui | |
| env: | |
| TUIOS_E2E: "1" | |
| # -count=1 is enforced on the command line too, but pointing the suite | |
| # at an explicit binary keeps the build step's output out of the test | |
| # log and makes the artifact obvious. | |
| TUIOS_E2E_BIN: ${{ runner.temp }}/tuios | |
| SHARD: ${{ matrix.shard }} | |
| # -count=1 defeats Go's test cache, which otherwise can replay a stale | |
| # result across a change of binary. -race is deliberately absent: the | |
| # program under test is a child process, so the detector would | |
| # instrument only the harness. | |
| # | |
| # The lanes write to a log each, and the logs print one after another | |
| # when every lane is done, so no two lanes' lines interleave. | |
| run: | | |
| set -euo pipefail | |
| start=$SECONDS | |
| # How busy the runner is while the lanes run, for tuning E2E_LANES. | |
| vmstat -n 5 > "$RUNNER_TEMP/vmstat.log" & | |
| vmstat_pid=$! | |
| # The test listing in the step before built the suite's packages, so | |
| # the lanes find them in the build cache and only link. | |
| pids=() | |
| for lane in $(seq 1 "$E2E_LANES"); do | |
| [ -s "$RUNNER_TEMP/names-$lane.txt" ] || continue | |
| names=$(paste -sd'|' "$RUNNER_TEMP/names-$lane.txt") | |
| ( | |
| lstart=$SECONDS | |
| lstatus=0 | |
| go test -count=1 -timeout 12m -v -run "^($names)$" ./... > "$RUNNER_TEMP/e2e-$lane.log" 2>&1 || lstatus=$? | |
| echo "$lstatus" > "$RUNNER_TEMP/status-$lane" | |
| echo "$((SECONDS - lstart))" > "$RUNNER_TEMP/took-$lane" | |
| ) & | |
| pids+=("$!") | |
| done | |
| wait "${pids[@]}" | |
| kill "$vmstat_pid" || true | |
| awk 'NR > 3 { n++; idle += $15; wa += $16; if ($1 > r) r = $1 } | |
| END { if (n) printf "runner while the lanes ran: %d%% idle, %d%% io wait, run queue up to %d\n", idle / n, wa / n, r }' "$RUNNER_TEMP/vmstat.log" | |
| # A failure of tests listed in .github/flaky-tests.txt, and only of | |
| # those, gets one retry of those tests. Anything else fails the shard. | |
| status=0 | |
| limit=$(( ${E2E_LANE_LIMIT%m} * 60 )) | |
| : > "$RUNNER_TEMP/e2e.log" | |
| for lane in $(seq 1 "$E2E_LANES"); do | |
| [ -s "$RUNNER_TEMP/names-$lane.txt" ] || continue | |
| took=$(cat "$RUNNER_TEMP/took-$lane") | |
| lstatus=$(cat "$RUNNER_TEMP/status-$lane") | |
| echo "=== lane $lane of shard $SHARD: $(wc -l < "$RUNNER_TEMP/names-$lane.txt") tests, ${took}s, exit $lstatus" | |
| cat "$RUNNER_TEMP/e2e-$lane.log" | |
| cat "$RUNNER_TEMP/e2e-$lane.log" >> "$RUNNER_TEMP/e2e.log" | |
| [ "$lstatus" = 0 ] || status=1 | |
| # A lane that ran over the limit still passes, but says so, so the | |
| # drift shows before it reaches the timeout. | |
| if [ "$took" -gt "$limit" ]; then | |
| echo "::warning title=Slow e2e lane::Lane $lane of shard $SHARD ran ${took}s, over the $E2E_LANE_LIMIT limit. Refresh e2e/tui/shardplan/durations.txt or add a shard." | |
| fi | |
| done | |
| echo "shard $SHARD ran in $((SECONDS - start))s" | |
| if [ "$status" -ne 0 ]; then | |
| ../../scripts/retry-flaky.sh "$RUNNER_TEMP/e2e.log" ./... | |
| fi | |
| - name: Reap any surviving daemons | |
| # The harness tears down its own process groups, but a hard job failure | |
| # can leave one behind, and a leaked daemon would poison a self-hosted | |
| # runner. Never fail the job on this. | |
| if: always() && env.TESTED_URL == '' && steps.plan.outputs.run == 'true' | |
| run: pkill -f "$RUNNER_TEMP/tuios" || true |