From f606f99bfe1fb4057abbd9a99bb6d2589ab2cbe7 Mon Sep 17 00:00:00 2001 From: nej1gotnochill Date: Sun, 15 Nov 2026 14:18:37 +0530 Subject: [PATCH 1/2] Migrate nightly benchmark result persistence to dedicated branch Nightly package-mode runs #37-#41 computed their results and then failed at persistence: the generated result JSONs were committed to `main`, which the Protect Main ruleset only accepts through a pull request. Generated benchmark data is not reviewable source, so it moves to a dedicated `benchmark-results` branch and Pages assembles history from both branches. - nightly-benchmark: dispatch benchmark.yml with runtime_source=package, trust only the returned workflow_run_id, download just the two result artifacts and validate them before staging (package runtime, run_id, and one identical identity.runtime.package on both arms). Stage outside every checkout, commit exactly two files with cmp -s idempotency and a two-path assertion, rebase before pushing, retry only when the remote tip moved, never force-push, and dispatch Pages only after a successful push. - pages: check out main for code and pinned reference data plus benchmark-results for generated history, assemble site-results from both, never copy main's frozen pre-split run trees, and fail loudly on a missing branch or empty history instead of publishing an empty report. - recovery: recover_nightly_results.py re-imports retained artifacts into the data branch, dry-run by default, refusing main collisions, partial, differing, mixed-package and duplicate run IDs; --apply commits per run and never pushes. - tests: recover_nightly_results.py unit tests plus two harnesses that execute the real workflow steps against a PATH-shadowed git double, registered in pr-checks. The five affected runs (#37-#41) were recovered separately onto benchmark-results as commits 4641bc0, 9bd9929, f5577d7, 328eb67 and 7ae1a8f. Co-authored-by: Codebuff Assisted-by: ChatGPT --- .github/workflows/nightly-benchmark.yml | 199 +++++- .github/workflows/pages.yml | 77 ++- .github/workflows/pr-checks.yml | 3 + README.md | 68 +- benchmark/recover_nightly_results.py | 725 ++++++++++++++++++++++ benchmark/test_recover_nightly_results.py | 596 ++++++++++++++++++ tests/test_pages_assembly.sh | 202 ++++++ tests/test_persist_step.sh | 440 +++++++++++++ 8 files changed, 2261 insertions(+), 49 deletions(-) create mode 100644 benchmark/recover_nightly_results.py create mode 100644 benchmark/test_recover_nightly_results.py create mode 100644 tests/test_pages_assembly.sh create mode 100644 tests/test_persist_step.sh diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index 1469b28..c411854 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -9,11 +9,18 @@ name: "Nightly Benchmark (routine package mode)" # -> wait for that run to finish (fail here if it failed) # -> download only its benchmark result artifacts (tiny JSONs) # -> extract the normalized result JSONs (fail loudly if missing) -# -> commit them to benchmark/results/ and push to main +# -> stage them under persist-staging/ (never inside a git checkout) +# -> commit them to the unprotected `benchmark-results` data branch # -> dispatch pages.yml explicitly: a GITHUB_TOKEN push does not fire # pages.yml's push trigger (GitHub never triggers push events for # GITHUB_TOKEN pushes), so the report would otherwise go stale # +# Why a separate branch: `main` is protected by the "Protect Main" ruleset, +# which requires a pull request for every change. Generated benchmark data is +# not reviewable source; it lives on `benchmark-results` instead, and the +# Pages workflow assembles `main`'s code/pinned data with that branch's run +# history. Nothing here weakens or bypasses the ruleset. +# # Why dispatch instead of adding a schedule to benchmark.yml itself: # benchmark.yml is dispatch-only today, its inputs have defaults tuned for # source mode, and scheduled runs receive empty inputs. Dispatching it with @@ -22,8 +29,11 @@ name: "Nightly Benchmark (routine package mode)" # # Permissions (minimum required): # actions: write - dispatch benchmark.yml + download its run artifacts -# contents: write - commit result JSONs and push to main +# contents: write - commit result JSONs and push to benchmark-results # Nothing else. The dispatched run itself keeps benchmark.yml's permissions. +# +# `benchmark-results` must already exist (created once by a maintainer); this +# workflow never creates branches and never pushes to `main`. on: schedule: @@ -51,8 +61,10 @@ jobs: - name: Checkout benchmarking repo uses: actions/checkout@v6 with: - # Cheap; also keeps the rebase-before-push below safe even if a - # commit lands on main while the benchmark is running. + # Workflow code always comes from main, independent of which ref + # triggered this run. + ref: main + # Cheap; also keeps any rebase-before-push safe. fetch-depth: 0 - name: Dispatch benchmark.yml (package mode) and wait for it @@ -145,11 +157,13 @@ jobs: unzip -o -q "artifact-$artifact_id.zip" -d downloaded done < artifact-ids.txt - # Persist exactly the two normalized result JSONs, renamed to this + # Stage exactly the two normalized result JSONs, renamed to this # dispatch's run ID (the report keys runs by run_id, so the name must - # match the ID inside the file), into the locations the committed - # results already use. Everything else in the artifacts (raw JSONs, - # resource logs, tools/) stays out of the repository. + # match the ID inside the file), under persist-staging/. Staging is + # deliberately outside every git checkout: the results-branch checkout + # is a separate worktree and must only ever receive the two final files. + # Everything else in the artifacts (raw JSONs, resource logs, tools/) + # stays out of the repository. - name: Extract result JSONs run: | set -euo pipefail @@ -170,6 +184,8 @@ jobs: python3 - "$smoke" "$openswath" "$BENCH_RUN_ID" <<'PY' import json, sys smoke_path, openswath_path, expected = sys.argv[1], sys.argv[2], sys.argv[3] + REQUIRED = ("filename", "sha256", "archive_dir", "date") + packages = {} for label, path in (("smoke", smoke_path), ("openswath", openswath_path)): with open(path, encoding="utf-8") as fh: data = json.load(fh) @@ -186,32 +202,163 @@ jobs: run_id = data.get("run", {}).get("run_id") or data.get("run_id") if str(run_id) != expected: sys.exit(f"{label}: run_id mismatch (file says {run_id!r}, expected {expected})") - print("result JSONs valid; package runtime and run_id verified") + # Provenance: a package run must carry its package identity, using the + # four fields report_generate already requires (empty/null counts as + # missing). identity.package is not populated; identity.runtime.package + # is the real path. + pkg = ((data.get("identity") or {}).get("runtime") or {}).get("package") + if not isinstance(pkg, dict): + sys.exit(f"{label}: identity.runtime.package is missing or not an object " + f"({pkg!r}); refusing to persist a package run without provenance") + # A field must be a non-empty string: a truthy non-string (true, + # 123, []) would otherwise pass and corrupt the report's + # provenance rows. + absent = [k for k in REQUIRED + if not isinstance(pkg.get(k), str) or not pkg.get(k)] + if absent: + sys.exit(f"{label}: identity.runtime.package is missing required metadata: " + f"{', '.join(absent)} (empty or not a string)") + packages[label] = pkg + # Both arms of one benchmark run must describe the same package: the run + # is the unit of comparison, so a mixed pair would present one nightly + # build as two. Compare the contract fields only, so additive schema + # changes (extra package metadata) stay compatible. + for field in REQUIRED: + if packages["smoke"][field] != packages["openswath"][field]: + sys.exit(f"package identity mismatch for run {expected}: {field} " + f"smoke={packages['smoke'][field]!r} " + f"openswath={packages['openswath'][field]!r} " + f"- refusing to persist a mixed-package pair") + print("result JSONs valid; package runtime, run_id and package-pair identity verified") PY - mkdir -p benchmark/results/openms benchmark/results/openswath_dia/openms - cp "$smoke" "benchmark/results/openms/smoke-$BENCH_RUN_ID.json" - cp "$openswath" "benchmark/results/openswath_dia/openms/openswath_dia-$BENCH_RUN_ID.json" + mkdir -p "persist-staging/openms" "persist-staging/openswath_dia/openms" + cp "$smoke" "persist-staging/openms/smoke-$BENCH_RUN_ID.json" + cp "$openswath" "persist-staging/openswath_dia/openms/openswath_dia-$BENCH_RUN_ID.json" + echo "staged results for benchmark run $BENCH_RUN_ID" - - name: Commit results to main + # continue-on-error turns a missing/renamed data branch into the explicit + # message below instead of GitHub's generic "pathspec did not match". + - name: Checkout benchmark-results branch + id: results + continue-on-error: true + uses: actions/checkout@v6 + with: + ref: benchmark-results + path: results-branch + # Full history keeps the rebase-before-push below robust. + fetch-depth: 0 + + - name: Verify benchmark-results checkout + if: steps.results.outcome != 'success' + run: | + echo "::error::could not check out branch 'benchmark-results'. Generated benchmark results live there; a maintainer must create it (see README 'Results and reporting') before nightly runs can persist results." + exit 1 + + # Persist the staged pair onto the unprotected data branch. Idempotent + # for an identical re-run, loud for any conflicting/incomplete pair, and + # retried only when the remote tip actually moved (a genuine race) - + # never blindly, and never with --force. + - name: Persist results to benchmark-results env: - GH_TOKEN: ${{ github.token }} + STAGE_DIR: ${{ github.workspace }}/persist-staging + BRANCH_DIR: ${{ github.workspace }}/results-branch + RESULTS_BRANCH: benchmark-results + MAX_PUSH_ATTEMPTS: "3" run: | set -euo pipefail + RESULTS_BRANCH="${RESULTS_BRANCH:-benchmark-results}" + MAX_PUSH_ATTEMPTS="${MAX_PUSH_ATTEMPTS:-3}" + PUSH_LOG="${PUSH_LOG:-${RUNNER_TEMP:-/tmp}/benchmark-results-push.log}" + : "${STAGE_DIR:?STAGE_DIR must point at the staged result files}" + : "${BRANCH_DIR:?BRANCH_DIR must point at the benchmark-results checkout}" + + smoke_rel="openms/smoke-$BENCH_RUN_ID.json" + openswath_rel="openswath_dia/openms/openswath_dia-$BENCH_RUN_ID.json" + smoke_src="$STAGE_DIR/$smoke_rel" + openswath_src="$STAGE_DIR/$openswath_rel" + + for src in "$smoke_src" "$openswath_src"; do + if [ ! -s "$src" ]; then + echo "::error::staged result is missing or empty: $src" + exit 1 + fi + done + + cd "$BRANCH_DIR" + + if [ -e "$smoke_rel" ] || [ -e "$openswath_rel" ]; then + if [ -f "$smoke_rel" ] && [ -f "$openswath_rel" ] \ + && cmp -s "$smoke_rel" "$smoke_src" && cmp -s "$openswath_rel" "$openswath_src"; then + echo "benchmark run $BENCH_RUN_ID is already persisted byte-identically on $RESULTS_BRANCH; nothing to do" + exit 0 + fi + echo "::error::benchmark run $BENCH_RUN_ID already exists on $RESULTS_BRANCH but the pair is incomplete or differs; refusing to overwrite ($smoke_rel, $openswath_rel)" + exit 1 + fi + + mkdir -p "$(dirname "$smoke_rel")" "$(dirname "$openswath_rel")" + cp "$smoke_src" "$smoke_rel" + cp "$openswath_src" "$openswath_rel" + git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add benchmark/results - if git diff --cached --quiet; then + + git add -- "$smoke_rel" "$openswath_rel" + staged="$(git diff --cached --name-only)" + if [ -z "$staged" ]; then echo "::error::nothing new to persist for benchmark run $BENCH_RUN_ID (unexpected for a fresh run)" exit 1 fi + expected="$(printf '%s\n%s\n' "$smoke_rel" "$openswath_rel" | sort)" + if [ "$(printf '%s\n' "$staged" | sort)" != "$expected" ]; then + echo "::error::staged paths are not exactly the two expected result files:" + printf '%s\n' "$staged" >&2 + exit 1 + fi git commit -m "Persist package-mode benchmark results (benchmark run $BENCH_RUN_ID)" - git fetch origin main - git rebase origin/main - git push origin HEAD:main - - # Regenerate the report explicitly: the GITHUB_TOKEN push above does - # not trigger pages.yml's push trigger, but workflow_dispatch is the - # documented exception GITHUB_TOKEN dispatches are allowed to run. - # Last command of this step under set -e: only reached after a - # successful commit AND push, and a failed dispatch fails the job. - gh workflow run pages.yml --repo "$GITHUB_REPOSITORY" --ref main + + attempt=1 + while :; do + git fetch origin "$RESULTS_BRANCH" + if ! git rev-parse --verify --quiet "refs/remotes/origin/$RESULTS_BRANCH" >/dev/null; then + echo "::error::branch '$RESULTS_BRANCH' does not exist on the remote; a maintainer must create it before this workflow can persist results" + exit 1 + fi + base="$(git rev-parse "refs/remotes/origin/$RESULTS_BRANCH")" + if ! git rebase "refs/remotes/origin/$RESULTS_BRANCH"; then + git rebase --abort || true + echo "::error::rebase onto origin/$RESULTS_BRANCH hit a conflict; resolve manually (result files for a run ID must never be rewritten)" + exit 1 + fi + if git push origin "HEAD:$RESULTS_BRANCH" >"$PUSH_LOG" 2>&1; then + echo "pushed benchmark run $BENCH_RUN_ID to $RESULTS_BRANCH" + break + fi + cat "$PUSH_LOG" >&2 || true + if [ "$attempt" -ge "$MAX_PUSH_ATTEMPTS" ]; then + echo "::error::push to $RESULTS_BRANCH failed after $attempt attempt(s); not retrying further" + exit 1 + fi + # Retry ONLY for a genuine race: the push failed and the remote + # tip moved since the base we rebased onto. Any other failure + # (credentials, ruleset, protection, network) fails immediately + # instead of being retried as if it were a lost update. + git fetch origin "$RESULTS_BRANCH" + new_tip="$(git rev-parse "refs/remotes/origin/$RESULTS_BRANCH")" + if [ "$new_tip" = "$base" ]; then + echo "::error::push to $RESULTS_BRANCH failed and the remote tip did not move; this is not a race (check token permissions, ruleset scope, or branch protection)" + exit 1 + fi + echo "origin/$RESULTS_BRANCH moved from $base to $new_tip; rebasing and retrying (attempt $((attempt + 1))/$MAX_PUSH_ATTEMPTS)" + attempt=$((attempt + 1)) + done + + # Regenerate the report explicitly: the GITHUB_TOKEN push above does not + # trigger pages.yml's push trigger, but workflow_dispatch is the + # documented exception GITHUB_TOKEN dispatches are allowed to run. + # Separate step, so it only runs after the persistence step succeeded: + # a failed push, conflict, or permission error fails the job before this. + - name: Dispatch Pages + env: + GH_TOKEN: ${{ github.token }} + run: gh workflow run pages.yml --repo "$GITHUB_REPOSITORY" --ref main diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index fd213b9..c5b421a 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,6 +1,7 @@ name: Pages -# Static benchmark-results site (ProteoBench external-context comparison). +# Static benchmark-results site (ProteoBench external-context comparison + +# benchmark report). # # Two views: # pinned.html - the DEFAULT/reference view, generated from the vendored @@ -10,6 +11,16 @@ name: Pages # clone of the upstream results repository at its HEAD. # Clearly labeled "not the reference" on the page itself. # +# Storage split (see README "Results storage"): +# * main - workflow/report code, the pinned ProteoBench +# snapshot, and the local tool-reference JSONs. +# * benchmark-results - only generated OpenMS benchmark run JSONs. +# The report renderer takes a single --results-dir, so this workflow copies +# the generated run families from the data branch plus main's tools/ into +# `site-results/` before rendering. It deliberately never copies main's +# benchmark/results/openms or openswath_dia trees: they are frozen history +# from before the split, and merging them would double-count runs. +# # Deployment uses the official Actions/Deploy pattern with an artifact # (not a gh-pages branch) so no force-pushes or extra branches are needed. # Runs on manual dispatch or pushes to main; the Pages site itself must be @@ -33,8 +44,28 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout code and pinned reference data (main) + uses: actions/checkout@v6 + with: + ref: main + + # Separate checkout: generated results live on an unprotected data + # branch. continue-on-error turns a missing/renamed branch into the + # explicit failure message below instead of an opaque checkout error. + - name: Checkout generated results (benchmark-results) + id: data + continue-on-error: true uses: actions/checkout@v6 + with: + ref: benchmark-results + path: benchmark-data + fetch-depth: 1 + + - name: Verify benchmark-results checkout + if: steps.data.outcome != 'success' + run: | + echo "::error::could not check out branch 'benchmark-results'. Generated benchmark results live there; if the branch was renamed or deleted, restore it (see README 'Results and reporting') before publishing the site." + exit 1 - name: Set up Python uses: actions/setup-python@v5 @@ -50,6 +81,42 @@ jobs: git clone --depth 1 --quiet \ https://github.com/Proteobench/Results_quant_ion_DDA upstream + # One results tree for the renderer: generated runs from the data + # branch, tool-reference JSONs from main. Nothing else is copied. + - name: Assemble combined results tree + env: + DATA_DIR: benchmark-data + OUT_DIR: site-results + run: | + set -euo pipefail + DATA_DIR="${DATA_DIR:-benchmark-data}" + OUT_DIR="${OUT_DIR:-site-results}" + + for d in openms openswath_dia/openms; do + if [ ! -d "$DATA_DIR/$d" ]; then + echo "::error::$DATA_DIR/$d is missing: branch 'benchmark-results' must contain generated results at $d/" + exit 1 + fi + done + if [ ! -d "benchmark/results/tools" ]; then + echo "::error::benchmark/results/tools is missing; the local tool-reference JSONs must stay on main" + exit 1 + fi + + rm -rf "$OUT_DIR" + mkdir -p "$OUT_DIR/openms" "$OUT_DIR/openswath_dia/openms" "$OUT_DIR/tools" + cp -a "$DATA_DIR/openms/." "$OUT_DIR/openms/" + cp -a "$DATA_DIR/openswath_dia/openms/." "$OUT_DIR/openswath_dia/openms/" + cp -a benchmark/results/tools/. "$OUT_DIR/tools/" + + smoke="$(find "$OUT_DIR/openms" -maxdepth 1 -name 'smoke-*.json' | wc -l)" + openswath="$(find "$OUT_DIR/openswath_dia/openms" -maxdepth 1 -name 'openswath_dia-*.json' | wc -l)" + if [ "$smoke" -eq 0 ] || [ "$openswath" -eq 0 ]; then + echo "::error::branch 'benchmark-results' holds no usable history (smoke=$smoke, openswath=$openswath); refusing to publish an empty report" + exit 1 + fi + echo "assembled $smoke smoke run(s), $openswath openswath run(s), $(find "$OUT_DIR/tools" -name '*.json' | wc -l) tool reference(s)" + - name: Generate site (pinned reference view + optional latest view) run: | ARGS=(--site-dir site) @@ -59,10 +126,10 @@ jobs: echo "::warning::upstream clone unavailable; publishing pinned reference view only" fi python benchmark/pages_generate.py "${ARGS[@]}" - # Benchmark report from the committed results tree (Stage 1 runs, - # incl. their runtime provenance). Same renderer the CI report uses. + # Benchmark report from the generated run history (data branch) plus + # main's tool-reference JSONs. Same renderer the CI report uses. python benchmark/report/report_generate.py render \ - --results-dir benchmark/results --out site/report.html + --results-dir site-results --out site/report.html - name: Upload Pages artifact uses: actions/upload-pages-artifact@v3 diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 58cad4e..c5f9b52 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -55,6 +55,7 @@ jobs: benchmark/test_openswath_metrics.py \ benchmark/test_prose_metrics.py \ benchmark/test_proteobench_compare.py \ + benchmark/test_recover_nightly_results.py \ benchmark/test_runner_timestamps.py \ benchmark/report/test_generic_renderer.py \ benchmark/report/test_v2_schema.py; do @@ -68,3 +69,5 @@ jobs: set -euo pipefail bash tests/test_id_validation.sh bash tests/test_dispatch_step.sh + bash tests/test_pages_assembly.sh + bash tests/test_persist_step.sh diff --git a/README.md b/README.md index 7be5585..61c32cf 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# OpenMS Benchmarking +# OpenMS Benchmarking [![CI](https://github.com/OpenMS/OpenMS-benchmarking/actions/workflows/benchmark.yml/badge.svg)](https://github.com/OpenMS/OpenMS-benchmarking/actions/workflows/benchmark.yml) @@ -73,15 +73,15 @@ SWATH window list). The vendored fixtures include OpenMS' reference outputs, which the benchmark uses to validate correctness alongside the performance measurements. It runs in CI as a separate job after the OpenMS build. Runner: `benchmark/run_openswath_benchmark.sh` (definition: `openswath.json`). Raw -results are normalized with `normalize openswath` and stored under -`benchmark/results/openswath_dia/openms/`, giving OpenSwath the same -baseline/history/comparison lifecycle as the smoke benchmark. +results are normalized with `normalize openswath` and stored on the +`benchmark-results` branch under `openswath_dia/openms/`, giving OpenSwath the +same baseline/history/comparison lifecycle as the smoke benchmark. ### ProteoBench reference results ProteoBench results - produced externally by a local ProteoBench scorer -(quantms on the PXD028735 dataset) - are stored under -`benchmark/results/tools/` and pulled into the HTML report as external +(quantms on the PXD028735 dataset) - stay on `main` under +`benchmark/results/tools/` and are pulled into the HTML report as external reference context. They are not an OpenMS version-to-version baseline, and the report does not rank tools or workflows. Definition: `proteobench.json` (no runner - results are generated externally and normalized locally). @@ -105,7 +105,8 @@ raw benchmark results smoke.json / openswath.json, stages.tsv, logs report_generate.py normalize convert raw output to the common schema | v -benchmark/results/ stored, schema-versioned results +benchmark-results/ (branch) generated run history (nightly commits) +benchmark/results/tools/ reference results from other tools (main) | v report_generate.py render compare current run vs stored baseline @@ -153,17 +154,46 @@ and pipeline outputs. Results and reporting -------- -Results are machine-readable JSON, stored under `benchmark/results/` -(schema `openms-benchmarking/report/v2`; older v1 files are promoted -automatically): +Results are machine-readable JSON (schema +`openms-benchmarking/report/v2`; older v1 files are promoted automatically): ``` -benchmark/results/ - openms/ smoke-.json # OpenMS benchmark runs (the comparison history) - openswath_dia/openms/ # stored OpenSwath DIA runs (comparison history) - tools/ proteobench-*.json # reference results from other tools/sources +benchmark/results/ # on main + proteobench/external/ # pinned ProteoBench snapshot + tools/ # reference results from other tools + +benchmark-results (branch) # generated run history (nightly commits) + openms/smoke-.json + openswath_dia/openms/openswath_dia-.json ``` +`main` keeps code, the pinned ProteoBench snapshot and the tool-reference +JSONs; generated run JSONs are committed by the nightly workflow to the +unprotected `benchmark-results` branch. The Pages workflow assembles a +temporary `site-results/` tree from that branch's run families plus `main`'s +`tools/` and renders it, so the report keeps its full history without merging +`main`'s frozen pre-split results twice. + +Recover a nightly run whose persistence failed with +`benchmark/recover_nightly_results.py`: it is dry-run by default and never +pushes. After reviewing the summary, check out `benchmark-results`, re-run it +with `--apply` (one commit per benchmark run), review `git log` and push the +branch yourself. Before applying it fetches `origin/main` and +`origin/benchmark-results` and refuses to continue if either cannot be +resolved or if the local branch is behind the remote tip, so a stale clone can +neither miss a collision nor commit onto the wrong base. Both arms of a run +must also carry the same package identity (`identity.runtime.package`: +`filename`, `sha256`, `archive_dir`, `date`, each a non-empty string) - the +same contract the nightly workflow enforces before staging - so a +mixed-package or pair missing the required package identity is refused rather than imported. +Re-running the same selection is safe: runs already present byte-identically +are reported as already present and skipped, a partially present or differing +pair fails closed, and a failure partway through restores the checkout so the command can +simply be run again. The preflight fetch bounds, but does not eliminate, the +race with a branch moving during the download: `main` is protected, so no +concurrent run can add a result file there, and all writes target the data +branch, whose tip the nightly workflow re-fetches right before pushing. + `benchmark/report/report_generate.py` has two stages: `normalize` converts a raw result into the common schema, adding run metadata the runner cannot know (build wall time, ccache state, run id); `render` discovers all stored @@ -251,8 +281,9 @@ that have actually executed - it is never inferred from binary availability. (`.github/workflows/nightly-benchmark.yml`) runs the package-mode pipeline daily (and on `workflow_dispatch`): it dispatches `benchmark.yml` with `runtime_source=package`, waits for the run, persists the resulting -Smoke/OpenSwath result JSONs to `benchmark/results/`, and pushes them to -`main`; the Pages workflow then publishes the updated history. No build or +Smoke/OpenSwath result JSONs to the unprotected `benchmark-results` branch +(never to `main`); the Pages workflow then assembles the report input from +that branch plus main's `tools/` and publishes the updated history. No build or runner logic is involved. Jobs, in dependency order: @@ -340,8 +371,9 @@ benchmark/ test_v2_schema.py # v1 -> v2 promotion + schema tests test_generic_renderer.py # generic renderer tests results/ - openms/ # stored OpenMS run results (comparison baseline) - tools/ # reference results from other tools + proteobench/external/ # pinned ProteoBench snapshot (stays on main) + tools/ # reference results from other tools (stays on main) + # generated run JSONs live on the `benchmark-results` branch reports/ # generated report.html ``` diff --git a/benchmark/recover_nightly_results.py b/benchmark/recover_nightly_results.py new file mode 100644 index 0000000..2550fad --- /dev/null +++ b/benchmark/recover_nightly_results.py @@ -0,0 +1,725 @@ +#!/usr/bin/env python3 +"""Recover nightly package-mode benchmark results onto ``benchmark-results``. + +Context +------- +The nightly workflow (``.github/workflows/nightly-benchmark.yml``) persists +generated result JSONs to the unprotected ``benchmark-results`` branch. Before +that change it pushed to protected ``main``, so nightly runs #37-#41 failed +*after* their benchmark run succeeded. The benchmark artifacts are still +retained (90 days by default), so the validated JSONs can be re-imported. + +What this tool does +------------------- +1. resolves each nested benchmark run's unexpired artifacts through ``gh``; +2. downloads and extracts the smoke and OpenSwath result JSONs, refusing ZIP + members that are absolute, contain ``..``, or are symlinks; +3. validates every JSON: object shape, schema ``openms-benchmarking/report/v2``, + ``run.run_id`` equal to the benchmark run ID, filename suffix equal to that + ID, ``identity.runtime.runtime_source == "package"``, ``identity.benchmark`` + matching the expected benchmark, and a timezone-aware ``run.run_at`` newer + than the last persisted run; +3b. requires the same package identity on both arms of a run - the same + four non-empty-string fields (``filename``, ``sha256``, ``archive_dir``, + ``date``) enforced by the nightly persistence guard, so a mixed-package + pair can never be imported; +4. refreshes ``origin/main`` and ``origin/`` first and refuses to + import unless both resolve (never checks a ref that may be stale); +5. refuses a destination that exists on ``main``, and on the data branch accepts + only a complete, byte-identical pair - so a partially completed import can be + re-run: finished runs are reported as already present and skipped; +6. in dry-run mode (the default) only stages the files and prints a summary. + +With ``--apply`` it writes the two files per run into the current checkout of +``benchmark-results`` and commits them, one commit per benchmark run. It never +pushes, never creates branches, never dispatches workflows, never changes +repository settings or branch protection. A failure partway through restores +the checkout (no stray copies, no staged paths) before exiting, so the same +command can simply be run again to resume with the remaining runs. + +Recovery procedure +------------------ + # 1. Validate and stage only (no repository changes): + python benchmark/recover_nightly_results.py + + # 2. Review the summary, then switch to the data branch: + git switch benchmark-results + + # 3. Import locally (one commit per run, no push): + python benchmark/recover_nightly_results.py --apply + + # 4. Review and push yourself: + git log --oneline -5 + git push origin benchmark-results + +Exit codes: 0 = ok, 1 = validation/import failure, 2 = usage/environment error. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import json +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import zipfile +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath + +SCHEMA = "openms-benchmarking/report/v2" +DEFAULT_REPO = "OpenMS/OpenMS-benchmarking" +DEFAULT_BRANCH = "benchmark-results" +DEFAULT_MIN_RUN_AT = "2026-11-06T03:21:53Z" +DEFAULT_STAGING = Path(tempfile.gettempdir()) / "openms-benchmark-recovery" + +# Nested benchmark run -> nightly run that failed to persist it. +NIGHTLY_BY_BENCH = { + "42333314804": 37, + "42442257487": 38, + "42555927417": 39, + "42748636332": 40, + "42951989482": 41, +} + +BENCHMARKS = ( + ("smoke", "benchmark-results", "smoke-", "openms"), + ("openswath_dia", "benchmark-openswath", "openswath_dia-", "openswath_dia/openms"), +) + + +class RecoveryError(RuntimeError): + """A recoverable, user-facing failure (validation, gh, git).""" + + +@dataclasses.dataclass +class Validated: + """One benchmark run's validated result pair.""" + + bench_id: str + nightly: int | None + run_at: datetime + files: dict[str, Path] # branch-relative path -> staged file + + +def _run(cmd, cwd=None, check=True): + proc = subprocess.run( # noqa: S603 - fixed argv, no shell + cmd, cwd=cwd, capture_output=True, text=True, check=False + ) + if check and proc.returncode != 0: + raise RecoveryError( + f"{' '.join(cmd)} failed ({proc.returncode}): " + f"{proc.stderr.strip() or proc.stdout.strip()}" + ) + return proc + + +# --------------------------------------------------------------------------- +# downloads +# --------------------------------------------------------------------------- + + +def list_artifacts(repo: str, run_id: str) -> list[dict]: + """All artifacts of a workflow run, following pagination.""" + artifacts: list[dict] = [] + page = 1 + while True: + url = f"repos/{repo}/actions/runs/{run_id}/artifacts?per_page=100&page={page}" + payload = json.loads(_run(["gh", "api", url]).stdout) + page_items = payload.get("artifacts") or [] + artifacts.extend(page_items) + total = int(payload.get("total_count") or 0) + if not page_items or len(artifacts) >= total: + return artifacts + page += 1 + + +def select_artifacts(artifacts: list[dict]) -> dict[str, dict]: + """Exactly one unexpired results artifact and one openswath artifact.""" + selected: dict[str, dict] = {} + for benchmark, prefix, _file_prefix, _dir in BENCHMARKS: + matches = [ + a + for a in artifacts + if str(a.get("name", "")).startswith(prefix + "-") and not a.get("expired") + ] + if len(matches) != 1: + names = [a.get("name") for a in matches] + raise RecoveryError( + f"expected exactly one unexpired '{prefix}-*' artifact, found {len(matches)}: {names}" + ) + selected[benchmark] = matches[0] + return selected + + +def _unsafe_member(name: str) -> bool: + """True for a ZIP member that would escape the extraction directory. + + ZIP members are POSIX paths whatever the host OS, so separators are + normalized first and interpreted with ``PurePosixPath``. Backslash-rooted + (``\\outside.json``), UNC (``//host/share``), drive-letter (``C:\\x``), + absolute and ``..`` members are all rejected before anything is written. + """ + normalized = name.replace("\\", "/") + if not normalized or normalized.startswith("/"): + return True # absolute, UNC root, or empty name + if PurePosixPath(normalized).is_absolute(): + return True + if re.match(r"^[A-Za-z]:", normalized): + return True # drive-letter path + return ".." in PurePosixPath(normalized).parts + + +def safe_extract(zip_path: Path, dest: Path) -> None: + """Extract an artifact ZIP, rejecting unsafe member paths and symlinks.""" + with zipfile.ZipFile(zip_path) as zf: + for info in zf.infolist(): + if _unsafe_member(info.filename): + raise RecoveryError( + f"refusing unsafe ZIP member {info.filename!r} in {zip_path.name}" + ) + mode = info.external_attr >> 16 + if stat.S_ISLNK(mode): + raise RecoveryError( + f"refusing symlink ZIP member {info.filename!r} in {zip_path.name}" + ) + dest.mkdir(parents=True, exist_ok=True) + zf.extractall(dest) + + + +def download_artifact(repo: str, run_id: str, artifact: dict, workdir: Path) -> Path: + """Download an artifact ZIP and safely extract it.""" + artifact_id = artifact.get("id") + if ( + isinstance(artifact_id, bool) + or not str(artifact_id).isdigit() + or int(artifact_id) <= 0 + ): + raise RecoveryError(f"invalid artifact ID: {artifact_id!r}") + + workdir.mkdir(parents=True, exist_ok=True) + + zip_path = workdir / f"artifact-{artifact_id}.zip" + dest = workdir / f"artifact-{artifact_id}" + + command = [ + "gh", + "api", + f"repos/{repo}/actions/artifacts/{artifact_id}/zip", + ] + + with zip_path.open("wb") as archive: + proc = subprocess.run( + command, + stdout=archive, + stderr=subprocess.PIPE, + check=False, + ) + + if proc.returncode != 0: + zip_path.unlink(missing_ok=True) + error = proc.stderr.decode("utf-8", errors="replace").strip() + raise RecoveryError( + f"artifact download failed ({proc.returncode}): {error}" + ) + + try: + safe_extract(zip_path, dest) + finally: + zip_path.unlink(missing_ok=True) + + return dest + + +def find_result_json(root: Path, file_prefix: str) -> Path: + """The single result JSON named ``.json``.""" + matches = [ + p + for p in sorted(root.rglob("*.json")) + if p.name.startswith(file_prefix) and "tools" not in p.parts and "reference" not in p.parts + ] + if len(matches) != 1: + raise RecoveryError( + f"expected exactly one '{file_prefix}*.json' under {root}, found {len(matches)}: " + f"{[str(m.relative_to(root)) for m in matches]}" + ) + return matches[0] + + +# --------------------------------------------------------------------------- +# validation +# --------------------------------------------------------------------------- + + +def parse_run_at(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise RecoveryError(f"run.run_at is not timezone-aware: {value!r}") + return parsed + + +def validate_result( + path: Path, bench_id: str, benchmark: str, min_run_at: datetime +) -> datetime: + """Validate one result JSON against the persistence contract.""" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise RecoveryError(f"{path.name}: not valid JSON ({exc})") from exc + if not isinstance(data, dict): + raise RecoveryError(f"{path.name}: top-level JSON value is not an object") + if data.get("schema") != SCHEMA: + raise RecoveryError(f"{path.name}: schema is {data.get('schema')!r}, expected {SCHEMA!r}") + + run = data.get("run") or {} + if str(run.get("run_id")) != bench_id: + raise RecoveryError(f"{path.name}: run.run_id is {run.get('run_id')!r}, expected {bench_id!r}") + if not path.name.endswith(f"-{bench_id}.json"): + raise RecoveryError(f"{path.name}: filename does not end with '-{bench_id}.json'") + + identity = data.get("identity") or {} + runtime_source = (identity.get("runtime") or {}).get("runtime_source") + if runtime_source != "package": + raise RecoveryError( + f"{path.name}: identity.runtime.runtime_source is {runtime_source!r}, expected 'package'" + ) + if identity.get("benchmark") != benchmark: + raise RecoveryError( + f"{path.name}: identity.benchmark is {identity.get('benchmark')!r}, expected {benchmark!r}" + ) + + run_at_raw = run.get("run_at") + if not isinstance(run_at_raw, str): + raise RecoveryError(f"{path.name}: run.run_at is missing") + run_at = parse_run_at(run_at_raw) + if run_at <= min_run_at: + raise RecoveryError( + f"{path.name}: run.run_at {run_at_raw} is not newer than the last persisted run ({min_run_at.isoformat()})" + ) + return run_at + + +REQUIRED_PACKAGE_FIELDS = ("filename", "sha256", "archive_dir", "date") + + +def package_identity(path: Path) -> object: + """The (unvalidated) ``identity.runtime.package`` value of one result JSON.""" + data = json.loads(path.read_text(encoding="utf-8")) + runtime = (data.get("identity") or {}).get("runtime") or {} + return runtime.get("package") + + +def validate_package_pair(smoke_path: Path, openswath_path: Path, bench_id: str) -> None: + """Both arms of one benchmark run must describe the same OpenMS package. + + Mirrors the nightly persistence guard exactly: each required field must be + a non-empty string on both arms and all four must match. Unknown extra + package keys are ignored, so additive schema changes stay compatible. + ``identity.software.version`` is deliberately not required, and + ``identity.package`` is never read (it is null; the populated path is + ``identity.runtime.package``). + """ + packages: dict[str, dict] = {} + for label, path in (("smoke", smoke_path), ("openswath", openswath_path)): + pkg = package_identity(path) + if not isinstance(pkg, dict): + raise RecoveryError( + f"{path.name}: identity.runtime.package is missing or not an object ({pkg!r})" + ) + absent = [ + k + for k in REQUIRED_PACKAGE_FIELDS + if not isinstance(pkg.get(k), str) or not pkg.get(k) + ] + if absent: + raise RecoveryError( + f"{path.name}: identity.runtime.package is missing required metadata: " + f"{', '.join(absent)} (empty or not a string)" + ) + packages[label] = pkg + + for field in REQUIRED_PACKAGE_FIELDS: + if packages["smoke"][field] != packages["openswath"][field]: + raise RecoveryError( + f"benchmark run {bench_id}: package identity mismatch between smoke and " + f"openswath: {field} smoke={packages['smoke'][field]!r} " + f"openswath={packages['openswath'][field]!r}" + ) + + +# --------------------------------------------------------------------------- +# git helpers (read-only except in --apply) +# --------------------------------------------------------------------------- + + +def git(repo_dir: Path, *args, check=True): + return _run(["git", *args], cwd=repo_dir, check=check) + + +def ref_exists(repo_dir: Path, ref: str) -> bool: + return git(repo_dir, "rev-parse", "--verify", "--quiet", ref, check=False).returncode == 0 + + +def path_on_ref(repo_dir: Path, ref: str, rel: str) -> bool: + """True if ``rel`` exists on ``ref``; never calls cat-file for a missing ref.""" + if not ref_exists(repo_dir, ref): + return False + return git(repo_dir, "cat-file", "-e", f"{ref}:{rel}", check=False).returncode == 0 + + +def main_refs(repo_dir: Path) -> list[str]: + """Refs that must not already hold a destination, freshest first. + + ``origin/main`` is preferred because a local ``main`` can be stale; the + local ref is still checked as a belt-and-braces guard. + """ + candidates = ("refs/remotes/origin/main", "refs/heads/main") + found = [ref for ref in candidates if ref_exists(repo_dir, ref)] + return found or ["refs/heads/main"] + + +def ensure_no_main_collision(repo_dir: Path, rel_paths: list[str]) -> None: + """Refuse a destination that already exists on ``main`` (any known ref).""" + refs = main_refs(repo_dir) + for rel in rel_paths: + for ref in refs: + if path_on_ref(repo_dir, ref, rel): + raise RecoveryError(f"refusing to overwrite existing {rel} on {ref}") + + +def ensure_no_collision(repo_dir: Path, rel_paths: list[str]) -> None: + """Strict check: absent on ``main`` and absent in the working tree.""" + ensure_no_main_collision(repo_dir, rel_paths) + for rel in rel_paths: + if (repo_dir / rel).exists(): + raise RecoveryError(f"refusing to overwrite existing working-tree file {rel}") + + +def fetch_remote_ref(repo_dir: Path, branch: str, remote: str = "origin") -> str: + """Fetch ``main`` and ``branch`` from ``remote``; return the data-branch ref. + + Fails closed: an unreachable remote or a missing/renamed data branch must + never be read as "no collision" or "up to date". + """ + spec = [ + f"+refs/heads/main:refs/remotes/{remote}/main", + f"+refs/heads/{branch}:refs/remotes/{remote}/{branch}", + ] + proc = git(repo_dir, "fetch", "--quiet", remote, *spec, check=False) + if proc.returncode != 0: + raise RecoveryError( + f"could not fetch '{remote}' (main and {branch}); refusing to import against " + f"possibly stale refs: {proc.stderr.strip() or proc.stdout.strip()}" + ) + ref = f"refs/remotes/{remote}/{branch}" + if not ref_exists(repo_dir, ref): + raise RecoveryError( + f"could not establish {ref}; '{branch}' may not exist on '{remote}' - " + f"refusing to import (create the branch first)" + ) + return ref + + +def ensure_on_remote_base(repo_dir: Path, remote_ref: str) -> None: + """Refuse to import onto a local branch that is not based on ``remote_ref``.""" + proc = git(repo_dir, "merge-base", "--is-ancestor", remote_ref, "HEAD", check=False) + if proc.returncode != 0: + raise RecoveryError( + f"local branch is not based on {remote_ref}; fetch and fast-forward " + f"(or reset onto it) before --apply instead of importing onto a stale branch" + ) + + +def destination_state(repo_dir: Path, item: Validated) -> str: + """``"import"`` if the pair is absent, ``"already-imported"`` if identical. + + Any other shape (only one file, or different bytes) fails closed, so a + resumed import can never rewrite or half-complete an existing run. + """ + present = {rel for rel in item.files if (repo_dir / rel).exists()} + if not present: + return "import" + missing = sorted(set(item.files) - present) + if missing: + raise RecoveryError( + f"benchmark run {item.bench_id} is only partially present on the data branch " + f"(missing {missing}); refusing to complete a foreign partial pair" + ) + for rel, src in sorted(item.files.items()): + if (repo_dir / rel).read_bytes() != Path(src).read_bytes(): + raise RecoveryError( + f"benchmark run {item.bench_id} already exists with different contents " + f"({rel}); refusing to overwrite" + ) + return "already-imported" + + +def staged_paths(repo_dir: Path) -> list[str]: + out = git(repo_dir, "diff", "--cached", "--name-only").stdout + return [line for line in out.splitlines() if line.strip()] + + +# --------------------------------------------------------------------------- +# discovery + import +# --------------------------------------------------------------------------- + + +def collect( + repo: str, + bench_ids: list[str], + staging: Path, + min_run_at: datetime, +) -> list[Validated]: + """Download and validate everything before any repository change.""" + staging.mkdir(parents=True, exist_ok=True) + validated: list[Validated] = [] + for bench_id in bench_ids: + nightly = NIGHTLY_BY_BENCH.get(bench_id) + workdir = Path(tempfile.mkdtemp(prefix=f"{bench_id}-", dir=staging)) + workdir.mkdir(parents=True, exist_ok=True) + artifacts = select_artifacts(list_artifacts(repo, bench_id)) + files: dict[str, Path] = {} + run_ats: list[datetime] = [] + result_paths: dict[str, Path] = {} + dest_dirs: dict[str, str] = {} + for benchmark, _prefix, file_prefix, dest_dir in BENCHMARKS: + extracted = download_artifact(repo, bench_id, artifacts[benchmark], workdir) + result = find_result_json(extracted, file_prefix) + run_ats.append(validate_result(result, bench_id, benchmark, min_run_at)) + result_paths[benchmark] = result + dest_dirs[benchmark] = dest_dir + # Both arms must describe one package before anything is staged: a mixed + # or incomplete pair must never leave a half-staged file behind. + validate_package_pair(result_paths["smoke"], result_paths["openswath_dia"], bench_id) + for benchmark, result in result_paths.items(): + rel = f"{dest_dirs[benchmark]}/{result.name}" + target = staging / "validated" / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(result, target) + files[rel] = target + # Both arms come from the same benchmark run; keep the newest stamp. + run_at = max(run_ats) + print( + f" OK benchmark run {bench_id}" + f" (nightly #{nightly if nightly is not None else '?'})" + f" run_at={run_at.isoformat()}" + ) + print(f" smoke -> {files[f'openms/smoke-{bench_id}.json']}") + print( + f" openswath -> {files[f'openswath_dia/openms/openswath_dia-{bench_id}.json']}" + ) + validated.append( + Validated( + bench_id=bench_id, + nightly=nightly, + run_at=run_at, + files=files, + ) + ) + validated.sort(key=lambda v: (v.run_at, int(v.bench_id))) + return validated + + +def import_run(repo_dir: Path, branch: str, item: Validated) -> None: + """Copy one run's pair into the checkout and commit exactly those files. + + On any failure the copies and the staged paths are removed again, leaving + the checkout exactly as it was found so a re-run can resume cleanly. + """ + rel_paths = sorted(item.files) + try: + for rel, src in item.files.items(): + dest = repo_dir / rel + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dest) + + git(repo_dir, "add", "--", *rel_paths) + staged = sorted(staged_paths(repo_dir)) + if staged != rel_paths: + raise RecoveryError( + f"staged paths are not exactly the two result files for run {item.bench_id}: {staged}" + ) + subject = f"Recover package-mode benchmark results (benchmark run {item.bench_id})" + body = ( + f"Nightly run #{item.nightly} failed to persist this pair while results " + f"still targeted protected main. Imported from the retained benchmark " + f"artifact; contents unmodified." + ) + git(repo_dir, "commit", "-m", subject, "-m", body) + except BaseException: + git(repo_dir, "reset", "-q", "--", *rel_paths, check=False) + for rel in rel_paths: + (repo_dir / rel).unlink(missing_ok=True) + raise + print(f" committed 2 file(s) for benchmark run {item.bench_id} on {branch}") + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + description="Recover nightly benchmark results onto the benchmark-results branch " + "(dry-run by default; never pushes).", + epilog="See the module docstring for the full recovery procedure.", + ) + parser.add_argument("--repo", default=DEFAULT_REPO, help="owner/name on GitHub") + parser.add_argument("--repo-dir", default=".", help="local checkout to import into") + parser.add_argument("--branch", default=DEFAULT_BRANCH, help="data branch name") + parser.add_argument( + "--runs", + default=",".join(NIGHTLY_BY_BENCH), + help="comma-separated nested benchmark run IDs (default: all known)", + ) + parser.add_argument( + "--nightly", + default=None, + help="comma-separated nightly run numbers to select instead of --runs", + ) + parser.add_argument("--apply", action="store_true", help="write files and commit locally") + parser.add_argument( + "--staging-dir", + default=str(DEFAULT_STAGING), + help=f"where validated downloads are staged (default: {DEFAULT_STAGING})", + ) + parser.add_argument( + "--min-run-at", + default=DEFAULT_MIN_RUN_AT, + help=f"require run.run_at newer than this (default: {DEFAULT_MIN_RUN_AT})", + ) + args = parser.parse_args(argv) + + repo_dir = Path(args.repo_dir).resolve() + staging = Path(args.staging_dir).resolve() + min_run_at = parse_run_at(args.min_run_at) + + if args.nightly: + wanted = {int(token) for token in args.nightly.split(",") if token.strip()} + bench_ids = [b for b, n in NIGHTLY_BY_BENCH.items() if n in wanted] + missing = sorted(wanted - {NIGHTLY_BY_BENCH[b] for b in bench_ids}) + if missing: + print(f"ERROR: no known benchmark run for nightly {missing}", file=sys.stderr) + return 2 + else: + bench_ids = [token.strip() for token in args.runs.split(",") if token.strip()] + duplicates = sorted({ + bench_id for bench_id in bench_ids + if bench_ids.count(bench_id) > 1 + }) + if duplicates: + print(f"ERROR: duplicate benchmark run id(s): {duplicates}", file=sys.stderr) + return 2 + + unknown = [b for b in bench_ids if b not in NIGHTLY_BY_BENCH] + if unknown: + print(f"ERROR: unknown benchmark run id(s): {unknown}", file=sys.stderr) + return 2 + + current_branch = git(repo_dir, "rev-parse", "--abbrev-ref", "HEAD").stdout.strip() + on_data_branch = current_branch == args.branch + + if args.apply: + if not ref_exists(repo_dir, f"refs/heads/{args.branch}"): + print( + f"ERROR: local branch '{args.branch}' does not exist; a maintainer must create it " + f"and check it out before --apply (this tool never creates branches).", + file=sys.stderr, + ) + return 2 + if not on_data_branch: + print(f"ERROR: --apply requires branch '{args.branch}' to be checked out (now: {current_branch})", + file=sys.stderr) + return 2 + if git(repo_dir, "status", "--porcelain").stdout.strip(): + print("ERROR: working tree is not clean; commit or stash changes before --apply", + file=sys.stderr) + return 2 + # Refresh before touching anything: a stale clone must never decide + # whether a destination already exists, nor which tip to build on. + # Residual window, by design: `main` can still move while the artifacts + # below are downloaded, so this bounds the race rather than closing it. + # That is acceptable because `main` is protected (no concurrent result + # commits can appear there) and every write goes to the data branch, + # whose tip the nightly workflow re-fetches immediately before pushing. + try: + remote_ref = fetch_remote_ref(repo_dir, args.branch) + ensure_on_remote_base(repo_dir, remote_ref) + except RecoveryError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + print(f"== recovering {len(bench_ids)} benchmark run(s) from {args.repo} ==") + print(f" staging: {staging}") + print(f" branch: {args.branch}{'' if args.apply else ' (dry run; nothing is written or committed)'}") + + # Classify every run before writing anything: main collisions are always + # fatal, and on the data branch only an identical existing pair is skipped. + states: dict[str, str] = {} + try: + validated = collect(args.repo, bench_ids, staging, min_run_at) + for item in validated: + ensure_no_main_collision(repo_dir, sorted(item.files)) + if on_data_branch: + states[item.bench_id] = destination_state(repo_dir, item) + else: + ensure_no_collision(repo_dir, sorted(item.files)) + states[item.bench_id] = "import" + except RecoveryError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + print() + print(f"validated {len(validated)} run(s); all JSONs passed schema/run-id/package/time checks") + for item in validated: + nightly = f"#{item.nightly}" if item.nightly is not None else "?" + print( + f" nightly {nightly:>4} benchmark {item.bench_id} " + f"run_at {item.run_at.isoformat()} [{states[item.bench_id]}]" + ) + + pending = [item for item in validated if states[item.bench_id] == "import"] + if not args.apply: + print() + print( + f"{len(pending)} run(s) would be imported, " + f"{len(validated) - len(pending)} already present on '{args.branch}'" + ) + print("dry run: no repository changes. Re-run with --apply on the " + f"'{args.branch}' branch to import.") + return 0 + + imported: list[str] = [] + skipped: list[str] = [] + failed: list[str] = [] + for item in validated: + if states[item.bench_id] == "already-imported": + print(f" already present, skipping benchmark run {item.bench_id}") + skipped.append(item.bench_id) + continue + try: + import_run(repo_dir, args.branch, item) + except RecoveryError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + failed.append(item.bench_id) + break + imported.append(item.bench_id) + + print() + print(f"imported: {imported if imported else 'none'}") + print(f"already present: {skipped if skipped else 'none'}") + print(f"failed: {failed if failed else 'none'}") + if failed: + print( + "stopped at the first failure and restored the working tree; re-run the " + "same command to resume with the remaining runs.", + file=sys.stderr, + ) + return 1 + print(f"{len(imported)} commit(s) on {args.branch}. Review with " + f"'git log' and push the branch yourself; this tool never pushes.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmark/test_recover_nightly_results.py b/benchmark/test_recover_nightly_results.py new file mode 100644 index 0000000..d82794a --- /dev/null +++ b/benchmark/test_recover_nightly_results.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python3 +"""Tests for benchmark/recover_nightly_results.py (no network, no real gh). + +Run: python benchmark/test_recover_nightly_results.py +""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import recover_nightly_results as rnr # noqa: E402 + +BENCH_ID = "42333314804" +MIN_RUN_AT = rnr.parse_run_at(rnr.DEFAULT_MIN_RUN_AT) + + +_DEFAULT_PACKAGE = { + "filename": "OpenMS-3.7.0-pre-nightly-2026-11-06-Debian-Linux-x86_64.deb", + "sha256": "0" * 64, + "archive_dir": "2026.11.07", + "date": "2026-11-06", +} +_OMIT = object() # identity.runtime.package absent altogether +_DEFAULT = object() # use the standard package object + + +def payload( + bench_id: str, + benchmark: str = "smoke", + run_at: str = "2026-11-07T03:17:21Z", + package=_DEFAULT, +) -> dict: + pkg = _DEFAULT_PACKAGE if package is _DEFAULT else package + runtime: dict = {"runtime_source": "package"} + if pkg is not _OMIT: + runtime["package"] = pkg + return { + "schema": rnr.SCHEMA, + "identity": { + "benchmark": benchmark, + "benchmark_version": 1, + "dataset": "fixture", + "software": {"name": "OpenMS", "version": "5e974cdef84a778717bc156c842c9d46baecb604"}, + "configuration": {}, + "runtime": runtime, + }, + "run": {"run_id": bench_id, "run_at": run_at, "cache": "none"}, + "performance": { + "wall_time_s": 1.0, + "cpu_time_s": 0.1, + "peak_rss_kb": 1024, + "build": {"wall_time_s": 0.0, "artifact_bytes": 1}, + "stages": [], + }, + "metrics": {"verdict": "pass"}, + } + + +def git(cwd: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], cwd=cwd, check=True, capture_output=True, text=True + ).stdout + + +class ZipSafetyTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(lambda: __import__("shutil").rmtree(self.tmp, ignore_errors=True)) + + def _zip(self, name: str, member: str, data: str = "{}") -> Path: + path = self.tmp / name + with zipfile.ZipFile(path, "w") as zf: + zf.writestr(member, data) + return path + + def test_rejects_parent_traversal(self) -> None: + with self.assertRaises(rnr.RecoveryError): + rnr.safe_extract(self._zip("a.zip", "../evil.json"), self.tmp / "a") + + def test_rejects_absolute_path(self) -> None: + with self.assertRaises(rnr.RecoveryError): + rnr.safe_extract(self._zip("b.zip", "/etc/evil.json"), self.tmp / "b") + + def test_rejects_symlink_member(self) -> None: + path = self.tmp / "c.zip" + with zipfile.ZipFile(path, "w") as zf: + info = zipfile.ZipInfo("link") + info.external_attr = (0o120777 << 16) # symlink mode + zf.writestr(info, "target") + with self.assertRaises(rnr.RecoveryError): + rnr.safe_extract(path, self.tmp / "c") + + def test_rejects_backslash_rooted_member(self) -> None: + with self.assertRaises(rnr.RecoveryError): + rnr.safe_extract(self._zip("e.zip", "\\outside.json"), self.tmp / "e") + + def test_rejects_unc_member(self) -> None: + with self.assertRaises(rnr.RecoveryError): + rnr.safe_extract(self._zip("f.zip", "//server/share/evil.json"), self.tmp / "f") + + def test_rejects_drive_letter_member(self) -> None: + with self.assertRaises(rnr.RecoveryError): + rnr.safe_extract(self._zip("g.zip", "C:\\temp\\evil.json"), self.tmp / "g") + + def test_rejects_backslash_traversal(self) -> None: + with self.assertRaises(rnr.RecoveryError): + rnr.safe_extract(self._zip("h.zip", "..\\evil.json"), self.tmp / "h") + + def test_extracts_nested_member(self) -> None: + rnr.safe_extract( + self._zip("d.zip", "smoke/openms/smoke-1.json", '{"ok": true}'), self.tmp / "d" + ) + self.assertTrue((self.tmp / "d" / "smoke" / "openms" / "smoke-1.json").is_file()) + + +class FindResultTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(lambda: __import__("shutil").rmtree(self.tmp, ignore_errors=True)) + + def test_finds_single_result_and_ignores_tools(self) -> None: + (self.tmp / "smoke" / "openms").mkdir(parents=True) + (self.tmp / "tools").mkdir() + (self.tmp / "smoke" / "openms" / "smoke-1.json").write_text("{}") + (self.tmp / "tools" / "smoke-1.json").write_text("{}") + found = rnr.find_result_json(self.tmp, "smoke-") + self.assertEqual(found.name, "smoke-1.json") + self.assertIn("openms", found.parts) + + def test_missing_and_ambiguous_fail(self) -> None: + with self.assertRaises(rnr.RecoveryError): + rnr.find_result_json(self.tmp, "smoke-") + (self.tmp / "a").mkdir() + (self.tmp / "b").mkdir() + (self.tmp / "a" / "smoke-1.json").write_text("{}") + (self.tmp / "b" / "smoke-1.json").write_text("{}") + with self.assertRaises(rnr.RecoveryError): + rnr.find_result_json(self.tmp, "smoke-") + + +class ValidationTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(lambda: __import__("shutil").rmtree(self.tmp, ignore_errors=True)) + + def _write(self, data, name=None) -> Path: + path = self.tmp / (name or f"smoke-{BENCH_ID}.json") + path.write_text(json.dumps(data), encoding="utf-8") + return path + + def test_accepts_valid_package_result(self) -> None: + stamp = rnr.validate_result(self._write(payload(BENCH_ID)), BENCH_ID, "smoke", MIN_RUN_AT) + self.assertEqual(stamp.year, 2026) + + def test_rejects_each_contract_violation(self) -> None: + cases = { + "schema": lambda d: d.__setitem__("schema", "v1"), + "run_id": lambda d: d["run"].__setitem__("run_id", "999"), + "runtime": lambda d: d["identity"]["runtime"].__setitem__("runtime_source", "source"), + "benchmark": lambda d: d["identity"].__setitem__("benchmark", "openswath_dia"), + "naive_time": lambda d: d["run"].__setitem__("run_at", "2026-11-07T03:17:21"), + "old_time": lambda d: d["run"].__setitem__("run_at", "2026-11-01T00:00:00Z"), + "not_an_object": None, + } + for label, mutate in cases.items(): + with self.subTest(label=label): + if mutate is None: + path = self._write([payload(BENCH_ID)]) + else: + data = payload(BENCH_ID) + mutate(data) + path = self._write(data) + with self.assertRaises(rnr.RecoveryError): + rnr.validate_result(path, BENCH_ID, "smoke", MIN_RUN_AT) + + def test_rejects_filename_that_does_not_match_run_id(self) -> None: + data = payload(BENCH_ID) + path = self._write(data, name="smoke-11111111111.json") + with self.assertRaises(rnr.RecoveryError): + rnr.validate_result(path, BENCH_ID, "smoke", MIN_RUN_AT) + + +class DownloadArtifactTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(lambda: __import__("shutil").rmtree(self.tmp, ignore_errors=True)) + + @staticmethod + def _zip_bytes(member: str, data: str = '{"ok": true}') -> bytes: + import io + + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr(member, data) + return buffer.getvalue() + + def test_downloads_and_safely_extracts_artifact(self) -> None: + archive_bytes = self._zip_bytes("smoke/openms/smoke-1.json") + + def fake_run(command, stdout, stderr, check): + stdout.write(archive_bytes) + return subprocess.CompletedProcess(command, 0, b"", b"") + + with mock.patch.object(rnr.subprocess, "run", side_effect=fake_run) as run: + result = rnr.download_artifact( + "OpenMS/OpenMS-benchmarking", + BENCH_ID, + {"id": 123, "name": "untrusted-name"}, + self.tmp, + ) + + self.assertEqual( + run.call_args.args[0], + [ + "gh", + "api", + "repos/OpenMS/OpenMS-benchmarking/actions/artifacts/123/zip", + ], + ) + self.assertEqual( + (result / "smoke/openms/smoke-1.json").read_text(encoding="utf-8"), + '{"ok": true}', + ) + self.assertFalse((self.tmp / "artifact-123.zip").exists()) + + def test_rejects_invalid_artifact_ids_before_download(self) -> None: + for artifact_id in (None, True, 0, -1, "../123", "invalid"): + with self.subTest(artifact_id=artifact_id): + with mock.patch.object(rnr.subprocess, "run") as run: + with self.assertRaises(rnr.RecoveryError): + rnr.download_artifact( + "OpenMS/OpenMS-benchmarking", + BENCH_ID, + {"id": artifact_id}, + self.tmp, + ) + run.assert_not_called() + + def test_download_rejects_unsafe_zip_before_extraction(self) -> None: + archive_bytes = self._zip_bytes("../outside.json") + + def fake_run(command, stdout, stderr, check): + stdout.write(archive_bytes) + return subprocess.CompletedProcess(command, 0, b"", b"") + + with mock.patch.object(rnr.subprocess, "run", side_effect=fake_run): + with self.assertRaises(rnr.RecoveryError): + rnr.download_artifact( + "OpenMS/OpenMS-benchmarking", + BENCH_ID, + {"id": 124}, + self.tmp, + ) + + self.assertFalse((self.tmp / "outside.json").exists()) + self.assertFalse((self.tmp / "artifact-124.zip").exists()) + + def test_download_failure_reports_error_and_cleans_zip(self) -> None: + def fake_run(command, stdout, stderr, check): + return subprocess.CompletedProcess( + command, 1, b"", b"authentication failed" + ) + + with mock.patch.object(rnr.subprocess, "run", side_effect=fake_run): + with self.assertRaisesRegex( + rnr.RecoveryError, "authentication failed" + ): + rnr.download_artifact( + "OpenMS/OpenMS-benchmarking", + BENCH_ID, + {"id": 125}, + self.tmp, + ) + + self.assertFalse((self.tmp / "artifact-125.zip").exists()) + +class ArtifactSelectionTest(unittest.TestCase): + def test_requires_exactly_one_of_each(self) -> None: + ok = [ + {"name": "benchmark-results-abc", "id": 1, "expired": False}, + {"name": "benchmark-openswath-abc", "id": 2, "expired": False}, + {"name": "benchmark-report-abc", "id": 3, "expired": False}, + ] + selected = rnr.select_artifacts(ok) + self.assertEqual(selected["smoke"]["id"], 1) + self.assertEqual(selected["openswath_dia"]["id"], 2) + + def test_expired_and_duplicate_fail(self) -> None: + expired = [ + {"name": "benchmark-results-abc", "id": 1, "expired": True}, + {"name": "benchmark-openswath-abc", "id": 2, "expired": False}, + ] + with self.assertRaises(rnr.RecoveryError): + rnr.select_artifacts(expired) + duplicate = [ + {"name": "benchmark-results-abc", "id": 1, "expired": False}, + {"name": "benchmark-results-def", "id": 2, "expired": False}, + {"name": "benchmark-openswath-abc", "id": 3, "expired": False}, + ] + with self.assertRaises(rnr.RecoveryError): + rnr.select_artifacts(duplicate) + + +class RecoveryFlowTest(unittest.TestCase): + """End-to-end dry run and apply against a temporary git repository.""" + + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(lambda: __import__("shutil").rmtree(self.tmp, ignore_errors=True)) + self.repo = self.tmp / "repo" + self.repo.mkdir() + git(self.repo, "init", "-q", "-b", "main") + git(self.repo, "config", "user.email", "t@example.com") + git(self.repo, "config", "user.name", "Test") + (self.repo / "README.md").write_text("seed\n", encoding="utf-8") + git(self.repo, "add", "README.md") + git(self.repo, "commit", "-qm", "init") + git(self.repo, "branch", "benchmark-results") + + # A local bare repository stands in for origin, so the fetch-based + # freshness checks run for real without any network access. + self.origin = self.tmp / "origin.git" + subprocess.run(["git", "init", "--bare", "-q", str(self.origin)], check=True) + git(self.repo, "remote", "add", "origin", str(self.origin)) + git(self.repo, "push", "-q", "origin", "main", "benchmark-results") + git(self.repo, "fetch", "-q", "origin") + self.staging = self.tmp / "staging" + # Per-benchmark package override, used by the package-pair tests below. + self.package_overrides: dict[str, object] = {} + + self._patches = [ + mock.patch.object(rnr, "list_artifacts", side_effect=self._artifacts), + mock.patch.object(rnr, "download_artifact", side_effect=self._download), + ] + for patcher in self._patches: + patcher.start() + self.addCleanup(patcher.stop) + + # -- fakes ------------------------------------------------------------- + @staticmethod + def _artifacts(repo, run_id): + return [ + {"name": "benchmark-results-abc", "id": 1, "expired": False}, + {"name": "benchmark-openswath-abc", "id": 2, "expired": False}, + {"name": "benchmark-report-abc", "id": 3, "expired": False}, + ] + + def _download(self, repo, run_id, artifact, workdir): + dest = Path(workdir) / artifact["name"] + dest.mkdir(parents=True, exist_ok=True) + if artifact["name"].startswith("benchmark-results"): + rel = Path("smoke") / "openms" / f"smoke-{run_id}.json" + benchmark = "smoke" + else: + rel = Path("openswath_dia") / "openms" / f"openswath_dia-{run_id}.json" + benchmark = "openswath_dia" + (dest / rel).parent.mkdir(parents=True, exist_ok=True) + (dest / rel).write_text( + json.dumps(payload(run_id, benchmark, package=self.package_overrides.get(benchmark, _DEFAULT))), + encoding="utf-8", + ) + # Decoy that must be ignored by discovery. + (dest / "tools").mkdir(exist_ok=True) + (dest / "tools" / f"smoke-{run_id}.json").write_text("{}", encoding="utf-8") + return dest + + def _args(self, *extra: str) -> list[str]: + return [ + "--repo-dir", + str(self.repo), + "--staging-dir", + str(self.staging), + "--runs", + BENCH_ID, + *extra, + ] + + # -- tests ------------------------------------------------------------- + def test_dry_run_stages_but_does_not_touch_the_repo(self) -> None: + before = git(self.repo, "rev-list", "--count", "HEAD").strip() + self.assertEqual(rnr.main(self._args()), 0) + self.assertEqual(git(self.repo, "rev-list", "--count", "HEAD").strip(), before) + self.assertEqual(git(self.repo, "status", "--porcelain").strip(), "") + staged = self.staging / "validated" + self.assertTrue((staged / "openms" / f"smoke-{BENCH_ID}.json").is_file()) + self.assertTrue( + (staged / "openswath_dia" / "openms" / f"openswath_dia-{BENCH_ID}.json").is_file() + ) + + # -- package-pair contract (parity with the nightly persistence guard) --- + def _assert_pair_rejected(self) -> None: + """A bad pair fails closed before staging, committing, or recovery.""" + before = git(self.repo, "rev-list", "--count", "HEAD").strip() + self.assertEqual(rnr.main(self._args()), 1) + self.assertEqual(git(self.repo, "rev-list", "--count", "HEAD").strip(), before) + self.assertEqual(git(self.repo, "status", "--porcelain").strip(), "") + self.assertFalse((self.repo / "openms" / f"smoke-{BENCH_ID}.json").exists()) + self.assertFalse( + (self.repo / "openswath_dia" / "openms" / f"openswath_dia-{BENCH_ID}.json").exists() + ) + staged = self.staging / "validated" + self.assertFalse((staged / "openms" / f"smoke-{BENCH_ID}.json").exists()) + self.assertFalse( + (staged / "openswath_dia" / "openms" / f"openswath_dia-{BENCH_ID}.json").exists() + ) + + def test_rejects_mismatched_package_pair(self) -> None: + self.package_overrides["openswath_dia"] = dict(_DEFAULT_PACKAGE, sha256="1" * 64) + self._assert_pair_rejected() + + def test_rejects_missing_package_object(self) -> None: + self.package_overrides["smoke"] = _OMIT + self._assert_pair_rejected() + + def test_rejects_null_package_object(self) -> None: + self.package_overrides["openswath_dia"] = None + self._assert_pair_rejected() + + def test_rejects_empty_package_field(self) -> None: + self.package_overrides["smoke"] = dict(_DEFAULT_PACKAGE, archive_dir="") + self._assert_pair_rejected() + + def test_rejects_missing_package_field(self) -> None: + partial = dict(_DEFAULT_PACKAGE) + del partial["sha256"] + self.package_overrides["openswath_dia"] = partial + self._assert_pair_rejected() + + def test_rejects_non_string_package_field(self) -> None: + self.package_overrides["openswath_dia"] = dict(_DEFAULT_PACKAGE, date=True) + self._assert_pair_rejected() + + def test_accepts_unknown_extra_package_keys(self) -> None: + extra = dict(_DEFAULT_PACKAGE, bytes=12345) + self.package_overrides["smoke"] = extra + self.package_overrides["openswath_dia"] = extra + self.assertEqual(rnr.main(self._args()), 0) + self.assertTrue( + (self.staging / "validated" / "openms" / f"smoke-{BENCH_ID}.json").is_file() + ) + + def test_apply_commits_one_commit_with_both_files(self) -> None: + git(self.repo, "switch", "-q", "benchmark-results") + self.assertEqual(rnr.main(self._args("--apply")), 0) + self.assertTrue((self.repo / "openms" / f"smoke-{BENCH_ID}.json").is_file()) + self.assertTrue( + (self.repo / "openswath_dia" / "openms" / f"openswath_dia-{BENCH_ID}.json").is_file() + ) + self.assertEqual(git(self.repo, "status", "--porcelain").strip(), "") + self.assertEqual(git(self.repo, "rev-list", "--count", "HEAD").strip(), "2") + subject = git(self.repo, "log", "-1", "--format=%s").strip() + self.assertIn(f"benchmark run {BENCH_ID}", subject) + + def test_apply_requires_the_data_branch_checked_out(self) -> None: + self.assertEqual(rnr.main(self._args("--apply")), 2) + + def test_apply_requires_a_clean_tree(self) -> None: + git(self.repo, "switch", "-q", "benchmark-results") + (self.repo / "dirty.txt").write_text("x", encoding="utf-8") + self.assertEqual(rnr.main(self._args("--apply")), 2) + + def test_missing_local_branch_is_a_usage_error(self) -> None: + git(self.repo, "branch", "-D", "benchmark-results") + git(self.repo, "switch", "-q", "-c", "other") + self.assertEqual(rnr.main(self._args("--apply")), 2) + + def test_collision_on_branch_refuses_to_overwrite(self) -> None: + git(self.repo, "switch", "-q", "benchmark-results") + target = self.repo / "openms" / f"smoke-{BENCH_ID}.json" + target.parent.mkdir(parents=True) + target.write_text('{"keep": "me"}', encoding="utf-8") + # Committed collision: the clean-tree requirement passes and the + # collision guard is what refuses the import. + git(self.repo, "add", f"openms/smoke-{BENCH_ID}.json") + git(self.repo, "commit", "-qm", "existing result") + self.assertEqual(rnr.main(self._args("--apply")), 1) + self.assertEqual(target.read_text(encoding="utf-8"), '{"keep": "me"}') + + def test_collision_on_main_ref_refuses(self) -> None: + # main still holds the frozen pre-split results; importing the same id + # must be refused even though the branch checkout is clean. + git(self.repo, "switch", "-q", "main") + target = self.repo / "openms" / "smoke-999.json" + target.parent.mkdir(parents=True) + target.write_text("{}", encoding="utf-8") + git(self.repo, "add", "openms/smoke-999.json") + git(self.repo, "commit", "-qm", "frozen") + rel = "openms/smoke-999.json" + self.assertTrue(rnr.path_on_ref(self.repo, "refs/heads/main", rel)) + with self.assertRaises(rnr.RecoveryError): + rnr.ensure_no_collision(self.repo, [rel]) + + def test_collision_on_fetched_origin_main_refuses_when_local_main_is_stale(self) -> None: + rel = "openms/smoke-777.json" + git(self.repo, "switch", "-q", "main") + target = self.repo / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("{}", encoding="utf-8") + git(self.repo, "add", rel) + git(self.repo, "commit", "-qm", "upstream only") + git(self.repo, "push", "-q", "origin", "main") + git(self.repo, "reset", "-q", "--hard", "HEAD~1") # local main now stale + git(self.repo, "fetch", "-q", "origin") + self.assertFalse(rnr.path_on_ref(self.repo, "refs/heads/main", rel)) + self.assertTrue(rnr.path_on_ref(self.repo, "refs/remotes/origin/main", rel)) + with self.assertRaises(rnr.RecoveryError): + rnr.ensure_no_main_collision(self.repo, [rel]) + + def test_partial_import_resumes_without_duplicating(self) -> None: + other = "42442257487" + git(self.repo, "switch", "-q", "benchmark-results") + args = [ + "--repo-dir", + str(self.repo), + "--staging-dir", + str(self.staging), + "--runs", + f"{BENCH_ID},{other}", + "--apply", + ] + + real_git = rnr.git + + def failing_git(repo_dir, *cmd, check=True): + if cmd and cmd[0] == "commit" and other in " ".join(cmd): + raise rnr.RecoveryError("simulated commit failure") + return real_git(repo_dir, *cmd, check=check) + + with mock.patch.object(rnr, "git", side_effect=failing_git): + self.assertEqual(rnr.main(args), 1) + + # The failed run was rolled back, so the checkout is clean and only the + # first run is committed. + self.assertEqual(git(self.repo, "status", "--porcelain").strip(), "") + self.assertEqual(git(self.repo, "rev-list", "--count", "HEAD").strip(), "2") + self.assertTrue((self.repo / "openms" / f"smoke-{BENCH_ID}.json").is_file()) + self.assertFalse((self.repo / "openms" / f"smoke-{other}.json").exists()) + + # Re-running the same selection skips the completed run and imports the + # remaining one exactly once. + self.assertEqual(rnr.main(args), 0) + self.assertEqual(git(self.repo, "rev-list", "--count", "HEAD").strip(), "3") + self.assertTrue((self.repo / "openms" / f"smoke-{other}.json").is_file()) + subjects = git(self.repo, "log", "--format=%s").splitlines() + self.assertEqual(sum(BENCH_ID in subject for subject in subjects), 1) + self.assertEqual(sum(other in subject for subject in subjects), 1) + + def test_unreachable_remote_fails_closed_before_writing(self) -> None: + git(self.repo, "switch", "-q", "benchmark-results") + commits = git(self.repo, "rev-list", "--count", "HEAD").strip() + git(self.repo, "remote", "set-url", "origin", str(self.tmp / "missing.git")) + self.assertEqual(rnr.main(self._args("--apply")), 1) + self.assertEqual(git(self.repo, "rev-list", "--count", "HEAD").strip(), commits) + self.assertEqual(git(self.repo, "status", "--porcelain").strip(), "") + + def test_stale_local_data_branch_is_refused(self) -> None: + git(self.repo, "switch", "-q", "benchmark-results") + git(self.repo, "switch", "-q", "main") + (self.repo / "upstream.txt").write_text("x", encoding="utf-8") + git(self.repo, "add", "upstream.txt") + git(self.repo, "commit", "-qm", "remote moved ahead") + git(self.repo, "push", "-q", "origin", "main:benchmark-results") + git(self.repo, "switch", "-q", "benchmark-results") + commits = git(self.repo, "rev-list", "--count", "HEAD").strip() + self.assertEqual(rnr.main(self._args("--apply")), 1) + self.assertEqual(git(self.repo, "rev-list", "--count", "HEAD").strip(), commits) + self.assertEqual(git(self.repo, "status", "--porcelain").strip(), "") + + +class CliValidationTest(unittest.TestCase): + def test_duplicate_run_ids_fail_before_artifact_discovery(self) -> None: + stderr = io.StringIO() + + with mock.patch.object(rnr, "list_artifacts") as list_artifacts: + with contextlib.redirect_stderr(stderr): + result = rnr.main(["--runs", f"{BENCH_ID},{BENCH_ID}"]) + + self.assertEqual(result, 2) + self.assertIn("duplicate benchmark run id(s)", stderr.getvalue()) + self.assertIn(BENCH_ID, stderr.getvalue()) + list_artifacts.assert_not_called() + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_pages_assembly.sh b/tests/test_pages_assembly.sh new file mode 100644 index 0000000..0d88f3e --- /dev/null +++ b/tests/test_pages_assembly.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# Executes the ACTUAL "Assemble combined results tree" step extracted from +# .github/workflows/pages.yml, proving the report input contract: +# * both generated run families come from the data branch +# * main's tools/ reference JSONs are retained +# * main's frozen benchmark/results/openms tree is NOT copied +# * missing data, empty history, or missing tools fail with a clear message +# * the assembled tree is exactly what the renderer expects (no duplicates) +# +# LOCAL ONLY: no network, no GH_TOKEN. +set -euo pipefail +IFS=$'\n\t' + +HERE="$(cd "$(dirname "$0")" && pwd)" +BENCH_DIR="$(cd "$HERE/.." && pwd)" +PAGES="$BENCH_DIR/.github/workflows/pages.yml" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +PYTHON="${PYTHON:-}" +if [[ -z "$PYTHON" ]]; then + for cand in python3 python py; do + cpath="$(command -v "$cand" 2>/dev/null || true)" + [[ -n "$cpath" ]] || continue + if "$cpath" -c 'import yaml' >/dev/null 2>&1; then PYTHON="$cpath"; break; fi + done +fi +[[ -n "$PYTHON" ]] || { + echo "ERROR: no working python with PyYAML found (tried python3, python, py; set PYTHON=...)" >&2 + exit 2 +} + +STEP="$TMP/step.sh" +WF_IN="$PAGES"; STEP_IN="$STEP" +if [[ "$("$PYTHON" -c 'import sys; print(sys.platform)')" == "win32" ]]; then + WF_IN="$(cygpath -w "$PAGES")" + STEP_IN="$(cygpath -w "$STEP")" +fi +"$PYTHON" - "$WF_IN" "$STEP_IN" <<'PY' +import sys, yaml +doc = yaml.safe_load(open(sys.argv[1], encoding="utf-8")) +steps = doc["jobs"]["build"]["steps"] +step = next(s for s in steps if s.get("name", "").startswith("Assemble combined results tree")) +with open(sys.argv[2], "w", encoding="utf-8", newline="\n") as fh: + fh.write(step["run"]) +PY + +# fixture_result +fixture_result() { + "$PYTHON" - "$1" "$2" "$3" <<'PY' +import json, sys +path, run_id, benchmark = sys.argv[1], sys.argv[2], sys.argv[3] +json.dump({ + "schema": "openms-benchmarking/report/v2", + "identity": { + "benchmark": benchmark, + "benchmark_version": 1, + "dataset": "fixture", + "software": {"name": "OpenMS", "version": "abc123"}, + "configuration": {}, + "runtime": {"runtime_source": "package"}, + }, + "run": {"run_id": run_id, "run_at": "2026-11-07T03:17:21Z", "cache": "none"}, + "performance": {"wall_time_s": 1.0, "cpu_time_s": 0.1, "peak_rss_kb": 1024, + "build": {"wall_time_s": 0.0, "artifact_bytes": 1}, "stages": []}, + "metrics": {"verdict": "pass"}, +}, open(path, "w", encoding="utf-8")) +PY +} + +# make_repo +make_repo() { + local with_data="$1" with_runs="$2" with_tools="$3" + REPO="$TMP/repo" + rm -rf "$REPO" + mkdir -p "$REPO/benchmark/results/openms" + # Frozen pre-split history on main: must never be copied into site-results. + fixture_result "$REPO/benchmark/results/openms/smoke-42155957466.json" 42155957466 smoke + if [[ "$with_tools" == "yes" ]]; then + mkdir -p "$REPO/benchmark/results/tools" + fixture_result "$REPO/benchmark/results/tools/proteobench-Exp-1.json" Exp-1 proteobench + fi + if [[ "$with_data" == "yes" ]]; then + mkdir -p "$REPO/benchmark-data/openms" "$REPO/benchmark-data/openswath_dia/openms" + if [[ "$with_runs" == "yes" ]]; then + fixture_result "$REPO/benchmark-data/openms/smoke-42333314804.json" 42333314804 smoke + fixture_result "$REPO/benchmark-data/openms/smoke-42951989482.json" 42951989482 smoke + fixture_result \ + "$REPO/benchmark-data/openswath_dia/openms/openswath_dia-42333314804.json" \ + 42333314804 openswath_dia + fixture_result \ + "$REPO/benchmark-data/openswath_dia/openms/openswath_dia-42951989482.json" \ + 42951989482 openswath_dia + fi + fi +} + +passed=0 +failed=0 +total=0 + +# run_step +run_step() { + local name="$1" expect_rc="$2" expect_message="$3" + total=$((total + 1)) + local rc=0 + ( cd "$REPO" && DATA_DIR=benchmark-data OUT_DIR=site-results bash "$STEP" ) \ + > "$TMP/out.log" 2> "$TMP/err.log" || rc=$? + local ok=1 why="" + if [[ "$rc" -ne "$expect_rc" ]]; then + ok=0; why="rc=$rc want=$expect_rc; stderr: $(head -c 300 "$TMP/err.log")" + fi + if [[ "$expect_message" != "-" ]] && ! grep -qF -- "$expect_message" "$TMP/out.log" "$TMP/err.log"; then + ok=0; why="$why; missing message '$expect_message'" + fi + if [[ "$ok" -eq 1 ]]; then + passed=$((passed + 1)); echo "OK: $name" + else + failed=$((failed + 1)); echo "FAIL: $name --$why" >&2 + fi +} + +echo "== Pages results-tree assembly tests ==" +echo + +make_repo yes yes yes +run_step "assembles both run families plus main's tools/" 0 "assembled 2 smoke run(s), 2 openswath run(s), 1 tool reference(s)" +total=$((total + 1)) +if [[ -f "$REPO/site-results/openms/smoke-42333314804.json" \ + && -f "$REPO/site-results/openswath_dia/openms/openswath_dia-42333314804.json" \ + && -f "$REPO/site-results/tools/proteobench-Exp-1.json" ]]; then + passed=$((passed + 1)); echo "OK: expected files land under site-results/" +else + failed=$((failed + 1)); echo "FAIL: assembled tree is missing expected files" >&2 +fi +total=$((total + 1)) +if [[ -e "$REPO/site-results/openms/smoke-42155957466.json" ]]; then + failed=$((failed + 1)); echo "FAIL: frozen main history was copied into site-results" >&2 +else + passed=$((passed + 1)); echo "OK: frozen main history is not copied" +fi +total=$((total + 1)) +counts="$("$PYTHON" - "$REPO/site-results" "$BENCH_DIR/benchmark/report" <<'PY' +import sys +sys.path.insert(0, sys.argv[2]) +from report_generate import load_results +runs, tools = load_results(sys.argv[1]) +print(f"{len(runs)} {len(tools)}") +PY +)" +if [[ "$counts" == "4 1" ]]; then + passed=$((passed + 1)); echo "OK: renderer sees exactly 4 runs and 1 tool result (no duplicates)" +else + failed=$((failed + 1)); echo "FAIL: renderer saw '$counts' (want '4 1')" >&2 +fi + +make_repo no yes yes +run_step "missing data checkout fails clearly" 1 "is missing" +make_repo yes no yes +run_step "empty run history fails clearly" 1 "no usable history" +make_repo yes yes no +run_step "missing tools/ on main fails clearly" 1 "tools is missing" + +echo +echo "== workflow-level guarantees ==" +total=$((total + 1)) +if "$PYTHON" - "$WF_IN" <<'PY' +import sys, yaml + +doc = yaml.safe_load(open(sys.argv[1], encoding="utf-8")) +steps = doc["jobs"]["build"]["steps"] +text = open(sys.argv[1], encoding="utf-8").read() + +assert "--results-dir site-results" in text +assert "--results-dir benchmark/results" not in text +assemble = next( + s for s in steps if s.get("name", "").startswith("Assemble combined results tree") +) +assemble_run = assemble["run"] +assert "benchmark/results/openms" not in assemble_run, "assembly must not copy the frozen run history" +assert "benchmark/results/tools" in assemble_run +code = [s for s in steps if s.get("name", "").startswith("Checkout code")] +data = [s for s in steps if s.get("name", "").startswith("Checkout generated results")] +assert code and code[0]["with"]["ref"] == "main" +assert data and data[0]["with"]["ref"] == "benchmark-results" +PY +then + passed=$((passed + 1)); echo "OK: data source, render argument, and no frozen-history copy" +else + failed=$((failed + 1)); echo "FAIL: workflow-level guarantees" >&2 +fi + +echo +echo "== summary ==" +echo "total: $total" +echo "passed: $passed" +echo "failed: $failed" +if [[ "$failed" -gt 0 ]]; then + echo "RESULT: FAIL" >&2 + exit 1 +fi +echo "RESULT: PASS" diff --git a/tests/test_persist_step.sh b/tests/test_persist_step.sh new file mode 100644 index 0000000..1eb0eeb --- /dev/null +++ b/tests/test_persist_step.sh @@ -0,0 +1,440 @@ +#!/usr/bin/env bash +# Executes the ACTUAL "Persist results to benchmark-results" step extracted +# from .github/workflows/nightly-benchmark.yml against a stubbed `git`, +# proving the persistence contract: +# * the push target is exactly HEAD:benchmark-results, never main +# * only the two expected result files are staged and committed +# * an identical re-run is a no-op; a differing/incomplete pair fails +# * a non-race push failure is not retried; a genuine race is retried +# * a rebase conflict fails before any push +# * no force-push anywhere +# +# LOCAL ONLY: `git` is a shell function; nothing contacts GitHub. +set -euo pipefail +IFS=$'\n\t' + +HERE="$(cd "$(dirname "$0")" && pwd)" +BENCH_DIR="$(cd "$HERE/.." && pwd)" +WORKFLOW="$BENCH_DIR/.github/workflows/nightly-benchmark.yml" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +PYTHON="${PYTHON:-}" +if [[ -z "$PYTHON" ]]; then + for cand in python3 python py; do + cpath="$(command -v "$cand" 2>/dev/null || true)" + [[ -n "$cpath" ]] || continue + if "$cpath" -c 'import yaml' >/dev/null 2>&1; then PYTHON="$cpath"; break; fi + done +fi +[[ -n "$PYTHON" ]] || { + echo "ERROR: no working python with PyYAML found (tried python3, python, py; set PYTHON=...)" >&2 + exit 2 +} + +STEP="$TMP/step.sh" +WF_IN="$WORKFLOW"; STEP_IN="$STEP" +if [[ "$("$PYTHON" -c 'import sys; print(sys.platform)')" == "win32" ]]; then + WF_IN="$(cygpath -w "$WORKFLOW")" + STEP_IN="$(cygpath -w "$TMP/step.sh")" +fi +"$PYTHON" - "$WF_IN" "$STEP_IN" <<'PY' +import sys, yaml +doc = yaml.safe_load(open(sys.argv[1], encoding="utf-8")) +steps = doc["jobs"]["persist-nightly-package-results"]["steps"] +step = next(s for s in steps if s.get("name", "").startswith("Persist results to")) +with open(sys.argv[2], "w", encoding="utf-8", newline="\n") as fh: + fh.write(step["run"]) +PY + +# The pre-persistence package-pair guard lives in the "Extract result JSONs" +# step, so the harness executes THAT step too instead of reimplementing it. +EXTRACT_STEP="$TMP/extract_step.sh" +EXTRACT_IN="$EXTRACT_STEP" +if [[ "$("$PYTHON" -c 'import sys; print(sys.platform)')" == "win32" ]]; then + EXTRACT_IN="$(cygpath -w "$EXTRACT_STEP")" +fi +"$PYTHON" - "$WF_IN" "$EXTRACT_IN" <<'PY' +import sys, yaml +doc = yaml.safe_load(open(sys.argv[1], encoding="utf-8")) +steps = doc["jobs"]["persist-nightly-package-results"]["steps"] +step = next(s for s in steps if s.get("name", "").startswith("Extract result JSONs")) +with open(sys.argv[2], "w", encoding="utf-8", newline="\n") as fh: + fh.write(step["run"]) +PY + +BENCH_RUN_ID="42333314804" +SMOKE_REL="openms/smoke-$BENCH_RUN_ID.json" +OW_REL="openswath_dia/openms/openswath_dia-$BENCH_RUN_ID.json" + +passed=0 +failed=0 +total=0 + +# case_setup +case_setup() { + local stage="$1" branch="$2" + WORK="$TMP/work" + rm -rf "$WORK" + mkdir -p "$WORK/stage/openms" "$WORK/stage/openswath_dia/openms" "$WORK/branch" + echo '{"smoke":1}' > "$WORK/stage/$SMOKE_REL" + if [[ "$stage" == "full" ]]; then + echo '{"openswath":1}' > "$WORK/stage/$OW_REL" + fi + case "$branch" in + identical) + mkdir -p "$WORK/branch/openms" "$WORK/branch/openswath_dia/openms" + cp "$WORK/stage/$SMOKE_REL" "$WORK/branch/$SMOKE_REL" + cp "$WORK/stage/$OW_REL" "$WORK/branch/$OW_REL" + ;; + differs) + mkdir -p "$WORK/branch/openms" "$WORK/branch/openswath_dia/openms" + echo '{"other":1}' > "$WORK/branch/$SMOKE_REL" + echo '{"other":1}' > "$WORK/branch/$OW_REL" + ;; + incomplete) + mkdir -p "$WORK/branch/openms" + cp "$WORK/stage/$SMOKE_REL" "$WORK/branch/$SMOKE_REL" + ;; + esac +} + +# run_case +# [unexpected_staged yes|no] +# The persistence step is executed as a child process (see run_case), so the +# git stub must be an executable rather than a shell function. It is placed +# first on PATH and never delegates to the real git. +mkdir -p "$TMP/bin" +cat > "$TMP/bin/git" <<'STUB' +#!/usr/bin/env bash +# Test double for git: logs every invocation, never runs a real git command. +{ printf 'git'; printf ' %s' "$@"; printf ' +'; } >> "${CALL_LOG:?CALL_LOG must be set}" +case "$1" in + config|add|commit|fetch) exit 0 ;; + diff) + if [ "${UNEXPECTED_STAGED:-no}" = "yes" ]; then + printf '%s +' "${SMOKE_REL:?}" "README.md" + else + printf '%s +' "${SMOKE_REL:?}" "${OW_REL:?}" + fi + exit 0 ;; + rev-parse) + case "$*" in + *--verify*) + [ "${GIT_FIXTURE:-ok}" = "remote_missing" ] && exit 1 + echo "revA"; exit 0 ;; + esac + n=$(( $(cat "${STATE:?}/rev.count" 2>/dev/null || echo 0) + 1 )) + echo "$n" > "${STATE}/rev.count" + if [ "${GIT_FIXTURE:-ok}" = "push_fail_then_race" ] && [ "$n" -ge 2 ]; then + echo "revB" + else + echo "revA" + fi + exit 0 ;; + rebase) + [ "${2:-}" = "--abort" ] && exit 0 + [ "${GIT_FIXTURE:-ok}" = "rebase_conflict" ] && exit 1 + exit 0 ;; + push) + p=$(( $(cat "${STATE:?}/push.count" 2>/dev/null || echo 0) + 1 )) + echo "$p" > "${STATE}/push.count" + echo "push-target: $*" >> "${STATE}/push.args" + [ "${GIT_FIXTURE:-ok}" = "push_fail_no_move" ] && exit 1 + if [ "${GIT_FIXTURE:-ok}" = "push_fail_then_race" ] && [ "$p" -eq 1 ]; then + exit 1 + fi + exit 0 ;; + *) exit 0 ;; +esac +STUB +chmod +x "$TMP/bin/git" + +# Hermeticity: the child process must resolve `git` to the stub above. If the +# stub were ever bypassed the step would run real git against the fixture +# directory, so this is asserted once, before any case runs. +total=$((total + 1)) +resolved_git="$(PATH="$TMP/bin:$PATH" TMP="$TMP" bash -c 'command -v git')" +if [[ ! -x "$TMP/bin/git" || "$resolved_git" != "$TMP/bin/git" ]]; then + failed=$((failed + 1)) + echo "FAIL: child environment resolves git to '$resolved_git', expected '$TMP/bin/git'" >&2 +else + passed=$((passed + 1)) + echo "OK: child process resolves git to the test stub" +fi + +run_case() { + local name="$1" fixture="$2" stage="$3" branch="$4" expect_rc="$5" + local expect_message="$6" expect_pushes="$7" unexpected="${8:-no}" + total=$((total + 1)) + case_setup "$stage" "$branch" + local call_log="$WORK/git.calls" push_count_file="$WORK/push.count" rev_file="$WORK/rev.count" + : > "$call_log"; : > "$push_count_file"; : > "$rev_file" + + local rc=0 + set +e + ( + export BENCH_RUN_ID STAGE_DIR="$WORK/stage" BRANCH_DIR="$WORK/branch" + export RESULTS_BRANCH="benchmark-results" MAX_PUSH_ATTEMPTS=3 + export GIT_FIXTURE="$fixture" CALL_LOG="$call_log" UNEXPECTED_STAGED="$unexpected" + export STATE="$WORK" SMOKE_REL OW_REL + export PATH="$TMP/bin:$PATH" + + bash -e -o pipefail "$STEP" + ) > "$WORK/out.log" 2> "$WORK/err.log" + rc=$? + set -e + + local ok=1 why="" pushes + pushes="$(cat "$push_count_file" 2>/dev/null || echo 0)" + if [[ "$rc" -ne "$expect_rc" ]]; then + ok=0; why="rc=$rc want=$expect_rc; stderr: $(head -c 300 "$WORK/err.log")" + fi + if [[ "$expect_message" != "-" ]] && ! grep -qF -- "$expect_message" "$WORK/out.log" "$WORK/err.log"; then + ok=0; why="$why; missing message '$expect_message'" + fi + if grep -qiE "HEAD:main|push origin main|--force" "$call_log"; then + ok=0; why="$why; call log contains a main/force push" + fi + if [[ "$pushes" -ne "$expect_pushes" ]]; then + ok=0; why="$why; expected $expect_pushes push(es), got $pushes" + fi + if [[ "$expect_pushes" -gt 0 ]] && ! grep -qF -- "push origin HEAD:benchmark-results" "$call_log"; then + ok=0; why="$why; missing expected push target; calls: $(tr ' +' '|' <"$call_log")" + fi + + if [[ "$ok" -eq 1 ]]; then + passed=$((passed + 1)); echo "OK: $name" + else + failed=$((failed + 1)); echo "FAIL: $name --$why" >&2 + fi +} + +echo "== nightly persistence contract tests (stubbed git, no network) ==" +echo + +run_case "fresh run: commits the pair and pushes to benchmark-results" \ + ok full empty 0 "pushed benchmark run $BENCH_RUN_ID" 1 +total=$((total + 1)) +if [[ "$(cat "$TMP/work/branch/$SMOKE_REL")" != '{"smoke":1}' ]]; then + failed=$((failed + 1)); echo "FAIL: persisted smoke file content mismatch" >&2 +else + passed=$((passed + 1)); echo "OK: persisted files land at the expected branch paths" +fi + +run_case "identical re-run is an idempotent no-op" \ + ok full identical 0 "already persisted byte-identically" 0 +total=$((total + 1)) +if [[ -s "$TMP/work/git.calls" ]]; then + failed=$((failed + 1)); echo "FAIL: no-op case invoked git: $(tr '\n' ' ' <"$TMP/work/git.calls")" >&2 +else + passed=$((passed + 1)); echo "OK: no-op case performs no git operation" +fi + +run_case "differing existing pair is refused" \ + ok full differs 1 "refusing to overwrite" 0 +run_case "incomplete existing pair is refused" \ + ok full incomplete 1 "incomplete or differs" 0 +run_case "missing staged file fails before touching the branch" \ + ok missing-openswath empty 1 "staged result is missing or empty" 0 +run_case "unexpected staged path fails" \ + ok full empty 1 "not exactly the two expected" 0 yes +run_case "non-race push failure is not retried" \ + push_fail_no_move full empty 1 "not a race" 1 +run_case "genuine race is retried once and succeeds" \ + push_fail_then_race full empty 0 "moved from revA to revB" 2 +run_case "rebase conflict fails without pushing" \ + rebase_conflict full empty 1 "hit a conflict" 0 +run_case "missing remote branch fails with a clear message" \ + remote_missing full empty 1 "does not exist on the remote" 0 + +echo +echo "== workflow-level guarantees ==" +total=$((total + 1)) +if "$PYTHON" - "$WF_IN" <<'PY' +import sys, yaml + +doc = yaml.safe_load(open(sys.argv[1], encoding="utf-8")) +steps = doc["jobs"]["persist-nightly-package-results"]["steps"] +names = [s.get("name", "") for s in steps] +text = open(sys.argv[1], encoding="utf-8").read() + +assert "HEAD:main" not in text, "workflow still pushes to main" +assert "push origin main" not in text, "workflow still pushes to main" +persist_run = next(s for s in steps if s.get("name", "").startswith("Persist results to"))["run"] +assert "--force" not in persist_run, "persist step force-pushes" + +pages = [s for s in steps if "gh workflow run pages.yml" in (s.get("run") or "")] +assert len(pages) == 1, f"expected exactly one Pages dispatch, got {[p['name'] for p in pages]}" +assert names.index(pages[0]["name"]) > names.index( + next(n for n in names if n.startswith("Persist results to")) +), "Pages is dispatched before persistence" + +persist = next(s for s in steps if s.get("name", "").startswith("Persist results to")) +assert "gh " not in (persist.get("run") or ""), "persist step must not dispatch workflows" +checkout = [s for s in steps if s.get("name", "").startswith("Checkout benchmark-results")] +assert checkout and checkout[0]["with"]["ref"] == "benchmark-results" +PY +then + passed=$((passed + 1)); echo "OK: dispatch order, push target, and no force-push" +else + failed=$((failed + 1)); echo "FAIL: workflow-level guarantees" >&2 +fi + +# --------------------------------------------------------------------------- +# Package-pair consistency guard: executes the ACTUAL "Extract result JSONs" +# step (the pre-persistence validation) against generated result JSONs. +# --------------------------------------------------------------------------- +FNAME_A="OpenMS-3.7.0-pre-nightly-2026-11-06-Debian-Linux-x86_64.deb" +SHA_A="3d56f94e9f4209b38e3e61e3bfffad896c55ddee21c5d946e39fdf3bda9e9a1e" +SHA_B="c0664adffdbf4591a796dc323004696b9b7df5164a56fcdb8afb286b7c3c821a" + +# pkg [extra json suffix] +pkg() { + printf '{"filename":"%s","sha256":"%s","archive_dir":"%s","date":"%s"%s}' \ + "$1" "$2" "$3" "$4" "${5:-}" +} + +PKG_OK="$(pkg "$FNAME_A" "$SHA_A" 2026.11.07 2026-11-06)" + +# The step invokes `python3`; keep it working even when only `python`/`py` is +# on PATH (the harness already resolved a working interpreter). +mkdir -p "$TMP/bin" +cat > "$TMP/bin/python3" < +make_result_json() { + "$PYTHON" - "$1" "$2" "$3" "$4" "$5" <<'PY' +import json, sys +path, bench, run_id, runtime_source, pkg = sys.argv[1:6] +doc = { + "schema": "openms-benchmarking/report/v2", + "identity": { + "benchmark": bench, + "software": {"name": "OpenMS", "version": "05518e68abaa58b39f66b841e08be7be207cdaa8"}, + "runtime": {"runtime_source": runtime_source, "package": json.loads(pkg)}, + }, + "run": {"run_id": run_id, "run_at": "2026-11-07T03:18:46Z"}, +} +with open(path, "w", encoding="utf-8") as fh: + json.dump(doc, fh) +PY +} + +staging_is_empty() { + local staging="$1/persist-staging" + [[ ! -e "$staging" ]] && return 0 + [[ -d "$staging" ]] || return 1 + [[ -z "$(find "$staging" -type f -print -quit)" ]] +} + +# run_extract_case \ +# [smoke_runtime] [ow_runtime] +run_extract_case() { + local name="$1" smoke_pkg="$2" ow_pkg="$3" expect_rc="$4" msg="$5" + local smoke_rt="${6:-package}" ow_rt="${7:-package}" + total=$((total + 1)) + local WORKX="$TMP/extract-work" + rm -rf "$WORKX" + mkdir -p "$WORKX/downloaded/openms" "$WORKX/downloaded/openswath_dia/openms" + make_result_json "$WORKX/downloaded/smoke-$BENCH_RUN_ID.json" \ + smoke "$BENCH_RUN_ID" "$smoke_rt" "$smoke_pkg" + make_result_json "$WORKX/downloaded/openswath_dia/openms/openswath_dia-$BENCH_RUN_ID.json" \ + openswath_dia "$BENCH_RUN_ID" "$ow_rt" "$ow_pkg" + + # Run the step the way GitHub Actions does (bash -e -o pipefail) rather than + # sourcing it: errexit is suppressed inside a subshell that is part of an + # `||` list, which would hide a non-zero exit from the step's python guard. + set +e + ( + cd "$WORKX" + export BENCH_RUN_ID + export PATH="$TMP/bin:$PATH" + bash -e -o pipefail "$EXTRACT_STEP" + ) > "$WORKX/out.log" 2> "$WORKX/err.log" + local rc=$? + set -e + + local ok=1 why="" + if [[ "$rc" -ne "$expect_rc" ]]; then + ok=0; why="rc=$rc want=$expect_rc; stderr: $(head -c 300 "$WORKX/err.log")" + fi + if [[ "$msg" != "-" ]] && ! grep -qF -- "$msg" "$WORKX/out.log" "$WORKX/err.log"; then + ok=0; why="$why; missing message '$msg'; stderr: $(head -c 300 "$WORKX/err.log")" + fi + if [[ "$expect_rc" -ne 0 ]]; then + if ! staging_is_empty "$WORKX"; then + ok=0; why="$why; persist-staging was populated despite the failure" + fi + else + if [[ ! -s "$WORKX/persist-staging/openms/smoke-$BENCH_RUN_ID.json" ]] \ + || [[ ! -s "$WORKX/persist-staging/openswath_dia/openms/openswath_dia-$BENCH_RUN_ID.json" ]]; then + ok=0; why="$why; expected staged pair is missing" + fi + fi + + if [[ "$ok" -eq 1 ]]; then + passed=$((passed + 1)); echo "OK: $name" + else + failed=$((failed + 1)); echo "FAIL: $name --$why" >&2 + fi +} + +echo "== package-pair consistency guard (actual Extract result JSONs step) ==" +echo + +run_extract_case "matching package metadata stages both JSONs" \ + "$PKG_OK" "$PKG_OK" 0 "package-pair identity verified" + +run_extract_case "differing sha256 fails" \ + "$PKG_OK" "$(pkg "$FNAME_A" "$SHA_B" 2026.11.07 2026-11-06)" \ + 1 "package identity mismatch for run $BENCH_RUN_ID: sha256" + +run_extract_case "differing filename fails" \ + "$PKG_OK" "$(pkg OpenMS-3.7.0-pre-nightly-2026-11-10-Debian-Linux-x86_64.deb "$SHA_A" 2026.11.07 2026-11-06)" \ + 1 "package identity mismatch for run $BENCH_RUN_ID: filename" + +run_extract_case "missing package object on one side fails" \ + "$PKG_OK" null 1 "identity.runtime.package is missing or not an object" + +run_extract_case "null package objects fail" \ + null null 1 "identity.runtime.package is missing or not an object" + +run_extract_case "empty required metadata fails" \ + "$PKG_OK" "$(pkg "$FNAME_A" "" 2026.11.07 2026-11-06)" \ + 1 "missing required metadata: sha256" + +run_extract_case "unknown additional package field is ignored" \ + "$PKG_OK" "$(pkg "$FNAME_A" "$SHA_A" 2026.11.07 2026-11-06 ',"bytes":12345')" \ + 0 "package-pair identity verified" + +run_extract_case "differing date fails" \ + "$PKG_OK" "$(pkg "$FNAME_A" "$SHA_A" 2026.11.07 2026-11-10)" \ + 1 "package identity mismatch for run $BENCH_RUN_ID: date" + +# A truthy non-string (here: boolean true) must be rejected, not compared. +run_extract_case "non-string required metadata fails" \ + "$PKG_OK" "{\"filename\":\"$FNAME_A\",\"sha256\":\"$SHA_A\",\"archive_dir\":\"2026.11.07\",\"date\":true}" \ + 1 "identity.runtime.package is missing required metadata: date (empty or not a string)" + +run_extract_case "runtime_source source still fails with the existing diagnostic" \ + "$PKG_OK" "$PKG_OK" 1 "identity.runtime.runtime_source is 'source'" package source + +echo +echo "== summary ==" +echo "total: $total" +echo "passed: $passed" +echo "failed: $failed" +if [[ "$failed" -gt 0 ]]; then + echo "RESULT: FAIL" >&2 + exit 1 +fi +echo "RESULT: PASS" From df7413a50e7740cbb0b3d3a0ef2605f4dc32cac6 Mon Sep 17 00:00:00 2001 From: nej1gotnochill Date: Tue, 17 Nov 2026 09:37:24 +0530 Subject: [PATCH 2/2] Fix benchmark persistence review findings Co-authored-by: Codebuff Assisted-by: ChatGPT --- .github/workflows/pages.yml | 108 ++++++++++--- README.md | 44 +++-- benchmark/recover_nightly_results.py | 61 +++++-- benchmark/test_recover_nightly_results.py | 51 ++++-- tests/test_pages_assembly.sh | 185 ++++++++++++++++------ 5 files changed, 348 insertions(+), 101 deletions(-) diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index c5b421a..4a7c50c 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -13,13 +13,18 @@ name: Pages # # Storage split (see README "Results storage"): # * main - workflow/report code, the pinned ProteoBench -# snapshot, and the local tool-reference JSONs. -# * benchmark-results - only generated OpenMS benchmark run JSONs. -# The report renderer takes a single --results-dir, so this workflow copies -# the generated run families from the data branch plus main's tools/ into -# `site-results/` before rendering. It deliberately never copies main's -# benchmark/results/openms or openswath_dia trees: they are frozen history -# from before the split, and merging them would double-count runs. +# snapshot, the local tool-reference JSONs, and the +# historical generated runs from before the split. +# * benchmark-results - generated OpenMS benchmark run JSONs from the split +# onwards. +# The report renderer takes a single --results-dir, so this workflow assembles +# `site-results/` as the UNION of two sources per run family: main's historical +# runs under benchmark/results// and the data branch's generated runs +# at /. Dropping either source would lose runs that only it holds, so +# neither is excluded. The same relative path may legitimately appear in both; +# identical copies are collapsed to one, while differing bytes at the same path +# fail the build instead of one copy silently winning. main's tools/ is copied +# as before. # # Deployment uses the official Actions/Deploy pattern with an artifact # (not a gh-pages branch) so no force-pushes or extra branches are needed. @@ -81,15 +86,18 @@ jobs: git clone --depth 1 --quiet \ https://github.com/Proteobench/Results_quant_ion_DDA upstream - # One results tree for the renderer: generated runs from the data - # branch, tool-reference JSONs from main. Nothing else is copied. + # One results tree for the renderer: main's historical runs plus the data + # branch's generated runs for both families, and main's tool-reference + # JSONs. Both run sources are kept, so no history is discarded. - name: Assemble combined results tree env: DATA_DIR: benchmark-data + HISTORY_DIR: benchmark/results OUT_DIR: site-results run: | set -euo pipefail DATA_DIR="${DATA_DIR:-benchmark-data}" + HISTORY_DIR="${HISTORY_DIR:-benchmark/results}" OUT_DIR="${OUT_DIR:-site-results}" for d in openms openswath_dia/openms; do @@ -98,24 +106,84 @@ jobs: exit 1 fi done - if [ ! -d "benchmark/results/tools" ]; then - echo "::error::benchmark/results/tools is missing; the local tool-reference JSONs must stay on main" + if [ ! -d "$HISTORY_DIR/tools" ]; then + echo "::error::$HISTORY_DIR/tools is missing; the local tool-reference JSONs must stay on main" exit 1 fi rm -rf "$OUT_DIR" mkdir -p "$OUT_DIR/openms" "$OUT_DIR/openswath_dia/openms" "$OUT_DIR/tools" - cp -a "$DATA_DIR/openms/." "$OUT_DIR/openms/" - cp -a "$DATA_DIR/openswath_dia/openms/." "$OUT_DIR/openswath_dia/openms/" - cp -a benchmark/results/tools/. "$OUT_DIR/tools/" - - smoke="$(find "$OUT_DIR/openms" -maxdepth 1 -name 'smoke-*.json' | wc -l)" - openswath="$(find "$OUT_DIR/openswath_dia/openms" -maxdepth 1 -name 'openswath_dia-*.json' | wc -l)" - if [ "$smoke" -eq 0 ] || [ "$openswath" -eq 0 ]; then - echo "::error::branch 'benchmark-results' holds no usable history (smoke=$smoke, openswath=$openswath); refusing to publish an empty report" + + history_total=0 + generated_total=0 + duplicates_identical=0 + generated_smoke=0 + generated_openswath=0 + + # place copies one result to its family-relative destination. A path + # already placed is accepted only when the bytes are identical (both + # sources can hold the same run); different contents fail the build, + # so one copy can never silently replace the other. + place() { + src="$1" + rel="$2" + dst="$OUT_DIR/$rel" + if [ -e "$dst" ]; then + if cmp -s "$src" "$dst"; then + duplicates_identical=$((duplicates_identical + 1)) + echo "duplicate result with identical contents at $rel; keeping one copy" + return 0 + fi + echo "::error::conflicting result at $rel: $src differs from the copy already assembled; refusing to publish a report that would silently pick one" + return 1 + fi + mkdir -p "$(dirname "$dst")" + cp -p "$src" "$dst" + return 0 + } + + # copy_family copies every JSON of one source tree, keeping the family + # prefix so both sources land at the same relative layout. + copy_family() { + family="$1" + source_dir="$2" + [ -d "$source_dir" ] || return 0 + while IFS= read -r -d '' file; do + place "$file" "$family/${file#"$source_dir"/}" || exit 1 + done < <(find "$source_dir" -type f -name '*.json' -print0) + } + + for family in openms openswath_dia/openms; do + history_dir="$HISTORY_DIR/$family" + generated_dir="$DATA_DIR/$family" + if [ -d "$history_dir" ]; then + history_count="$(find "$history_dir" -type f -name '*.json' | wc -l | tr -d ' ')" + else + history_count=0 + echo "$history_dir holds no historical results on main; $family keeps the data branch's runs only" + fi + generated_count="$(find "$generated_dir" -type f -name '*.json' | wc -l | tr -d ' ')" + case "$family" in + openms) generated_smoke="$generated_count" ;; + openswath_dia/openms) generated_openswath="$generated_count" ;; + esac + echo "source counts for $family: main history=$history_count, benchmark-results=$generated_count" + history_total=$((history_total + history_count)) + generated_total=$((generated_total + generated_count)) + copy_family "$family" "$history_dir" + copy_family "$family" "$generated_dir" + done + + cp -a "$HISTORY_DIR/tools/." "$OUT_DIR/tools/" + + smoke="$(find "$OUT_DIR/openms" -maxdepth 1 -name 'smoke-*.json' | wc -l | tr -d ' ')" + openswath="$(find "$OUT_DIR/openswath_dia/openms" -maxdepth 1 -name 'openswath_dia-*.json' | wc -l | tr -d ' ')" + if [ "$generated_smoke" -eq 0 ] || [ "$generated_openswath" -eq 0 ]; then + echo "::error::branch 'benchmark-results' holds no usable history (smoke=$generated_smoke, openswath=$generated_openswath); refusing to publish an empty report" exit 1 fi - echo "assembled $smoke smoke run(s), $openswath openswath run(s), $(find "$OUT_DIR/tools" -name '*.json' | wc -l) tool reference(s)" + echo "assembled $smoke smoke run(s), $openswath openswath run(s), $(find "$OUT_DIR/tools" -name '*.json' | wc -l | tr -d ' ') tool reference(s)" + echo "sources: main history=$history_total file(s), benchmark-results=$generated_total file(s), identical duplicate(s) collapsed=$duplicates_identical" - name: Generate site (pinned reference view + optional latest view) run: | diff --git a/README.md b/README.md index 61c32cf..d3801c9 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,8 @@ Results are machine-readable JSON (schema ``` benchmark/results/ # on main + openms/ # historical runs from before the split + openswath_dia/openms/ proteobench/external/ # pinned ProteoBench snapshot tools/ # reference results from other tools @@ -167,18 +169,42 @@ benchmark-results (branch) # generated run history (nightly commits) openswath_dia/openms/openswath_dia-.json ``` -`main` keeps code, the pinned ProteoBench snapshot and the tool-reference -JSONs; generated run JSONs are committed by the nightly workflow to the -unprotected `benchmark-results` branch. The Pages workflow assembles a -temporary `site-results/` tree from that branch's run families plus `main`'s -`tools/` and renders it, so the report keeps its full history without merging -`main`'s frozen pre-split results twice. +`main` keeps code, the pinned ProteoBench snapshot, the tool-reference JSONs +and the historical run trees produced before the split; generated run JSONs are +committed by the nightly workflow to the unprotected `benchmark-results` +branch. The Pages workflow assembles a temporary `site-results/` tree as the +**union** of `main`'s historical runs (`benchmark/results/openms/`, +`benchmark/results/openswath_dia/openms/`) and the data branch's generated +runs, plus `main`'s `tools/`, and renders it. A run present in both trees is +kept once when the bytes are identical; differing bytes at the same relative +path fail the build instead of letting one copy silently replace the other. +Keeping only one source would drop runs that exist nowhere else. Recover a nightly run whose persistence failed with `benchmark/recover_nightly_results.py`: it is dry-run by default and never -pushes. After reviewing the summary, check out `benchmark-results`, re-run it -with `--apply` (one commit per benchmark run), review `git log` and push the -branch yourself. Before applying it fetches `origin/main` and +pushes. The data branch holds only result JSONs, so it has no copy of the +script: keep the code checkout on `main` and give the script a separate +worktree of `benchmark-results` through `--repo-dir`. + +```bash +# Once, from the main checkout that contains the script: +# (omit -b when a local benchmark-results branch already exists) +git fetch origin +git worktree add ../openms-benchmarking-data -b benchmark-results origin/benchmark-results + +# Dry run: validates and stages, writes and commits nothing. +python benchmark/recover_nightly_results.py --repo-dir ../openms-benchmarking-data + +# Import: one commit per benchmark run, no push. +python benchmark/recover_nightly_results.py --repo-dir ../openms-benchmarking-data --apply + +# Review the commits in the data worktree, then push the branch yourself. +git -C ../openms-benchmarking-data log --oneline -5 +git -C ../openms-benchmarking-data push origin benchmark-results +``` + +`--apply` requires that worktree to be checked out on `benchmark-results` with +a clean working tree. Before applying it fetches `origin/main` and `origin/benchmark-results` and refuses to continue if either cannot be resolved or if the local branch is behind the remote tip, so a stale clone can neither miss a collision nor commit onto the wrong base. Both arms of a run diff --git a/benchmark/recover_nightly_results.py b/benchmark/recover_nightly_results.py index 2550fad..953b218 100644 --- a/benchmark/recover_nightly_results.py +++ b/benchmark/recover_nightly_results.py @@ -25,9 +25,10 @@ pair can never be imported; 4. refreshes ``origin/main`` and ``origin/`` first and refuses to import unless both resolve (never checks a ref that may be stale); -5. refuses a destination that exists on ``main``, and on the data branch accepts - only a complete, byte-identical pair - so a partially completed import can be - re-run: finished runs are reported as already present and skipped; +5. refuses a destination whose historical result already exists on ``main`` + under ``benchmark/results/``, and on the data branch accepts only a complete, + byte-identical pair - so a partially completed import can be re-run: finished + runs are reported as already present and skipped; 6. in dry-run mode (the default) only stages the files and prints a summary. With ``--apply`` it writes the two files per run into the current checkout of @@ -39,18 +40,26 @@ Recovery procedure ------------------ - # 1. Validate and stage only (no repository changes): - python benchmark/recover_nightly_results.py +``benchmark-results`` holds only result JSONs, so it has no copy of this +script. Run the script from a ``main`` checkout and point ``--repo-dir`` at a +separate worktree of the data branch: - # 2. Review the summary, then switch to the data branch: - git switch benchmark-results + # 1. From the main checkout that contains this script, create the data + # worktree (skip this if it already exists): + git fetch origin + git worktree add ../openms-benchmarking-data -b benchmark-results origin/benchmark-results + # (or, when a local benchmark-results branch already exists:) + # git worktree add ../openms-benchmarking-data benchmark-results - # 3. Import locally (one commit per run, no push): - python benchmark/recover_nightly_results.py --apply + # 2. Dry run: validates and stages, writes and commits nothing: + python benchmark/recover_nightly_results.py --repo-dir ../openms-benchmarking-data - # 4. Review and push yourself: - git log --oneline -5 - git push origin benchmark-results + # 3. Review the summary, then import locally - one commit per run, no push: + python benchmark/recover_nightly_results.py --repo-dir ../openms-benchmarking-data --apply + + # 4. Review the commits in the data worktree and push the branch yourself: + git -C ../openms-benchmarking-data log --oneline -5 + git -C ../openms-benchmarking-data push origin benchmark-results Exit codes: 0 = ok, 1 = validation/import failure, 2 = usage/environment error. """ @@ -90,6 +99,17 @@ ("openswath_dia", "benchmark-openswath", "openswath_dia-", "openswath_dia/openms"), ) +# Where the same run lives on ``main``: before the storage split, generated +# results were committed under the results tree, so the data branch's +# ``openms/smoke-.json`` is ``benchmark/results/openms/smoke-.json`` +# there. +MAIN_RESULTS_PREFIX = "benchmark/results/" + + +def main_result_path(rel: str) -> str: + """The ``main`` path holding the historical result for data path ``rel``.""" + return f"{MAIN_RESULTS_PREFIX}{rel}" + class RecoveryError(RuntimeError): """A recoverable, user-facing failure (validation, gh, git).""" @@ -383,12 +403,23 @@ def main_refs(repo_dir: Path) -> list[str]: def ensure_no_main_collision(repo_dir: Path, rel_paths: list[str]) -> None: - """Refuse a destination that already exists on ``main`` (any known ref).""" + """Refuse a destination whose historical result already exists on ``main``. + + ``rel_paths`` are data-branch destinations (``openms/smoke-.json``). + Their historical counterparts on ``main`` sit under + ``benchmark/results/``, so that is the path each destination is checked + against: a file at the data layout on ``main`` is not this guard's concern, + and a historical result must never be imported a second time. + """ refs = main_refs(repo_dir) for rel in rel_paths: + historical = main_result_path(rel) for ref in refs: - if path_on_ref(repo_dir, ref, rel): - raise RecoveryError(f"refusing to overwrite existing {rel} on {ref}") + if path_on_ref(repo_dir, ref, historical): + raise RecoveryError( + f"refusing to overwrite existing {historical} on {ref} " + f"(destination {rel} on the data branch)" + ) def ensure_no_collision(repo_dir: Path, rel_paths: list[str]) -> None: diff --git a/benchmark/test_recover_nightly_results.py b/benchmark/test_recover_nightly_results.py index d82794a..f9c1a93 100644 --- a/benchmark/test_recover_nightly_results.py +++ b/benchmark/test_recover_nightly_results.py @@ -489,35 +489,64 @@ def test_collision_on_branch_refuses_to_overwrite(self) -> None: self.assertEqual(target.read_text(encoding="utf-8"), '{"keep": "me"}') def test_collision_on_main_ref_refuses(self) -> None: - # main still holds the frozen pre-split results; importing the same id - # must be refused even though the branch checkout is clean. + # Historical results live under benchmark/results/ on main (not at the + # data branch's layout), so that is the path the guard must check. + historical = "benchmark/results/openms/smoke-999.json" git(self.repo, "switch", "-q", "main") - target = self.repo / "openms" / "smoke-999.json" + target = self.repo / historical target.parent.mkdir(parents=True) target.write_text("{}", encoding="utf-8") - git(self.repo, "add", "openms/smoke-999.json") - git(self.repo, "commit", "-qm", "frozen") + git(self.repo, "add", historical) + git(self.repo, "commit", "-qm", "historical result") rel = "openms/smoke-999.json" - self.assertTrue(rnr.path_on_ref(self.repo, "refs/heads/main", rel)) - with self.assertRaises(rnr.RecoveryError): + self.assertEqual(rnr.main_result_path(rel), historical) + self.assertTrue(rnr.path_on_ref(self.repo, "refs/heads/main", historical)) + with self.assertRaises(rnr.RecoveryError) as ctx: rnr.ensure_no_collision(self.repo, [rel]) + self.assertIn(historical, str(ctx.exception)) def test_collision_on_fetched_origin_main_refuses_when_local_main_is_stale(self) -> None: rel = "openms/smoke-777.json" + historical = "benchmark/results/openms/smoke-777.json" git(self.repo, "switch", "-q", "main") - target = self.repo / rel + target = self.repo / historical target.parent.mkdir(parents=True, exist_ok=True) target.write_text("{}", encoding="utf-8") - git(self.repo, "add", rel) + git(self.repo, "add", historical) git(self.repo, "commit", "-qm", "upstream only") git(self.repo, "push", "-q", "origin", "main") git(self.repo, "reset", "-q", "--hard", "HEAD~1") # local main now stale git(self.repo, "fetch", "-q", "origin") - self.assertFalse(rnr.path_on_ref(self.repo, "refs/heads/main", rel)) - self.assertTrue(rnr.path_on_ref(self.repo, "refs/remotes/origin/main", rel)) + self.assertFalse(rnr.path_on_ref(self.repo, "refs/heads/main", historical)) + self.assertTrue( + rnr.path_on_ref(self.repo, "refs/remotes/origin/main", historical) + ) with self.assertRaises(rnr.RecoveryError): rnr.ensure_no_main_collision(self.repo, [rel]) + def test_wrong_path_on_main_is_not_mistaken_for_the_guard(self) -> None: + # A file at the data-branch layout on main is not a historical result, + # so it must neither satisfy nor trigger the benchmark/results/ guard. + git(self.repo, "switch", "-q", "main") + wrong = self.repo / "openms" / "smoke-888.json" + wrong.parent.mkdir(parents=True) + wrong.write_text("{}", encoding="utf-8") + git(self.repo, "add", "openms/smoke-888.json") + git(self.repo, "commit", "-qm", "same name, wrong path") + rnr.ensure_no_main_collision(self.repo, ["openms/smoke-888.json"]) + + # The correctly placed historical result for another run is still caught. + historical = "benchmark/results/openms/smoke-889.json" + target = self.repo / historical + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("{}", encoding="utf-8") + git(self.repo, "add", historical) + git(self.repo, "commit", "-qm", "historical result") + rnr.ensure_no_main_collision(self.repo, ["openms/smoke-888.json"]) + with self.assertRaises(rnr.RecoveryError) as ctx: + rnr.ensure_no_main_collision(self.repo, ["openms/smoke-889.json"]) + self.assertIn(historical, str(ctx.exception)) + def test_partial_import_resumes_without_duplicating(self) -> None: other = "42442257487" git(self.repo, "switch", "-q", "benchmark-results") diff --git a/tests/test_pages_assembly.sh b/tests/test_pages_assembly.sh index 0d88f3e..02300d9 100644 --- a/tests/test_pages_assembly.sh +++ b/tests/test_pages_assembly.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash # Executes the ACTUAL "Assemble combined results tree" step extracted from # .github/workflows/pages.yml, proving the report input contract: -# * both generated run families come from the data branch +# * both run families are the UNION of main's historical tree +# (benchmark/results/) and the data branch's generated runs +# * a run held by only one source is still published (disjoint history) +# * a duplicated relative path is kept once when byte-identical +# * a conflicting duplicate fails the build instead of silently winning # * main's tools/ reference JSONs are retained -# * main's frozen benchmark/results/openms tree is NOT copied -# * missing data, empty history, or missing tools fail with a clear message +# * missing data, empty generated history, or missing tools fail clearly # * the assembled tree is exactly what the renderer expects (no duplicates) # # LOCAL ONLY: no network, no GH_TOKEN. @@ -68,14 +71,27 @@ json.dump({ PY } -# make_repo +# make_repo +# history: yes = main holds its historical runs, no = empty historical trees +# duplicate: none | identical | conflict -- a data-branch run that also exists +# in main's history at the same relative path make_repo() { local with_data="$1" with_runs="$2" with_tools="$3" + local history="$4" duplicate="$5" REPO="$TMP/repo" rm -rf "$REPO" - mkdir -p "$REPO/benchmark/results/openms" - # Frozen pre-split history on main: must never be copied into site-results. - fixture_result "$REPO/benchmark/results/openms/smoke-42155957466.json" 42155957466 smoke + mkdir -p "$REPO/benchmark/results" + if [[ "$history" == "yes" ]]; then + mkdir -p "$REPO/benchmark/results/openms" \ + "$REPO/benchmark/results/openswath_dia/openms" + # Historical runs committed to main before the storage split: same relative + # layout as the data branch, under benchmark/results/. + fixture_result "$REPO/benchmark/results/openms/smoke-42155957466.json" \ + 42155957466 smoke + fixture_result \ + "$REPO/benchmark/results/openswath_dia/openms/openswath_dia-42155957466.json" \ + 42155957466 openswath_dia + fi if [[ "$with_tools" == "yes" ]]; then mkdir -p "$REPO/benchmark/results/tools" fixture_result "$REPO/benchmark/results/tools/proteobench-Exp-1.json" Exp-1 proteobench @@ -92,6 +108,29 @@ make_repo() { "$REPO/benchmark-data/openswath_dia/openms/openswath_dia-42951989482.json" \ 42951989482 openswath_dia fi + case "$duplicate" in + none) ;; + identical) + cp "$REPO/benchmark/results/openms/smoke-42155957466.json" \ + "$REPO/benchmark-data/openms/smoke-42155957466.json" + cp "$REPO/benchmark/results/openswath_dia/openms/openswath_dia-42155957466.json" \ + "$REPO/benchmark-data/openswath_dia/openms/openswath_dia-42155957466.json" + ;; + conflict) + # Same relative path, different contents: main's history and the data + # branch disagree about one run. + fixture_result "$REPO/benchmark-data/openms/smoke-42155957466.json" \ + 42155957466 smoke + "$PYTHON" - "$REPO/benchmark-data/openms/smoke-42155957466.json" <<'PY' +import json, sys +path = sys.argv[1] +data = json.load(open(path, encoding="utf-8")) +data["performance"]["wall_time_s"] = 99.0 +json.dump(data, open(path, "w", encoding="utf-8")) +PY + ;; + *) echo "unknown duplicate mode: $duplicate" >&2; exit 2 ;; + esac fi } @@ -99,20 +138,25 @@ passed=0 failed=0 total=0 -# run_step +# run_step [] run_step() { - local name="$1" expect_rc="$2" expect_message="$3" + local name="$1" expect_rc="$2" expect_message="$3" forbidden="${4:-}" total=$((total + 1)) local rc=0 - ( cd "$REPO" && DATA_DIR=benchmark-data OUT_DIR=site-results bash "$STEP" ) \ + ( cd "$REPO" && DATA_DIR=benchmark-data HISTORY_DIR=benchmark/results \ + OUT_DIR=site-results bash "$STEP" ) \ > "$TMP/out.log" 2> "$TMP/err.log" || rc=$? local ok=1 why="" if [[ "$rc" -ne "$expect_rc" ]]; then ok=0; why="rc=$rc want=$expect_rc; stderr: $(head -c 300 "$TMP/err.log")" fi - if [[ "$expect_message" != "-" ]] && ! grep -qF -- "$expect_message" "$TMP/out.log" "$TMP/err.log"; then + if [[ "$expect_message" != "-" ]] \ + && ! grep -qF -- "$expect_message" "$TMP/out.log" "$TMP/err.log"; then ok=0; why="$why; missing message '$expect_message'" fi + if [[ -n "$forbidden" ]] && grep -qF -- "$forbidden" "$TMP/out.log" "$TMP/err.log"; then + ok=0; why="$why; unexpected message '$forbidden'" + fi if [[ "$ok" -eq 1 ]]; then passed=$((passed + 1)); echo "OK: $name" else @@ -120,45 +164,88 @@ run_step() { fi } -echo "== Pages results-tree assembly tests ==" -echo - -make_repo yes yes yes -run_step "assembles both run families plus main's tools/" 0 "assembled 2 smoke run(s), 2 openswath run(s), 1 tool reference(s)" -total=$((total + 1)) -if [[ -f "$REPO/site-results/openms/smoke-42333314804.json" \ - && -f "$REPO/site-results/openswath_dia/openms/openswath_dia-42333314804.json" \ - && -f "$REPO/site-results/tools/proteobench-Exp-1.json" ]]; then - passed=$((passed + 1)); echo "OK: expected files land under site-results/" -else - failed=$((failed + 1)); echo "FAIL: assembled tree is missing expected files" >&2 -fi -total=$((total + 1)) -if [[ -e "$REPO/site-results/openms/smoke-42155957466.json" ]]; then - failed=$((failed + 1)); echo "FAIL: frozen main history was copied into site-results" >&2 -else - passed=$((passed + 1)); echo "OK: frozen main history is not copied" -fi -total=$((total + 1)) -counts="$("$PYTHON" - "$REPO/site-results" "$BENCH_DIR/benchmark/report" <<'PY' +# renderer_counts -> " " as report_generate.load_results sees it +renderer_counts() { + "$PYTHON" - "$REPO/site-results" "$BENCH_DIR/benchmark/report" <<'PY' import sys sys.path.insert(0, sys.argv[2]) from report_generate import load_results runs, tools = load_results(sys.argv[1]) print(f"{len(runs)} {len(tools)}") PY -)" -if [[ "$counts" == "4 1" ]]; then - passed=$((passed + 1)); echo "OK: renderer sees exactly 4 runs and 1 tool result (no duplicates)" -else - failed=$((failed + 1)); echo "FAIL: renderer saw '$counts' (want '4 1')" >&2 -fi +} + +# check +check() { + local name="$1" why="$2" + shift 2 + total=$((total + 1)) + if "$@"; then + passed=$((passed + 1)); echo "OK: $name" + else + failed=$((failed + 1)); echo "FAIL: $name -- $why" >&2 + fi +} + +echo "== Pages results-tree assembly tests ==" +echo + +# --- disjoint history: main's old runs plus the data branch's new runs ------- +make_repo yes yes yes yes none +run_step "unions historical and generated runs plus main's tools/" 0 \ + "assembled 3 smoke run(s), 3 openswath run(s), 1 tool reference(s)" +run_step "reports source counts for both sources" 0 \ + "main history=2 file(s), benchmark-results=4 file(s), identical duplicate(s) collapsed=0" +check "historical result from main is published" \ + "main's historical run was dropped from site-results" \ + test -f "$REPO/site-results/openms/smoke-42155957466.json" +check "historical openswath result from main is published" \ + "main's historical openswath run was dropped from site-results" \ + test -f "$REPO/site-results/openswath_dia/openms/openswath_dia-42155957466.json" +check "generated runs from the data branch are published" \ + "the data branch's runs were not assembled" \ + test -f "$REPO/site-results/openms/smoke-42333314804.json" +check "main's tools/ reference result is published" \ + "the tool reference JSON is missing" \ + test -f "$REPO/site-results/tools/proteobench-Exp-1.json" +counts="$(renderer_counts)" +check "renderer sees exactly 6 runs and 1 tool result (no duplicates)" \ + "renderer saw '$counts' (want '6 1')" \ + test "$counts" = "6 1" + +# --- identical duplicate: the same run on both sides is kept once ----------- +make_repo yes yes yes yes identical +run_step "identical duplicate at the same relative path is collapsed" 0 \ + "duplicate result with identical contents at openms/smoke-42155957466.json" +run_step "identical duplicates are counted in the log" 0 \ + "main history=2 file(s), benchmark-results=6 file(s), identical duplicate(s) collapsed=2" +counts="$(renderer_counts)" +check "identical duplicates do not double-count runs" \ + "renderer saw '$counts' (want '6 1')" \ + test "$counts" = "6 1" + +# --- conflicting duplicate: differing bytes fail the build ------------------ +make_repo yes yes yes yes conflict +run_step "conflicting duplicate at the same relative path fails loudly" 1 \ + "conflicting result at openms/smoke-42155957466.json" +check "a conflicting duplicate never overwrites main's historical copy" \ + "the conflicting data copy replaced main's historical result" \ + cmp -s "$REPO/site-results/openms/smoke-42155957466.json" \ + "$REPO/benchmark/results/openms/smoke-42155957466.json" + +# --- main has no historical runs: the data branch alone is still assembled -- +make_repo yes yes yes no none +run_step "empty historical tree on main is reported, not fatal" 0 \ + "holds no historical results on main" +run_step "assembles the data branch's runs without main history" 0 \ + "assembled 2 smoke run(s), 2 openswath run(s), 1 tool reference(s)" -make_repo no yes yes +# --- failure modes ---------------------------------------------------------- +make_repo no yes yes yes none run_step "missing data checkout fails clearly" 1 "is missing" -make_repo yes no yes -run_step "empty run history fails clearly" 1 "no usable history" -make_repo yes yes no +make_repo yes no yes yes none +run_step "empty generated history fails clearly" 1 "no usable history" +make_repo yes yes no yes none run_step "missing tools/ on main fails clearly" 1 "tools is missing" echo @@ -176,16 +263,22 @@ assert "--results-dir benchmark/results" not in text assemble = next( s for s in steps if s.get("name", "").startswith("Assemble combined results tree") ) -assemble_run = assemble["run"] -assert "benchmark/results/openms" not in assemble_run, "assembly must not copy the frozen run history" -assert "benchmark/results/tools" in assemble_run +run = assemble["run"] +# Both run sources must be assembled: main's history and the data branch. +assert assemble.get("env", {}).get("HISTORY_DIR") == "benchmark/results" +assert 'HISTORY_DIR="$HISTORY_DIR/$family"' in run or "$HISTORY_DIR/$family" in run +assert "$DATA_DIR/$family" in run +assert "$HISTORY_DIR/tools" in run +assert "cmp -s" in run, "duplicate relative paths must be compared, not overwritten" +assert "::error::conflicting result at" in run, "conflicting duplicates must fail" +assert "frozen" not in text, "comments must not claim main's result trees are frozen" code = [s for s in steps if s.get("name", "").startswith("Checkout code")] data = [s for s in steps if s.get("name", "").startswith("Checkout generated results")] assert code and code[0]["with"]["ref"] == "main" assert data and data[0]["with"]["ref"] == "benchmark-results" PY then - passed=$((passed + 1)); echo "OK: data source, render argument, and no frozen-history copy" + passed=$((passed + 1)); echo "OK: both run sources, duplicate guard, and checkout refs" else failed=$((failed + 1)); echo "FAIL: workflow-level guarantees" >&2 fi