From 7749f894383227c78dc7b72f09fe712c55258802 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 8 Nov 2026 19:15:31 +0100 Subject: [PATCH] Kill hung tensor tests with pytest-timeout in the oneMath workflow The oneMath workflow hits the same tensor-test hang at 99% (https://github.com/IntelPython/dpctl/issues/1954) as the conda package workflows: since 2026-10-01, 55 of 577 test jobs hit the 10 min retry timeout of the tensor step, 8 of them twice, about 10.5 h of runner time spent waiting. The tensor tests now run with `--timeout=120 --timeout-method=thread`, as in `conda-package.yml`, and with `-n auto`: without xdist the thread method exits the whole pytest process on the first timeout, while with xdist only the stuck worker is killed and replaced, the run finishes and the hung test is reported by name. `pytest-timeout` and `pytest-xdist` are added to `environments/test.yml`, which the coverage workflow also merges; it passes neither option, so its run doesn't change. --- .github/workflows/check-onemath.yaml | 11 +++++++---- environments/test.yml | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index 544367ae8197..b652ae8a448b 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -31,6 +31,9 @@ env: rerun-tests-timeout: 30 rerun-tensor-tests-timeout: 10 rerun-tensor-tests-max-attempts: 3 + # Per-test limit to kill a test stuck at 99%: https://github.com/IntelPython/dpctl/issues/1954 + # The thread method exits the process, so -n is needed for xdist to replace the worker + tensor-test-timeout: 120 jobs: build_env_file: @@ -171,7 +174,7 @@ jobs: - name: Run tensor tests if: env.rerun-tests-on-failure != 'true' run: | - python -m pytest -ra --pyargs dpnp.tests.tensor + python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 @@ -206,7 +209,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --pyargs dpnp.tests.tensor + python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 @@ -307,7 +310,7 @@ jobs: - name: Run tensor tests if: env.rerun-tests-on-failure != 'true' run: | - python -m pytest -ra --pyargs dpnp.tests.tensor + python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 @@ -342,6 +345,6 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --pyargs dpnp.tests.tensor + python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 diff --git a/environments/test.yml b/environments/test.yml index db446a7d0d16..2ae7ea6d2468 100644 --- a/environments/test.yml +++ b/environments/test.yml @@ -3,4 +3,6 @@ channels: - conda-forge dependencies: - pytest + - pytest-timeout + - pytest-xdist - scipy