diff --git a/.github/AGENTS.md b/.github/AGENTS.md index 6b807a17df70..904bb903dadd 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -8,6 +8,7 @@ These keep zizmor (`zizmor.yml`, fails on medium or higher findings) and the Ope - Never inline untrusted contexts (`github.event.*`, `github.head_ref`, `inputs.*`) in `run:`; pass them through `env:`. Prefer `env:` for every value in new code. - Every job sets `timeout-minutes`; for a job that calls a reusable workflow, set it inside the called workflow. - `actionlint` runs in pre-commit. +- `workflows/intel-ci.yml` is Intel CI: its conda matrix is `conda-recipe/conda_build_config.yaml`, its wheel matrix the Python classifiers in `pyproject.toml`, and its upload and cleanup jobs run only for pushes to `master` and tags in `IntelPython/dpnp`; see `ci/README.md`. - `workflows/array-api-skips.txt` lists array-api-tests node IDs skipped by the conformance job; precede each with a comment naming the upstream issue. - `bump-sycl-deps.yml` rewrites the `DRIVER_PATH`, `OCLCPUEXP_FN`, `TBB_URL`, `TBB_INSTALL_DIR` and `TBB_FN` pins in `.github/sycl-build-deps.env` with `sed`, and `os-llvm-sycl-build.yml` reads exactly those five; keep one unquoted `KEY=VALUE` per line, starting at column 0. - Dependabot (`dependabot.yml`) bumps actions, `environments/` pins and pre-commit hooks weekly with a 7-day cooldown. diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml deleted file mode 100644 index 77a934a15981..000000000000 --- a/.github/workflows/conda-build-test.yml +++ /dev/null @@ -1,303 +0,0 @@ -name: Reusable conda build and test - -on: - workflow_call: - inputs: - runs-on: - description: Runner image to execute the jobs on - required: true - type: string - conda-subdir: - description: Conda platform subdir (e.g. linux-64, win-64) - required: true - type: string - channels-list: - description: Conda channels to build and test against - required: true - type: string - recipe-dir: - description: Path to the conda recipe directory - required: true - type: string - -permissions: - contents: read - -env: - # Pinned: Miniforge3 26.9.4-0 fails 'conda init' on Windows runners - miniforge-version: '26.7.2-1' - package-name: dpnp - module-name: dpnp - ver-script-part1: "import json; f = open('version.json', 'r'); j = json.load(f); f.close(); " - ver-script-part2: "d = j['dpnp'][0]; print('='.join((d[s] for s in ('version', 'build'))))" - fetch-depth: 1 - python-ver-test-all-dtypes: '3.14' - test-env-name: 'test' - rerun-tests-max-attempts: 2 - rerun-tests-timeout: 45 - rerun-tensor-tests-timeout: 15 - rerun-tensor-tests-max-attempts: 3 - # Per-test limit to kill a test stuck at 99%: https://github.com/IntelPython/dpctl/issues/1954 - tensor-test-timeout: 120 - -jobs: - build: - name: Build - - runs-on: ${{ inputs.runs-on }} - timeout-minutes: 90 - - defaults: - run: - shell: bash -el {0} - - strategy: - fail-fast: false - matrix: - python: ['3.12', '3.13'] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - # TODO: add free-threaded builds ('3.14.* *_cp314t') once a - # free-threaded dpctl is available on conda-forge - # TODO: add Python 3.15 ('3.15.* *_cp315') once dpctl with - # Python 3.15 support is available on conda-forge - - env: - build-conda-pkg-env: 'environments/build_conda_pkg.yml' - build-env-name: 'build' - python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} - python-conda-spec: ${{ matrix.python_spec || matrix.python }} - - steps: - - name: Checkout DPNP repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - # Blobless: all commits and tags for git describe, no gh-pages blobs - filter: blob:none - - - name: Setup miniconda - id: setup_miniconda - continue-on-error: true - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.build-conda-pkg-env }} - activate-environment: ${{ env.build-env-name }} - - - name: ReSetup miniconda - if: steps.setup_miniconda.outcome == 'failure' - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.build-conda-pkg-env }} - activate-environment: ${{ env.build-env-name }} - - - name: List installed packages - run: conda list - - - name: Store conda paths as envs - env: - CONDA_SUBDIR: ${{ inputs.conda-subdir }} - run: | - echo "CONDA_BLD=$CONDA_PREFIX/conda-bld/$CONDA_SUBDIR/" | tr "\\\\" '/' >> "$GITHUB_ENV" - - - name: Build conda package - id: build_conda_pkg - continue-on-error: true - env: - PYTHON_CONDA_SPEC: ${{ env.python-conda-spec }} - CHANNELS_LIST: ${{ inputs.channels-list }} - RECIPE_DIR: ${{ inputs.recipe-dir }} - run: | - # CHANNELS_LIST must stay unquoted to split into separate CLI args - # shellcheck disable=SC2086 - conda-build --no-test --python "$PYTHON_CONDA_SPEC" --numpy 2.0 $CHANNELS_LIST "$RECIPE_DIR" - - - name: ReBuild conda package - if: steps.build_conda_pkg.outcome == 'failure' - env: - PYTHON_CONDA_SPEC: ${{ env.python-conda-spec }} - CHANNELS_LIST: ${{ inputs.channels-list }} - RECIPE_DIR: ${{ inputs.recipe-dir }} - run: | - # CHANNELS_LIST must stay unquoted to split into separate CLI args - # shellcheck disable=SC2086 - conda-build --no-test --python "$PYTHON_CONDA_SPEC" --numpy 2.0 $CHANNELS_LIST "$RECIPE_DIR" - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Python ${{ env.python-label }} - path: ${{ env.CONDA_BLD }}${{ env.package-name }}-*.conda - - test: - name: Test - - needs: build - - runs-on: ${{ inputs.runs-on }} - timeout-minutes: 120 - - defaults: - run: - shell: bash -el {0} - - strategy: - fail-fast: false - matrix: - python: ['3.12', '3.13'] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - # TODO: add free-threaded builds ('3.14.* *_cp314t') once a - # free-threaded dpctl is available on conda-forge - # TODO: add Python 3.15 ('3.15.* *_cp315') once dpctl with - # Python 3.15 support is available on conda-forge - - env: - dpnp-repo-path: './source' - create-conda-channel-env: 'source/environments/create_conda_channel.yml' - channel-path: './channel' - pkg-path-in-channel: './channel/${{ inputs.conda-subdir }}' - ver-json-path: './version.json' - python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} - python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}" - - steps: - - name: Set Swap Space - if: runner.os == 'Linux' - uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c # v1.0 - with: - swap-size-gb: 8 - - - name: Checkout DPNP repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: ${{ env.fetch-depth }} - path: ${{ env.dpnp-repo-path }} - - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Python ${{ env.python-label }} - path: ${{ env.pkg-path-in-channel }} - - - name: Setup miniconda - id: setup_miniconda - continue-on-error: true - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.create-conda-channel-env }} - activate-environment: ${{ env.test-env-name }} - - - name: ReSetup miniconda - if: steps.setup_miniconda.outcome == 'failure' - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.create-conda-channel-env }} - activate-environment: ${{ env.test-env-name }} - - - name: Create conda channel - run: | - python -m conda_index ${{ env.channel-path }} - - - name: Test conda channel - run: | - conda search ${{ env.package-name }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} - cat ${{ env.ver-json-path }} - - - name: Get package version - run: | - PACKAGE_VERSION=$(python -c "${{ env.ver-script-part1 }} ${{ env.ver-script-part2 }}") - export PACKAGE_VERSION - - echo "PACKAGE_VERSION=${PACKAGE_VERSION}" - echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> "$GITHUB_ENV" - - - name: Install dpnp - id: install_dpnp - continue-on-error: true - run: | - conda install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest pytest-timeout pytest-xdist "${{ env.python-install-spec }}" ${{ env.TEST_CHANNELS }} - env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ inputs.channels-list }}' - - - name: ReInstall dpnp - if: steps.install_dpnp.outcome == 'failure' - run: | - conda install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest pytest-timeout pytest-xdist "${{ env.python-install-spec }}" ${{ env.TEST_CHANNELS }} - env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ inputs.channels-list }}' - - - name: List installed packages - run: conda list - - - name: Register the OpenCL CPU driver with the ICD loader - if: runner.os == 'Windows' - shell: pwsh - run: | - # conda-forge intel-opencl-rt does not ship a registration helper, - # so register the OpenCL CPU driver (intelocl64.dll) with the ICD loader manually - $vendors = "HKLM:\SOFTWARE\Khronos\OpenCL\Vendors" - $drivers = @(Get-ChildItem -Recurse -ErrorAction SilentlyContinue -Filter intelocl64.dll -Path "$env:CONDA_PREFIX\Library\bin") - if ($drivers.Count -eq 0) { throw "No OpenCL CPU driver (intelocl64.dll) found under $env:CONDA_PREFIX" } - if (-not (Test-Path $vendors)) { New-Item -Path $vendors -Force | Out-Null } - foreach ($driver in $drivers) { - New-ItemProperty -Path $vendors -Name $driver.FullName -Value 0 -PropertyType DWord -Force | Out-Null - } - Get-ItemProperty -Path $vendors - - - name: Smoke test - run: | - python -c "import dpctl; dpctl.lsplatform()" - python -c "import dpnp; print(dpnp.__version__)" - - - name: Run tests - id: run_tests - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 - with: - timeout_minutes: ${{ env.rerun-tests-timeout }} - max_attempts: ${{ env.rerun-tests-max-attempts }} - retry_on: any - shell: bash - command: | - CONDA_UNIX=$(cygpath -u "$CONDA" 2>/dev/null || echo "$CONDA") - . "$CONDA_UNIX/etc/profile.d/conda.sh" - conda activate ${{ env.test-env-name }} - export SKIP_TENSOR_TESTS=1 - - if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then - export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.module-name }}.tests - else - python -m pytest -n auto -ra --pyargs ${{ env.module-name }}.tests - fi - - - name: Run tensor tests - id: run_tests_tensor - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 - with: - timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} - max_attempts: ${{ env.rerun-tensor-tests-max-attempts }} - retry_on: any - shell: bash - command: | - CONDA_UNIX=$(cygpath -u "$CONDA" 2>/dev/null || echo "$CONDA") - . "$CONDA_UNIX/etc/profile.d/conda.sh" - conda activate ${{ env.test-env-name }} - - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor diff --git a/.github/workflows/conda-package-cf.yml b/.github/workflows/conda-package-cf.yml deleted file mode 100644 index 9356b40da0d8..000000000000 --- a/.github/workflows/conda-package-cf.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Conda package (conda-forge) - -on: - push: - branches: - - master - pull_request: - -permissions: - contents: read - -# Cancel superseded PR runs only; other runs get a unique group and always complete -concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - linux: - name: linux - uses: ./.github/workflows/conda-build-test.yml - with: - runs-on: ubuntu-latest - conda-subdir: linux-64 - channels-list: '-c conda-forge --override-channels' - recipe-dir: conda-recipe-cf - - windows: - name: windows - uses: ./.github/workflows/conda-build-test.yml - with: - runs-on: windows-2022 - conda-subdir: win-64 - channels-list: '-c conda-forge --override-channels' - recipe-dir: conda-recipe-cf diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml deleted file mode 100644 index 4fc22b5b4676..000000000000 --- a/.github/workflows/conda-package.yml +++ /dev/null @@ -1,960 +0,0 @@ -name: Conda package - -on: - push: - branches: - - master - pull_request: - -permissions: - contents: read - -# Cancel superseded PR runs only; other runs get a unique group and always complete -concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -env: - # Pinned: Miniforge3 26.9.4-0 fails 'conda init' on Windows runners - miniforge-version: '26.7.2-1' - package-name: dpnp - # Follow oneAPI installation instruction for conda, since intel channel is not longer available - # channels-list: '-c dppy/label/dev -c intel -c conda-forge --override-channels' - channels-list: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels' - ver-script-part1: "import json; f = open('version.json', 'r'); j = json.load(f); f.close(); " - ver-script-part2: "d = j['dpnp'][0]; print('='.join((d[s] for s in ('version', 'build'))))" - fetch-depth: 1 - python-ver-test-all-dtypes: '3.14' - test-env-name: 'test' - rerun-tests-on-failure: 'true' - rerun-tests-max-attempts: 2 - rerun-tests-timeout: 45 - rerun-tensor-tests-timeout: 15 - rerun-tensor-tests-max-attempts: 3 - # Per-test limit to kill a test stuck at 99%: https://github.com/IntelPython/dpctl/issues/1954 - tensor-test-timeout: 120 - -jobs: - build: - name: Build - - strategy: - fail-fast: false - matrix: - python: ['3.13'] - os: [ubuntu-latest, windows-2022] - include: - - python: '3.14' - os: ubuntu-latest - python_spec: '3.14.* *_cp314' - - python: '3.14' - os: windows-2022 - python_spec: '3.14.* *_cp314' - - python: '3.14' - os: ubuntu-latest - python_spec: '3.14.* *_cp314t' - - python: '3.14' - os: windows-2022 - python_spec: '3.14.* *_cp314t' - - python: '3.15' - os: ubuntu-latest - python_spec: '3.15.* *_cp315' - - python: '3.15' - os: windows-2022 - python_spec: '3.15.* *_cp315' - # TODO: add free-threaded builds ('3.15.* *_cp315t') once NumPy and - # dpctl cp315t builds are available - - runs-on: ${{ matrix.os }} - timeout-minutes: 90 - - defaults: - run: - shell: ${{ matrix.os == 'windows-2022' && 'cmd /C CALL {0}' || 'bash -el {0}' }} - - env: - python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} - python-conda-spec: ${{ matrix.python_spec || matrix.python }} - - steps: - - name: Checkout DPNP repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - # Blobless: all commits and tags for git describe, no gh-pages blobs - filter: blob:none - - # rattler-build is a standalone binary with no conda/Python dependency, - # so building conda packages with it doesn't need a full conda - # distribution on the runner - only rattler-build itself needs to be on - # PATH - - name: Setup rattler-build - uses: prefix-dev/rattler-build-action@1ca5f45832f419a46d1326ccc5861d7e14d67c44 # v0.2.39 - with: - setup-only: true - - - name: Check rattler-build version - run: rattler-build --version - - - name: Store build output paths - shell: bash -el {0} - run: | - echo "CONDA_BLD=$GITHUB_WORKSPACE/output/${{ runner.os == 'Linux' && 'linux-64' || 'win-64' }}/" >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE${{ runner.os == 'Linux' && '/' || '\\' }}" >> "$GITHUB_ENV" - - - name: Compute version from git describe - shell: bash -el {0} - run: | - DESCRIBE=$(git describe --tags --long) - echo "GIT_DESCRIBE_TAG=$(echo "$DESCRIBE" | sed -E 's/-[0-9]+-g[0-9a-f]+$//')" >> "$GITHUB_ENV" - echo "GIT_DESCRIBE_NUMBER=$(echo "$DESCRIBE" | sed -E 's/^.*-([0-9]+)-g[0-9a-f]+$/\1/')" >> "$GITHUB_ENV" - - - name: Compute rattler-build arguments - shell: bash -el {0} - run: | - read -ra RATTLER_CHANNELS <<< "$(echo "${{ env.channels-list }}" | sed 's/--override-channels//')" - echo "RATTLER_BUILD_ARGS=--no-include-recipe --test skip --channel-priority disabled --target-platform ${{ runner.os == 'Linux' && 'linux-64' || 'win-64' }} --variant-config conda-recipe/conda_build_config.yaml --variant \"python=${{ env.python-conda-spec }}\" --variant numpy=2 ${RATTLER_CHANNELS[*]}" >> "$GITHUB_ENV" - - - name: Build conda package - id: build_conda_pkg - continue-on-error: true - uses: prefix-dev/rattler-build-action@1ca5f45832f419a46d1326ccc5861d7e14d67c44 # v0.2.39 - with: - recipe-path: conda-recipe/rattler_recipe.yaml - build-args: ${{ env.RATTLER_BUILD_ARGS }} - upload-artifact: false - env: - MAX_BUILD_CMPL_MKL_VERSION: '2027.0a0' - - - name: ReBuild conda package - if: steps.build_conda_pkg.outcome == 'failure' - uses: prefix-dev/rattler-build-action@1ca5f45832f419a46d1326ccc5861d7e14d67c44 # v0.2.39 - with: - recipe-path: conda-recipe/rattler_recipe.yaml - build-args: ${{ env.RATTLER_BUILD_ARGS }} - upload-artifact: false - env: - MAX_BUILD_CMPL_MKL_VERSION: '2027.0a0' - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Python ${{ env.python-label }} - path: ${{ env.CONDA_BLD }}${{ env.package-name }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Wheels Python ${{ env.python-label }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.package-name }}-*.whl - - test_linux: - name: Test - - needs: build - - runs-on: ${{ matrix.os }} - timeout-minutes: 120 - - defaults: - run: - shell: bash -el {0} - - strategy: - fail-fast: false - matrix: - python: ['3.13'] - os: [ubuntu-latest] - include: - - python: '3.14' - os: ubuntu-latest - python_spec: '3.14.* *_cp314' - - python: '3.14' - os: ubuntu-latest - python_spec: '3.14.* *_cp314t' - - python: '3.15' - os: ubuntu-latest - python_spec: '3.15.* *_cp315' - # TODO: add free-threaded builds ('3.15.* *_cp315t') once NumPy and - # dpctl cp315t builds are available - - env: - dpnp-repo-path: '${{ github.workspace }}/source/' - create-conda-channel-env: 'source/environments/create_conda_channel.yml' - channel-path: '${{ github.workspace }}/channel/' - pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/' - ver-json-path: '${{ github.workspace }}/version.json' - python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} - python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}" - - steps: - - name: Set Swap Space - uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c # v1.0 - with: - swap-size-gb: 8 - - - name: Checkout DPNP repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: ${{ env.fetch-depth }} - path: ${{ env.dpnp-repo-path }} - - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Python ${{ env.python-label }} - path: ${{ env.pkg-path-in-channel }} - - - name: Setup miniconda - id: setup_miniconda - continue-on-error: true - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.create-conda-channel-env }} - activate-environment: ${{ env.test-env-name }} - - - name: ReSetup miniconda - if: steps.setup_miniconda.outcome == 'failure' - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.create-conda-channel-env }} - activate-environment: ${{ env.test-env-name }} - - - name: Create conda channel - run: | - python -m conda_index ${{ env.channel-path }} - - - name: Test conda channel - run: | - conda search ${{ env.package-name }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} - cat ${{ env.ver-json-path }} - - - name: Get package version - run: | - PACKAGE_VERSION=$(python -c "${{ env.ver-script-part1 }} ${{ env.ver-script-part2 }}") - export PACKAGE_VERSION - - echo "PACKAGE_VERSION=${PACKAGE_VERSION}" - echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> "$GITHUB_ENV" - - - name: Install dpnp - id: install_dpnp - continue-on-error: true - run: | - mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest pytest-timeout pytest-xdist "${{ env.python-install-spec }}" ${{ env.TEST_CHANNELS }} - env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - - - name: ReInstall dpnp - if: steps.install_dpnp.outcome == 'failure' - run: | - mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest pytest-timeout pytest-xdist "${{ env.python-install-spec }}" ${{ env.TEST_CHANNELS }} - env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - - - name: List installed packages - run: mamba list - - - name: Smoke test - run: | - python -c "import dpctl; dpctl.lsplatform()" - python -c "import dpnp; print(dpnp.__version__)" - - - name: Run tests - if: env.rerun-tests-on-failure != 'true' - run: | - export SKIP_TENSOR_TESTS=1 - if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then - export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.package-name }}.tests - else - python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests - fi - - - name: Run tests - if: env.rerun-tests-on-failure == 'true' - id: run_tests_linux - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 - with: - timeout_minutes: ${{ env.rerun-tests-timeout }} - max_attempts: ${{ env.rerun-tests-max-attempts }} - retry_on: any - command: | - . $CONDA/etc/profile.d/conda.sh - . $CONDA/etc/profile.d/mamba.sh - mamba activate ${{ env.test-env-name }} - export SKIP_TENSOR_TESTS=1 - - if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then - export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.package-name }}.tests - else - python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests - fi - - - name: Run tensor tests - if: env.rerun-tests-on-failure != 'true' - run: | - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor - - - name: Run tensor tests - if: env.rerun-tests-on-failure == 'true' - id: run_tests_tensor_linux - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 - with: - timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} - max_attempts: ${{ env.rerun-tensor-tests-max-attempts }} - retry_on: any - command: | - . $CONDA/etc/profile.d/conda.sh - . $CONDA/etc/profile.d/mamba.sh - mamba activate ${{ env.test-env-name }} - - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor - - test_examples_linux: - name: Test examples - - needs: build - - runs-on: ubuntu-latest - timeout-minutes: 60 - - defaults: - run: - shell: bash -el {0} - - strategy: - fail-fast: false - matrix: - python: ['3.14'] - python_spec: ['3.14.* *_cp314', '3.14.* *_cp314t'] - - env: - dpnp-repo-path: '${{ github.workspace }}/source/' - examples-conda-pkg-env: 'source/environments/examples.yml' - channel-path: '${{ github.workspace }}/channel/' - pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/' - ver-json-path: '${{ github.workspace }}/version.json' - examples-env-name: 'examples_test' - python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} - python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}" - - steps: - - name: Checkout DPNP repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: ${{ env.fetch-depth }} - path: ${{ env.dpnp-repo-path }} - - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Python ${{ env.python-label }} - path: ${{ env.pkg-path-in-channel }} - - - name: Setup miniconda - id: setup_miniconda - continue-on-error: true - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.examples-conda-pkg-env }} - activate-environment: ${{ env.examples-env-name }} - - - name: ReSetup miniconda - if: steps.setup_miniconda.outcome == 'failure' - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.examples-conda-pkg-env }} - activate-environment: ${{ env.examples-env-name }} - - - name: Create conda channel - run: | - python -m conda_index ${{ env.channel-path }} - - - name: Test conda channel - run: | - conda search ${{ env.package-name }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} - cat ${{ env.ver-json-path }} - - - name: Get package version - run: | - PACKAGE_VERSION=$(python -c "${{ env.ver-script-part1 }} ${{ env.ver-script-part2 }}") - echo "PACKAGE_VERSION=${PACKAGE_VERSION}" >> "$GITHUB_ENV" - - - name: Install dpnp - id: install_dpnp - continue-on-error: true - run: | - mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} "${{ env.python-install-spec }}" ${{ env.TEST_CHANNELS }} - env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - - - name: ReInstall dpnp - if: steps.install_dpnp.outcome == 'failure' - run: | - mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} "${{ env.python-install-spec }}" ${{ env.TEST_CHANNELS }} - env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - - - name: List installed packages - run: mamba list - - - name: Smoke test - run: | - python -c "import dpctl; dpctl.lsplatform()" - python -c "import dpnp; print(dpnp.__version__)" - - - name: Build and run pybind11 examples - run: | - cd ${{ env.dpnp-repo-path }}/examples/pybind11 - for d in */; do - pushd "$d" > /dev/null - CC=icx CXX=icpx python setup.py build_ext --inplace -G Ninja || exit 1 - if [ -d tests ]; then - python -m pytest tests || exit 1 - fi - popd > /dev/null - done - - test_windows: - name: Test - - needs: build - - runs-on: ${{ matrix.os }} - timeout-minutes: 120 - - defaults: - run: - shell: cmd /C CALL {0} - - strategy: - fail-fast: false - matrix: - python: ['3.13'] - os: [windows-2022] - include: - - python: '3.14' - os: windows-2022 - python_spec: '3.14.* *_cp314' - - python: '3.14' - os: windows-2022 - python_spec: '3.14.* *_cp314t' - - python: '3.15' - os: windows-2022 - python_spec: '3.15.* *_cp315' - # TODO: add free-threaded builds ('3.15.* *_cp315t') once NumPy and - # dpctl cp315t builds are available - - env: - dpnp-repo-path: '${{ github.workspace }}\source' - create-conda-channel-env: 'source\environments\create_conda_channel.yml' - channel-path: '${{ github.workspace }}\channel\' - pkg-path-in-channel: '${{ github.workspace }}\channel\win-64\' - ver-json-path: '${{ github.workspace }}\version.json' - python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} - python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}" - - steps: - - name: Checkout DPNP repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: ${{ env.fetch-depth }} - path: ${{ env.dpnp-repo-path }} - - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Python ${{ env.python-label }} - path: ${{ env.pkg-path-in-channel }} - - - name: Store a path to package archive - run: | - @echo on - - set SEARCH_SCRIPT="DIR ${{ env.pkg-path-in-channel }} /s/b | FINDSTR /r "dpnp-.*\.conda"" - FOR /F "tokens=* USEBACKQ" %%F IN (`%SEARCH_SCRIPT%`) DO ( - SET FULL_PACKAGE_PATH=%%F - ) - echo FULL_PACKAGE_PATH: %FULL_PACKAGE_PATH% - (echo FULL_PACKAGE_PATH=%FULL_PACKAGE_PATH%) >> %GITHUB_ENV% - - - name: Setup miniconda - id: setup_miniconda - continue-on-error: true - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.create-conda-channel-env }} - activate-environment: ${{ env.test-env-name }} - - - name: ReSetup miniconda - if: steps.setup_miniconda.outcome == 'failure' - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.create-conda-channel-env }} - activate-environment: ${{ env.test-env-name }} - - - name: Store conda paths as envs - run: | - @echo on - (echo CONDA_LIB_PATH=%CONDA_PREFIX%\Library\lib\) >> %GITHUB_ENV% - (echo CONDA_LIB_BIN_PATH=%CONDA_PREFIX%\Library\bin\) >> %GITHUB_ENV% - - - name: Create conda channel - run: | - @echo on - python -m conda_index ${{ env.channel-path }} - - - name: Test conda channel - run: | - @echo on - conda search ${{ env.package-name }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} - - - name: Dump version.json - run: more ${{ env.ver-json-path }} - - - name: Get package version - run: | - @echo on - set "SCRIPT=${{ env.ver-script-part1 }} ${{ env.ver-script-part2 }}" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - set PACKAGE_VERSION=%%F - ) - echo PACKAGE_VERSION: %PACKAGE_VERSION% - (echo PACKAGE_VERSION=%PACKAGE_VERSION%) >> %GITHUB_ENV% - - - name: Install dpnp - run: | - @echo on - mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest pytest-timeout pytest-xdist "${{ env.python-install-spec }}" ${{ env.TEST_CHANNELS }} - env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - MAMBA_NO_LOW_SPEED_LIMIT: 1 - - - name: Install OCL CPU RT from Intel channel - run: mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} intel-opencl-rt=*=intel_* ${{ env.channels-list }} - - - name: List installed packages - run: mamba list - - - name: Activate OCL CPU RT - shell: pwsh - run: | - $script_path="$env:CONDA_PREFIX\Scripts\set-intel-ocl-icd-registry.ps1" - if (Test-Path $script_path) { - &$script_path - } else { - Write-Warning "File $script_path was NOT found!" - } - # Check the variable assisting OpenCL CPU driver to find TBB DLLs which are not located where it expects them by default - $cl_cfg="$env:CONDA_PREFIX\Library\bin\cl.cfg" - Get-Content -Tail 5 -Path $cl_cfg - - - name: Smoke test - run: | - python -c "import dpctl; dpctl.lsplatform()" - python -c "import dpnp; print(dpnp.__version__)" - - - name: Run tests - if: env.rerun-tests-on-failure != 'true' - shell: pwsh - run: | - $env:SKIP_TENSOR_TESTS=1 - if ("${{ env.python-label }}" -eq "${{ env.python-ver-test-all-dtypes }}") { - $env:DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.package-name }}.tests - } else { - python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests - } - - - name: Run tests - if: env.rerun-tests-on-failure == 'true' - id: run_tests_win - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 - with: - timeout_minutes: ${{ env.rerun-tests-timeout }} - max_attempts: ${{ env.rerun-tests-max-attempts }} - retry_on: any - shell: pwsh - command: | - $env:SKIP_TENSOR_TESTS=1 - if ( "${{ env.python-label }}" -eq "${{ env.python-ver-test-all-dtypes }}" ) { - $env:DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.package-name }}.tests - } else { - python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests - } - - - name: Run tensor tests - if: env.rerun-tests-on-failure != 'true' - shell: pwsh - run: | - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor - - - name: Run tensor tests - if: env.rerun-tests-on-failure == 'true' - id: run_tests_tensor_win - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 - with: - timeout_minutes: ${{ env.rerun-tensor-tests-timeout }} - max_attempts: ${{ env.rerun-tensor-tests-max-attempts }} - retry_on: any - shell: pwsh - command: | - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor - - upload: - name: Upload - - needs: [test_linux, test_windows, test_examples_linux] - - strategy: - fail-fast: false - matrix: - python: ['3.13'] - os: [ubuntu-latest, windows-2022] - include: - - python: '3.14' - os: ubuntu-latest - python_spec: '3.14.* *_cp314' - - python: '3.14' - os: windows-2022 - python_spec: '3.14.* *_cp314' - - python: '3.14' - os: ubuntu-latest - python_spec: '3.14.* *_cp314t' - - python: '3.14' - os: windows-2022 - python_spec: '3.14.* *_cp314t' - - python: '3.15' - os: ubuntu-latest - python_spec: '3.15.* *_cp315' - - python: '3.15' - os: windows-2022 - python_spec: '3.15.* *_cp315' - # TODO: add free-threaded builds ('3.15.* *_cp315t') once NumPy and - # dpctl cp315t builds are available - - runs-on: ${{ matrix.os }} - timeout-minutes: 10 - - defaults: - run: - shell: bash -el {0} - - env: - upload-conda-pkg-env: 'environments/upload_cleanup_conda_pkg.yml' - upload-env-name: 'upload' - python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} - - if: | - (github.repository == 'IntelPython/dpnp') && - (github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')) - - steps: - - name: Checkout DPNP repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: ${{ env.fetch-depth }} - - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Python ${{ env.python-label }} - - - name: Download wheels artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Wheels Python ${{ env.python-label }} - - - name: Setup miniconda - id: setup_miniconda - continue-on-error: true - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.upload-conda-pkg-env }} - activate-environment: ${{ env.upload-env-name }} - - - name: ReSetup miniconda - if: steps.setup_miniconda.outcome == 'failure' - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.upload-conda-pkg-env }} - activate-environment: ${{ env.upload-env-name }} - - - name: Package version - run: | - echo "PACKAGE_VERSION=$(basename ${{ env.package-name }}-*.conda | sed 's/^${{ env.package-name }}-\([^-]*\).*/\1/')" >> "$GITHUB_ENV" - - - name: Upload - run: | - anaconda --token "${ANACONDA_TOKEN}" upload --user dppy --label dev ${{ env.package-name }}-*.conda - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - - - name: Upload wheels - run: | - anaconda --token "${ANACONDA_TOKEN}" upload --user dppy --label dev ${{ env.package-name }}-*.whl --version "${PACKAGE_VERSION}" - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - - array-api-conformity: - name: Array API conformity - - needs: build - - permissions: - # Needed to add a comment to a pull request's issue - pull-requests: write - - runs-on: ubuntu-latest - timeout-minutes: 15 - - defaults: - run: - shell: bash -el {0} - - continue-on-error: true - - env: - array-api-tests-path: '${{ github.workspace }}/array-api-tests/' - json-report-file: '${{ github.workspace }}/.report.json' - dpnp-repo-path: '${{ github.workspace }}/source/' - array-api-skips-file: '${{ github.workspace }}/source/.github/workflows/array-api-skips.txt' - create-conda-channel-env: 'source/environments/create_conda_channel.yml' - python-ver: '3.14' - conda-env-name: 'array-api-conformity' - channel-path: '${{ github.workspace }}/channel/' - pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/' - ver-json-path: '${{ github.workspace }}/version.json' - - steps: - - name: Checkout DPNP repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: ${{ env.fetch-depth }} - path: ${{ env.dpnp-repo-path }} - - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.package-name }} ${{ runner.os }} Python ${{ env.python-ver }} - path: ${{ env.pkg-path-in-channel }} - - - name: Setup miniconda - id: setup_miniconda - continue-on-error: true - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.create-conda-channel-env }} - activate-environment: ${{ env.conda-env-name }} - - - name: ReSetup miniconda - if: steps.setup_miniconda.outcome == 'failure' - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.create-conda-channel-env }} - activate-environment: ${{ env.conda-env-name }} - - - name: Create conda channel - run: | - python -m conda_index ${{ env.channel-path }} - - - name: Test conda channel - run: | - conda search ${{ env.package-name }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }} - cat ${{ env.ver-json-path }} - - - name: Get package version - run: | - PACKAGE_VERSION=$(python -c "${{ env.ver-script-part1 }} ${{ env.ver-script-part2 }}") - export PACKAGE_VERSION - - echo "PACKAGE_VERSION=${PACKAGE_VERSION}" - echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> "$GITHUB_ENV" - - - name: Install dpnp - id: install_dpnp - continue-on-error: true - run: | - mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest python=${{ env.python-ver }} ${{ env.TEST_CHANNELS }} - env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - - - name: ReInstall dpnp - if: steps.install_dpnp.outcome == 'failure' - run: | - mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} pytest python=${{ env.python-ver }} ${{ env.TEST_CHANNELS }} - env: - TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}' - - - name: List installed packages - run: mamba list - - - name: Smoke test - run: | - python -c "import dpctl; dpctl.lsplatform()" - python -c "import dpnp; print(dpnp.__version__)" - - - name: Clone array API tests repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - repository: 'data-apis/array-api-tests' - path: ${{ env.array-api-tests-path }} - fetch-depth: ${{ env.fetch-depth }} - submodules: 'recursive' - - - name: Install array API test dependencies - run: | - pip install -r requirements.txt - working-directory: ${{ env.array-api-tests-path }} - - - name: Install jq - run: | - sudo apt-get install jq - - - name: List installed packages - run: mamba list - - - name: Smoke test - run: | - python -c "import dpctl; dpctl.lsplatform()" - python -c "import dpnp; print(dpnp.__version__)" - - - name: Run array API conformance tests - run: | - python -m pytest --json-report --json-report-file=${{ env.json-report-file }} --disable-deadline --skips-file ${{ env.array-api-skips-file }} array_api_tests || true - env: - ARRAY_API_TESTS_MODULE: 'dpnp' - ARRAY_API_TESTS_VERSION: '2025.12' - SYCL_CACHE_PERSISTENT: 1 - working-directory: ${{ env.array-api-tests-path }} - - - name: Set GitHub environment variables - run: | - FILE=${{ env.json-report-file }} - if test -f "$FILE"; then - PASSED_TESTS=$(jq '.summary | .passed // 0' $FILE) - FAILED_TESTS=$(jq '.summary | .failed // 0' $FILE) - SKIPPED_TESTS=$(jq '.summary | .skipped // 0' $FILE) - MESSAGE="Array API standard conformance tests for dpnp=$PACKAGE_VERSION ran successfully. - Passed: $PASSED_TESTS - Failed: $FAILED_TESTS - Skipped: $SKIPPED_TESTS" - { - echo "MESSAGE<> "$GITHUB_ENV" - else - echo "Array API standard conformance tests failed to run for dpnp=$PACKAGE_VERSION." - exit 1 - fi - - - name: Output API summary - run: echo "::notice ${{ env.MESSAGE }}" - - - name: Post result to PR - if: ${{ github.event.pull_request && !github.event.pull_request.head.repo.fork }} - uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12.0 - with: - message-id: array_api_results - message: | - ${{ env.MESSAGE }} - - cleanup_packages: - name: Clean up anaconda packages - - needs: [upload] - - runs-on: ubuntu-latest - timeout-minutes: 10 - - defaults: - run: - shell: bash -el {0} - - env: - cleanup-conda-pkg-env: 'environments/upload_cleanup_conda_pkg.yml' - cleanup-env-name: 'cleanup' - - steps: - - name: Checkout DPNP repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: ${{ env.fetch-depth }} - - - name: Setup miniconda - id: setup_miniconda - continue-on-error: true - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.cleanup-conda-pkg-env }} - activate-environment: ${{ env.cleanup-env-name }} - - - name: ReSetup miniconda - if: steps.setup_miniconda.outcome == 'failure' - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: ${{ env.miniforge-version }} - use-mamba: 'true' - conda-remove-defaults: 'true' - environment-file: ${{ env.cleanup-conda-pkg-env }} - activate-environment: ${{ env.cleanup-env-name }} - - - name: Checkout repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - repository: IntelPython/devops-tools - fetch-depth: ${{ env.fetch-depth }} - - - name: Cleanup old packages - run: | - python scripts/cleanup-old-packages.py \ - --verbose --force --token "${ANACONDA_TOKEN}" \ - --package dppy/${{ env.package-name }} --label dev - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} diff --git a/.github/workflows/intel-ci.yml b/.github/workflows/intel-ci.yml new file mode 100644 index 000000000000..5ea97a4bc2a6 --- /dev/null +++ b/.github/workflows/intel-ci.yml @@ -0,0 +1,483 @@ +# Builds and tests dpnp's conda packages and wheels, see ci/README.md +name: Intel CI + +on: + push: + branches: [master] + tags: ['*'] + pull_request: + +permissions: + contents: read + +# A push or tag run gets a group of its own, so a queued master push is never dropped. +concurrency: + group: intel-ci-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + matrix: + name: Platforms and Python versions + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + outputs: + matrix: ${{ steps.declared.outputs.matrix }} + wheel: ${{ steps.declared.outputs.wheel }} + examples: ${{ steps.declared.outputs.examples }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Read the declared platforms and Python versions + id: declared + run: | + python ci/matrix.py --event "$GITHUB_EVENT_NAME" --github-output matrix + python ci/matrix.py --system wheel --event "$GITHUB_EVENT_NAME" --github-output wheel + python ci/matrix.py --newest --platform linux --event "$GITHUB_EVENT_NAME" --github-output examples + + # macOS runs on Intel runners, as MKL has no Apple silicon build. + build: + name: ${{ matrix.name }} Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The recipe takes its version from git describe: all commits and tags, no blobs. + fetch-depth: 0 + filter: blob:none + + # Miniforge 26.9.4-0 fails setup-miniconda's conda init on Windows runners; 26.7.2-0 works. + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install conda-build + run: conda install -y -q -n base --override-channels -c conda-forge conda-build "py-lief<0.17" + + - name: Build and run the recipe tests + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dpnp ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.conda + if-no-files-found: error + # Tags keep them 30 days, pull requests and pushes to master 3. + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test: + name: Test ${{ matrix.name }} Python ${{ matrix.python }} + needs: [matrix, build] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + # The suites use more memory than a runner has; without swap it is shut down. + - uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c # v1.0 + if: runner.os == 'Linux' + with: + swap-size-gb: 8 + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dpnp ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the package and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + build-wheel: + name: ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The wheel takes its version from git describe: all commits and tags, no blobs. + fetch-depth: 0 + filter: blob:none + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install cibuildwheel + run: python -m pip install --no-input cibuildwheel==4.3.5 + + - name: Build the wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --system wheel --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dpnp ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.whl + if-no-files-found: error + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test-wheel: + name: Test ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: [matrix, build-wheel] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash -el {0} + steps: + # The suites use more memory than a runner has; without swap it is shut down. + - uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c # v1.0 + if: runner.os == 'Linux' + with: + swap-size-gb: 8 + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dpnp ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the wheel and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --kind wheel --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + examples: + name: Test examples Python ${{ matrix.python }} + needs: [matrix, build] + # Needs only the build of its own package, not the tests + if: ${{ !cancelled() && needs.matrix.result == 'success' }} + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.examples) }} + + defaults: + run: + shell: bash -el {0} + + env: + channel-path: '${{ github.workspace }}/channel' + python-ver: ${{ matrix.python }} + + steps: + # Not in the workspace root, where its dpnp would shadow the installed one + - name: Checkout DPNP repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + path: source + + - name: Download artifact + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dpnp Linux Python ${{ env.python-ver }} + path: ${{ env.channel-path }}/linux-64 + + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: '26.7.2-1' + use-mamba: 'true' + conda-remove-defaults: 'true' + environment-file: source/environments/examples.yml + activate-environment: examples_test + + - name: Create conda channel + run: python -m conda_index "$CHANNEL_PATH" + env: + CHANNEL_PATH: ${{ env.channel-path }} + + - name: Install dpnp + # Strict priority keeps conda-forge's dpnp out; the ABI picks 3.14t's free-threaded build + run: mamba install -y --strict-channel-priority dpnp "python ${PYTHON_VER%t}.* *_cp${PYTHON_VER//./}" -c "$CHANNEL_PATH" -c conda-forge --override-channels + env: + CHANNEL_PATH: ${{ env.channel-path }} + PYTHON_VER: ${{ env.python-ver }} + + - name: List installed packages + run: mamba list + + - name: Smoke test + run: | + python -c "import dpctl; dpctl.lsplatform()" + python -c "import dpnp; print(dpnp.__version__)" + + - name: Build and run pybind11 examples + working-directory: source/examples/pybind11 + run: | + for d in */; do + pushd "$d" > /dev/null + CC=icx CXX=icpx python setup.py build_ext --inplace -G Ninja || exit 1 + if [ -d tests ]; then + python -m pytest tests || exit 1 + fi + popd > /dev/null + done + + array-api-conformity: + name: Array API conformity + needs: [matrix, build] + # Needs only the build of its own package, not the tests + if: ${{ !cancelled() && needs.matrix.result == 'success' }} + + permissions: + # Needed to add a comment to a pull request's issue + pull-requests: write + + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 15 + + defaults: + run: + shell: bash -el {0} + + continue-on-error: true + + env: + array-api-tests-path: '${{ github.workspace }}/array-api-tests' + json-report-file: '${{ github.workspace }}/.report.json' + channel-path: '${{ github.workspace }}/channel' + # The newest regular version, which every event builds + python-ver: ${{ fromJSON(needs.matrix.outputs.examples).include[0].python }} + + steps: + # Not in the workspace root, where its dpnp would shadow the installed one + - name: Checkout DPNP repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + path: source + + - name: Download artifact + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dpnp Linux Python ${{ env.python-ver }} + path: ${{ env.channel-path }}/linux-64 + + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: '26.7.2-1' + use-mamba: 'true' + conda-remove-defaults: 'true' + environment-file: source/environments/create_conda_channel.yml + activate-environment: array-api-conformity + + - name: Create conda channel + run: python -m conda_index "$CHANNEL_PATH" + env: + CHANNEL_PATH: ${{ env.channel-path }} + + - name: Install dpnp + run: mamba install -y --strict-channel-priority dpnp pytest "python $PYTHON_VER.* *_cp${PYTHON_VER//./}" -c "$CHANNEL_PATH" -c conda-forge --override-channels + env: + CHANNEL_PATH: ${{ env.channel-path }} + PYTHON_VER: ${{ env.python-ver }} + + - name: List installed packages + run: mamba list + + - name: Smoke test + run: | + python -c "import dpctl; dpctl.lsplatform()" + python -c "import dpnp; print(dpnp.__version__)" + + - name: Clone array API tests repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: 'data-apis/array-api-tests' + path: ${{ env.array-api-tests-path }} + submodules: 'recursive' + + - name: Install array API test dependencies + run: pip install -r requirements.txt + working-directory: ${{ env.array-api-tests-path }} + + - name: Install jq + run: sudo apt-get install jq + + - name: Run array API conformance tests + run: python -m pytest --json-report --json-report-file="$JSON_REPORT_FILE" --disable-deadline --skips-file "$SKIPS_FILE" array_api_tests || true + env: + ARRAY_API_TESTS_MODULE: 'dpnp' + ARRAY_API_TESTS_VERSION: '2025.12' + SYCL_CACHE_PERSISTENT: 1 + JSON_REPORT_FILE: ${{ env.json-report-file }} + SKIPS_FILE: ${{ github.workspace }}/source/.github/workflows/array-api-skips.txt + working-directory: ${{ env.array-api-tests-path }} + + - name: Set GitHub environment variables + env: + JSON_REPORT_FILE: ${{ env.json-report-file }} + run: | + PACKAGE_VERSION=$(python -c "import dpnp; print(dpnp.__version__)") + if test -f "$JSON_REPORT_FILE"; then + PASSED_TESTS=$(jq '.summary | .passed // 0' "$JSON_REPORT_FILE") + FAILED_TESTS=$(jq '.summary | .failed // 0' "$JSON_REPORT_FILE") + SKIPPED_TESTS=$(jq '.summary | .skipped // 0' "$JSON_REPORT_FILE") + MESSAGE="Array API standard conformance tests for dpnp=$PACKAGE_VERSION ran successfully. + Passed: $PASSED_TESTS + Failed: $FAILED_TESTS + Skipped: $SKIPPED_TESTS" + { + echo "MESSAGE<> "$GITHUB_ENV" + else + echo "Array API standard conformance tests failed to run for dpnp=$PACKAGE_VERSION." + exit 1 + fi + + - name: Output API summary + env: + SUMMARY: ${{ env.MESSAGE }} + run: echo "::notice $SUMMARY" + + - name: Post result to PR + if: ${{ github.event.pull_request && !github.event.pull_request.head.repo.fork }} + uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12.0 + with: + message-id: array_api_results + message: | + ${{ env.MESSAGE }} + + # Dev builds for anaconda.org, from pushes to master and tags in this repository only. + upload: + name: Upload to anaconda.org + needs: [test, test-wheel] + if: github.repository == 'IntelPython/dpnp' && github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: dpnp * + merge-multiple: true + path: ${{ runner.temp }}/packages + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + conda-remove-defaults: "true" + environment-file: environments/upload_cleanup_conda_pkg.yml + activate-environment: upload + + - name: Upload the conda packages + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + run: anaconda --token "$ANACONDA_TOKEN" upload --user dppy --label dev "$RUNNER_TEMP"/packages/dpnp-*.conda + + - name: Upload the wheels + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + run: | + conda_packages=("$RUNNER_TEMP"/packages/dpnp-*.conda) + version=$(basename "${conda_packages[0]}" | sed 's/^dpnp-\([^-]*\).*/\1/') + anaconda --token "$ANACONDA_TOKEN" upload --user dppy --label dev "$RUNNER_TEMP"/packages/dpnp-*.whl --version "$version" + + cleanup-packages: + name: Clean up anaconda.org + needs: upload + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + conda-remove-defaults: "true" + environment-file: environments/upload_cleanup_conda_pkg.yml + activate-environment: cleanup + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: IntelPython/devops-tools + ref: 08ffddf88627957e8b094817aac4ce0bde7cff63 + path: devops-tools + + - name: Clean up old dev packages + working-directory: devops-tools + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + run: python scripts/cleanup-old-packages.py --verbose --force --token "$ANACONDA_TOKEN" --package dppy/dpnp --label dev diff --git a/AGENTS.md b/AGENTS.md index e0c6f423ea7d..fa29ec0e87d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,11 +46,10 @@ Append to the end of the matching `### Added` / `Changed` / `Deprecated` / `Remo | Check | Workflow | Required on master | | --- | --- | --- | | pre-commit (`Check`) | `.github/workflows/pre-commit.yml` | yes | -| conda build and tests, Linux and Windows, py3.13-3.15 and 3.14t (`Test (...)`) | `conda-package.yml` | yes | +| conda packages and wheels built and tested, Linux and Windows, py3.11-3.15 and 3.14t for conda, py3.11-3.14 and 3.14t for wheels (`Test Python `, `Test Wheels Python `); the conda matrix is `conda-recipe/conda_build_config.yaml`, the wheel matrix the Python classifiers in `pyproject.toml`; see `ci/README.md` | `intel-ci.yml` | no | | Internal Jenkins (`jenkins-test-dpnp-3.13-linux-dev`, `...-3.14-...`) | external | yes | -| Array API conformance of `dpnp`, not yet `dpnp.tensor` (array-api-tests; never fails, results posted as a PR comment on non-fork PRs) | `conda-package.yml` | no | -| pybind11 examples | `conda-package.yml` | no | -| conda-forge build and tests | `conda-package-cf.yml`, matrix in `conda-build-test.yml` | no | +| Array API conformance of `dpnp`, not yet `dpnp.tensor` (array-api-tests; never fails, results posted as a PR comment on non-fork PRs) | `intel-ci.yml` | no | +| pybind11 examples | `intel-ci.yml` | no | | oneMath build, open-source LLVM SYCL build | `check-onemath.yaml`, `os-llvm-sycl-build.yml` | no | | Sphinx docs (`make spelling`, `make html`) | `build-sphinx.yml` | no | | Coverage (Coveralls), zizmor | `generate_coverage.yaml`, `zizmor.yml` | no | diff --git a/CHANGELOG.md b/CHANGELOG.md index 1757174c8cc6..97ebffd7f98d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ This release drops support for Python 3.10, making Python 3.11 the minimum requi * Improved performance of `dpnp.scipy.sparse.csr_matrix.sort_indices` by sorting once on a combined row and column key instead of in two passes [#3134](https://github.com/IntelPython/dpnp/pull/3134) * Reduced the overhead of `dpnp.ndarray.flags` and `dpnp.tensor.usm_ndarray.flags` [#3168](https://github.com/IntelPython/dpnp/pull/3168) * Changed `dpnp.linalg.LinAlgError` to derive from `numpy.linalg.LinAlgError`, so `except numpy.linalg.LinAlgError` also catches errors raised by dpnp linear algebra functions [#3184](https://github.com/IntelPython/dpnp/pull/3184) +* Changed the CI to build and test the conda packages of `conda-recipe` and the wheels with Intel CI (`ci/`, `.github/workflows/intel-ci.yml`); the `dppy/label/dev` packages are now built against conda-forge only, and the wheels declare the oneAPI runtime wheels in their metadata [#3151](https://github.com/IntelPython/dpnp/pull/3151) ### Deprecated @@ -36,6 +37,7 @@ This release drops support for Python 3.10, making Python 3.11 the minimum requi * Removed the unused `FourZeroOffsets_Indexer`, `FixedDimStridedIndexer` and `ThreeOffsets_FixedDimStridedIndexer` from the libtensor header `utils/offset_utils.hpp` [#3131](https://github.com/IntelPython/dpnp/pull/3131) * Dropped support for Python 3.10 [#3173](https://github.com/IntelPython/dpnp/pull/3173) * Removed the `USE_GROUP_LOAD_STORE` macro from the libtensor header `utils/sycl_utils.hpp`, so `sub_group_load` and `sub_group_store` always use the `sycl_ext_oneapi_group_load_store` extension; the `sycl::sub_group` member load/store fallback it selected does not compile with DPC++ 2026.0, the minimum supported compiler [#3174](https://github.com/IntelPython/dpnp/pull/3174) +* Removed the Intel channel conda recipe, with `rattler_recipe.yaml`, and the `conda-package.yml` workflow; `conda-recipe/` is now the conda-forge recipe that was `conda-recipe-cf/` [#3151](https://github.com/IntelPython/dpnp/pull/3151) ### Fixed diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e45698b116b..65b489617b18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,6 +133,15 @@ execute_process( set(CYTHON_FLAGS "-t -w \"${CMAKE_SOURCE_DIR}\" -I \"${Dpctl_Cython_INCLUDE_DIR}\"") find_package(Cython REQUIRED) +# dpctl from a wheel keeps its CMake files in the package, not in the prefix +if(NOT DEFINED Dpctl_ROOT AND NOT DEFINED ENV{Dpctl_ROOT}) + execute_process( + COMMAND "${Python_EXECUTABLE}" -m dpctl --cmakedir + OUTPUT_VARIABLE Dpctl_ROOT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) +endif() find_package(Dpctl REQUIRED) message(STATUS "Dpctl_INCLUDE_DIR=" ${Dpctl_INCLUDE_DIR}) diff --git a/README.md b/README.md index 28cdf3fdd4c6..e607b8bac1e1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) [![Pre-commit](https://github.com/IntelPython/dpnp/actions/workflows/pre-commit.yml/badge.svg?branch=master&event=push)](https://github.com/IntelPython/dpnp/actions/workflows/pre-commit.yml) -[![Conda package](https://github.com/IntelPython/dpnp/actions/workflows/conda-package.yml/badge.svg?branch=master&event=push)](https://github.com/IntelPython/dpnp/actions/workflows/conda-package.yml) +[![Intel CI](https://github.com/IntelPython/dpnp/actions/workflows/intel-ci.yml/badge.svg?branch=master&event=push)](https://github.com/IntelPython/dpnp/actions/workflows/intel-ci.yml) [![Coverage Status](https://coveralls.io/repos/github/IntelPython/dpnp/badge.svg?branch=master)](https://coveralls.io/github/IntelPython/dpnp?branch=master) [![Build Sphinx](https://github.com/IntelPython/dpnp/workflows/Build%20Sphinx/badge.svg)](https://intelpython.github.io/dpnp) [![Coverity Scan Build Status](https://scan.coverity.com/projects/intelpython-dpnp/badge.svg)](https://scan.coverity.com/projects/intelpython-dpnp) diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 000000000000..68fa5ce29b72 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,39 @@ +# How Intel CI tests dpnp + +`.github/workflows/intel-ci.yml` builds dpnp's conda packages and wheels in GitHub Actions and tests each one. It needs this directory, the recipe, and `pyproject.toml`. + +| File | Role | +|---|---| +| `../conda-recipe/` | The conda recipe, built against conda-forge only. `conda_build_config.yaml` lists the Python versions. The recipe skips macOS. Its own tests are `pip check` and an import; the test suite runs in `test.py`. | +| `../pyproject.toml` | What the wheels are built from. Its Python classifiers list the wheels' Python versions, and its Operating System classifiers their platforms; 3.15 is not classified because dpctl has no 3.15 wheel to build against yet. `[tool.cibuildwheel]` holds the wheel build settings, and `[tool.gh-ci]` holds the run-time requirements the wheels need on top of `[project]`'s, which the conda packages get from the recipe. | +| `../.github/workflows/intel-ci.yml` | One job reads both matrices. One job per platform and Python version builds and uploads `dpnp Python `, and one builds `dpnp Wheels Python `. Then one job per package tests it. Pushes to `master` and tags then upload the packages to the `dev` label of `dppy` on anaconda.org and remove old ones; no other event does. | +| `matrix.py` | Reads the platforms and Python versions from the recipe, or with `--system wheel` from `pyproject.toml`. A pull request gets the oldest and the newest regular versions and the newest free-threaded one. | +| `build.py` | Builds one Python version: with conda-build, which runs the recipe's own tests, or with `--system wheel` with cibuildwheel, in a manylinux container on Linux. | +| `hooks.py` | The workflow's test jobs: installs one package into a clean environment with the requirements in `config.yml`, and runs `test.py`. A conda package's environment comes from conda-forge; a wheel's has only Python from conda-forge and everything else from PyPI. | +| `install_icx.py` | Installs Intel's icx and the oneMKL, oneDPL and oneTBB development packages from conda-forge for the Windows wheel builds: `[tool.cibuildwheel.windows]` in `../pyproject.toml` runs it first. The Linux wheel builds get icpx from Intel's yum repository and the libraries from conda-forge with micromamba. | +| `config.yml` | Names the workflow that builds the packages, its artifact names, and what the tests and benchmarks need. | +| `test.py` | Runs the installed package's test suites with pytest: `dpnp.tests`, then `dpnp.tests.tensor` on its own, which needs a lot of memory. It sets the SYCL and OpenCL variables itself, because the test environment has none. | +| `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package. | + +The scripts need only the Python standard library. + +Nothing in `intel-ci.yml` reads `workflow`, `artifacts` or `benchmark` from `config.yml`, runs `benchmark.py`, or keeps the JUnit reports `test.py` writes; they are there to run the same build, tests and benchmarks outside this workflow. The scripts also handle rattler-build recipes, macOS and a `conda-recipe-cf/` directory, which dpnp doesn't use. + +**Devices.** GitHub's runners have no GPU, so `test.py` runs there on the OpenCL CPU device of `intel-opencl-rt`. On Windows it registers that runtime's `intelocl64.dll` with the OpenCL loader in the machine's registry for the length of the tests, only if none is registered, as the runner has none and the loader ignores `OCL_ICD_FILENAMES`; on Linux it merges the environment's and the system's ICD files for the loader. Neither happens in a GPU run on Windows, which uses the registered driver. `test.py` prints the SYCL devices, fails if there is none, and with `GH_CI_DEVICE=gpu` fails if none is a GPU, so a GPU run that falls back to the CPU cannot pass. Workers of `pytest-xdist` run only on a CPU device, as they would share one GPU. + +**Wheels.** They are built against the dpctl wheel on PyPI, with icpx 2026.2. They bundle none of the SYCL, oneMKL or compiler libraries: `[tool.gh-ci] wheel-requires` adds the wheels that provide them to the metadata, as the wheels on PyPI declare. Nothing is repaired: Linux only retags the wheels to `manylinux_2_28` without an `auditwheel` check, and Windows has no repair step. The Linux wheels link GLIBCXX_3.4.32, so they need the libstdc++ of GCC 13 or newer, as the dpctl wheels need 3.4.26. A wheel's test environment adds `intel-opencl-rt` from PyPI for the CPU device. + +**A pull request.** Actions builds the oldest and the newest regular Python versions and the newest free-threaded one on Linux and Windows, as a conda package and as a wheel, and tests each package. + +**A push to `master` or a tag.** Actions builds and tests every declared version. + +To support another Python, add it to `conda-recipe/conda_build_config.yaml` for the conda packages and as a classifier in `pyproject.toml` for the wheels; nothing else lists versions. Try it locally from a conda base environment with conda-build, or with cibuildwheel for wheels: + +```bash +python ci/matrix.py # every platform and Python version +python ci/matrix.py --event pull_request # what a pull request builds +python ci/build.py --python VERSION --output /tmp/out # VERSION: one matrix.py lists +python ci/hooks.py --python VERSION --artifacts /tmp/out --work /tmp/hooks +python ci/build.py --system wheel --python VERSION --output /tmp/wheels +python ci/hooks.py --kind wheel --python VERSION --artifacts /tmp/wheels --work /tmp/wheel-hooks +``` diff --git a/ci/benchmark.py b/ci/benchmark.py new file mode 100644 index 000000000000..7be7cdb86aeb --- /dev/null +++ b/ci/benchmark.py @@ -0,0 +1,112 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +""" +Benchmarks the installed package. + +Runs the asv suite in benchmarks/ against the package installed in this +environment, from an empty working directory, without a build of its own, and +records the results under the commit being tested, in $GH_CI_RESULTS. A pull +request samples each benchmark once, enough to show that every one still runs; +a tag measures them properly. A benchmark that fails fails the run; a slower one +does not, as nothing compares the timings with a baseline yet. +""" + +import json +import os +import pathlib +import subprocess +import sys + +# asv's own dependency, which reads the comments in its configuration. +import json5 + +ASV_CONFIG = pathlib.Path("benchmarks", "asv.conf.json") + + +def asv_config(source: pathlib.Path, results: pathlib.Path) -> pathlib.Path: + """Writes this run's asv configuration into the working directory. + + It is the repository's own, with absolute paths, since asv reads them from + where the configuration is, and with results under GH_CI_RESULTS. It lists + one branch, HEAD: the checkout is the commit under test, detached, and asv + cannot resolve the repository's branches there. + """ + path = source / ASV_CONFIG + config = json5.loads(path.read_text(encoding="utf-8")) + work = pathlib.Path.cwd() + config.update( + repo=str((path.parent / config.get("repo", "..")).resolve()), + benchmark_dir=str( + (path.parent / config.get("benchmark_dir", "benchmarks")).resolve() + ), + branches=["HEAD"], + results_dir=str(results / "asv"), + html_dir=str(work / "html"), + env_dir=str(work / "env"), + ) + written = work / "asv.conf.json" + written.write_text(json.dumps(config, indent=2), encoding="utf-8") + return written + + +def main() -> int: + """Runs the asv suite against the installed package.""" + config = asv_config( + pathlib.Path(os.environ["GH_CI_SOURCE"]), + pathlib.Path(os.environ["GH_CI_RESULTS"]), + ) + asv = [sys.executable, "-m", "asv"] + # asv keeps its machine description in ~/.asv-machine.json; this one stays + # with the run rather than with the machine's user. + home = pathlib.Path.cwd() / "home" + home.mkdir(exist_ok=True) + env = dict(os.environ, HOME=str(home)) + described = subprocess.run( + asv + ["machine", "--yes", "--config", str(config)], env=env + ) + if described.returncode: + return described.returncode + run = [ + "run", + "--config", + str(config), + f"--environment=existing:{sys.executable}", + f"--set-commit-hash={os.environ['GH_CI_COMMIT']}", + "--no-pull", + "--show-stderr", + "--launch-method=spawn", + ] + if os.environ.get("GH_CI_EVENT") != "tag": + run.append("--quick") + # asv exits 2 when a benchmark failed. + return subprocess.run(asv + run, env=env).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/build.py b/ci/build.py new file mode 100644 index 000000000000..0cbb1717aec0 --- /dev/null +++ b/ci/build.py @@ -0,0 +1,394 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +""" +Builds this repository's packages for one Python version. + +For the intel-ci.yml GitHub Actions workflow. + +This file and ci/matrix.py work out the build system from the files present, +and the Python versions from that build system's own declaration: + + rattler-build .conda recipe.yaml or recipe/recipe.yaml, versions in + variants.yaml + conda-build .conda conda-recipe/meta.yaml or meta.yaml, versions in + conda_build_config.yaml + cibuildwheel .whl pyproject.toml, versions from its classifiers + +Usage: + python ci/build.py --python 3.14 --output OUT [--system rattler-build] + [--channel conda-forge] + +--python must be one of the declared versions; `python ci/matrix.py` lists them. + +Packages land directly in OUT, ready for actions/upload-artifact. A conda +recipe's own tests run during the build. Needs only the Python standard library. +rattler-build is used from PATH, or downloaded at a pinned version for Linux or +Windows and checked against its published sha256; conda-build, or cibuildwheel +for wheels, must already be installed for this interpreter, so run it with the +conda base install's Python to build with conda-build. + +Wheels build with cibuildwheel, in a manylinux container on Linux, as +pyproject.toml's [tool.cibuildwheel] says. Run-time requirements that +pyproject.toml cannot declare, because the conda package gets them from its +recipe instead, such as MKL's libraries, go in [tool.gh-ci] wheel-requires: +build.py adds them to each wheel's metadata. +""" + +import argparse +import base64 +import hashlib +import os +import platform +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +import matrix + +ROOT = matrix.ROOT +PACKAGE_SUFFIXES = (".conda", ".tar.bz2", ".whl") +# cibuildwheel's platform part of a build identifier, per system. +WHEEL_PLATFORMS = { + "Linux": "manylinux_x86_64", + "Windows": "win_amd64", + "Darwin": "macosx_x86_64", +} + +# The rattler-build release fetched when none is on PATH, per system, with the +# file name it goes under and the sha256 its release publishes. +RATTLER_BUILD_VERSION = "v0.77.0" +RATTLER_BUILD_RELEASES = { + "Linux": ( + "rattler-build-x86_64-unknown-linux-musl", + "rattler-build", + "5a5e8150c703b21e88a01f9130ea49a7a680198e9ac9414170c0a293fed7e5af", + ), + "Windows": ( + "rattler-build-x86_64-pc-windows-msvc.exe", + "rattler-build.exe", + "e144334f782cee9aa23e26eb8d095c143e1696fbbec7ba7534a156663d4abb2c", + ), +} + + +class BuildError(Exception): + """Raised when the build cannot start or produced nothing.""" + + +def ensure_rattler_build(tools: Path) -> None: + """ + Puts rattler-build on PATH, downloading the pinned release when none is + there. + """ + if shutil.which("rattler-build"): + return + release = RATTLER_BUILD_RELEASES.get(platform.system()) + if release is None or platform.machine().lower() not in ("x86_64", "amd64"): + raise BuildError( + "no rattler-build on PATH, and only the Linux and Windows x86_64 " + "releases are pinned here" + ) + asset, name, sha256 = release + print(f"build: fetching rattler-build {RATTLER_BUILD_VERSION}", flush=True) + url = ( + "https://github.com/prefix-dev/rattler-build/releases/download/" + f"{RATTLER_BUILD_VERSION}/{asset}" + ) + with urllib.request.urlopen(url, timeout=300) as reply: + binary = reply.read() + if hashlib.sha256(binary).hexdigest() != sha256: + raise BuildError( + f"rattler-build {RATTLER_BUILD_VERSION} does not match its pinned " + "sha256" + ) + tools.mkdir(parents=True, exist_ok=True) + target = tools / name + target.write_bytes(binary) + target.chmod(0o755) + os.environ["PATH"] = f'{tools}{os.pathsep}{os.environ.get("PATH", "")}' + + +def cibuildwheel_identifier(python: matrix.Python) -> str: + """The one build cibuildwheel runs, such as cp314t-manylinux_x86_64.""" + tag = WHEEL_PLATFORMS.get(platform.system()) + if tag is None: + raise BuildError( + f"wheels build on Linux, Windows or macOS, not {platform.system()}" + ) + minor, threaded = matrix.version_key(python.version) + return f'cp3{minor}{"t" if threaded else ""}-{tag}' + + +def wheel_requires() -> list[str]: + """[tool.gh-ci] wheel-requires from pyproject.toml.""" + if matrix.TOMLLIB is None: + raise BuildError("reading pyproject.toml needs Python 3.11 or newer") + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + tool = matrix.TOMLLIB.loads(text).get("tool", {}).get("gh-ci", {}) + requires = tool.get("wheel-requires", []) + if not isinstance(requires, list) or not all( + isinstance(r, str) for r in requires + ): + raise BuildError("[tool.gh-ci] wheel-requires must be a list of specs") + return requires + + +def record_hash(data: bytes) -> str: + """A file's hash as a wheel's RECORD spells it.""" + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + return "sha256=" + digest.rstrip(b"=").decode("ascii") + + +def add_requirements(wheel: Path, requirements: list[str]) -> None: + """Adds Requires-Dist lines to a built wheel, and updates its RECORD.""" + with zipfile.ZipFile(wheel) as source: + members = [(info, source.read(info)) for info in source.infolist()] + names = [info.filename for info, _ in members] + metadata = next(n for n in names if n.endswith(".dist-info/METADATA")) + record = metadata.removesuffix("METADATA") + "RECORD" + for index, (info, data) in enumerate(members): + if info.filename == metadata: + text = data.decode("utf-8") + # Some Windows backends write CRLF; take the line ending from the + # headers, as the description can keep the README's. + headers = re.split(r"\r?\n\r?\n", text, maxsplit=1)[0] + eol = "\r\n" if "\r\n" in headers else "\n" + head, blank, body = text.partition(eol * 2) + head = head.rstrip("\r\n") + "".join( + f"{eol}Requires-Dist: {r}" for r in requirements + ) + text = head + (eol * 2 + body if blank else eol) + new_metadata = text.encode("utf-8") + members[index] = (info, new_metadata) + for index, (info, data) in enumerate(members): + if info.filename == record: + lines = [ + ( + f"{metadata},{record_hash(new_metadata)}," + f"{len(new_metadata)}" + if line.split(",")[0] == metadata + else line + ) + for line in data.decode("utf-8").splitlines() + ] + members[index] = (info, ("\n".join(lines) + "\n").encode()) + rewritten = wheel.with_suffix(".tmp") + with zipfile.ZipFile(rewritten, "w", zipfile.ZIP_DEFLATED) as target: + for info, data in members: + target.writestr(info, data) + rewritten.replace(wheel) + + +def commands( + system: str, + selector: Path, + python: matrix.Python, + channels: list[str], + scratch: Path, +) -> list[list[str]]: + """ + The commands that build one Python version's packages into scratch. + + Each build system gets the version back exactly as its own file declares it, + and keeps every other variant setting from that file. + """ + channel_args = [arg for channel in channels for arg in ("-c", channel)] + # The commit a conda package came from, in its info/about.json + github = [ + os.environ.get(name, "") + for name in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_SHA") + ] + extra_meta = ( + [ + "--extra-meta", + f"remote_url={github[0]}/{github[1]}", + f"sha={github[2]}", + ] + if all(github) + else [] + ) + if system == "rattler-build": + return [ + [ + "rattler-build", + "build", + "-r", + str(selector), + "--variant", + f"python={python.spec}", + "--output-dir", + str(scratch / "conda"), + *channel_args, + ] + ] + if system == "conda-build": + # conda runs as a module of this interpreter, so Windows needs no + # conda.bat on PATH. + return [ + [ + sys.executable, + "-m", + "conda", + "build", + str(selector.parent), + "--python", + python.spec, + "--no-anaconda-upload", + "--output-folder", + str(scratch / "conda"), + *extra_meta, + "--override-channels", + *channel_args, + ] + ] + return [ + [ + sys.executable, + "-m", + "cibuildwheel", + "--only", + cibuildwheel_identifier(python), + "--output-dir", + str(scratch / "wheel"), + str(ROOT), + ] + ] + + +def collect(scratch: Path, output: Path) -> list[Path]: + """ + Moves built packages into output, flat, and lists the ones it moved. + Two packages of one name are an error: flat, one would overwrite the other. + """ + output.mkdir(parents=True, exist_ok=True) + sources: dict[str, Path] = {} + for package in sorted(scratch.rglob("*")): + if ( + package.is_file() + and package.name.endswith(PACKAGE_SUFFIXES) + and "broken" not in package.parts + ): + if package.name in sources: + raise BuildError( + f"two packages named {package.name}: " + f"{sources[package.name]} and {package}" + ) + sources[package.name] = package + shutil.move(str(package), output / package.name) + built = [output / name for name in sorted(sources)] + if not built: + raise BuildError(f"the build left no package in {output}") + return built + + +def main(argv: list[str] | None = None) -> int: + """Builds one Python version.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--output", required=True, type=Path, help="where the packages go" + ) + parser.add_argument( + "--system", + default="", + choices=("", *matrix.SYSTEMS), + help="build system to use instead of the detected one", + ) + parser.add_argument( + "--channel", + action="append", + default=None, + help="conda channel to build against, repeatable; default conda-forge", + ) + default_tools = ( + Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) / "ci-tools" + ) + parser.add_argument( + "--tools", + type=Path, + default=default_tools, + help="where a downloaded rattler-build goes", + ) + args = parser.parse_args(argv) + + system, selector = matrix.detect(ROOT, args.system) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise BuildError( + f"Python {args.python} is not declared for {system}; " + f'declared: {", ".join(declared)}' + ) + python = declared[args.python] + print( + f"build: {system} from {selector.relative_to(ROOT)} " + f"for Python {python.version} ({python.spec})", + flush=True, + ) + if system == "rattler-build": + ensure_rattler_build(args.tools) + output = args.output.resolve() + with tempfile.TemporaryDirectory() as scratch: + for command in commands( + system, + selector, + python, + args.channel or ["conda-forge"], + Path(scratch), + ): + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True) + built = collect(Path(scratch), output) + if system == "wheel" and (requires := wheel_requires()): + for wheel in built: + add_requirements(wheel, requires) + print("build: added Requires-Dist " + ", ".join(requires)) + print("build: " + ", ".join(path.name for path in built)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + BuildError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"build: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/config.yml b/ci/config.yml new file mode 100644 index 000000000000..1c4cb135bc8d --- /dev/null +++ b/ci/config.yml @@ -0,0 +1,15 @@ +# Intel CI settings for dpnp, see ci/README.md +version: 1 +workflow: intel-ci.yml +artifacts: + conda: "dpnp {platform} Python {python}" + wheel: "dpnp {platform} Wheels Python {python}" +test: + requires: [pytest, pytest-timeout, pytest-xdist, scipy] + # The wheels' requirements have no OpenCL CPU runtime of their own, which the tests need + wheel_requires: [pytest, pytest-timeout, pytest-xdist, scipy, intel-opencl-rt] +benchmark: + # PyYAML, which asv needs, has no PyPI wheel for every Python the recipe declares. + requires: [scipy, pyyaml] + # conda-forge lacks asv for some Python the recipe declares; the wheel on PyPI fits all. + pip_requires: [asv] diff --git a/ci/hooks.py b/ci/hooks.py new file mode 100644 index 000000000000..05d161aeb953 --- /dev/null +++ b/ci/hooks.py @@ -0,0 +1,467 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +""" +Runs ci/test.py against one built artifact. + +For the GitHub Actions test job. + +Each Python version's artifact is installed into a fresh conda environment from +a local channel plus conda-forge, pinned to its exact build, with the test +requirements ci/config.yml lists, and the test runs with that environment's +Python from an empty working directory, on Linux or Windows. A wheel's +environment has only Python from conda-forge: the wheel and the test +requirements install with pip, as in the wheel's users' environments. The test +gets the variables ci/test.py reads, and nothing else from the runner. + +Usage, from any Python 3.10 or newer; it runs itself again with the conda base +install's Python, from $CONDA, which setup-miniconda sets, or the conda on PATH: + python ci/hooks.py --python 3.14 --artifacts DIR --work DIR [--kind wheel] + +Needs conda, conda-package-handling and ruamel.yaml, which every conda base has, +and installs nothing into it. +""" + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import matrix + +# Dependencies of conda itself; YAML is read with PyYAML, else ruamel.yaml +try: + from conda_package_handling import api as CPH_API +except ModuleNotFoundError: + CPH_API = None +try: + import yaml as PYYAML +except ModuleNotFoundError: + PYYAML = None +try: + from ruamel.yaml import YAML as RUAMEL_YAML +except ModuleNotFoundError: + RUAMEL_YAML = None + +ROOT = matrix.ROOT +IS_WINDOWS = os.name == "nt" +if IS_WINDOWS: + PLATFORM, HOST_SUBDIR = "windows", "win-64" +elif sys.platform == "darwin": + PLATFORM, HOST_SUBDIR = "macos", "osx-64" +else: + PLATFORM, HOST_SUBDIR = "linux", "linux-64" +CONDA_SUFFIXES = (".conda", ".tar.bz2") +# All the test sees of the runner's environment, so no tokens reach it +KEEP = ( + "PATH", + "HOME", + "USER", + "LANG", + "TERM", + "TMPDIR", + "TZ", + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CONDA_PKGS_DIRS", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "TEMP", + "TMP", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", + "GH_CI_DEVICE", + "ONEAPI_DEVICE_SELECTOR", +) + + +class HookError(Exception): + """Raised when the artifact cannot be installed or ci/test.py is missing.""" + + +def run(command: list[str], **options: object) -> None: + """Echoes and runs a command, never through a shell.""" + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True, **options) + + +def conda(*args: str) -> None: + """Runs conda from the base environment running this script.""" + run([sys.executable, "-m", "conda", *args]) + + +def empty_repodata(subdir: str) -> dict: + """The repodata.json of a channel subdir with no packages.""" + return { + "info": {"subdir": subdir}, + "packages": {}, + "packages.conda": {}, + "repodata_version": 1, + } + + +def package_record(package: Path) -> tuple[str, dict]: + """A conda package's subdir and repodata record, from its index.json.""" + if CPH_API is None: + raise HookError( + "run this with a conda base Python, " + "which has conda-package-handling" + ) + with tempfile.TemporaryDirectory() as scratch: + CPH_API.extract(str(package), scratch, components="info") + record = json.loads( + (Path(scratch) / "info" / "index.json").read_text(encoding="utf-8") + ) + data = package.read_bytes() + record.update( + { + "md5": hashlib.md5(data, usedforsecurity=False).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), + "size": len(data), + } + ) + subdir = str(record.get("subdir") or "noarch") + if not re.fullmatch(r"[a-z0-9]+(-[a-z0-9_]+)?", subdir): + raise HookError( + f"{package.name} declares an unusable subdir {subdir!r}" + ) + return subdir, record + + +def stage_channel(packages: list[Path], channel: Path) -> None: + """ + Lays the packages out as a local channel, by the subdir each one declares, + and writes its repodata.json itself, without conda-index. + """ + repodata = { + subdir: empty_repodata(subdir) for subdir in ("noarch", HOST_SUBDIR) + } + for package in packages: + subdir, record = package_record(package) + (channel / subdir).mkdir(parents=True, exist_ok=True) + shutil.copy2(package, channel / subdir / package.name) + section = ( + "packages.conda" if package.name.endswith(".conda") else "packages" + ) + repodata.setdefault(subdir, empty_repodata(subdir))[section][ + package.name + ] = record + for subdir, data in repodata.items(): + (channel / subdir).mkdir(parents=True, exist_ok=True) + (channel / subdir / "repodata.json").write_text( + json.dumps(data, indent=2) + "\n", encoding="utf-8" + ) + + +def test_python(version: str) -> str: + """The conda spec for the test environment's interpreter.""" + minor, threaded = matrix.version_key(version) + # The .* matters: conda reads a bare 3.11 as exactly 3.11.0 + if minor < matrix.FREE_THREADED_MINOR: + return f"3.{minor}.*" + # conda-forge marks the ABI from 3.13, so 3.14 can't resolve to 3.14t + return f'3.{minor}.* *_cp3{minor}{"t" if threaded else ""}' + + +def test_requirements() -> tuple[list[str], list[str], list[str] | None]: + """ + The conda-forge and pip packages ci/config.yml installs next to the + artifact, and a wheel's PyPI requirements, None when it gives none. + """ + text = (ROOT / "ci" / "config.yml").read_text(encoding="utf-8") + if PYYAML is not None: + config = PYYAML.safe_load(text) + elif RUAMEL_YAML is not None: + config = RUAMEL_YAML(typ="safe", pure=True).load(text) + else: + raise HookError("reading ci/config.yml needs PyYAML or ruamel.yaml") + test = (config or {}).get("test") or {} + lists = [] + for key in ("requires", "pip_requires", "wheel_requires"): + values = test.get(key) + if values is not None and ( + not isinstance(values, list) + or not all(isinstance(v, str) for v in values) + ): + raise HookError( + f"ci/config.yml: test.{key} must be a list of package specs" + ) + lists.append(values) + return lists[0] or [], lists[1] or [], lists[2] + + +def exact_spec(package: Path) -> str: + """name=version=build from a conda package file name.""" + stem = package.name + for suffix in CONDA_SUFFIXES: + stem = stem.removesuffix(suffix) + name, version, build = stem.rsplit("-", 2) + return f"{name}={version}={build}" + + +def env_python(env_dir: Path) -> Path: + """The interpreter of a conda environment.""" + return env_dir / "python.exe" if IS_WINDOWS else env_dir / "bin" / "python" + + +def env_path(env_dir: Path) -> list[Path]: + """ + The directories conda's activation would put first on PATH for an + environment. + """ + if not IS_WINDOWS: + return [env_dir / "bin"] + library = env_dir / "Library" + return [ + env_dir, + library / "mingw-w64" / "bin", + library / "usr" / "bin", + library / "bin", + env_dir / "Scripts", + env_dir / "bin", + ] + + +def test_env( + env_dir: Path, artifacts: Path, results: Path, args: argparse.Namespace +) -> dict: + """The variables ci/test.py reads, and no runner tokens.""" + env = {key: value for key, value in os.environ.items() if key in KEEP} + env["PATH"] = os.pathsep.join( + [*map(str, env_path(env_dir)), env.get("PATH", "")] + ) + env.update( + { + "CONDA_PREFIX": str(env_dir), + "GH_CI_ARTIFACTS": str(artifacts), + "GH_CI_SOURCE": str(ROOT), + "GH_CI_RESULTS": str(results), + "GH_CI_KIND": args.kind, + "GH_CI_PYTHON": args.python, + "GH_CI_PLATFORM": PLATFORM, + "GH_CI_EVENT": args.event, + "GH_CI_COMMIT": args.commit, + } + ) + return env + + +def summarize(lines: list[str]) -> None: + """Adds the results to the job summary page, in GitHub Actions.""" + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as page: + page.write("\n".join(lines) + "\n") + + +def install(args: argparse.Namespace) -> Path: + """ + Creates the test environment with the artifact, pinned to its exact build; + returns it. + """ + system, selector = matrix.detect( + ROOT, "wheel" if args.kind == "wheel" else "" + ) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise HookError( + f"Python {args.python} is not declared; " + f'declared: {", ".join(declared)}' + ) + suffixes = (".whl",) if args.kind == "wheel" else CONDA_SUFFIXES + packages = sorted( + path + for path in args.artifacts.rglob("*") + if path.name.endswith(suffixes) + ) + if not packages: + raise HookError(f"{args.artifacts} holds no {args.kind} package") + requires, pip_requires, wheel_requires = test_requirements() + env_dir = args.work / "env" + pip = [str(env_python(env_dir)), "-m", "pip", "install", "--no-input"] + python = f"python {test_python(args.python)}" + if args.kind == "wheel": + conda_create(env_dir, python, "pip") + wheel_specs = requires if wheel_requires is None else wheel_requires + run([*pip, *map(str, packages), *wheel_specs]) + else: + channel = args.work / "channel" + stage_channel(packages, channel) + conda_create( + env_dir, + python, + *requires, + *map(exact_spec, packages), + channels=(channel.as_uri(),), + ) + if pip_requires: + run([*pip, *pip_requires]) + return env_dir + + +def conda_create( + env_dir: Path, *specs: str, channels: tuple[str, ...] = () +) -> None: + """Creates a conda environment from conda-forge and channels.""" + conda( + "create", + "-y", + "-q", + "-p", + str(env_dir), + "--override-channels", + *(arg for channel in channels for arg in ("-c", channel)), + "-c", + "conda-forge", + *specs, + ) + + +def run_test(env_dir: Path, args: argparse.Namespace) -> int: + """Runs ci/test.py with the environment's Python; returns its status.""" + script = ROOT / "ci" / "test.py" + if not script.is_file(): + raise HookError("ci/test.py is required and missing") + results, scratch = ( + args.work / "results" / "test", + args.work / "scratch" / "test", + ) + for directory in (results, scratch): + directory.mkdir(parents=True, exist_ok=True) + command = [str(env_python(env_dir)), str(script)] + print(f"+ {shlex.join(command)}", flush=True) + status = subprocess.run( + command, + cwd=scratch, + check=False, + env=test_env(env_dir, args.artifacts, results, args), + ).returncode + result = "passed" if status == 0 else f"failed, exit {status}" + summarize( + [ + f"### {matrix.PLATFORMS[PLATFORM]} Python {args.python}" + f" {args.kind}", + "", + "| Test | Result |", + "|---|---|", + f"| ci/test.py | {result} |", + ] + ) + return status + + +def conda_base_python() -> Path: + """The conda base install's Python: from $CONDA, else the conda on PATH.""" + root = os.environ.get("CONDA", "") + if not root: + found = shutil.which("conda") + if not found: + raise HookError( + "no conda base install: set CONDA or put conda on PATH" + ) + # conda lives in bin, Scripts or condabin, one level below the install. + root = str(Path(found).resolve().parent.parent) + python = env_python(Path(root)) + if not python.is_file(): + raise HookError( + f"{root} is not a conda base install; it has no {python.name}" + ) + return python + + +def main(argv: list[str] | None = None) -> int: + """Installs the artifact, then runs ci/test.py against it.""" + if CPH_API is None: + # Re-run with base's Python, which has conda-package-handling + command = [ + str(conda_base_python()), + str(Path(__file__).resolve()), + *(sys.argv[1:] if argv is None else argv), + ] + return subprocess.run(command, check=False).returncode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--artifacts", required=True, type=Path, help="the downloaded artifact" + ) + parser.add_argument( + "--work", required=True, type=Path, help="an empty scratch directory" + ) + parser.add_argument( + "--event", default="push", choices=("pr", "tag", "push") + ) + parser.add_argument("--commit", default="") + parser.add_argument( + "--kind", + default="conda", + choices=("conda", "wheel"), + help="what the artifact holds", + ) + args = parser.parse_args(argv) + args.artifacts, args.work = args.artifacts.resolve(), args.work.resolve() + if args.work.exists() and ( + not args.work.is_dir() or any(args.work.iterdir()) + ): + raise HookError(f"--work {args.work} must be an empty directory") + return run_test(install(args), args) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + HookError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"hooks: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/install_icx.py b/ci/install_icx.py new file mode 100644 index 000000000000..bc6fa8e40f65 --- /dev/null +++ b/ci/install_icx.py @@ -0,0 +1,96 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +""" +Installs Intel's icx and the oneAPI libraries for dpnp's Windows wheel builds. + +cibuildwheel runs this before the builds, as pyproject.toml says. It fetches +micromamba, checked against its published sha256, and creates PREFIX with +conda-forge's dpcpp_win-64, the compiler the conda recipe builds with, and the +oneMKL, oneDPL and oneTBB development packages it builds against. + +Usage: + python ci/install_icx.py PREFIX +""" + +import hashlib +import subprocess +import sys +import tempfile +import urllib.request +from pathlib import Path + +MICROMAMBA = ( + "https://github.com/mamba-org/micromamba-releases/releases/download/" + "2.10.0-0/micromamba-win-64" +) +MICROMAMBA_SHA256 = ( + "f8b4284bd4eb1134122eaf93b3ea6c7be567e596de5829e1a6f478e34e41974a" +) +PACKAGES = [ + "dpcpp_win-64=2026.2.0", + "mkl-devel-dpcpp>=2026.0,<2027", + "onedpl-devel", + "tbb-devel", +] + + +def main() -> int: + """Creates PREFIX with the compiler and the libraries.""" + prefix = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as scratch: + micromamba = Path(scratch) / "micromamba.exe" + with urllib.request.urlopen(MICROMAMBA, timeout=300) as response: + micromamba.write_bytes(response.read()) + digest = hashlib.sha256(micromamba.read_bytes()).hexdigest() + if digest != MICROMAMBA_SHA256: + print( + f"install_icx: micromamba has sha256 {digest}, " + f"not {MICROMAMBA_SHA256}", + file=sys.stderr, + ) + return 1 + command = [ + str(micromamba), + "create", + "-y", + "-r", + str(Path(scratch) / "root"), + "-p", + str(prefix), + "--override-channels", + "-c", + "conda-forge", + *PACKAGES, + ] + print("+ " + " ".join(command), flush=True) + return subprocess.run(command, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/matrix.py b/ci/matrix.py new file mode 100644 index 000000000000..4f3b3c0f0801 --- /dev/null +++ b/ci/matrix.py @@ -0,0 +1,715 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +""" +The platforms and Python versions this repository builds. + +Both are read from the repository's own build files. + +Every build system already has a standard place for both, so each is written +once, where the build reads it, and the GitHub Actions matrix follows it: + + rattler-build, selected by recipe.yaml + Python versions python: in variants.yaml next to the recipe + platforms build: skip: in the recipe + conda-build, selected by meta.yaml + Python versions python: in conda_build_config.yaml next to the recipe + platforms skip: true # [selector] lines in the recipe + wheel, selected by pyproject.toml + Python versions the Programming Language :: Python :: 3.N classifiers; + any Free Threading classifier adds 3.Nt for each from 3.14 + platforms the Operating System :: classifiers + +A recipe builds on every platform in PLATFORMS, Linux, Windows and macOS on +Intel, unless a skip that depends on the platform alone rules one out, such as +`skip: [win]` or `skip: true # [osx]`; a skip that also depends on something +else, such as `win and py < 311` or `osx and arm64`, skips some builds, not the +platform. A pyproject.toml without Operating System classifiers, or with OS +Independent, builds on all three. Of the declared platforms, the matrix holds +those GitHub Actions has runners for, BUILT_PLATFORMS. + +meson-python, scikit-build-core and every other PEP 517 backend declare these in +pyproject.toml too, so the wheel row covers them. Another convention, such as +pixi.toml, is one more reader in READERS and PLATFORM_READERS and, if it is a +new build system, one more entry in SYSTEMS. + +Usage: + python ci/matrix.py [--system SYSTEM] [--event EVENT] [--github-output NAME] + [--newest] [--platform PLATFORM] + +Prints the build matrix as JSON for strategy.matrix: {"include": [{"platform": +"linux", "name": "Linux", "python": "3.14"}, ...]}, every platform with every +Python version. For a pull_request event only the oldest and the newest regular +versions and the newest free-threaded one are listed. --github-output also +writes NAME= to $GITHUB_OUTPUT, for later jobs. Standard library only; +reading pyproject.toml needs Python 3.11 or newer for tomllib. +""" + +import argparse +import ast +import json +import os +import re +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +# Only the wheel reader needs it, and only it needs Python 3.11 or newer. +try: + import tomllib as TOMLLIB +except ModuleNotFoundError: + TOMLLIB = None + +ROOT = Path(__file__).resolve().parent.parent + +# Build system -> files that select it, in the order they are looked for. Where +# a repository still has conda-recipe-cf/ next to conda-recipe/, the conda-forge +# one comes first: the tests use conda-forge alone. +SYSTEMS = { + "rattler-build": ("recipe.yaml", "recipe/recipe.yaml"), + "conda-build": ( + "conda-recipe-cf/meta.yaml", + "meta.yaml", + "conda-recipe/meta.yaml", + "recipe/meta.yaml", + ), + "wheel": ("pyproject.toml",), +} + +# The platforms CI builds and tests, and how artifact names spell them. +PLATFORMS = {"linux": "Linux", "windows": "Windows", "macos": "macOS"} +# The platforms the workflow can build on: GitHub-hosted runners have all +# three. A declared platform missing here is reported and left out. +BUILT_PLATFORMS = ("linux", "windows", "macos") +# What a platform-only selector means on each platform, for both conda-build and +# rattler-build. +SELECTORS = { + "linux": { + "linux": True, + "linux64": True, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + "windows": { + "linux": False, + "linux64": False, + "unix": False, + "win": True, + "win64": True, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + # Intel Macs only, as MKL has no Apple silicon build. + "macos": { + "linux": False, + "linux64": False, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": True, + "osx64": True, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, +} +CONDA_SKIP_RE = re.compile(r"\s*skip\s*:\s*(?:true|True|yes)\s*#\s*\[(.+)\]\s*") +OS_CLASSIFIERS = { + "Operating System :: OS Independent": tuple(PLATFORMS), + "Operating System :: POSIX": ("linux",), + "Operating System :: Unix": ("linux",), + "Operating System :: POSIX :: Linux": ("linux",), + "Operating System :: Microsoft :: Windows": ("windows",), + "Operating System :: MacOS": ("macos",), +} + +PULL_REQUEST_EVENTS = ("pull_request",) +OLDEST_MINOR = 10 +FREE_THREADED_MINOR = 13 +# A Free Threading classifier adds 3.Nt from 3.14, the first supported one, +# which is also the oldest cibuildwheel builds. +CLASSIFIED_FREE_THREADED_MINOR = 14 + +# A conda python variant: 3.12, 3.12.*, 3.12.* *_cpython, 3.13.* *_cp313 or +# 3.14.* *_cp314t. +CONDA_SPEC_RE = re.compile( + r"3\.(\d+)(?:\.\*)?(?:\s+\*_(?:cpython|cp3(\d+)(t?)))?" +) +CLASSIFIER_RE = re.compile(r"Programming Language :: Python :: 3\.(\d+)") +FREE_THREADING_CLASSIFIER = "Programming Language :: Python :: Free Threading" +LOWER_BOUND_RE = re.compile(r">=\s*3\.(\d+)") + + +class DeclarationError(Exception): + """ + Raised when the build files declare no Python versions, or ones this cannot + read. + """ + + +@dataclass(frozen=True) +class Python: + """One declared Python version.""" + + version: str # as artifact names spell it: 3.14 or 3.14t + spec: str # as the build system spells it, passed back to it unchanged + + +def version_key(version: str) -> tuple[int, bool]: + """(minor, free-threaded); also the sort order.""" + return int(version.rstrip("t").split(".")[1]), version.endswith("t") + + +def checked(minor: int, threaded: bool, where: str) -> str: + """The version string, once it is one CI can test.""" + if minor < OLDEST_MINOR or (threaded and minor < FREE_THREADED_MINOR): + raise DeclarationError( + f'{where}: 3.{minor}{"t" if threaded else ""} is not supported; ' + f"the oldest is 3.{OLDEST_MINOR}, free-threaded from " + f"3.{FREE_THREADED_MINOR}" + ) + return f'3.{minor}{"t" if threaded else ""}' + + +def _strip_comment(line: str, where: str) -> str: + """ + The line without a trailing comment; conda-build selectors are refused, not + guessed at. + """ + in_quote = "" + for index, char in enumerate(line): + if char in "\"'" and in_quote in ("", char): + in_quote = "" if in_quote else char + elif ( + char == "#" + and not in_quote + and (index == 0 or line[index - 1].isspace()) + ): + if line[index:].lstrip("# ").startswith("["): + raise DeclarationError( + f"{where}: selectors in the python list are not supported" + ) + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str, where: str) -> str: + """A plain or quoted YAML scalar.""" + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + if not value or value[0] in "{[&*!|>" or ":" in value: + raise DeclarationError( + f"{where}: write each python entry as a plain value, not {value!r}" + ) + return value + + +def yaml_python_list(text: str, where: str) -> list[str]: + """ + The top-level python: list of a variants file, without a YAML library. + + Accepts a block list or a one-line [a, b] list of plain or quoted values. + Anything richer, such as if/then selectors or templating, is refused so the + matrix is never guessed. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + if not re.match(r"python\s*:", line): + continue + rest = _strip_comment(line.split(":", 1)[1], where).strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + raise DeclarationError(f"{where}: python must be a list") + values = [] + for item in lines[number + 1 :]: + # Checked before the comment, as the next key may have a selector + if item[:1] not in ("", " ", "\t", "-", "#"): + break # the next top-level key + entry = _strip_comment(item, where).lstrip() + if not entry: + continue + if not entry.startswith("- "): + raise DeclarationError( + f"{where}: python must be a plain list of versions" + ) + values.append(_unquote(entry[2:], where)) + return values + raise DeclarationError(f"{where} has no top-level python: list") + + +def from_conda_variants(path: Path) -> list[Python]: + """The python: list of a variants.yaml or a conda_build_config.yaml.""" + where = ( + path.relative_to(ROOT).as_posix() + if path.is_relative_to(ROOT) + else str(path) + ) + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as ex: + raise DeclarationError( + f"{where} is missing; list the Python versions to build there" + ) from ex + pythons = [] + for spec in yaml_python_list(text, where): + match = CONDA_SPEC_RE.fullmatch(spec) + if not match or (match[2] and match[2] != match[1]): + raise DeclarationError( + f"{where}: {spec!r} is not a CPython variant such as " + '"3.13.* *_cp313" or "3.14.* *_cp314t"' + ) + pythons.append( + Python(checked(int(match[1]), bool(match[3]), where), spec) + ) + return pythons + + +def from_pyproject(path: Path) -> list[Python]: + """ + The 3.N classifiers in pyproject.toml, plus 3.Nt from 3.14 if free-threading + is declared. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + project = TOMLLIB.loads(path.read_text(encoding="utf-8")).get("project", {}) + classifiers = project.get("classifiers", []) + minors = sorted( + { + int(match[1]) + for classifier in classifiers + if (match := CLASSIFIER_RE.fullmatch(classifier)) + } + ) + if not minors: + raise DeclarationError( + "pyproject.toml lists no Programming Language :: Python :: 3.N " + "classifiers" + ) + bound = LOWER_BOUND_RE.search(project.get("requires-python", "")) + if bound and minors[0] < int(bound[1]): + raise DeclarationError( + f"pyproject.toml classifies 3.{minors[0]}, below requires-python " + f'{project["requires-python"]}' + ) + threaded = any( + classifier.startswith(FREE_THREADING_CLASSIFIER) + for classifier in classifiers + ) + versions = [checked(minor, False, "pyproject.toml") for minor in minors] + versions += [ + f"3.{minor}t" + for minor in minors + if threaded and minor >= CLASSIFIED_FREE_THREADED_MINOR + ] + return [Python(version, version) for version in versions] + + +def skips_platform(expression: str, platform: str) -> bool | None: + """ + Whether a skip expression rules out a whole platform: True or False when it + depends on the platform alone, None when it also depends on anything else, + such as the Python version. + """ + try: + tree = ast.parse( + expression.strip().removeprefix("${{").removesuffix("}}").strip(), + mode="eval", + ).body + except SyntaxError: + return None + + def value(node: ast.AST) -> bool: + if isinstance(node, ast.BoolOp): + values = [value(item) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not value(node.operand) + if isinstance(node, ast.Name) and node.id in SELECTORS[platform]: + return SELECTORS[platform][node.id] + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return node.value + raise ValueError(ast.dump(node)) + + try: + return value(tree) + except ValueError: + return None + + +def platforms_without(skips: list[str], where: str) -> list[str]: + """Every platform in PLATFORMS, but those a platform-only skip rules out.""" + kept = [ + platform + for platform in PLATFORMS + if not any(skips_platform(skip, platform) for skip in skips) + ] + if not kept: + raise DeclarationError( + f"{where} skips every platform CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return kept + + +def yaml_build_skips(text: str, where: str) -> list[str]: + """ + The skip: entries directly under a rattler-build recipe's top-level build:, + as a list of plain or quoted values, a one-line [a, b] list, or one value; + none when there is no skip. + """ + lines = text.splitlines() + starts = [ + number + for number, line in enumerate(lines) + if re.match(r"build\s*:\s*(#.*)?$", line) + ] + if not starts: + return [] + child = "" + for number in range(starts[0] + 1, len(lines)): + line = ( + _strip_comment(lines[number], where) + if "#" in lines[number] + else lines[number] + ) + if not line.strip(): + continue + if not line[0].isspace(): + return [] # the next top-level key + indent = line[: len(line) - len(line.lstrip())] + child = child or indent + match = ( + re.match(r"skip\s*:(.*)", line.lstrip()) + if indent == child + else None + ) + if not match: + continue + rest = match[1].strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + return [_unquote(rest, where)] + values = [] + for item in lines[number + 1 :]: + entry = item.strip() + if not entry or entry.startswith("#"): + continue + if not entry.startswith("- "): + break + values.append(_unquote(entry[2:].split(" #")[0], where)) + return values + return [] + + +def from_rattler_recipe(recipe: Path) -> list[str]: + """The platforms a rattler-build recipe builds on, from its build: skip:.""" + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + return platforms_without( + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), where + ) + + +def from_conda_recipe(recipe: Path) -> list[str]: + """ + The platforms a conda-build meta.yaml builds on, from its skip: true # + [selector] lines. + """ + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + skips = [ + match[1] + for line in recipe.read_text(encoding="utf-8").splitlines() + if (match := CONDA_SKIP_RE.fullmatch(line)) + ] + return platforms_without(skips, where) + + +def from_os_classifiers(path: Path) -> list[str]: + """ + The platforms pyproject.toml's Operating System classifiers name; all of + them when it names none. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + classifiers = ( + TOMLLIB.loads(path.read_text(encoding="utf-8")) + .get("project", {}) + .get("classifiers", []) + ) + named = [ + classifier + for classifier in classifiers + if classifier.startswith("Operating System ::") + ] + if not named: + return list(PLATFORMS) + found = { + platform + for classifier in named + for prefix, platforms in OS_CLASSIFIERS.items() + if classifier == prefix or classifier.startswith(prefix + " ::") + for platform in platforms + } + if not found: + raise DeclarationError( + "pyproject.toml names no Operating System CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return [platform for platform in PLATFORMS if platform in found] + + +# Build system -> where it declares Python versions, given the file that +# selected it. +READERS: dict[str, Callable[[Path], list[Python]]] = { + "rattler-build": lambda recipe: from_conda_variants( + recipe.parent / "variants.yaml" + ), + "conda-build": lambda recipe: from_conda_variants( + recipe.parent / "conda_build_config.yaml" + ), + "wheel": from_pyproject, +} + +# Build system -> where it declares the platforms it builds on, given the file +# that selected it. +PLATFORM_READERS: dict[str, Callable[[Path], list[str]]] = { + "rattler-build": from_rattler_recipe, + "conda-build": from_conda_recipe, + "wheel": from_os_classifiers, +} + + +def detect(root: Path, system: str = "") -> tuple[str, Path]: + """The build system and the file that selects it.""" + for name, candidates in SYSTEMS.items(): + if system and name != system: + continue + for candidate in candidates: + if (root / candidate).is_file(): + return name, root / candidate + wanted = system or " or ".join(SYSTEMS) + raise DeclarationError( + f"no {wanted} input found; looked for " + + ", ".join(c for name, cs in SYSTEMS.items() for c in cs) + ) + + +def declared(system: str, selector: Path) -> list[Python]: + """The build system's Python versions, sorted, each listed once.""" + pythons = READERS[system](selector) + versions = [python.version for python in pythons] + if not pythons: + raise DeclarationError(f"{system} declares no Python versions") + if len(set(versions)) != len(versions): + raise DeclarationError( + f"{system} declares a Python version twice: {versions}" + ) + return sorted(pythons, key=lambda python: version_key(python.version)) + + +def declared_platforms(system: str, selector: Path) -> list[str]: + """The platforms the build system builds on, in PLATFORMS order.""" + return PLATFORM_READERS[system](selector) + + +def for_event(pythons: list[Python], event: str) -> list[Python]: + """ + All of them, or for a pull request the oldest and newest regular ones and + the newest free-threaded one. + """ + if event not in PULL_REQUEST_EVENTS: + return pythons + regular = [python for python in pythons if not python.version.endswith("t")] + threaded = [python for python in pythons if python.version.endswith("t")] + regular = regular or pythons + return list(dict.fromkeys(regular[:1] + regular[-1:] + threaded[-1:])) + + +def newest(pythons: list[Python]) -> list[Python]: + """The newest regular and the newest free-threaded version.""" + regular = [python for python in pythons if not python.version.endswith("t")] + threaded = [python for python in pythons if python.version.endswith("t")] + return list(dict.fromkeys(regular[-1:] + threaded[-1:])) + + +def build_matrix(system: str, selector: Path, event: str) -> dict: + """ + Every declared platform Actions can build on, with every Python version the + event builds. + """ + pythons = for_event(declared(system, selector), event) + platforms = declared_platforms(system, selector) + waiting = [ + PLATFORMS[platform] + for platform in platforms + if platform not in BUILT_PLATFORMS + ] + if waiting: + print( + f'matrix: {", ".join(waiting)} declared but not built: ' + "no runners for it yet", + file=sys.stderr, + ) + built = [platform for platform in platforms if platform in BUILT_PLATFORMS] + if not built: + raise DeclarationError( + "the build files declare only platforms there are no runners " + "for yet: " + ", ".join(waiting) + ) + return { + "include": [ + { + "platform": platform, + "name": PLATFORMS[platform], + "python": python.version, + } + for platform in built + for python in pythons + ] + } + + +def main(argv: list[str] | None = None) -> int: + """Prints the matrix to build; writes it for GitHub Actions if asked.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--system", + default="", + choices=("", *SYSTEMS), + help="build system to read instead of the detected one", + ) + parser.add_argument( + "--event", default="", help="GitHub event name, such as pull_request" + ) + parser.add_argument( + "--github-output", + default="", + metavar="NAME", + help="also write NAME= to $GITHUB_OUTPUT", + ) + parser.add_argument( + "--newest", + action="store_true", + help="only the newest regular and free-threaded versions, which " + "every event builds", + ) + parser.add_argument( + "--platform", + default="", + choices=("", *PLATFORMS), + help="only this platform", + ) + args = parser.parse_args(argv) + if args.github_output: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", args.github_output): + raise DeclarationError( + f"{args.github_output!r} is not an output name" + ) + if not os.environ.get("GITHUB_OUTPUT"): + raise DeclarationError( + "--github-output needs $GITHUB_OUTPUT, which Actions sets" + ) + system, selector = detect(ROOT, args.system) + matrix = build_matrix(system, selector, args.event) + keep = [python.version for python in newest(declared(system, selector))] + matrix["include"] = [ + cell + for cell in matrix["include"] + if (not args.newest or cell["python"] in keep) + and (not args.platform or cell["platform"] == args.platform) + ] + if args.newest: + # The regular version first, for a job that needs just one + matrix["include"].sort(key=lambda cell: keep.index(cell["python"])) + if not matrix["include"]: + raise DeclarationError("--newest and --platform leave no build") + cells = ", ".join( + f'{cell["name"]} {cell["python"]}' for cell in matrix["include"] + ) + print( + f"matrix: {system} from {selector.relative_to(ROOT)}: {cells}", + file=sys.stderr, + ) + text = json.dumps(matrix) + print(text) + if args.github_output: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"{args.github_output}={text}\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except DeclarationError as error: + print(f"matrix: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/test.py b/ci/test.py new file mode 100644 index 000000000000..61e9a8778602 --- /dev/null +++ b/ci/test.py @@ -0,0 +1,326 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +""" +Tests dpnp's installed package. + +Runs with one built package installed in a clean environment, as ci/hooks.py +sets up, from an empty working directory, so the tests import the installed +dpnp rather than the source tree. GH_CI_DEVICE=gpu requires a GPU. The test +requirements come from ci/config.yml. Writes JUnit reports to $GH_CI_RESULTS. + +It runs in a scrubbed environment, so it sets those the SYCL runtime, the OpenCL +loader and the test suite need itself. +""" + +import contextlib +import importlib.util +import json +import os +import pathlib +import signal +import subprocess +import sys +import sysconfig +import tempfile + +if os.name == "nt": + import winreg + +PACKAGE = "dpnp" +# pytest's exit status when it collects nothing +NO_TESTS = 5 +# Tests now and then fail or hang, and a rerun passes +MAIN_ATTEMPTS, TENSOR_ATTEMPTS = 2, 3 +# A tensor test stuck at 99%: https://github.com/IntelPython/dpctl/issues/1954 +TENSOR_TEST_TIMEOUT = 120 +MAIN_TIMEOUT = 60 * 60 +TENSOR_TIMEOUT = 20 * 60 +# The Python version whose main suite runs every integer type +ALL_INT_TYPES_PYTHON = "3.14" +PROBE = """ +import json +import dpctl + +devices = dpctl.get_devices() +try: + default = "gpu" if dpctl.select_default_device().is_gpu else "cpu" +except dpctl.SyclDeviceCreationError: + default = "none" +print(json.dumps({ + "cpu": any(device.is_cpu for device in devices), + "gpu": any(device.is_gpu for device in devices), + "default": default, +})) +""" +# What the PyPI wheel of the runtime has for the prefix, which conda replaces on +# install +CONDA_PLACEHOLDER = "/opt/anaconda1anaconda2anaconda3" +VENDORS_KEY = r"SOFTWARE\Khronos\OpenCL\Vendors" + + +def running_python() -> str: + """This interpreter's version as artifact names spell it, such as 3.14t.""" + version = f"{sys.version_info.major}.{sys.version_info.minor}" + threaded = sysconfig.get_config_var("Py_GIL_DISABLED") + return version + ("t" if threaded else "") + + +def merged_icds(scratch: pathlib.Path) -> str | None: + """ + The directory of OpenCL ICD files for OCL_ICD_VENDORS, if there are any. + + It holds the environment's CPU runtime, with the path of its library, and + the system's, such as a GPU driver's, so the loader sees both. + """ + prefix = pathlib.Path(sys.prefix) + system = pathlib.Path("/etc/OpenCL/vendors") + merged = scratch / "vendors" + merged.mkdir() + for directory in (prefix / "etc" / "OpenCL" / "vendors", system): + for icd in sorted(directory.glob("*.icd")): + library = ( + icd.read_text(encoding="utf-8") + .strip() + .replace(CONDA_PLACEHOLDER, str(prefix)) + ) + local = prefix / "lib" / library + if directory != system and local.is_file(): + library = str(local) + if not (merged / icd.name).exists(): + (merged / icd.name).write_text(library + "\n", encoding="utf-8") + return str(merged) if any(merged.iterdir()) else None + + +def runtime_environment(scratch: pathlib.Path) -> dict[str, str]: + """The variables the SYCL runtime and the OpenCL loader need.""" + env = {"SYCL_CACHE_PERSISTENT": "1"} + if os.name != "nt" and (icds := merged_icds(scratch)): + env["OCL_ICD_VENDORS"] = icds + return env + + +@contextlib.contextmanager +def registered_cpu_driver(): + """ + Registers the environment's OpenCL CPU runtime; that needs an administrator. + + The Windows loader finds drivers only in the registry, and the runtime + does not register itself. The entry goes when the tests end. A GPU run + or a machine with drivers registered already changes nothing. + """ + bin_dir = pathlib.Path(sys.prefix, "Library", "bin") + driver = next(bin_dir.rglob("intelocl64.dll"), None) + if os.name != "nt" or os.environ.get("GH_CI_DEVICE") == "gpu": + yield + elif driver is None: + print(f"no OpenCL CPU runtime under {bin_dir}") + yield + else: + with winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, VENDORS_KEY) as key: + if winreg.QueryInfoKey(key)[1]: + print("OpenCL drivers are already registered; adding none") + yield + return + winreg.SetValueEx(key, str(driver), 0, winreg.REG_DWORD, 0) + try: + yield + finally: + winreg.DeleteValue(key, str(driver)) + + +def probe_devices(env: dict[str, str]) -> dict: + """Lists the SYCL devices, and tells which kinds there are.""" + subprocess.run([sys.executable, "-I", "-m", "dpctl", "-f"], env=env) + probe = subprocess.run( + [sys.executable, "-I", "-c", PROBE], + env=env, + capture_output=True, + text=True, + ) + if probe.returncode: + print(probe.stderr) + return {"cpu": False, "gpu": False, "default": "none"} + return json.loads(probe.stdout.splitlines()[-1]) + + +def device_problem(devices: dict) -> str | None: + """Why the tests cannot say anything on these devices, if they cannot.""" + if not (devices["cpu"] or devices["gpu"]): + return "there is no SYCL device" + if os.environ.get("GH_CI_DEVICE") == "gpu" and not devices["gpu"]: + return "GH_CI_DEVICE asks for a GPU, and there is none" + return None + + +def pytest_command( + module: str, + report: pathlib.Path, + suite: str, + parallel: bool, + extra: list[str], +) -> list[str]: + """The pytest command for one module of the installed package.""" + command = [ + sys.executable, + "-m", + "pytest", + "-ra", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + "-o", + f"junit_suite_name={suite}", + *extra, + "--pyargs", + module, + ] + return command + (["-n", "auto"] if parallel else []) + + +def run_suite( + command: list[str], env: dict[str, str], timeout: int, attempts: int +) -> int | None: + """ + Runs a pytest command until it passes or collects nothing, up to attempts + times, killing it and its workers if it takes too long. + + Returns the last exit status, or None if the last attempt hung. + """ + code = None + for attempt in range(1, attempts + 1): + with subprocess.Popen( + command, env=env, start_new_session=os.name != "nt" + ) as process: + try: + code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + code = None + print(f"hung for {timeout} s, attempt {attempt}", flush=True) + if os.name == "nt": + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(process.pid)], + check=False, + ) + else: + os.killpg(process.pid, signal.SIGKILL) + if code in (0, NO_TESTS): + return code + if code is not None: + print( + f"failed with exit status {code}, attempt {attempt}", flush=True + ) + return code + + +def main() -> int: + """Runs the installed package's test suites.""" + expected = os.environ["GH_CI_PYTHON"] + if running_python() != expected: + print(f"running Python {running_python()}, expected {expected}") + return 1 + with tempfile.TemporaryDirectory() as scratch, registered_cpu_driver(): + env = dict(os.environ, **runtime_environment(pathlib.Path(scratch))) + devices = probe_devices(env) + print(f"devices: {devices}", flush=True) + if problem := device_problem(devices): + print(f"error: {problem}") + trace = dict(env, SYCL_UR_TRACE="2", OCL_ICD_DEBUG="7") + print(f"OpenCL ICDs: {env.get('OCL_ICD_VENDORS')}") + subprocess.run( + [sys.executable, "-I", "-m", "dpctl", "-f"], env=trace + ) + return 1 + # Workers would share one GPU, and its memory. + workers = ( + devices["default"] == "cpu" + and importlib.util.find_spec("xdist") is not None + ) + # The suite name says which platform and Python ran. + cell = f"{os.environ.get('GH_CI_PLATFORM', 'linux')}.py{expected}" + results = pathlib.Path(os.environ["GH_CI_RESULTS"]) + main_env = dict(env, SKIP_TENSOR_TESTS="1") + # Every integer type in parallel exhausts a runner's memory. + main_workers = workers and expected != ALL_INT_TYPES_PYTHON + if expected == ALL_INT_TYPES_PYTHON: + main_env["DPNP_TEST_ALL_INT_TYPES"] = "1" + # The tensor tests need a lot of memory, so they run on their own. + tensor_env = { + name: value + for name, value in env.items() + if name != "SKIP_TENSOR_TESTS" + } + status = 0 + tensor_extra = [ + f"--timeout={TENSOR_TEST_TIMEOUT}", + "--timeout-method=thread", + ] + for module, suite_env, name, parallel, timeout, attempts, extra in ( + ( + f"{PACKAGE}.tests", + main_env, + PACKAGE, + main_workers, + MAIN_TIMEOUT, + MAIN_ATTEMPTS, + [], + ), + ( + f"{PACKAGE}.tests.tensor", + tensor_env, + f"{PACKAGE}-tensor", + workers, + TENSOR_TIMEOUT, + TENSOR_ATTEMPTS, + tensor_extra, + ), + ): + command = pytest_command( + module, + results / f"{name}.xml", + f"{name} {cell}", + parallel, + extra, + ) + print("+ " + " ".join(command), flush=True) + code = run_suite(command, suite_env, timeout, attempts) + if code is None: + status = 1 + continue + if code == NO_TESTS: + # Say why nothing was collected. + subprocess.run( + [sys.executable, "-I", "-c", f"import {module}"], + env=suite_env, + ) + status |= code + return 1 if status else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conda-recipe-cf/bld.bat b/conda-recipe-cf/bld.bat deleted file mode 100644 index 2962c9ae0cc1..000000000000 --- a/conda-recipe-cf/bld.bat +++ /dev/null @@ -1,49 +0,0 @@ -:: conda-forge folds the Intel compiler runtime libs (libircmt.lib, ...) into the -:: build-only dpcpp_impl package (not visible from the host env), so they live -:: in the build prefix and are off the linker's default LIB -set "LIB=%BUILD_PREFIX%\Library\lib;%LIB%" - -"%PYTHON%" setup.py clean --all - -set "MKLROOT=%PREFIX%/Library" -set "TBB_ROOT_HINT=%PREFIX%/Library" -set "DPL_ROOT_HINT=%PREFIX%/Library" -:: useful for building in resources constrained VMs (public CI) -set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE" - -:: try to detect SYCL resource/include dir from icpx -for /f "usebackq delims=" %%R in (`icpx --print-resource-dir 2^>NUL`) do ( - set "SYCL_RESOURCE_DIR=%%R" -) - -if defined SYCL_RESOURCE_DIR ( - if exist "%SYCL_RESOURCE_DIR%\include" ( - set "SYCL_INCLUDE_DIR_HINT=%SYCL_RESOURCE_DIR%" - ) -) - -set "CC=icx" -set "CXX=icx" - -set "CMAKE_GENERATOR=Ninja" -:: make CMake verbose -set "VERBOSE=1" - -:: set CMAKE to use less threads to avoid OOM -set CMAKE_BUILD_PARALLEL_LEVEL=%CPU_COUNT% - -:: flags mean: --wheel --no-isolation --skip-dependency-check -%PYTHON% -m build -w -n -x -if %errorlevel% neq 0 exit 1 - -:: wheel file was renamed -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m pip install %%f ^ - --no-build-isolation ^ - --no-deps ^ - --only-binary :all: ^ - --no-index ^ - --prefix %PREFIX% ^ - -vv - if errorlevel 1 exit 1 -) diff --git a/conda-recipe-cf/build.sh b/conda-recipe-cf/build.sh deleted file mode 100755 index f5c4935f7251..000000000000 --- a/conda-recipe-cf/build.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# conda-forge folds the Intel compiler runtime libs (libircmt.lib, ...) into the -# build-only dpcpp_impl package (not visible from the host env), so they live -# in the build prefix and are off the linker's default LIBRARY_PATH -export LIBRARY_PATH="$LIBRARY_PATH:${BUILD_PREFIX}/lib" - -# Intel LLVM must cooperate with compiler and sysroot from conda -echo "--gcc-toolchain=${BUILD_PREFIX} --sysroot=${BUILD_PREFIX}/${HOST}/sysroot -target ${HOST}" > icpx_for_conda.cfg - -ICPXCFG="$(pwd)/icpx_for_conda.cfg" -export ICPXCFG - -ICXCFG="$(pwd)/icpx_for_conda.cfg" -export ICXCFG - -if [ -e "_skbuild" ]; then - ${PYTHON} setup.py clean --all -fi - -export CC=icx -export CXX=icpx - -# conda-forge's gcc/g++ activation injects -fno-merge-constants into CFLAGS/CXXFLAGS -# when CONDA_BUILD==1, while icx/icpx doesn't support it and emit the warning. -# See CMPLRLLVM-19167 / CMPLRLLVM-28729 for more context and PR which added the flag: -# https://github.com/conda-forge/ctng-compiler-activation-feedstock/pull/193 -export CFLAGS="${CFLAGS//-fno-merge-constants/}" -export CXXFLAGS="${CXXFLAGS//-fno-merge-constants/}" - -export CMAKE_GENERATOR=Ninja -# Make CMake verbose -export VERBOSE=1 - -# set CMAKE to use less threads to avoid OOM -export CMAKE_BUILD_PARALLEL_LEVEL=${CPU_COUNT} - -CMAKE_ARGS="${CMAKE_ARGS} -DDPNP_WITH_REDIST:BOOL=ON" - -# -wnx flags mean: --wheel --no-isolation --skip-dependency-check -${PYTHON} -m build -w -n -x - -${PYTHON} -m pip install dist/dpnp*.whl \ - --no-build-isolation \ - --no-deps \ - --only-binary :all: \ - --no-index \ - --prefix "${PREFIX}" \ - -vv diff --git a/conda-recipe-cf/conda_build_config.yaml b/conda-recipe-cf/conda_build_config.yaml deleted file mode 100644 index 408d05b438a9..000000000000 --- a/conda-recipe-cf/conda_build_config.yaml +++ /dev/null @@ -1,18 +0,0 @@ -numpy: - - '2' -c_compiler: # [linux] - - gcc # [linux] -cxx_compiler: # [linux] - - gxx # [linux] -cxx_compiler_version: # [linux] - - '15' # [linux] -c_stdlib: # [linux] - - sysroot # [linux] -c_stdlib_version: # [linux] - - '2.28' # [linux] -c_stdlib: # [win] - - vs # [win] -cxx_compiler: # [win] - - vs2022 # [win] -c_compiler: # [win] - - vs2022 # [win] diff --git a/conda-recipe-cf/meta.yaml b/conda-recipe-cf/meta.yaml deleted file mode 100644 index 07b9e86a268b..000000000000 --- a/conda-recipe-cf/meta.yaml +++ /dev/null @@ -1,92 +0,0 @@ -{% set required_compiler_and_mkl_version = "2026.0.0" %} -{% set required_dpctl_version = "0.22.1" %} - -package: - name: dpnp - version: {{ GIT_DESCRIBE_TAG }} - -source: - path: .. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports: - - dpctl - - numpy - - python - # building with Intel DPC++ and do not import the MSVC runtime DLLs - - vc14_runtime # [win] - - ucrt # [win] - missing_dso_whitelist: - - '*DPCTLSyclInterface*' - - '*/dpnp_backend_c.dll' # [win] - skip: true # [osx or py<39] - -requirements: - build: - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - {{ compiler('dpcpp') }} >={{ required_compiler_and_mkl_version }} - - ninja >=1.11.1 # [linux] - - cmake >=3.31.6 - host: - - python - - pip >=25.0 - - pybind11 >=3.0.2 - - setuptools >=79.0.1 - - wheel >=0.45.1 - - python-build >=1.2.2 - - scikit-build >=0.18.1 - - cython >=3.1.0 - - numpy >=1.26.0 - - versioneer ==0.29 - - dpctl >={{ required_dpctl_version }} - - mkl-devel-dpcpp >={{ required_compiler_and_mkl_version }} - - onedpl-devel - - tbb-devel - run: - - python - - {{ pin_compatible('dpctl', min_pin='x.x.x', max_pin=None) }} - - {{ pin_compatible('dpcpp-cpp-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('mkl', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-blas', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-dft', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-lapack', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-rng', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-vm', min_pin='x.x', max_pin='x') }} - - numpy - - ocl-icd-system # [linux] - - khronos-opencl-icd-loader # [win] - -test: - requires: - - pip - - pytest - - setuptools - - scipy # [py<314] - commands: - - pip check - - python -c "import dpnp; print(dpnp.__version__)" - -about: - home: https://github.com/IntelPython/dpnp - license: BSD-3-Clause - license_file: LICENSE.txt - summary: 'Data Parallel Extension for NumPy' - description: | - LEGAL NOTICE: Use of this software package is subject to the - software license agreement (as set forth above, in the license section of - the installed Conda package and/or the README file) and all notices, - disclaimers or license terms for third party or open source software - included in or with the software. -

- EULA: BSD-3-Clause -

- -extra: - recipe-maintainers: - - antonwolfy - - vlad-perevezentsev - - ndgrigorian diff --git a/conda-recipe-cf/run_test.bat b/conda-recipe-cf/run_test.bat deleted file mode 100644 index ee2e40b16077..000000000000 --- a/conda-recipe-cf/run_test.bat +++ /dev/null @@ -1,16 +0,0 @@ -@echo on - -:: Skip tensor tests by default to avoid OOM in conda builds. -:: Set SKIP_TENSOR_TESTS=0 to run them on machines with enough memory. -if not defined SKIP_TENSOR_TESTS ( - set "SKIP_TENSOR_TESTS=1" -) - -"%PYTHON%" -c "import dpnp; print(dpnp.__version__)" -if %errorlevel% neq 0 exit 1 - -"%PYTHON%" -m dpctl -f -if %errorlevel% neq 0 exit 1 - -"%PYTHON%" -m pytest -ra --pyargs dpnp -if %errorlevel% neq 0 exit 1 diff --git a/conda-recipe-cf/run_test.sh b/conda-recipe-cf/run_test.sh deleted file mode 100755 index 7493ee5ea25e..000000000000 --- a/conda-recipe-cf/run_test.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Skip tensor tests by default to avoid OOM in conda builds. -# Set SKIP_TENSOR_TESTS=0 to run them on machines with enough memory. -if [ -z "${SKIP_TENSOR_TESTS}" ]; then - export SKIP_TENSOR_TESTS=1 -fi - -set -e - -$PYTHON -c "import dpnp; print(dpnp.__version__)" -$PYTHON -m dpctl -f -$PYTHON -m pytest -ra --pyargs dpnp diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index 4fd2b29b4d4a..2962c9ae0cc1 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -1,18 +1,24 @@ +:: conda-forge folds the Intel compiler runtime libs (libircmt.lib, ...) into the +:: build-only dpcpp_impl package (not visible from the host env), so they live +:: in the build prefix and are off the linker's default LIB +set "LIB=%BUILD_PREFIX%\Library\lib;%LIB%" + "%PYTHON%" setup.py clean --all set "MKLROOT=%PREFIX%/Library" set "TBB_ROOT_HINT=%PREFIX%/Library" set "DPL_ROOT_HINT=%PREFIX%/Library" +:: useful for building in resources constrained VMs (public CI) +set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE" -REM Overriding IPO is useful for building in resources constrained VMs (public CI) -if not "%OVERRIDE_INTEL_IPO%"=="" ( - set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE" +:: try to detect SYCL resource/include dir from icpx +for /f "usebackq delims=" %%R in (`icpx --print-resource-dir 2^>NUL`) do ( + set "SYCL_RESOURCE_DIR=%%R" ) -FOR %%V IN (17.0.0 17 18.0.0 18 19.0.0 19 20.0.0 20 21.0.0 21 22.0.0 22) DO @( - REM set DIR_HINT if directory exists - IF EXIST "%BUILD_PREFIX%\Library\lib\clang\%%V\" ( - set "SYCL_INCLUDE_DIR_HINT=%BUILD_PREFIX%\Library\lib\clang\%%V" +if defined SYCL_RESOURCE_DIR ( + if exist "%SYCL_RESOURCE_DIR%\include" ( + set "SYCL_INCLUDE_DIR_HINT=%SYCL_RESOURCE_DIR%" ) ) @@ -20,18 +26,15 @@ set "CC=icx" set "CXX=icx" set "CMAKE_GENERATOR=Ninja" -:: Make CMake verbose +:: make CMake verbose set "VERBOSE=1" -%PYTHON% -m build -w -n -x -if %ERRORLEVEL% neq 0 exit 1 +:: set CMAKE to use less threads to avoid OOM +set CMAKE_BUILD_PARALLEL_LEVEL=%CPU_COUNT% -:: `pip install dist\numpy*.whl` does not work on windows, -:: so use a loop; there's only one wheel in dist/ anyway -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m wheel tags --remove --build %GIT_DESCRIBE_NUMBER% %%f - if %ERRORLEVEL% neq 0 exit 1 -) +:: flags mean: --wheel --no-isolation --skip-dependency-check +%PYTHON% -m build -w -n -x +if %errorlevel% neq 0 exit 1 :: wheel file was renamed for /f %%f in ('dir /b /S .\dist') do ( @@ -42,11 +45,5 @@ for /f %%f in ('dir /b /S .\dist') do ( --no-index ^ --prefix %PREFIX% ^ -vv - if %ERRORLEVEL% neq 0 exit 1 -) - -:: Copy wheel package -if NOT "%WHEELS_OUTPUT_FOLDER%"=="" ( - copy dist\dpnp*.whl %WHEELS_OUTPUT_FOLDER% - if %ERRORLEVEL% neq 0 exit 1 + if errorlevel 1 exit 1 ) diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 7028c8830bb6..f5c4935f7251 100755 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -1,6 +1,8 @@ #!/bin/bash -# This is necessary to help DPC++ find Intel libraries such as SVML, IRNG, etc in build prefix +# conda-forge folds the Intel compiler runtime libs (libircmt.lib, ...) into the +# build-only dpcpp_impl package (not visible from the host env), so they live +# in the build prefix and are off the linker's default LIBRARY_PATH export LIBRARY_PATH="$LIBRARY_PATH:${BUILD_PREFIX}/lib" # Intel LLVM must cooperate with compiler and sysroot from conda @@ -12,16 +14,6 @@ export ICPXCFG ICXCFG="$(pwd)/icpx_for_conda.cfg" export ICXCFG -# Get sysroot glibc version from the build env metadata (no conda CLI needed) -SYSROOT_META=("${BUILD_PREFIX}"/conda-meta/sysroot_linux-64-*.json) -if [[ "${SYSROOT_META[0]##*/}" =~ ^sysroot_linux-64-([0-9]+)\.([0-9]+) ]]; then - GLIBC_MAJOR="${BASH_REMATCH[1]}" - GLIBC_MINOR="${BASH_REMATCH[2]}" -else - echo "Error: unable to detect sysroot_linux-64 version in ${BUILD_PREFIX}" >&2 - exit 1 -fi - if [ -e "_skbuild" ]; then ${PYTHON} setup.py clean --all fi @@ -40,15 +32,14 @@ export CMAKE_GENERATOR=Ninja # Make CMake verbose export VERBOSE=1 +# set CMAKE to use less threads to avoid OOM +export CMAKE_BUILD_PARALLEL_LEVEL=${CPU_COUNT} + CMAKE_ARGS="${CMAKE_ARGS} -DDPNP_WITH_REDIST:BOOL=ON" # -wnx flags mean: --wheel --no-isolation --skip-dependency-check ${PYTHON} -m build -w -n -x -${PYTHON} -m wheel tags --remove --build "$GIT_DESCRIBE_NUMBER" \ - --platform-tag "manylinux_${GLIBC_MAJOR}_${GLIBC_MINOR}_x86_64" \ - dist/dpnp*.whl - ${PYTHON} -m pip install dist/dpnp*.whl \ --no-build-isolation \ --no-deps \ @@ -56,8 +47,3 @@ ${PYTHON} -m pip install dist/dpnp*.whl \ --no-index \ --prefix "${PREFIX}" \ -vv - -# Copy wheel package -if [[ -d "${WHEELS_OUTPUT_FOLDER}" ]]; then - cp dist/dpnp*.whl "${WHEELS_OUTPUT_FOLDER[@]}" -fi diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index c5df1a7f2e29..521c7b42ff49 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,11 +1,19 @@ +# Python versions for ci/matrix.py, one per line, without selectors +python: + - "3.11" + - "3.12" + - "3.13" + - "3.14.* *_cp314" + - "3.14.* *_cp314t" + - "3.15.* *_cp315" numpy: - - '1.26' + - '2' c_compiler: # [linux] - gcc # [linux] cxx_compiler: # [linux] - gxx # [linux] cxx_compiler_version: # [linux] - - '14' # [linux] + - '15' # [linux] c_stdlib: # [linux] - sysroot # [linux] c_stdlib_version: # [linux] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index b290a2dc4cfb..d3e8bbc7c1d7 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,123 +1,93 @@ -{% set max_compiler_and_mkl_version = environ.get("MAX_BUILD_CMPL_MKL_VERSION", "2027.0a0") %} -{% set required_compiler_version = environ.get("MIN_BUILD_CMPL_VERSION", "2026.1.1") %} -{% set required_mkl_version = "2026.0" %} -{% set required_dpctl_version = "0.23.0dev0" %} - -{# DPNP does not support a compiler older than 2026.0, so reject any #} -{# MIN_BUILD_CMPL_VERSION override below that by calling an undefined macro. #} -{% if required_compiler_version.split('.') | map('int') | list < [2026, 0] %} -{{ MIN_BUILD_CMPL_VERSION_must_be_at_least_2026_0() }} -{% endif %} - -{# The minimum compiler version must stay strictly below the (exclusive) maximum, #} -{# otherwise the ">=min,= max_compiler_and_mkl_version.split('.') | map('int') | list %} -{{ MIN_BUILD_CMPL_VERSION_must_be_below_MAX_BUILD_CMPL_MKL_VERSION() }} -{% endif %} - -{# The conda-forge OpenCL ICD loaders (ocl-icd-system / khronos-opencl-icd-loader) #} -{# require DPC++ RT of 2026.1.1 or newer when installing from the Intel channel. #} -{# So fall back to intel-gpu-ocl-icd-system for older Intel channel DPC++ RTs. #} -{# Building and installing solely with conda-forge channel works with any 2026.0+ compiler. #} -{% set use_conda_forge_ocl_icd = required_compiler_version.split('.') | map('int') | list >= [2026, 1, 1] %} - -{% set pyproject = load_file_data('pyproject.toml') %} -{% set py_build_deps = pyproject.get('build-system', {}).get('requires', []) %} +{% set required_compiler_and_mkl_version = "2026.0.0" %} +{% set max_compiler_and_mkl_version = "2027.0a0" %} +{% set required_dpctl_version = "0.23.0" %} package: - name: dpnp - version: {{ GIT_DESCRIBE_TAG }} -source: - path: .. + name: dpnp + version: {{ GIT_DESCRIBE_TAG }} -requirements: - host: - - python - - pip >=25.0 - - pybind11 >=3.0.2 - {% for dep in py_build_deps %} - {% if dep.startswith('dpctl') %} - - dpctl >={{ required_dpctl_version }} - {% elif dep.startswith('ninja') %} - - {{ dep.split(';')[0] }} # [not win] - {% elif dep.startswith('cmake') %} - - {{ dep }} - {% elif dep.startswith('build>=') %} - - {{ 'python-' ~ dep }} - {% else %} - - {{ dep|replace('_','-') }} - {% endif %} - {% endfor %} - # Do not use pyproject.toml for setting dependencies on OneAPI packages - - mkl-devel-dpcpp >={{ required_mkl_version }},<{{ max_compiler_and_mkl_version }} - - onedpl-devel - - tbb-devel - - opencl-headers >=2025.06.13 # [win] - build: - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - {{ compiler('dpcpp') }} >={{ required_compiler_version }},<{{ max_compiler_and_mkl_version }} - run: - - python - - {{ pin_compatible('dpctl', min_pin='x.x.x', max_pin=None) }} - - {{ pin_compatible('dpcpp-cpp-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('mkl', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-blas', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-dft', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-lapack', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-rng', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-vm', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('onemkl-sycl-sparse', min_pin='x.x', max_pin='x') }} - - numpy - {% if use_conda_forge_ocl_icd %} - - ocl-icd-system # [linux] - - khronos-opencl-icd-loader # [win] - {% else %} - - intel-gpu-ocl-icd-system - {% endif %} +source: + path: .. build: - number: {{ GIT_DESCRIBE_NUMBER }} - include_recipe: False - script_env: - - WHEELS_OUTPUT_FOLDER - - OVERRIDE_INTEL_IPO # [win] - ignore_run_exports: - - dpctl - - numpy - - python - # building with Intel DPC++ and do not import the MSVC runtime DLLs - - vc14_runtime # [win] - - ucrt # [win] - missing_dso_whitelist: - - '*DPCTLSyclInterface*' - - '*/dpnp_backend_c.dll' # [win] + number: {{ GIT_DESCRIBE_NUMBER }} + ignore_run_exports: + - dpctl + # Pulls in oneMKL's datafitting and stats, which dpnp doesn't use + - mkl-dpcpp + - numpy + - python + # building with Intel DPC++ and do not import the MSVC runtime DLLs + - vc14_runtime # [win] + - ucrt # [win] + missing_dso_whitelist: + - '*DPCTLSyclInterface*' + - '*/dpnp_backend_c.dll' # [win] + skip: true # [osx] + +requirements: + build: + - {{ compiler('cxx') }} + - {{ stdlib('c') }} + - {{ compiler('dpcpp') }} >={{ required_compiler_and_mkl_version }},<{{ max_compiler_and_mkl_version }} + - ninja >=1.11.1 # [linux] + - cmake >=3.31.6 + host: + - python + - pip >=25.0 + - pybind11 >=3.0.2 + - setuptools >=79.0.1 + - wheel >=0.45.1 + - python-build >=1.2.2 + - scikit-build >=0.18.1 + - cython >=3.1.0 + - numpy >=1.26.0 + - versioneer ==0.29 + - dpctl >={{ required_dpctl_version }} + - mkl-devel-dpcpp >={{ required_compiler_and_mkl_version }},<{{ max_compiler_and_mkl_version }} + - onedpl-devel + - tbb-devel + run: + - python + - {{ pin_compatible('dpctl', min_pin='x.x.x', max_pin=None) }} + - {{ pin_compatible('dpcpp-cpp-rt', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('mkl', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('onemkl-sycl-blas', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('onemkl-sycl-dft', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('onemkl-sycl-lapack', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('onemkl-sycl-rng', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('onemkl-sycl-sparse', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('onemkl-sycl-vm', min_pin='x.x', max_pin='x') }} + - numpy + - ocl-icd-system # [linux] + - khronos-opencl-icd-loader # [win] test: - requires: - - pytest - - setuptools - - scipy # [py<314] + requires: + - pip + commands: + - pip check + - python -c "import dpnp; print(dpnp.__version__)" about: - home: https://github.com/IntelPython/dpnp - license: BSD-3-Clause - license_file: LICENSE.txt - summary: 'Data Parallel Extension for NumPy' - description: | - LEGAL NOTICE: Use of this software package is subject to the - software license agreement (as set forth above, in the license section of - the installed Conda package and/or the README file) and all notices, - disclaimers or license terms for third party or open source software - included in or with the software. -

- EULA: BSD-3-Clause -

+ home: https://github.com/IntelPython/dpnp + license: BSD-3-Clause + license_file: LICENSE.txt + summary: 'Data Parallel Extension for NumPy' + description: | + LEGAL NOTICE: Use of this software package is subject to the + software license agreement (as set forth above, in the license section of + the installed Conda package and/or the README file) and all notices, + disclaimers or license terms for third party or open source software + included in or with the software. +

+ EULA: BSD-3-Clause +

extra: - recipe-maintainers: - - antonwolfy - - vlad-perevezentsev - - ndgrigorian + recipe-maintainers: + - antonwolfy + - vlad-perevezentsev + - ndgrigorian diff --git a/conda-recipe/rattler_recipe.yaml b/conda-recipe/rattler_recipe.yaml deleted file mode 100644 index 35889fc38ee2..000000000000 --- a/conda-recipe/rattler_recipe.yaml +++ /dev/null @@ -1,128 +0,0 @@ -schema_version: 1 - -context: - max_compiler_and_mkl_version: ${{ env.get("MAX_BUILD_CMPL_MKL_VERSION", default="2027.0a0") }} - required_compiler_version: ${{ env.get("MIN_BUILD_CMPL_VERSION", default="2026.1.1") }} - required_mkl_version: "2026.0" - required_dpctl_version: 0.23.0dev0 - -package: - name: dpnp - version: ${{ env.get("GIT_DESCRIBE_TAG") }} - -source: - path: .. - -build: - number: ${{ env.get("GIT_DESCRIBE_NUMBER") }} - script: - file: ${{ "bld.bat" if win else "build.sh" }} - env: - WHEELS_OUTPUT_FOLDER: ${{ env.get("WHEELS_OUTPUT_FOLDER", default="") }} - OVERRIDE_INTEL_IPO: ${{ env.get("OVERRIDE_INTEL_IPO", default="") if win else "" }} - # conda-build used to forward GIT_DESCRIBE_NUMBER into the build script - # automatically; rattler-build does not, so bld.bat's - # `wheel tags --remove --build %GIT_DESCRIBE_NUMBER%` was silently - # getting an empty value on Windows, which made that command fail and - # broke the wheel install step further down the script. - GIT_DESCRIBE_NUMBER: ${{ env.get("GIT_DESCRIBE_NUMBER") }} - dynamic_linking: - missing_dso_allowlist: - - "*DPCTLSyclInterface*" - - if: win - then: "*/dpnp_backend_c.dll" - -requirements: - host: - - python - - pip >=25.0 - - pybind11 >=3.0.2 - - python-build>=1.2.2 - - cmake>=3.31.6 - - cython>=3.1.0 - - dpctl >=${{ required_dpctl_version }} - - if: not win - then: ninja>=1.11.1 - - numpy - - scikit-build>=0.18.1 - - setuptools>=79.0.1 - - wheel>=0.45.1 - - versioneer ==0.29 - # Do not use pyproject.toml for setting dependencies on OneAPI packages - - mkl-devel-dpcpp >=${{ required_mkl_version }},<${{ max_compiler_and_mkl_version }} - - onedpl-devel - - tbb-devel - - if: win - then: opencl-headers >=2025.06.13 - build: - - ${{ compiler('cxx') }} - - ${{ stdlib('c') }} - - ${{ compiler('dpcpp') }} >=${{ required_compiler_version }},<${{ max_compiler_and_mkl_version }} - run: - - python - - ${{ pin_compatible('dpctl', lower_bound='x.x.x', upper_bound=None) }} - - ${{ pin_compatible('intel-sycl-rt', lower_bound='x.x', upper_bound='x') }} - - ${{ pin_compatible('intel-cmplr-lib-rt', lower_bound='x.x', upper_bound='x') }} - - ${{ pin_compatible('mkl', lower_bound='x.x', upper_bound='x') }} - - ${{ pin_compatible('onemkl-sycl-blas', lower_bound='x.x', upper_bound='x') }} - - ${{ pin_compatible('onemkl-sycl-dft', lower_bound='x.x', upper_bound='x') }} - - ${{ pin_compatible('onemkl-sycl-lapack', lower_bound='x.x', upper_bound='x') }} - - ${{ pin_compatible('onemkl-sycl-rng', lower_bound='x.x', upper_bound='x') }} - - ${{ pin_compatible('onemkl-sycl-vm', lower_bound='x.x', upper_bound='x') }} - - ${{ pin_compatible('onemkl-sycl-sparse', lower_bound='x.x', upper_bound='x') }} - - numpy - - if: linux - then: ocl-icd-system - - if: win - then: khronos-opencl-icd-loader - ignore_run_exports: - by_name: - - dpctl - - numpy - - python - # leaked from mkl-devel-dpcpp; drags in unused onemkl-sycl-{datafitting,stats} - - mkl-dpcpp - # building with Intel DPC++ and do not import the MSVC runtime DLLs - - if: win - then: vc14_runtime - - if: win - then: ucrt - -tests: - - script: - - pip check - - python -c "import dpnp; print(dpnp.__version__)" - - python -m dpctl -f - requirements: - run: - - pip - - script: ${{ "run_test.bat" if win else "run_test.sh" }} - requirements: - run: - - pytest - - setuptools - - if: match(python, "<3.14") - then: scipy - -about: - license: BSD-3-Clause - license_file: LICENSE.txt - summary: Data Parallel Extension for NumPy - description: | - LEGAL NOTICE: Use of this software package is subject to the - software license agreement (as set forth above, in the license section of - the installed Conda package and/or the README file) and all notices, - disclaimers or license terms for third party or open source software - included in or with the software. -

- EULA: BSD-3-Clause -

- - - homepage: https://github.com/IntelPython/dpnp - -extra: - recipe-maintainers: - - antonwolfy - - vlad-perevezentsev - - ndgrigorian diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat deleted file mode 100644 index 41464cd43341..000000000000 --- a/conda-recipe/run_test.bat +++ /dev/null @@ -1,47 +0,0 @@ -@echo on - -REM if ONEAPI_ROOT is specified (use all from it) -if defined ONEAPI_ROOT ( - set "DPCPPROOT=%ONEAPI_ROOT%\compiler\latest" - set "MKLROOT=%ONEAPI_ROOT%\mkl\latest" - set "TBBROOT=%ONEAPI_ROOT%\tbb\latest" - set "DPLROOT=%ONEAPI_ROOT%\dpl\latest" -) - -REM if DPCPPROOT is specified (work with custom DPCPP) -if defined DPCPPROOT ( - call "%DPCPPROOT%\env\vars.bat" -) - -REM if MKLROOT is specified (work with custom math library) -if defined MKLROOT ( - call "%MKLROOT%\env\vars.bat" -) - -REM have to activate while SYCL CPU device/driver needs paths -REM if TBBROOT is specified -if defined TBBROOT ( - call "%TBBROOT%\env\vars.bat" -) - -REM If PYTHON is not set -REM assign it to the Python interpreter from the testing environment -if not defined PYTHON ( - for %%I in (python.exe) do set PYTHON=%%~$PATH:I -) - - -REM Skip tensor tests by default to avoid OOM in conda builds. -REM Set SKIP_TENSOR_TESTS=0 to run them on machines with enough memory. -if not defined SKIP_TENSOR_TESTS ( - set "SKIP_TENSOR_TESTS=1" -) - -"%PYTHON%" -c "import dpnp; print(dpnp.__version__)" -if %errorlevel% neq 0 exit 1 - -"%PYTHON%" -m dpctl -f -if %errorlevel% neq 0 exit 1 - -"%PYTHON%" -m pytest -ra --pyargs dpnp -if %errorlevel% neq 0 exit 1 diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh deleted file mode 100755 index 5ffeaf2ec1b0..000000000000 --- a/conda-recipe/run_test.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# if ONEAPI_ROOT is specified (use all from it) -if [ -n "${ONEAPI_ROOT}" ]; then - export DPCPPROOT=${ONEAPI_ROOT}/compiler/latest - export MKLROOT=${ONEAPI_ROOT}/mkl/latest - export TBBROOT=${ONEAPI_ROOT}/tbb/latest - export DPLROOT=${ONEAPI_ROOT}/dpl/latest -fi - -# if DPCPPROOT is specified (work with custom DPCPP) -if [ -n "${DPCPPROOT}" ]; then - # shellcheck source=/dev/null - . "${DPCPPROOT}"/env/vars.sh -fi - -# if MKLROOT is specified (work with custom math library) -if [ -n "${MKLROOT}" ]; then - # shellcheck source=/dev/null - . "${MKLROOT}"/env/vars.sh -fi - -# have to activate while SYCL CPU device/driver needs paths -# if TBBROOT is specified -if [ -n "${TBBROOT}" ]; then - # shellcheck source=/dev/null - . "${TBBROOT}"/env/vars.sh -fi - -# If PYTHON is not set -# assign it to the Python interpreter from the testing environment -if [ -z "${PYTHON}" ]; then - PYTHON=$PREFIX/bin/python -fi - -# Skip tensor tests by default to avoid OOM in conda builds. -# Set SKIP_TENSOR_TESTS=0 to run them on machines with enough memory. -if [ -z "${SKIP_TENSOR_TESTS}" ]; then - export SKIP_TENSOR_TESTS=1 -fi - -set -e - -$PYTHON -c "import dpnp; print(dpnp.__version__)" -$PYTHON -m dpctl -f -$PYTHON -m pytest -ra --pyargs dpnp diff --git a/doc/quick_start_guide.rst b/doc/quick_start_guide.rst index 8c5a2571edc3..f6fc9f85c57d 100644 --- a/doc/quick_start_guide.rst +++ b/doc/quick_start_guide.rst @@ -74,7 +74,7 @@ And to build dpnp package from the sources: .. code-block:: bash - conda-build conda-recipe -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels + conda-build conda-recipe -c conda-forge --override-channels --python "3.14.* *_cp314" Finally, to install the result package: diff --git a/dpnp/AGENTS.md b/dpnp/AGENTS.md index b7d3956d4134..ed45d0255515 100644 --- a/dpnp/AGENTS.md +++ b/dpnp/AGENTS.md @@ -21,7 +21,7 @@ - Use `dpnp.default_float_type(...)` or `map_dtype_to_device(...)` for the default float type. - Deprecated ufuncs use `DPNPDeprecatedUnaryFunc(..., deprecated_msg=...)`. - Unsupported NumPy keywords raise `NotImplementedError` (`dpnp.check_limitations`) and are listed under `Limitations` in the docstring. -- The Array API spec version is set in three places that change together: `__array_api_version__` in `dpnp/tensor/_array_api.py`, `ARRAY_API_TESTS_VERSION` in `.github/workflows/conda-package.yml` and `doc/reference/array_api.rst` (plus the version test in `dpnp/tests/test_ndarray.py`). Inspection utilities come from `__array_namespace_info__` in `dpnp_array_api_info.py`. +- The Array API spec version is set in three places that change together: `__array_api_version__` in `dpnp/tensor/_array_api.py`, `ARRAY_API_TESTS_VERSION` in `.github/workflows/intel-ci.yml` and `doc/reference/array_api.rst` (plus the version test in `dpnp/tests/test_ndarray.py`). Inspection utilities come from `__array_namespace_info__` in `dpnp_array_api_info.py`. - `dpnp.tensor` replaces `dpctl.tensor`, which no longer exists here; `SyclQueue`, `SyclDevice`, `dpctl.memory` and `SequentialOrderManager` still come from `dpctl`. ## Docstrings diff --git a/dpnp/tensor/AGENTS.md b/dpnp/tensor/AGENTS.md index 2e82ce427795..67cbc52558bc 100644 --- a/dpnp/tensor/AGENTS.md +++ b/dpnp/tensor/AGENTS.md @@ -21,4 +21,4 @@ ```bash python -m pytest -n auto -ra --pyargs dpnp.tests.tensor ``` -The Array API conformance job in `conda-package.yml` tests only `dpnp` (`ARRAY_API_TESTS_MODULE: 'dpnp'`); a check for `dpnp.tensor` itself is not added yet, so verify spec compliance of `dpnp.tensor` changes against the spec. A libtensor change can still move the `dpnp` results, and the job never fails the build, so read its PR comment (posted for PRs from IntelPython/dpnp branches only, and only when the tests produced a report) or its log. +The Array API conformance job in `intel-ci.yml` tests only `dpnp` (`ARRAY_API_TESTS_MODULE: 'dpnp'`); a check for `dpnp.tensor` itself is not added yet, so verify spec compliance of `dpnp.tensor` changes against the spec. A libtensor change can still move the `dpnp` results, and the job never fails the build, so read its PR comment (posted for PRs from IntelPython/dpnp branches only, and only when the tests produced a report) or its log. diff --git a/pyproject.toml b/pyproject.toml index fd440911feae..9c206d1fc485 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", - "Programming Language :: Python :: 3.15", + # No 3.15: dpctl has no cp315 wheel yet, which the wheels build against "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Free Threading :: 2 - Beta", @@ -109,6 +109,35 @@ Repository = "https://github.com/IntelPython/dpnp.git" line-length = 80 target-version = ['py311', 'py312', 'py313', 'py314', 'py315'] +[tool.cibuildwheel] +build-verbosity = 1 + +[tool.cibuildwheel.linux] +# Intel's icpx, from Intel's repository, and oneMKL, oneDPL and oneTBB from conda-forge; +# dpctl's wheel, imported at build time, needs a libstdc++ newer than the image's +before-all = [ + "git config --global --add safe.directory '*'", + "rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB", + "printf '[oneAPI]\\nname=Intel oneAPI\\nbaseurl=https://yum.repos.intel.com/oneapi\\nenabled=1\\ngpgcheck=1\\nrepo_gpgcheck=1\\ngpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\\n' > /etc/yum.repos.d/oneAPI.repo", + "dnf install -y intel-oneapi-compiler-dpcpp-cpp-2026.2", + "curl -fsSL -o /usr/local/bin/micromamba https://github.com/mamba-org/micromamba-releases/releases/download/2.10.0-0/micromamba-linux-64", + "echo '701f39c06f8e6a670a0e186c8479b9e5c0502a2f0cda6cef65d83bdd2875e8bd /usr/local/bin/micromamba' | sha256sum -c -", + "chmod +x /usr/local/bin/micromamba", + "micromamba create -y -r /tmp/micromamba -p /opt/sdk --override-channels -c conda-forge 'mkl-devel-dpcpp>=2026.0,<2027' onedpl-devel tbb-devel", + "mkdir /opt/stdcxx && cp -a /opt/sdk/lib/libstdc++.so.6* /opt/stdcxx/" +] +environment = { LD_LIBRARY_PATH = "/opt/stdcxx", CPATH = "/opt/sdk/include", CC = "/opt/intel/oneapi/compiler/2026.2/bin/icx", CXX = "/opt/intel/oneapi/compiler/2026.2/bin/icpx", CMAKE_GENERATOR = "Ninja", CMAKE_ARGS = "-DDPNP_WITH_REDIST:BOOL=ON", CMAKE_BUILD_PARALLEL_LEVEL = "4", CMAKE_PREFIX_PATH = "/opt/sdk", MKLROOT = "/opt/sdk", LDFLAGS = "-Wl,-z,relro -Wl,-z,now" } +# The SYCL runtime, oneMKL and the compiler's libraries come from wheels at run time, see [tool.gh-ci]. +# libstdc++ is as new as dpctl's wheel needs, which auditwheel's policy refuses, so the tag is set here. +repair-wheel-command = "python -m pip install -q wheel && python -m wheel tags --remove --platform-tag manylinux_2_28_x86_64 {wheel} && mv \"$(dirname {wheel})\"/*-manylinux_2_28_x86_64.whl {dest_dir}" + +[tool.cibuildwheel.windows] +# Intel's icx, and oneMKL, oneDPL and oneTBB, from conda-forge; DLLs come from wheels at run time, see [tool.gh-ci] +before-all = "python {project}/ci/install_icx.py C:/sdk" +environment = { CC = "C:/sdk/Library/bin/icx.exe", CXX = "C:/sdk/Library/bin/icx.exe", CMAKE_GENERATOR = "Ninja", CMAKE_ARGS = "-DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE", CMAKE_BUILD_PARALLEL_LEVEL = "4", CMAKE_PREFIX_PATH = "C:/sdk/Library", MKLROOT = "C:/sdk/Library", TBB_ROOT_HINT = "C:/sdk/Library", DPL_ROOT_HINT = "C:/sdk/Library", LIB = "C:/sdk/Library/lib;$LIB", INCLUDE = "C:/sdk/Library/include;$INCLUDE", PATH = "C:/sdk/Library/bin;$PATH" } +# The DLLs come from wheels at run time, so there is nothing to bundle +repair-wheel-command = "" + [tool.codespell] builtin = "clear,rare,informal,names" check-filenames = true @@ -142,6 +171,21 @@ source = [ [tool.cython-lint] max-line-length = 80 +[tool.gh-ci] +# What the wheels need at run time, which conda packages get from the recipe +wheel-requires = [ + "dpcpp-cpp-rt >=2026.2,<2027", + "intel-cmplr-lib-rt >=2026.2,<2027", + "intel-sycl-rt >=2026.2,<2027", + "mkl >=2026.2,<2027", + "onemkl-sycl-blas >=2026.2,<2027", + "onemkl-sycl-dft >=2026.2,<2027", + "onemkl-sycl-lapack >=2026.2,<2027", + "onemkl-sycl-rng >=2026.2,<2027", + "onemkl-sycl-sparse >=2026.2,<2027", + "onemkl-sycl-vm >=2026.2,<2027" +] + [tool.isort] ensure_newline_before_comments = true force_grid_wrap = 0