diff --git a/.github/workflows/conda-package-cf.yml b/.github/workflows/conda-package-cf.yml index e75623a3fa..b1c8e76d58 100644 --- a/.github/workflows/conda-package-cf.yml +++ b/.github/workflows/conda-package-cf.yml @@ -13,7 +13,7 @@ env: PACKAGE_NAME: dpctl MODULE_NAME: dpctl TEST_ENV_NAME: test_dpctl - RECIPE_DIR: conda-recipe-cf + RECIPE_DIR: conda-recipe CHANNELS: "-c conda-forge --override-channels" VER_SCRIPT1: "import json; f = open('ver.json', 'r'); j = json.load(f); f.close(); d = j['dpctl'][0];" VER_SCRIPT2: "print('='.join((d[s] for s in ('version', 'build'))))" @@ -60,7 +60,7 @@ jobs: with: path: ~/.conda/pkgs key: - ${{ runner.os }}-conda-cf-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('conda-recipe-cf/meta.yaml') }} + ${{ runner.os }}-conda-cf-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('conda-recipe/meta.yaml') }} restore-keys: | ${{ runner.os }}-conda-cf-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- ${{ runner.os }}-conda-cf-${{ env.CACHE_NUMBER }}- @@ -405,7 +405,7 @@ jobs: EXAMPLES_ENV_NAME: examples BUILD_ENV_NAME: build_env DPCPP_CMPLR: "dpcpp_linux-64>=2026.0" - # keep in sync with conda-recipe-cf/conda_build_config.yaml + # keep in sync with conda-recipe/conda_build_config.yaml GCC_CMPLR_VERSION: '15' steps: diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml deleted file mode 100644 index bd630c9db7..0000000000 --- a/.github/workflows/conda-package.yml +++ /dev/null @@ -1,995 +0,0 @@ -name: Conda package - -on: - push: - branches: - - master - pull_request: - -permissions: - contents: read - -env: - PACKAGE_NAME: dpctl - MODULE_NAME: dpctl - TEST_ENV_NAME: test_dpctl - VER_SCRIPT1: "import json; f = open('ver.json', 'r'); j = json.load(f); f.close(); d = j['dpctl'][0];" - VER_SCRIPT2: "print('='.join((d[s] for s in ('version', 'build'))))" - VER_SCRIPT3: "print(' '.join(map(lambda s: chr(34) + s + chr(34), [comp for comp in d['depends'] if 'dpcpp' in comp][1:])))" - INTEL_CHANNEL: "https://software.repos.intel.com/python/conda/" - CONDA_BUILD_VERSION: 26.3.0 - CONDA_INDEX_VERSION: 0.11.0 - -jobs: - build_linux: - # Pinned: ubuntu-latest is moving to Ubuntu 26.04, whose image has no - # preinstalled Miniconda ($CONDA is empty), and Linux jobs below rely on it - runs-on: ubuntu-24.04 - timeout-minutes: 90 - - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - - python: '3.14' - python_spec: '3.14.* *_cp314t' - - python: '3.15' - python_spec: '3.15.* *_cp315' - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - name: Set pkgs_dirs - run: | - echo "pkgs_dirs: [~/.conda/pkgs]" >> ~/.condarc - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - name: Add conda to system path - run: echo "$CONDA/bin" >> "$GITHUB_PATH" - - name: Update conda - run: | - conda update -n base --all - - name: Install conda-build - run: | - conda install -n base conda-build=${{ env.CONDA_BUILD_VERSION }} -c conda-forge --override-channels - - name: Show Conda info - run: | - conda info --all - - name: List base environment packages - run: | - conda list -n base - - name: Store conda paths as envs - shell: bash -l {0} - run: | - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE${{ runner.os == 'Linux' && '/' || '\\' }}" >> "$GITHUB_ENV" - - name: Build conda package - run: | - # use bootstrap channel to pull NumPy linked with OpenBLAS - if [ -n "${{ matrix.python_spec }}" ]; then - conda build --no-test --python "${{ matrix.python_spec }}" --numpy 2.0 -c conda-forge --override-channels conda-recipe - else - conda build --no-test --python ${{ matrix.python }} --numpy 2.0 -c conda-forge --override-channels conda-recipe - fi - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - path: /usr/share/miniconda/conda-bld/linux-64/${{ env.PACKAGE_NAME }}-*.conda - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - build_windows: - runs-on: windows-latest - timeout-minutes: 150 - defaults: - run: - shell: cmd /C CALL {0} - - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - - python: '3.14' - python_spec: '3.14.* *_cp314t' - - python: '3.15' - python_spec: '3.15.* *_cp315' - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - activate-environment: build - channels: conda-forge - python-version: ${{ matrix.python }} - conda-build-version: ${{ env.CONDA_BUILD_VERSION }} - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - name: Store conda paths as envs - shell: bash -l {0} - run: | - echo "CONDA_BLD=$CONDA/conda-bld/win-64/" | tr "\\\\" '/' >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE${{ runner.os == 'Linux' && '/' || '\\' }}" >> "$GITHUB_ENV" - - name: Show Conda info - run: | - conda info --all - - name: List base environment packages - run: | - conda list -n base - - name: Build conda package - env: - OVERRIDE_INTEL_IPO: 1 # IPO requires more resources that GH actions VM provides - run: | - set "PYTHON_VER=${{ matrix.python_spec }}" - if "%PYTHON_VER%" NEQ "" ( - conda build --no-test --python "%PYTHON_VER%" --numpy 2.0 -c conda-forge --override-channels conda-recipe - ) else ( - conda build --no-test --python ${{ matrix.python }} --numpy 2.0 -c conda-forge --override-channels conda-recipe - ) - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_linux: - needs: build_linux - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - experimental: false - runner: ubuntu-24.04 - - python: '3.14' - python_spec: '3.14.* *_cp314t' - experimental: false - runner: ubuntu-24.04 - - python: '3.15' - python_spec: '3.15.* *_cp315' - experimental: false - runner: ubuntu-24.04 - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - experimental: [false] - runner: [ubuntu-24.04] - continue-on-error: ${{ matrix.experimental }} - - steps: - - name: Construct channels line - run: | - echo "CHANNELS=-c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" >> "$GITHUB_ENV" - - name: Display channels line - run: | - echo "${{ env.CHANNELS }}" - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - name: Add conda to system path - run: echo "$CONDA/bin" >> "$GITHUB_PATH" - - name: Update conda - run: | - conda update -n base --all - - name: Install conda-index - run: | - conda install -n base conda-index=${{ env.CONDA_INDEX_VERSION }} -c conda-forge --override-channels - - name: Show Conda info - run: | - conda info --all - - name: List base environment packages - run: | - conda list -n base - - name: Create conda channel - run: | - mkdir -p "$GITHUB_WORKSPACE/channel/linux-64" - conda index "$GITHUB_WORKSPACE/channel" || exit 1 - mv "${PACKAGE_NAME}"-*.conda "$GITHUB_WORKSPACE/channel/linux-64" || exit 1 - conda index "$GITHUB_WORKSPACE/channel" || exit 1 - # Test channel - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE/channel" --override-channels --info --json > "$GITHUB_WORKSPACE/ver.json" - cat ver.json - - name: Collect dependencies - run: | - CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" - export CHANNELS - PACKAGE_VERSION=$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}") - export PACKAGE_VERSION - PYTHON_VER="${{ matrix.python_spec }}" - if [ -z "${PYTHON_VER}" ]; then - PYTHON_VER="${{ matrix.python }}" - fi - # shellcheck disable=SC2086 - conda create -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=${PACKAGE_VERSION} python="${PYTHON_VER}" $CHANNELS --only-deps --dry-run > lockfile - cat lockfile - - name: Set pkgs_dirs - run: | - echo "pkgs_dirs: [~/.conda/pkgs]" >> ~/.condarc - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - name: Install dpctl - run: | - CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" - export CHANNELS - TEST_DEPENDENCIES="pytest cython setuptools" - export TEST_DEPENDENCIES - PACKAGE_VERSION=$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}") - PYTHON_VER="${{ matrix.python_spec }}" - if [ -z "${PYTHON_VER}" ]; then - PYTHON_VER="${{ matrix.python }}" - fi - export PACKAGE_VERSION - # shellcheck disable=SC2086 - conda create -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=${PACKAGE_VERSION} $TEST_DEPENDENCIES python="${PYTHON_VER}" $CHANNELS - # Test installed packages - conda list -n "${{ env.TEST_ENV_NAME }}" - - name: Smoke test - run: | - . "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.TEST_ENV_NAME }}" - python -c "import dpctl; dpctl.lsplatform(verbosity=2)" - - name: Create test temp dir - # create temporary empty folder to runs tests from - # https://github.com/pytest-dev/pytest/issues/11904 - run: mkdir -p "${GITHUB_WORKSPACE}/test_tmp" - - name: Run tests - working-directory: ${{ github.workspace }}/test_tmp - env: - SYCL_CACHE_PERSISTENT: 1 - run: | - . "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.TEST_ENV_NAME }}" - python -m pytest -v --pyargs "${{ env.MODULE_NAME }}" - - test_windows: - needs: build_windows - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - experimental: false - runner: windows-latest - - python: '3.14' - python_spec: '3.14.* *_cp314t' - experimental: false - runner: windows-latest - - python: '3.15' - python_spec: '3.15.* *_cp315' - experimental: false - runner: windows-latest - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - experimental: [false] - runner: [windows-latest] - continue-on-error: ${{ matrix.experimental }} - env: - workdir: '${{ github.workspace }}' - - steps: - - name: Construct channels line - shell: pwsh - run: | - echo "CHANNELS=-c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" >> $env:GITHUB_ENV - - - name: Display channels line - run: | - echo "${{ env.CHANNELS }}" - - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - channels: conda-forge - activate-environment: ${{ env.TEST_ENV_NAME }} - python-version: ${{ matrix.python }} - - - name: Install conda-index - run: | - conda install -n base conda-index - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Create conda channel with the artifact bit - shell: cmd /C CALL {0} - run: | - @echo on - echo ${{ env.workdir }} - mkdir ${{ env.workdir }}\channel - mkdir ${{ env.workdir }}\channel\win-64 - move ${{ env.PACKAGE_NAME }}-*.conda ${{ env.workdir }}\channel\win-64 - dir ${{ env.workdir }}\channel\win-64\ - - - name: Index the channel - shell: cmd /C CALL {0} - run: | - @echo on - conda index ${{ env.workdir }}\channel - - - name: List content of the channels - shell: cmd /C CALL {0} - run: | - dir ${{ env.workdir }}\channel - dir ${{ env.workdir }}\channel\win-64 - - - name: Dump dpctl version info from created channel into ver.json - shell: cmd /C CALL {0} - run: | - @echo on - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json > ${{ env.workdir }}\ver.json - dir ${{ env.workdir }} - - - name: Output content of produced ver.json - shell: pwsh - run: Get-Content -Path ${{ env.workdir }}\ver.json - - - name: Collect dependencies - shell: cmd /C CALL {0} - run: | - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - SET "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - SET "PYTHON_VER=${{ matrix.python_spec }}" - IF "%PYTHON_VER%"=="" SET "PYTHON_VER=${{ matrix.python }}" - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% python="%PYTHON_VER%" -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} --only-deps --dry-run > lockfile - - - name: Display lockfile content - shell: pwsh - run: Get-Content -Path .\lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install opencl_rt - shell: cmd /C CALL {0} - run: conda install -n ${{ env.TEST_ENV_NAME }} opencl_rt -c ${{ env.INTEL_CHANNEL }} --override-channels - - - name: Install dpctl - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - set "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - SET "PYTHON_VER=${{ matrix.python_spec }}" - IF "%PYTHON_VER%"=="" SET "PYTHON_VER=${{ matrix.python }}" - SET TEST_DEPENDENCIES=pytest"<8" cython setuptools - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% %TEST_DEPENDENCIES% python="%PYTHON_VER%" -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} - - - name: Report content of test environment - shell: cmd /C CALL {0} - run: | - echo "Value of CONDA environment variable was: " %CONDA% - echo "Value of CONDA_PREFIX environment variable was: " %CONDA_PREFIX% - conda info && conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Configure Intel OpenCL CPU RT - shell: pwsh - run: | - $script_path="$env:CONDA_PREFIX\Scripts\set-intel-ocl-icd-registry.ps1" - if (Test-Path $script_path) { - &$script_path - } else { - Write-Warning "File $script_path was NOT found!" - } - # Check the variable assisting OpenCL CPU driver to find TBB DLLs which are not located where it expects them by default - $cl_cfg="$env:CONDA_PREFIX\Library\bin\cl.cfg" - Get-Content -Tail 5 -Path $cl_cfg - - - name: Smoke test, step 1 - shell: cmd /C CALL {0} - run: >- - conda activate ${{ env.TEST_ENV_NAME }} && python -c "import sys; print(sys.executable)" - - - name: Smoke test, step 2 - shell: cmd /C CALL {0} - run: >- - conda activate ${{ env.TEST_ENV_NAME }} && python -m dpctl -f - - - name: Create empty temporary directory to run tests from - shell: cmd /C CALL {0} - # create temporary empty folder to runs tests from - # https://github.com/pytest-dev/pytest/issues/11904 - run: >- - mkdir "${{ env.workdir }}\test_tmp" - - - name: List content of workdir folder - shell: cmd /C CALL {0} - run: dir "${{ env.workdir }}" - - - name: Run tests - shell: cmd /C CALL {0} - env: - SYCL_CACHE_PERSISTENT: 1 - working-directory: ${{ env.workdir }}\test_tmp - run: >- - conda activate ${{ env.TEST_ENV_NAME }} && python -m pytest -v -s --pyargs ${{ env.MODULE_NAME }} - - upload_linux: - needs: test_linux - if: | - (github.repository == 'IntelPython/dpctl') && - (github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')) - runs-on: ubuntu-24.04 - timeout-minutes: 20 - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - - python: '3.14' - python_spec: '3.14.* *_cp314t' - - python: '3.15' - python_spec: '3.15.* *_cp315' - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - steps: - - name: Download conda artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - name: Download wheel artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - name: Install anaconda-client - run: conda install anaconda-client -c conda-forge --override-channels - - name: Add conda to system path - run: echo "$CONDA/bin" >> "$GITHUB_PATH" - - name: Package version - run: echo "PACKAGE_VERSION=$(basename ${{ env.PACKAGE_NAME }}-*.conda | sed 's/^${{ env.PACKAGE_NAME }}-\([^-]*\).*/\1/')" >> "$GITHUB_ENV" - - - name: Upload - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - if: ${{ env.ANACONDA_TOKEN != '' }} - run: | - anaconda --token "${{ env.ANACONDA_TOKEN }}" upload --user dppy --label dev "${{ env.PACKAGE_NAME }}"-*.conda - - - name: Upload Wheels - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - run: anaconda --token "${{ env.ANACONDA_TOKEN }}" upload --user dppy --label dev "${{ env.PACKAGE_NAME }}"-*.whl --version "${{ env.PACKAGE_VERSION }}" - - upload_windows: - needs: test_windows - if: | - (github.repository == 'IntelPython/dpctl') && - (github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')) - runs-on: windows-latest - timeout-minutes: 20 - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - - python: '3.14' - python_spec: '3.14.* *_cp314t' - - python: '3.15' - python_spec: '3.15.* *_cp315' - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - name: Download wheel artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - channels: conda-forge - activate-environment: upload - - - name: Install anaconda-client - run: conda install anaconda-client -c conda-forge --override-channels - - - name: Package version - shell: bash -el {0} - run: echo "PACKAGE_VERSION=$(basename ${{ env.PACKAGE_NAME }}-*.conda | sed 's/^${{ env.PACKAGE_NAME }}-\([^-]*\).*/\1/')" >> "$GITHUB_ENV" - - - name: Upload - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - run: | - anaconda --token "${{ env.ANACONDA_TOKEN }}" upload --user dppy --label dev "${{ env.PACKAGE_NAME }}"-*.conda - - - name: Upload Wheels - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - run: anaconda --token "${{ env.ANACONDA_TOKEN }}" upload --user dppy --label dev "${{ env.PACKAGE_NAME }}"-*.whl --version "${{ env.PACKAGE_VERSION }}" - - test_examples_linux: - needs: build_linux - runs-on: ${{ matrix.runner }} - strategy: - matrix: - python: ['3.15'] - python_spec: ['3.15.* *_cp315'] - experimental: [false] - runner: [ubuntu-24.04] - include: - # TODO: switch to 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - - python: '3.14' - python_spec: '3.14.* *_cp314t' - experimental: false - runner: ubuntu-24.04 - continue-on-error: ${{ matrix.experimental }} - timeout-minutes: 60 - env: - EXAMPLES_ENV_NAME: examples - BUILD_ENV_NAME: build_env - - steps: - - name: Construct channels line - run: | - echo "CHANNELS=-c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" >> "$GITHUB_ENV" - - name: Display channels line - run: | - echo ${{ env.CHANNELS }} - - name: Install conda-index - # Needed to be able to run conda index - run: | - conda update -n base --all - conda install conda-index=${{ env.CONDA_INDEX_VERSION }} -c conda-forge --override-channels - - name: Checkout dpctl repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - name: Add conda to system path - run: echo "$CONDA/bin" >> "$GITHUB_PATH" - - name: Create conda channel - run: | - mkdir -p "$GITHUB_WORKSPACE/channel/linux-64" - conda index "$GITHUB_WORKSPACE/channel" || exit 1 - mv "${{ env.PACKAGE_NAME }}"-*.conda "$GITHUB_WORKSPACE/channel/linux-64" || exit 1 - conda index "$GITHUB_WORKSPACE/channel" || exit 1 - # Test channel - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE/channel" --override-channels --info --json > "$GITHUB_WORKSPACE/ver.json" - cat ver.json - - name: Collect dependencies - run: | - CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" - export CHANNELS - PACKAGE_VERSION=$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}") - export PACKAGE_VERSION - PYTHON_VER="${{ matrix.python_spec }}" - if [ -z "${PYTHON_VER}" ]; then - PYTHON_VER="${{ matrix.python }}" - fi - export PYTHON_VER - # shellcheck disable=SC2086 - conda create -n ${{ env.EXAMPLES_ENV_NAME }} ${{ env.PACKAGE_NAME }}=${PACKAGE_VERSION} python="${PYTHON_VER}" $CHANNELS --only-deps --dry-run > lockfile - cat lockfile - - name: Set pkgs_dirs - run: | - echo "pkgs_dirs: [~/.conda/pkgs]" >> ~/.condarc - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - name: Install example requirements - shell: bash -ex -l {0} - env: - DPCPP_CMPLR: "dpcpp_linux-64>=2026.0" - run: | - export CHANNELS="${{ env.CHANNELS }}" - . "$CONDA/etc/profile.d/conda.sh" - DPCTL_DEPENDS="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT3}")" - export DPCTL_DEPENDS - echo "Dpctl dependencies: ${DPCTL_DEPENDS}" - PYTHON_VER="${{ matrix.python_spec }}" - if [ -z "${PYTHON_VER}" ]; then - PYTHON_VER="${{ matrix.python }}" - fi - # shellcheck disable=SC2086 - conda create -n ${{ env.EXAMPLES_ENV_NAME }} -y pytest python="${PYTHON_VER}" "setuptools<72.2.0" $CHANNELS - echo "Environment created" - # shellcheck disable=SC2086 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y cmake ninja $CHANNELS || exit 1 - echo "Cmake and Ninja installed" - # shellcheck disable=SC2086 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y pybind11 cython scikit-build $CHANNELS || exit 1 - echo "scikit-build installed" - # shellcheck disable=SC2086 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y mkl-dpcpp \ - mkl-devel-dpcpp dpcpp_cpp_rt "${DPCTL_DEPENDS}" \ - $CHANNELS || exit 1 - echo "IPL installed" - # shellcheck disable=SC2086 - conda create -y -n ${{ env.BUILD_ENV_NAME }} $CHANNELS gcc_linux-64 gxx_linux-64 \ - "${{ env.DPCPP_CMPLR }}" "${DPCTL_DEPENDS}" \ - "sysroot_linux-64>=2.28" - echo "Compiler installed" - conda list -n "${{ env.BUILD_ENV_NAME }}" - - name: Install dpctl - shell: bash -l {0} - run: | - source "$CONDA/etc/profile.d/conda.sh" - CHANNELS="-c $GITHUB_WORKSPACE/channel -c dppy/label/dev -c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" - export CHANNELS - PACKAGE_VERSION=$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}") - export PACKAGE_VERSION - # shellcheck disable=SC2086 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y $CHANNELS ${{ env.PACKAGE_NAME }}=${PACKAGE_VERSION} || exit 1 - - name: Build and run examples of pybind11 extensions - shell: bash -l {0} - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.EXAMPLES_ENV_NAME }}" - conda list - cd examples/pybind11 - while IFS= read -r d; do - pushd "$d" > /dev/null - conda activate --stack build_env - CC=icx CXX=icpx python setup.py build_ext --inplace -G Ninja || exit 1 - conda deactivate - if [ -e tests ]; then - LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" python -m pytest tests || exit 1 - else - LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" python example.py || exit 1 - fi - popd > /dev/null - done < <(find . -maxdepth 1 -type d -not -path ".") - - name: Build and run examples of Cython extensions - shell: bash -l {0} - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.EXAMPLES_ENV_NAME }}" - conda list - cd examples/cython - while IFS= read -r d; do - pushd "$d" > /dev/null - conda activate --stack "${{ env.BUILD_ENV_NAME }}" - CC=icx CXX=icpx python setup.py build_ext --inplace -G Ninja || exit 1 - conda deactivate - python -m pytest tests || exit 1 - popd > /dev/null - done < <(find . -maxdepth 1 -type d -not -path ".") - - name: Build and run examples of C-extensions - shell: bash -l {0} - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.EXAMPLES_ENV_NAME }}" - conda list - cd examples/c - while IFS= read -r d; do - pushd "$d" > /dev/null - conda activate --stack "${{ env.BUILD_ENV_NAME }}" - python setup.py build_ext --inplace || exit 1 - conda deactivate - python -m pytest tests || exit 1 - popd > /dev/null - done < <(find . -maxdepth 1 -type d -not -path ".") - - name: Run Python examples - shell: bash -l {0} - run: | - cd examples/python - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.EXAMPLES_ENV_NAME }}" - while IFS= read -r script; do - echo "Executing ${script}" - python "${script}" --run all || exit 1 - done < <(find . \( -not -name "_*" -and -name "*.py" \)) - - test_examples_windows: - needs: build_windows - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - python: ['3.14'] - python_spec: ['3.14.* *_cp314'] - experimental: [false] - runner: [windows-latest] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314t' - experimental: false - runner: windows-latest - continue-on-error: ${{ matrix.experimental }} - timeout-minutes: 120 - env: - EXAMPLES_ENV_NAME: examples - BUILD_ENV_NAME: build_env - DPCPP_CMPLR: "dpcpp_win-64>=2026.0" - workdir: '${{ github.workspace }}' - - steps: - - name: Construct channels line - shell: pwsh - run: | - echo "CHANNELS=-c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" >> $env:GITHUB_ENV - - name: Display channels line - run: | - echo "${{ env.CHANNELS }}" - - name: Checkout dpctl repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - channels: conda-forge - activate-environment: ${{ env.EXAMPLES_ENV_NAME }} - python-version: ${{ matrix.python }} - - name: Install conda-index - run: | - conda install -n base conda-index=${{ env.CONDA_INDEX_VERSION }} - - name: Create conda channel with the artifact bit - run: | - @echo on - mkdir ${{ env.workdir }}\channel\win-64 - move ${{ env.PACKAGE_NAME }}-*.conda ${{ env.workdir }}\channel\win-64 - conda index ${{ env.workdir }}\channel - dir ${{ env.workdir }}\channel\win-64\ - - name: Dump dpctl version info from created channel into ver.json - run: | - @echo on - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json > ${{ env.workdir }}\ver.json - - name: Output content of produced ver.json - shell: pwsh - run: Get-Content -Path ${{ env.workdir }}\ver.json - - name: Install example requirements - run: | - @echo on - SET "PYTHON_VER=${{ matrix.python_spec }}" - IF "%PYTHON_VER%"=="" SET "PYTHON_VER=${{ matrix.python }}" - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y python="%PYTHON_VER%" pytest ^ - "setuptools<72.2.0" cmake ninja pybind11 cython scikit-build ^ - mkl-dpcpp mkl-devel-dpcpp ${{ env.CHANNELS }} || exit /b 1 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y opencl_rt -c ${{ env.INTEL_CHANNEL }} --override-channels || exit /b 1 - REM the compiler lives in its own environment, stacked on top of the - REM examples environment while extensions are being built - conda create -n ${{ env.BUILD_ENV_NAME }} -y "${{ env.DPCPP_CMPLR }}" vs2022_win-64 ${{ env.CHANNELS }} || exit /b 1 - conda list -n ${{ env.BUILD_ENV_NAME }} - - name: Install dpctl - run: | - @echo on - set "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% ^ - -c ${{ env.workdir }}/channel -c dppy/label/dev ${{ env.CHANNELS }} || exit /b 1 - conda list -n ${{ env.EXAMPLES_ENV_NAME }} - - name: Configure Intel OpenCL CPU RT - shell: pwsh - run: | - $script_path="$env:CONDA_PREFIX\Scripts\set-intel-ocl-icd-registry.ps1" - if (Test-Path $script_path) { - &$script_path - } else { - Write-Warning "File $script_path was NOT found!" - } - # Check the variable assisting OpenCL CPU driver to find TBB DLLs which are not located where it expects them by default - $cl_cfg="$env:CONDA_PREFIX\Library\bin\cl.cfg" - Get-Content -Tail 5 -Path $cl_cfg - - name: Smoke test - # dpctl source directory shadows the package, so don't run from the - # checkout root - working-directory: ${{ env.workdir }}\examples - run: >- - conda activate ${{ env.EXAMPLES_ENV_NAME }} && python -m dpctl -f - - name: Build and run examples of pybind11 extensions - working-directory: ${{ env.workdir }}\examples\pybind11 - env: - CC: icx - CXX: icx - run: | - @echo on - call conda activate ${{ env.EXAMPLES_ENV_NAME }} || exit /b 1 - conda list - for /d %%d in (*) do ( - pushd %%d || exit /b 1 - call conda activate --stack ${{ env.BUILD_ENV_NAME }} || exit /b 1 - python setup.py build_ext --inplace -G Ninja || exit /b 1 - call conda deactivate || exit /b 1 - if exist tests ( - python -m pytest -v tests || exit /b 1 - ) else ( - python example.py || exit /b 1 - ) - popd - ) - - name: Build and run examples of Cython extensions - working-directory: ${{ env.workdir }}\examples\cython - env: - CC: icx - CXX: icx - run: | - @echo on - call conda activate ${{ env.EXAMPLES_ENV_NAME }} || exit /b 1 - conda list - for /d %%d in (*) do ( - pushd %%d || exit /b 1 - call conda activate --stack ${{ env.BUILD_ENV_NAME }} || exit /b 1 - python setup.py build_ext --inplace -G Ninja || exit /b 1 - call conda deactivate || exit /b 1 - python -m pytest -v tests || exit /b 1 - popd - ) - - name: Build and run examples of C-extensions - working-directory: ${{ env.workdir }}\examples\c - run: | - @echo on - call conda activate ${{ env.EXAMPLES_ENV_NAME }} || exit /b 1 - conda list - for /d %%d in (*) do ( - pushd %%d || exit /b 1 - python setup.py build_ext --inplace || exit /b 1 - python -m pytest -v tests || exit /b 1 - popd - ) - - name: Run Python examples - working-directory: ${{ env.workdir }}\examples\python - run: | - @echo on - call conda activate ${{ env.EXAMPLES_ENV_NAME }} || exit /b 1 - for /f "delims=" %%f in ('dir /b *.py ^| findstr /v "^_"') do ( - echo Executing %%f - python %%f --run all || exit /b 1 - ) - - cleanup_packages: - name: Clean up anaconda packages - needs: [upload_linux, upload_windows] - runs-on: 'ubuntu-latest' - timeout-minutes: 30 - defaults: - run: - shell: bash -el {0} - steps: - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - run-post: false - channel-priority: "disabled" - channels: conda-forge - python-version: '3.11' - - - name: Install anaconda-client - run: conda install anaconda-client -c conda-forge --override-channels - - - name: Checkout repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - repository: IntelPython/devops-tools - fetch-depth: 0 - - - name: Cleanup old packages - run: | - python scripts/cleanup-old-packages.py \ - --verbose --force --token ${{ secrets.ANACONDA_TOKEN }} \ - --package dppy/${{ env.PACKAGE_NAME }} --label dev diff --git a/.github/workflows/intel-ci.yml b/.github/workflows/intel-ci.yml new file mode 100644 index 0000000000..66d1ff9959 --- /dev/null +++ b/.github/workflows/intel-ci.yml @@ -0,0 +1,271 @@ +# Builds and tests dpctl's conda packages and wheels, see ci/README.md +name: Intel CI + +on: + push: + branches: [master] + tags: ['*'] + pull_request: + +permissions: + contents: read + +# A push or tag run gets a group of its own, so a queued master push is never dropped. +concurrency: + group: intel-ci-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + matrix: + name: Platforms and Python versions + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + outputs: + matrix: ${{ steps.declared.outputs.matrix }} + wheel: ${{ steps.declared.outputs.wheel }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Read the declared platforms and Python versions + id: declared + run: | + python ci/matrix.py --event "$GITHUB_EVENT_NAME" --github-output matrix + python ci/matrix.py --system wheel --event "$GITHUB_EVENT_NAME" --github-output wheel + + # The recipe and pyproject.toml declare no macOS, as SYCL has none. + build: + name: ${{ matrix.name }} Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The recipe takes its version from git describe. + fetch-depth: 0 + + # Miniforge 26.9.4-0 fails setup-miniconda's conda init on Windows runners; 26.7.2-0 works. + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install conda-build + run: conda install -y -q -n base --override-channels -c conda-forge conda-build + + - name: Build and run the recipe tests + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dpctl ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.conda + if-no-files-found: error + # Tags keep them 30 days, pull requests and pushes to master 3. + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test: + name: Test ${{ matrix.name }} Python ${{ matrix.python }} + needs: [matrix, build] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dpctl ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the package and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + build-wheel: + name: ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # versioneer takes the version from git describe. + fetch-depth: 0 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + # CMake builds with Ninja and icx, which need MSVC's environment. + - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + if: matrix.platform == 'windows' + + - name: Install cibuildwheel + run: python -m pip install --no-input cibuildwheel==4.3.5 + + - name: Build the wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --system wheel --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dpctl ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.whl + if-no-files-found: error + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test-wheel: + name: Test ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: [matrix, build-wheel] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dpctl ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the wheel and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --kind wheel --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + # The dev channel on anaconda.org, which docs and the nightly run-tests-from-dppy-bits.yaml read. + upload: + name: Upload to anaconda.org + needs: [test, test-wheel] + if: | + github.repository == 'IntelPython/dpctl' && + github.event_name == 'push' && + (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 20 + defaults: + run: + shell: bash + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: dpctl * + merge-multiple: true + path: ${{ runner.temp }}/packages + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install anaconda-client + run: python -m pip install --no-input anaconda-client + + - name: Upload the conda packages and the wheels + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + PACKAGES: ${{ runner.temp }}/packages + run: | + cd "$PACKAGES" + shopt -s nullglob + conda_packages=(dpctl-*.conda) + wheels=(dpctl-*.whl) + if (( ${#conda_packages[@]} == 0 || ${#wheels[@]} == 0 )); then + echo "::error::no dpctl conda package or wheel to upload" + exit 1 + fi + anaconda --token "$ANACONDA_TOKEN" upload --user dppy --label dev "${conda_packages[@]}" + for wheel in "${wheels[@]}"; do + # The version is the file name's second field (PEP 427) + version=${wheel#dpctl-} + version=${version%%-*} + anaconda --token "$ANACONDA_TOKEN" upload --user dppy --label dev "$wheel" --version "$version" + done + + cleanup: + name: Clean up anaconda.org + needs: upload + if: | + github.repository == 'IntelPython/dpctl' && + github.event_name == 'push' && + (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: IntelPython/devops-tools + ref: 08ffddf88627957e8b094817aac4ce0bde7cff63 # main + fetch-depth: 0 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Install anaconda-client + run: python -m pip install --no-input anaconda-client + + - name: Remove the old dev packages + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + run: python scripts/cleanup-old-packages.py --verbose --force --token "$ANACONDA_TOKEN" --package dppy/dpctl --label dev diff --git a/CHANGELOG.md b/CHANGELOG.md index a9eb1d2f1f..7bda5793e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Maintenance * Removed the `-fno-sycl-id-queries-fit-in-int` compile option, which none of the `dpctl` extensions need since `dpctl.tensor` moved to `dpnp` [gh-2413](https://github.com/IntelPython/dpctl/pull/2413) +* Added Intel CI (`.github/workflows/intel-ci.yml` and `ci/`), which builds and tests the conda packages and wheels; `conda-recipe/` is now the only, conda-forge, recipe and `conda-package.yml` is removed [gh-2409](https://github.com/IntelPython/dpctl/pull/2409) + ## [0.23.0] - Oct. 22, 2026 This release extends the SYCL interface exposed by `dpctl` in several directions: `dpctl.compiler.SyclKernelBundle` objects can now be built from SYCL source strings via the DPC++ `kernel_compiler` extension, SPIR-V specialization constants are supported, USM-device allocations can be shared between processes using the SYCL IPC memory extension, and `dpctl.SyclQueue` gained `copy`, `memset`, and `fill` methods along with their asynchronous counterparts. The package can now be used with free-threaded builds of CPython 3.14 and supports Python 3.15. The `dpctl.program` submodule is deprecated in favor of `dpctl.compiler`, which better describes its purpose. diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000000..a2c9287119 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,50 @@ +# How Intel CI tests dpctl + +`.github/workflows/intel-ci.yml` builds dpctl's conda packages and wheels in GitHub Actions and tests each one. It needs this directory, the recipe, and `pyproject.toml`. + +| File | Role | +|---|---| +| `../conda-recipe/` | The conda recipe, built against conda-forge only. `conda_build_config.yaml` lists the Python versions, which has no 3.15t as conda-forge has no NumPy for it. The recipe skips macOS in `meta.yaml`, and its own tests only import dpctl and list the devices, so they need no device. | +| `../pyproject.toml` | What the wheels are built from. Its Python classifiers list the wheels' Python versions, and its Operating System classifiers their platforms. `[tool.cibuildwheel]` holds the wheel build settings, and `[tool.gh-ci]` holds the run-time requirements the wheels need on top of `[project]`'s. | +| `../.github/workflows/intel-ci.yml` | One job reads both matrices. One job per platform and Python version builds and uploads `dpctl Python `, and one builds `dpctl Wheels Python `. Then one job per package tests it. On a push to `master` or a tag, two more jobs upload the tested packages to the `dev` label of `dppy` on anaconda.org and remove its old ones. | +| `matrix.py` | Reads the platforms and Python versions from the recipe, or with `--system wheel` from `pyproject.toml`. A pull request gets the oldest and the newest regular versions and the newest free-threaded one. | +| `build.py` | Builds one Python version: with conda-build, which runs the recipe's own tests, or with `--system wheel` with cibuildwheel, in a manylinux container on Linux. | +| `hooks.py` | The workflow's test jobs: installs one package into a clean environment with the requirements in `config.yml`, and runs `test.py`. A conda package's environment comes from conda-forge; a wheel's has only Python from conda-forge and everything else from PyPI. | +| `install_icx.py` | Installs Intel's icx, which the conda recipe builds with too, for the Windows wheel builds: `[tool.cibuildwheel.windows]` in `../pyproject.toml` runs it first. The Linux wheel builds get icx from Intel's yum repository. Both get the Level Zero headers from a pinned tag of its GitHub repository. | +| `config.yml` | Names the workflow that builds the packages, its artifact names, and what the tests and benchmarks need. | +| `test.py` | Runs the installed package's test suite with pytest. | +| `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package. | + +The scripts need only the Python standard library. `build.py` also handles the CRLF `METADATA` that setuptools writes on Windows. + +## Devices + +dpctl's tests need a SYCL device, and GitHub's runners have none but a CPU. `test.py` prints the devices with `python -m dpctl -f` and fails if there is none, so a run that skipped every device test cannot pass. + +- **In Actions** the device is the OpenCL CPU runtime of `intel-opencl-rt`, which `config.yml` installs from conda-forge for a conda package and from PyPI for a wheel. `test.py` points the OpenCL loader at it with `OCL_ICD_FILENAMES`, and on Windows also registers it under `HKLM\SOFTWARE\Khronos\OpenCL\Vendors`, as the loader in `System32` reads only that, which needs the administrator rights a GitHub runner gives. +- **With `GH_CI_DEVICE=gpu`** `test.py` leaves the loader alone, apart from letting a Linux environment's loader see the system's ICDs too, and fails unless a GPU device is present. + +Two tests compile an extension when they run, `test_c_headers` and `test_cython_api`. `test.py` deselects them, and says so, when setuptools finds no C and C++ compiler. A conda package's environment gets `c-compiler` and `cxx-compiler` from `config.yml`; a wheel's environment uses the machine's. + +## Wheels + +The wheels link the SYCL runtime that `intel-sycl-rt` and `intel-cmplr-lib-rt` ship, and find it through their RPATH, or the DLL search path on Windows, so `auditwheel` and `delvewheel` leave those libraries out. `[tool.gh-ci] wheel-requires` adds the two as requirements to each wheel, as the wheels on PyPI have. icx is pinned to 2026.2 to match. + +## Uploads + +A push to `master` or a tag, in `IntelPython/dpctl` only and after every test passed, uploads the conda packages and the wheels the workflow built to `dppy` on anaconda.org with the label `dev`, with the repository secret `ANACONDA_TOKEN`, and then removes the old dev packages. No other event or branch gets the secret. + +**A pull request.** Actions builds the oldest and the newest regular Python versions and the newest free-threaded one on Linux and Windows, as a conda package and as a wheel, and tests each package. + +**A push to `master` or a tag.** Actions builds and tests every declared version. + +To support another Python, add it to `conda-recipe/conda_build_config.yaml` for the conda packages and as a classifier in `pyproject.toml` for the wheels; nothing else lists versions. Try it locally from a conda base environment with conda-build, or with cibuildwheel for wheels: + +```bash +python ci/matrix.py # every platform and Python version +python ci/matrix.py --event pull_request # what a pull request builds +python ci/build.py --python VERSION --output /tmp/out # VERSION: one matrix.py lists +python ci/hooks.py --python VERSION --artifacts /tmp/out --work /tmp/hooks +python ci/build.py --system wheel --python VERSION --output /tmp/wheels +python ci/hooks.py --kind wheel --python VERSION --artifacts /tmp/wheels --work /tmp/wheel-hooks +``` diff --git a/ci/benchmark.py b/ci/benchmark.py new file mode 100644 index 0000000000..eebdda3666 --- /dev/null +++ b/ci/benchmark.py @@ -0,0 +1,84 @@ +""" +Benchmarks the installed package. + +Runs the asv suite in benchmarks/ against the package installed in this +environment, from an empty working directory, without a build of its own, and +records the results under the commit being tested, in $GH_CI_RESULTS. A pull +request samples each benchmark once, enough to show that every one still runs; +a tag measures them properly. A benchmark that fails fails the run; a slower one +does not, as nothing compares the timings with a baseline yet. +""" + +import json +import os +import pathlib +import subprocess +import sys + +# asv's own dependency, which reads the comments in its configuration. +import json5 + +ASV_CONFIG = pathlib.Path("benchmarks", "asv.conf.json") + + +def asv_config(source: pathlib.Path, results: pathlib.Path) -> pathlib.Path: + """Writes this run's asv configuration into the working directory. + + It is the repository's own, with absolute paths, since asv reads them from + where the configuration is, and with results under GH_CI_RESULTS. It lists + one branch, HEAD: the checkout is the commit under test, detached, and asv + cannot resolve the repository's branches there. + """ + path = source / ASV_CONFIG + config = json5.loads(path.read_text(encoding="utf-8")) + work = pathlib.Path.cwd() + config.update( + repo=str((path.parent / config.get("repo", "..")).resolve()), + benchmark_dir=str( + (path.parent / config.get("benchmark_dir", "benchmarks")).resolve() + ), + branches=["HEAD"], + results_dir=str(results / "asv"), + html_dir=str(work / "html"), + env_dir=str(work / "env"), + ) + written = work / "asv.conf.json" + written.write_text(json.dumps(config, indent=2), encoding="utf-8") + return written + + +def main() -> int: + """Runs the asv suite against the installed package.""" + config = asv_config( + pathlib.Path(os.environ["GH_CI_SOURCE"]), + pathlib.Path(os.environ["GH_CI_RESULTS"]), + ) + asv = [sys.executable, "-m", "asv"] + # asv keeps its machine description in ~/.asv-machine.json; this one stays + # with the run rather than with the machine's user. + home = pathlib.Path.cwd() / "home" + home.mkdir(exist_ok=True) + env = dict(os.environ, HOME=str(home), USERPROFILE=str(home)) + described = subprocess.run( + asv + ["machine", "--yes", "--config", str(config)], env=env + ) + if described.returncode: + return described.returncode + run = [ + "run", + "--config", + str(config), + f"--environment=existing:{sys.executable}", + f"--set-commit-hash={os.environ['GH_CI_COMMIT']}", + "--no-pull", + "--show-stderr", + "--launch-method=spawn", + ] + if os.environ.get("GH_CI_EVENT") != "tag": + run.append("--quick") + # asv exits 2 when a benchmark failed. + return subprocess.run(asv + run, env=env).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/build.py b/ci/build.py new file mode 100644 index 0000000000..0dee720d2b --- /dev/null +++ b/ci/build.py @@ -0,0 +1,365 @@ +""" +Builds this repository's packages for one Python version. + +For the intel-ci.yml GitHub Actions workflow. + +This file and ci/matrix.py work out the build system from the files present, +and the Python versions from that build system's own declaration: + + rattler-build .conda recipe.yaml or recipe/recipe.yaml, versions in + variants.yaml + conda-build .conda conda-recipe/meta.yaml or meta.yaml, versions in + conda_build_config.yaml + cibuildwheel .whl pyproject.toml, versions from its classifiers + +Usage: + python ci/build.py --python 3.14 --output OUT [--system rattler-build] + [--channel conda-forge] + +--python must be one of the declared versions; `python ci/matrix.py` lists them. + +Packages land directly in OUT, ready for actions/upload-artifact. A conda +recipe's own tests run during the build. Needs only the Python standard library. +rattler-build is used from PATH, or downloaded at a pinned version for Linux or +Windows and checked against its published sha256; conda-build, or cibuildwheel +for wheels, must already be installed for this interpreter, so run it with the +conda base install's Python to build with conda-build. + +Wheels build with cibuildwheel, in a manylinux container on Linux, as +pyproject.toml's [tool.cibuildwheel] says. Run-time requirements that +pyproject.toml cannot declare, because the conda package gets them from its +recipe instead, such as MKL's libraries, go in [tool.gh-ci] wheel-requires: +build.py adds them to each wheel's metadata. +""" + +import argparse +import base64 +import hashlib +import os +import platform +import shlex +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +import matrix + +ROOT = matrix.ROOT +PACKAGE_SUFFIXES = (".conda", ".tar.bz2", ".whl") +# cibuildwheel's platform part of a build identifier, per system. +WHEEL_PLATFORMS = { + "Linux": "manylinux_x86_64", + "Windows": "win_amd64", + "Darwin": "macosx_x86_64", +} + +# The rattler-build release fetched when none is on PATH, per system, with the +# file name it goes under and the sha256 its release publishes. +RATTLER_BUILD_VERSION = "v0.77.0" +RATTLER_BUILD_RELEASES = { + "Linux": ( + "rattler-build-x86_64-unknown-linux-musl", + "rattler-build", + "5a5e8150c703b21e88a01f9130ea49a7a680198e9ac9414170c0a293fed7e5af", + ), + "Windows": ( + "rattler-build-x86_64-pc-windows-msvc.exe", + "rattler-build.exe", + "e144334f782cee9aa23e26eb8d095c143e1696fbbec7ba7534a156663d4abb2c", + ), +} + + +class BuildError(Exception): + """Raised when the build cannot start or produced nothing.""" + + +def ensure_rattler_build(tools: Path) -> None: + """ + Puts rattler-build on PATH, downloading the pinned release when none is + there. + """ + if shutil.which("rattler-build"): + return + release = RATTLER_BUILD_RELEASES.get(platform.system()) + if release is None or platform.machine().lower() not in ("x86_64", "amd64"): + raise BuildError( + "no rattler-build on PATH, and only the Linux and Windows x86_64 " + "releases are pinned here" + ) + asset, name, sha256 = release + print(f"build: fetching rattler-build {RATTLER_BUILD_VERSION}", flush=True) + url = ( + "https://github.com/prefix-dev/rattler-build/releases/download/" + f"{RATTLER_BUILD_VERSION}/{asset}" + ) + with urllib.request.urlopen(url, timeout=300) as reply: + binary = reply.read() + if hashlib.sha256(binary).hexdigest() != sha256: + raise BuildError( + f"rattler-build {RATTLER_BUILD_VERSION} does not match its pinned " + "sha256" + ) + tools.mkdir(parents=True, exist_ok=True) + target = tools / name + target.write_bytes(binary) + target.chmod(0o755) + os.environ["PATH"] = f'{tools}{os.pathsep}{os.environ.get("PATH", "")}' + + +def cibuildwheel_identifier(python: matrix.Python) -> str: + """The one build cibuildwheel runs, such as cp314t-manylinux_x86_64.""" + tag = WHEEL_PLATFORMS.get(platform.system()) + if tag is None: + raise BuildError( + f"wheels build on Linux, Windows or macOS, not {platform.system()}" + ) + minor, threaded = matrix.version_key(python.version) + return f'cp3{minor}{"t" if threaded else ""}-{tag}' + + +def wheel_requires() -> list[str]: + """[tool.gh-ci] wheel-requires from pyproject.toml.""" + if matrix.TOMLLIB is None: + raise BuildError("reading pyproject.toml needs Python 3.11 or newer") + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + tool = matrix.TOMLLIB.loads(text).get("tool", {}).get("gh-ci", {}) + requires = tool.get("wheel-requires", []) + if not isinstance(requires, list) or not all( + isinstance(r, str) for r in requires + ): + raise BuildError("[tool.gh-ci] wheel-requires must be a list of specs") + return requires + + +def record_hash(data: bytes) -> str: + """A file's hash as a wheel's RECORD spells it.""" + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + return "sha256=" + digest.rstrip(b"=").decode("ascii") + + +def add_requirements(wheel: Path, requirements: list[str]) -> None: + """Adds Requires-Dist lines to a built wheel, and updates its RECORD.""" + with zipfile.ZipFile(wheel) as source: + members = [(info, source.read(info)) for info in source.infolist()] + names = [info.filename for info, _ in members] + metadata = next(n for n in names if n.endswith(".dist-info/METADATA")) + record = metadata.removesuffix("METADATA") + "RECORD" + for index, (info, data) in enumerate(members): + if info.filename == metadata: + metadata_text = data.decode("utf-8").replace("\r\n", "\n") + head, blank, body = metadata_text.partition("\n\n") + head = head.rstrip("\n") + "".join( + f"\nRequires-Dist: {r}" for r in requirements + ) + text = head + ("\n\n" + body if blank else "\n") + new_metadata = text.encode("utf-8") + members[index] = (info, new_metadata) + for index, (info, data) in enumerate(members): + if info.filename == record: + lines = [ + ( + f"{metadata},{record_hash(new_metadata)}," + f"{len(new_metadata)}" + if line.split(",")[0] == metadata + else line + ) + for line in data.decode("utf-8").splitlines() + ] + members[index] = (info, ("\n".join(lines) + "\n").encode()) + rewritten = wheel.with_suffix(".tmp") + with zipfile.ZipFile(rewritten, "w", zipfile.ZIP_DEFLATED) as target: + for info, data in members: + target.writestr(info, data) + rewritten.replace(wheel) + + +def commands( + system: str, + selector: Path, + python: matrix.Python, + channels: list[str], + scratch: Path, +) -> list[list[str]]: + """ + The commands that build one Python version's packages into scratch. + + Each build system gets the version back exactly as its own file declares it, + and keeps every other variant setting from that file. + """ + channel_args = [arg for channel in channels for arg in ("-c", channel)] + # The commit a conda package came from, in its info/about.json + github = [ + os.environ.get(name, "") + for name in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_SHA") + ] + extra_meta = ( + [ + "--extra-meta", + f"remote_url={github[0]}/{github[1]}", + f"sha={github[2]}", + ] + if all(github) + else [] + ) + if system == "rattler-build": + return [ + [ + "rattler-build", + "build", + "-r", + str(selector), + "--variant", + f"python={python.spec}", + "--output-dir", + str(scratch / "conda"), + *channel_args, + ] + ] + if system == "conda-build": + # conda runs as a module of this interpreter, so Windows needs no + # conda.bat on PATH. + return [ + [ + sys.executable, + "-m", + "conda", + "build", + str(selector.parent), + "--python", + python.spec, + "--no-anaconda-upload", + "--output-folder", + str(scratch / "conda"), + *extra_meta, + "--override-channels", + *channel_args, + ] + ] + return [ + [ + sys.executable, + "-m", + "cibuildwheel", + "--only", + cibuildwheel_identifier(python), + "--output-dir", + str(scratch / "wheel"), + str(ROOT), + ] + ] + + +def collect(scratch: Path, output: Path) -> list[Path]: + """ + Moves built packages into output, flat, and lists them. + Two packages of one name are an error: flat, one would overwrite the other. + """ + output.mkdir(parents=True, exist_ok=True) + sources: dict[str, Path] = {} + for package in sorted(scratch.rglob("*")): + if ( + package.is_file() + and package.name.endswith(PACKAGE_SUFFIXES) + and "broken" not in package.parts + ): + if package.name in sources: + raise BuildError( + f"two packages named {package.name}: " + f"{sources[package.name]} and {package}" + ) + sources[package.name] = package + shutil.move(str(package), output / package.name) + built = sorted( + path + for path in output.iterdir() + if path.name.endswith(PACKAGE_SUFFIXES) + ) + if not built: + raise BuildError(f"the build left no package in {output}") + return built + + +def main(argv: list[str] | None = None) -> int: + """Builds one Python version.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--output", required=True, type=Path, help="where the packages go" + ) + parser.add_argument( + "--system", + default="", + choices=("", *matrix.SYSTEMS), + help="build system to use instead of the detected one", + ) + parser.add_argument( + "--channel", + action="append", + default=None, + help="conda channel to build against, repeatable; default conda-forge", + ) + default_tools = ( + Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) / "ci-tools" + ) + parser.add_argument( + "--tools", + type=Path, + default=default_tools, + help="where a downloaded rattler-build goes", + ) + args = parser.parse_args(argv) + + system, selector = matrix.detect(ROOT, args.system) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise BuildError( + f"Python {args.python} is not declared for {system}; " + f'declared: {", ".join(declared)}' + ) + python = declared[args.python] + print( + f"build: {system} from {selector.relative_to(ROOT)} " + f"for Python {python.version} ({python.spec})", + flush=True, + ) + if system == "rattler-build": + ensure_rattler_build(args.tools) + output = args.output.resolve() + with tempfile.TemporaryDirectory() as scratch: + for command in commands( + system, + selector, + python, + args.channel or ["conda-forge"], + Path(scratch), + ): + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True) + built = collect(Path(scratch), output) + if system == "wheel" and (requires := wheel_requires()): + for wheel in built: + add_requirements(wheel, requires) + print("build: added Requires-Dist " + ", ".join(requires)) + print("build: " + ", ".join(path.name for path in built)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + BuildError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"build: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/config.yml b/ci/config.yml new file mode 100644 index 0000000000..d95962ef6c --- /dev/null +++ b/ci/config.yml @@ -0,0 +1,16 @@ +# Intel CI settings for dpctl, see ci/README.md +version: 1 +workflow: intel-ci.yml +artifacts: + conda: "dpctl {platform} Python {python}" + wheel: "dpctl {platform} Wheels Python {python}" +test: + # The compilers build the extensions two tests compile; intel-opencl-rt is the OpenCL CPU device. + requires: [pytest, cython, setuptools, intel-opencl-rt, c-compiler, cxx-compiler] + # A wheel's environment has the system's compilers only, and the SYCL runtime comes with the wheel. + wheel_requires: [pytest, cython, setuptools, intel-opencl-rt] +benchmark: + # PyYAML, which asv needs, has no PyPI wheel for every Python the recipe declares. + requires: [numpy, intel-opencl-rt, pyyaml] + # conda-forge lacks asv for some Python the recipe declares; the abi3 wheel on PyPI fits all. + pip_requires: [asv] diff --git a/ci/hooks.py b/ci/hooks.py new file mode 100644 index 0000000000..ef6bc591df --- /dev/null +++ b/ci/hooks.py @@ -0,0 +1,437 @@ +""" +Runs ci/test.py against one built artifact. + +For the GitHub Actions test job. + +Each Python version's artifact is installed into a fresh conda environment from +a local channel plus conda-forge, pinned to its exact build, with the test +requirements ci/config.yml lists, and the test runs with that environment's +Python from an empty working directory, on Linux or Windows. A wheel's +environment has only Python from conda-forge: the wheel and the test +requirements install with pip, as in the wheel's users' environments. The test +gets the variables ci/test.py reads, and nothing else from the runner. + +Usage, from any Python 3.10 or newer; it runs itself again with the conda base +install's Python, from $CONDA, which setup-miniconda sets, or the conda on PATH: + python ci/hooks.py --python 3.14 --artifacts DIR --work DIR [--kind wheel] + +Needs conda, conda-package-handling and ruamel.yaml, which every conda base has, +and installs nothing into it. +""" + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import matrix + +# Dependencies of conda itself; YAML is read with PyYAML, else ruamel.yaml +try: + from conda_package_handling import api as CPH_API +except ModuleNotFoundError: + CPH_API = None +try: + import yaml as PYYAML +except ModuleNotFoundError: + PYYAML = None +try: + from ruamel.yaml import YAML as RUAMEL_YAML +except ModuleNotFoundError: + RUAMEL_YAML = None + +ROOT = matrix.ROOT +IS_WINDOWS = os.name == "nt" +if IS_WINDOWS: + PLATFORM, HOST_SUBDIR = "windows", "win-64" +elif sys.platform == "darwin": + PLATFORM, HOST_SUBDIR = "macos", "osx-64" +else: + PLATFORM, HOST_SUBDIR = "linux", "linux-64" +CONDA_SUFFIXES = (".conda", ".tar.bz2") +# All the test sees of the runner's environment, so no tokens reach it +KEEP = ( + "PATH", + "HOME", + "USER", + "LANG", + "TERM", + "TMPDIR", + "TZ", + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CONDA_PKGS_DIRS", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "TEMP", + "TMP", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", +) + + +class HookError(Exception): + """Raised when the artifact cannot be installed or ci/test.py is missing.""" + + +def run(command: list[str], **options: object) -> None: + """Echoes and runs a command, never through a shell.""" + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True, **options) + + +def conda(*args: str) -> None: + """Runs conda from the base environment running this script.""" + run([sys.executable, "-m", "conda", *args]) + + +def empty_repodata(subdir: str) -> dict: + """The repodata.json of a channel subdir with no packages.""" + return { + "info": {"subdir": subdir}, + "packages": {}, + "packages.conda": {}, + "repodata_version": 1, + } + + +def package_record(package: Path) -> tuple[str, dict]: + """A conda package's subdir and repodata record, from its index.json.""" + if CPH_API is None: + raise HookError( + "run this with a conda base Python, " + "which has conda-package-handling" + ) + with tempfile.TemporaryDirectory() as scratch: + CPH_API.extract(str(package), scratch, components="info") + record = json.loads( + (Path(scratch) / "info" / "index.json").read_text(encoding="utf-8") + ) + data = package.read_bytes() + record.update( + { + "md5": hashlib.md5(data, usedforsecurity=False).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), + "size": len(data), + } + ) + subdir = str(record.get("subdir") or "noarch") + if not re.fullmatch(r"[a-z0-9]+(-[a-z0-9_]+)?", subdir): + raise HookError( + f"{package.name} declares an unusable subdir {subdir!r}" + ) + return subdir, record + + +def stage_channel(packages: list[Path], channel: Path) -> None: + """ + Lays the packages out as a local channel, by the subdir each one declares, + and writes its repodata.json itself, without conda-index. + """ + repodata = { + subdir: empty_repodata(subdir) for subdir in ("noarch", HOST_SUBDIR) + } + for package in packages: + subdir, record = package_record(package) + (channel / subdir).mkdir(parents=True, exist_ok=True) + shutil.copy2(package, channel / subdir / package.name) + section = ( + "packages.conda" if package.name.endswith(".conda") else "packages" + ) + repodata.setdefault(subdir, empty_repodata(subdir))[section][ + package.name + ] = record + for subdir, data in repodata.items(): + (channel / subdir).mkdir(parents=True, exist_ok=True) + (channel / subdir / "repodata.json").write_text( + json.dumps(data, indent=2) + "\n", encoding="utf-8" + ) + + +def test_python(version: str) -> str: + """The conda spec for the test environment's interpreter.""" + minor, threaded = matrix.version_key(version) + # The .* matters: conda reads a bare 3.11 as exactly 3.11.0 + if minor < matrix.FREE_THREADED_MINOR: + return f"3.{minor}.*" + # conda-forge marks the ABI from 3.13, so 3.14 can't resolve to 3.14t + return f'3.{minor}.* *_cp3{minor}{"t" if threaded else ""}' + + +def test_requirements() -> tuple[list[str], list[str], list[str] | None]: + """ + The conda-forge and pip packages ci/config.yml installs next to the + artifact, and a wheel's PyPI requirements, None when it gives none. + """ + text = (ROOT / "ci" / "config.yml").read_text(encoding="utf-8") + if PYYAML is not None: + config = PYYAML.safe_load(text) + elif RUAMEL_YAML is not None: + config = RUAMEL_YAML(typ="safe", pure=True).load(text) + else: + raise HookError("reading ci/config.yml needs PyYAML or ruamel.yaml") + test = (config or {}).get("test") or {} + lists = [] + for key in ("requires", "pip_requires", "wheel_requires"): + values = test.get(key) + if values is not None and ( + not isinstance(values, list) + or not all(isinstance(v, str) for v in values) + ): + raise HookError( + f"ci/config.yml: test.{key} must be a list of package specs" + ) + lists.append(values) + return lists[0] or [], lists[1] or [], lists[2] + + +def exact_spec(package: Path) -> str: + """name=version=build from a conda package file name.""" + stem = package.name + for suffix in CONDA_SUFFIXES: + stem = stem.removesuffix(suffix) + name, version, build = stem.rsplit("-", 2) + return f"{name}={version}={build}" + + +def env_python(env_dir: Path) -> Path: + """The interpreter of a conda environment.""" + return env_dir / "python.exe" if IS_WINDOWS else env_dir / "bin" / "python" + + +def env_path(env_dir: Path) -> list[Path]: + """ + The directories conda's activation would put first on PATH for an + environment. + """ + if not IS_WINDOWS: + return [env_dir / "bin"] + library = env_dir / "Library" + return [ + env_dir, + library / "mingw-w64" / "bin", + library / "usr" / "bin", + library / "bin", + env_dir / "Scripts", + env_dir / "bin", + ] + + +def test_env( + env_dir: Path, artifacts: Path, results: Path, args: argparse.Namespace +) -> dict: + """The variables ci/test.py reads, and no runner tokens.""" + env = {key: value for key, value in os.environ.items() if key in KEEP} + env["PATH"] = os.pathsep.join( + [*map(str, env_path(env_dir)), env.get("PATH", "")] + ) + env.update( + { + "CONDA_PREFIX": str(env_dir), + "GH_CI_ARTIFACTS": str(artifacts), + "GH_CI_SOURCE": str(ROOT), + "GH_CI_RESULTS": str(results), + "GH_CI_KIND": args.kind, + "GH_CI_PYTHON": args.python, + "GH_CI_PLATFORM": PLATFORM, + "GH_CI_EVENT": args.event, + "GH_CI_COMMIT": args.commit, + } + ) + return env + + +def summarize(lines: list[str]) -> None: + """Adds the results to the job summary page, in GitHub Actions.""" + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as page: + page.write("\n".join(lines) + "\n") + + +def install(args: argparse.Namespace) -> Path: + """ + Creates the test environment with the artifact, pinned to its exact build; + returns it. + """ + system, selector = matrix.detect( + ROOT, "wheel" if args.kind == "wheel" else "" + ) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise HookError( + f"Python {args.python} is not declared; " + f'declared: {", ".join(declared)}' + ) + suffixes = (".whl",) if args.kind == "wheel" else CONDA_SUFFIXES + packages = sorted( + path + for path in args.artifacts.rglob("*") + if path.name.endswith(suffixes) + ) + if not packages: + raise HookError(f"{args.artifacts} holds no {args.kind} package") + requires, pip_requires, wheel_requires = test_requirements() + env_dir = args.work / "env" + pip = [str(env_python(env_dir)), "-m", "pip", "install", "--no-input"] + python = f"python {test_python(args.python)}" + if args.kind == "wheel": + conda_create(env_dir, python, "pip") + wheel_specs = requires if wheel_requires is None else wheel_requires + run([*pip, *map(str, packages), *wheel_specs]) + else: + channel = args.work / "channel" + stage_channel(packages, channel) + conda_create( + env_dir, + python, + *requires, + *map(exact_spec, packages), + channels=(channel.as_uri(),), + ) + if pip_requires: + run([*pip, *pip_requires]) + return env_dir + + +def conda_create( + env_dir: Path, *specs: str, channels: tuple[str, ...] = () +) -> None: + """Creates a conda environment from conda-forge and channels.""" + conda( + "create", + "-y", + "-q", + "-p", + str(env_dir), + "--override-channels", + *(arg for channel in channels for arg in ("-c", channel)), + "-c", + "conda-forge", + *specs, + ) + + +def run_test(env_dir: Path, args: argparse.Namespace) -> int: + """Runs ci/test.py with the environment's Python; returns its status.""" + script = ROOT / "ci" / "test.py" + if not script.is_file(): + raise HookError("ci/test.py is required and missing") + results, scratch = ( + args.work / "results" / "test", + args.work / "scratch" / "test", + ) + for directory in (results, scratch): + directory.mkdir(parents=True, exist_ok=True) + command = [str(env_python(env_dir)), str(script)] + print(f"+ {shlex.join(command)}", flush=True) + status = subprocess.run( + command, + cwd=scratch, + check=False, + env=test_env(env_dir, args.artifacts, results, args), + ).returncode + result = "passed" if status == 0 else f"failed, exit {status}" + summarize( + [ + f"### {matrix.PLATFORMS[PLATFORM]} Python {args.python}" + f" {args.kind}", + "", + "| Test | Result |", + "|---|---|", + f"| ci/test.py | {result} |", + ] + ) + return status + + +def conda_base_python() -> Path: + """The conda base install's Python: from $CONDA, else the conda on PATH.""" + root = os.environ.get("CONDA", "") + if not root: + found = shutil.which("conda") + if not found: + raise HookError( + "no conda base install: set CONDA or put conda on PATH" + ) + # conda lives in bin, Scripts or condabin, one level below the install. + root = str(Path(found).resolve().parent.parent) + python = env_python(Path(root)) + if not python.is_file(): + raise HookError( + f"{root} is not a conda base install; it has no {python.name}" + ) + return python + + +def main(argv: list[str] | None = None) -> int: + """Installs the artifact, then runs ci/test.py against it.""" + if CPH_API is None: + # Re-run with base's Python, which has conda-package-handling + command = [ + str(conda_base_python()), + str(Path(__file__).resolve()), + *(sys.argv[1:] if argv is None else argv), + ] + return subprocess.run(command, check=False).returncode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--artifacts", required=True, type=Path, help="the downloaded artifact" + ) + parser.add_argument( + "--work", required=True, type=Path, help="an empty scratch directory" + ) + parser.add_argument( + "--event", default="push", choices=("pr", "tag", "push") + ) + parser.add_argument("--commit", default="") + parser.add_argument( + "--kind", + default="conda", + choices=("conda", "wheel"), + help="what the artifact holds", + ) + args = parser.parse_args(argv) + args.artifacts, args.work = args.artifacts.resolve(), args.work.resolve() + if args.work.exists() and ( + not args.work.is_dir() or any(args.work.iterdir()) + ): + raise HookError(f"--work {args.work} must be an empty directory") + return run_test(install(args), args) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + HookError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"hooks: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/install_icx.py b/ci/install_icx.py new file mode 100644 index 0000000000..cbdfaa3cc6 --- /dev/null +++ b/ci/install_icx.py @@ -0,0 +1,78 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Installs Intel's icx for dpctl's Windows wheel builds. + +cibuildwheel runs this before the builds, as pyproject.toml says. It fetches +micromamba, checked against its published sha256, and creates PREFIX with +conda-forge's dpcpp_win-64, the compiler the conda recipe builds with. + +Usage: + python ci/install_icx.py PREFIX +""" + +import hashlib +import subprocess +import sys +import tempfile +import urllib.request +from pathlib import Path + +MICROMAMBA = ( + "https://github.com/mamba-org/micromamba-releases/releases/download/" + "2.10.0-0/micromamba-win-64" +) +MICROMAMBA_SHA256 = ( + "f8b4284bd4eb1134122eaf93b3ea6c7be567e596de5829e1a6f478e34e41974a" +) +COMPILER = "dpcpp_win-64=2026.2.0" + + +def main() -> int: + """Creates PREFIX with the compiler.""" + prefix = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as scratch: + micromamba = Path(scratch) / "micromamba.exe" + with urllib.request.urlopen(MICROMAMBA, timeout=300) as response: + micromamba.write_bytes(response.read()) + digest = hashlib.sha256(micromamba.read_bytes()).hexdigest() + if digest != MICROMAMBA_SHA256: + print( + f"install_icx: micromamba has sha256 {digest}, " + f"not {MICROMAMBA_SHA256}", + file=sys.stderr, + ) + return 1 + command = [ + str(micromamba), + "create", + "-y", + "-r", + str(Path(scratch) / "root"), + "-p", + str(prefix), + "--override-channels", + "-c", + "conda-forge", + COMPILER, + ] + print("+ " + " ".join(command), flush=True) + return subprocess.run(command, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/matrix.py b/ci/matrix.py new file mode 100644 index 0000000000..781d506288 --- /dev/null +++ b/ci/matrix.py @@ -0,0 +1,655 @@ +""" +The platforms and Python versions this repository builds. + +Both are read from the repository's own build files. + +Every build system already has a standard place for both, so each is written +once, where the build reads it, and the GitHub Actions matrix follows it: + + rattler-build, selected by recipe.yaml + Python versions python: in variants.yaml next to the recipe + platforms build: skip: in the recipe + conda-build, selected by meta.yaml + Python versions python: in conda_build_config.yaml next to the recipe + platforms skip: true # [selector] lines in the recipe + wheel, selected by pyproject.toml + Python versions the Programming Language :: Python :: 3.N classifiers; + any Free Threading classifier adds 3.Nt for each from 3.14 + platforms the Operating System :: classifiers + +A recipe builds on every platform in PLATFORMS, Linux, Windows and macOS on +Intel, unless a skip that depends on the platform alone rules one out, such as +`skip: [win]` or `skip: true # [osx]`; a skip that also depends on something +else, such as `win and py < 311` or `osx and arm64`, skips some builds, not the +platform. A pyproject.toml without Operating System classifiers, or with OS +Independent, builds on all three. Of the declared platforms, the matrix holds +those GitHub Actions has runners for, BUILT_PLATFORMS. + +meson-python, scikit-build-core and every other PEP 517 backend declare these in +pyproject.toml too, so the wheel row covers them. Another convention, such as +pixi.toml, is one more reader in READERS and PLATFORM_READERS and, if it is a +new build system, one more entry in SYSTEMS. + +Usage: + python ci/matrix.py [--system SYSTEM] [--event EVENT] [--github-output NAME] + +Prints the build matrix as JSON for strategy.matrix: {"include": [{"platform": +"linux", "name": "Linux", "python": "3.14"}, ...]}, every platform with every +Python version. For a pull_request event only the oldest and the newest regular +versions and the newest free-threaded one are listed. --github-output also +writes NAME= to $GITHUB_OUTPUT, for later jobs. Standard library only; +reading pyproject.toml needs Python 3.11 or newer for tomllib. +""" + +import argparse +import ast +import json +import os +import re +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +# Only the wheel reader needs it, and only it needs Python 3.11 or newer. +try: + import tomllib as TOMLLIB +except ModuleNotFoundError: + TOMLLIB = None + +ROOT = Path(__file__).resolve().parent.parent + +# Build system -> files that select it, in the order they are looked for. Where +# a repository still has conda-recipe-cf/ next to conda-recipe/, the conda-forge +# one comes first: the tests use conda-forge alone. +SYSTEMS = { + "rattler-build": ("recipe.yaml", "recipe/recipe.yaml"), + "conda-build": ( + "conda-recipe-cf/meta.yaml", + "meta.yaml", + "conda-recipe/meta.yaml", + "recipe/meta.yaml", + ), + "wheel": ("pyproject.toml",), +} + +# The platforms CI builds and tests, and how artifact names spell them. +PLATFORMS = {"linux": "Linux", "windows": "Windows", "macos": "macOS"} +# The platforms the workflow can build on: GitHub-hosted runners have all +# three. A declared platform missing here is reported and left out. +BUILT_PLATFORMS = ("linux", "windows", "macos") +# What a platform-only selector means on each platform, for both conda-build and +# rattler-build. +SELECTORS = { + "linux": { + "linux": True, + "linux64": True, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + "windows": { + "linux": False, + "linux64": False, + "unix": False, + "win": True, + "win64": True, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + # Intel Macs only, as MKL has no Apple silicon build. + "macos": { + "linux": False, + "linux64": False, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": True, + "osx64": True, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, +} +CONDA_SKIP_RE = re.compile(r"\s*skip\s*:\s*(?:true|True|yes)\s*#\s*\[(.+)\]\s*") +OS_CLASSIFIERS = { + "Operating System :: OS Independent": tuple(PLATFORMS), + "Operating System :: POSIX": ("linux",), + "Operating System :: Unix": ("linux",), + "Operating System :: POSIX :: Linux": ("linux",), + "Operating System :: Microsoft :: Windows": ("windows",), + "Operating System :: MacOS": ("macos",), +} + +PULL_REQUEST_EVENTS = ("pull_request",) +OLDEST_MINOR = 10 +FREE_THREADED_MINOR = 13 +# A Free Threading classifier adds 3.Nt from 3.14, the first supported one, +# which is also the oldest cibuildwheel builds. +CLASSIFIED_FREE_THREADED_MINOR = 14 + +# A conda python variant: 3.12, 3.12.*, 3.12.* *_cpython, 3.13.* *_cp313 or +# 3.14.* *_cp314t. +CONDA_SPEC_RE = re.compile( + r"3\.(\d+)(?:\.\*)?(?:\s+\*_(?:cpython|cp3(\d+)(t?)))?" +) +CLASSIFIER_RE = re.compile(r"Programming Language :: Python :: 3\.(\d+)") +FREE_THREADING_CLASSIFIER = "Programming Language :: Python :: Free Threading" +LOWER_BOUND_RE = re.compile(r">=\s*3\.(\d+)") + + +class DeclarationError(Exception): + """ + Raised when the build files declare no Python versions, or ones this cannot + read. + """ + + +@dataclass(frozen=True) +class Python: + """One declared Python version.""" + + version: str # as artifact names spell it: 3.14 or 3.14t + spec: str # as the build system spells it, passed back to it unchanged + + +def version_key(version: str) -> tuple[int, bool]: + """(minor, free-threaded); also the sort order.""" + return int(version.rstrip("t").split(".")[1]), version.endswith("t") + + +def checked(minor: int, threaded: bool, where: str) -> str: + """The version string, once it is one CI can test.""" + if minor < OLDEST_MINOR or (threaded and minor < FREE_THREADED_MINOR): + raise DeclarationError( + f'{where}: 3.{minor}{"t" if threaded else ""} is not supported; ' + f"the oldest is 3.{OLDEST_MINOR}, free-threaded from " + f"3.{FREE_THREADED_MINOR}" + ) + return f'3.{minor}{"t" if threaded else ""}' + + +def _strip_comment(line: str, where: str) -> str: + """ + The line without a trailing comment; conda-build selectors are refused, not + guessed at. + """ + in_quote = "" + for index, char in enumerate(line): + if char in "\"'" and in_quote in ("", char): + in_quote = "" if in_quote else char + elif ( + char == "#" + and not in_quote + and (index == 0 or line[index - 1].isspace()) + ): + if line[index:].lstrip("# ").startswith("["): + raise DeclarationError( + f"{where}: selectors in the python list are not supported" + ) + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str, where: str) -> str: + """A plain or quoted YAML scalar.""" + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + if not value or value[0] in "{[&*!|>" or ":" in value: + raise DeclarationError( + f"{where}: write each python entry as a plain value, not {value!r}" + ) + return value + + +def yaml_python_list(text: str, where: str) -> list[str]: + """ + The top-level python: list of a variants file, without a YAML library. + + Accepts a block list or a one-line [a, b] list of plain or quoted values. + Anything richer, such as if/then selectors or templating, is refused so the + matrix is never guessed. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + if not re.match(r"python\s*:", line): + continue + rest = _strip_comment(line.split(":", 1)[1], where).strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + raise DeclarationError(f"{where}: python must be a list") + values = [] + for item in lines[number + 1 :]: + # Checked before the comment, as the next key may have a selector + if item[:1] not in ("", " ", "\t", "-", "#"): + break # the next top-level key + entry = _strip_comment(item, where).lstrip() + if not entry: + continue + if not entry.startswith("- "): + raise DeclarationError( + f"{where}: python must be a plain list of versions" + ) + values.append(_unquote(entry[2:], where)) + return values + raise DeclarationError(f"{where} has no top-level python: list") + + +def from_conda_variants(path: Path) -> list[Python]: + """The python: list of a variants.yaml or a conda_build_config.yaml.""" + where = ( + path.relative_to(ROOT).as_posix() + if path.is_relative_to(ROOT) + else str(path) + ) + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as ex: + raise DeclarationError( + f"{where} is missing; list the Python versions to build there" + ) from ex + pythons = [] + for spec in yaml_python_list(text, where): + match = CONDA_SPEC_RE.fullmatch(spec) + if not match or (match[2] and match[2] != match[1]): + raise DeclarationError( + f"{where}: {spec!r} is not a CPython variant such as " + '"3.13.* *_cp313" or "3.14.* *_cp314t"' + ) + pythons.append( + Python(checked(int(match[1]), bool(match[3]), where), spec) + ) + return pythons + + +def from_pyproject(path: Path) -> list[Python]: + """ + The 3.N classifiers in pyproject.toml, plus 3.Nt from 3.14 if free-threading + is declared. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + project = TOMLLIB.loads(path.read_text(encoding="utf-8")).get("project", {}) + classifiers = project.get("classifiers", []) + minors = sorted( + { + int(match[1]) + for classifier in classifiers + if (match := CLASSIFIER_RE.fullmatch(classifier)) + } + ) + if not minors: + raise DeclarationError( + "pyproject.toml lists no Programming Language :: Python :: 3.N " + "classifiers" + ) + bound = LOWER_BOUND_RE.search(project.get("requires-python", "")) + if bound and minors[0] < int(bound[1]): + raise DeclarationError( + f"pyproject.toml classifies 3.{minors[0]}, below requires-python " + f'{project["requires-python"]}' + ) + threaded = any( + classifier.startswith(FREE_THREADING_CLASSIFIER) + for classifier in classifiers + ) + versions = [checked(minor, False, "pyproject.toml") for minor in minors] + versions += [ + f"3.{minor}t" + for minor in minors + if threaded and minor >= CLASSIFIED_FREE_THREADED_MINOR + ] + return [Python(version, version) for version in versions] + + +def skips_platform(expression: str, platform: str) -> bool | None: + """ + Whether a skip expression rules out a whole platform: True or False when it + depends on the platform alone, None when it also depends on anything else, + such as the Python version. + """ + try: + tree = ast.parse( + expression.strip().removeprefix("${{").removesuffix("}}").strip(), + mode="eval", + ).body + except SyntaxError: + return None + + def value(node: ast.AST) -> bool: + if isinstance(node, ast.BoolOp): + values = [value(item) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not value(node.operand) + if isinstance(node, ast.Name) and node.id in SELECTORS[platform]: + return SELECTORS[platform][node.id] + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return node.value + raise ValueError(ast.dump(node)) + + try: + return value(tree) + except ValueError: + return None + + +def platforms_without(skips: list[str], where: str) -> list[str]: + """Every platform in PLATFORMS, but those a platform-only skip rules out.""" + kept = [ + platform + for platform in PLATFORMS + if not any(skips_platform(skip, platform) for skip in skips) + ] + if not kept: + raise DeclarationError( + f"{where} skips every platform CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return kept + + +def yaml_build_skips(text: str, where: str) -> list[str]: + """ + The skip: entries directly under a rattler-build recipe's top-level build:, + as a list of plain or quoted values, a one-line [a, b] list, or one value; + none when there is no skip. + """ + lines = text.splitlines() + starts = [ + number + for number, line in enumerate(lines) + if re.match(r"build\s*:\s*(#.*)?$", line) + ] + if not starts: + return [] + child = "" + for number in range(starts[0] + 1, len(lines)): + line = ( + _strip_comment(lines[number], where) + if "#" in lines[number] + else lines[number] + ) + if not line.strip(): + continue + if not line[0].isspace(): + return [] # the next top-level key + indent = line[: len(line) - len(line.lstrip())] + child = child or indent + match = ( + re.match(r"skip\s*:(.*)", line.lstrip()) + if indent == child + else None + ) + if not match: + continue + rest = match[1].strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + return [_unquote(rest, where)] + values = [] + for item in lines[number + 1 :]: + entry = item.strip() + if not entry or entry.startswith("#"): + continue + if not entry.startswith("- "): + break + values.append(_unquote(entry[2:].split(" #")[0], where)) + return values + return [] + + +def from_rattler_recipe(recipe: Path) -> list[str]: + """The platforms a rattler-build recipe builds on, from its build: skip:.""" + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + return platforms_without( + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), where + ) + + +def from_conda_recipe(recipe: Path) -> list[str]: + """ + The platforms a conda-build meta.yaml builds on, from its skip: true # + [selector] lines. + """ + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + skips = [ + match[1] + for line in recipe.read_text(encoding="utf-8").splitlines() + if (match := CONDA_SKIP_RE.fullmatch(line)) + ] + return platforms_without(skips, where) + + +def from_os_classifiers(path: Path) -> list[str]: + """ + The platforms pyproject.toml's Operating System classifiers name; all of + them when it names none. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + classifiers = ( + TOMLLIB.loads(path.read_text(encoding="utf-8")) + .get("project", {}) + .get("classifiers", []) + ) + named = [ + classifier + for classifier in classifiers + if classifier.startswith("Operating System ::") + ] + if not named: + return list(PLATFORMS) + found = { + platform + for classifier in named + for prefix, platforms in OS_CLASSIFIERS.items() + if classifier == prefix or classifier.startswith(prefix + " ::") + for platform in platforms + } + if not found: + raise DeclarationError( + "pyproject.toml names no Operating System CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return [platform for platform in PLATFORMS if platform in found] + + +# Build system -> where it declares Python versions, given the file that +# selected it. +READERS: dict[str, Callable[[Path], list[Python]]] = { + "rattler-build": lambda recipe: from_conda_variants( + recipe.parent / "variants.yaml" + ), + "conda-build": lambda recipe: from_conda_variants( + recipe.parent / "conda_build_config.yaml" + ), + "wheel": from_pyproject, +} + +# Build system -> where it declares the platforms it builds on, given the file +# that selected it. +PLATFORM_READERS: dict[str, Callable[[Path], list[str]]] = { + "rattler-build": from_rattler_recipe, + "conda-build": from_conda_recipe, + "wheel": from_os_classifiers, +} + + +def detect(root: Path, system: str = "") -> tuple[str, Path]: + """The build system and the file that selects it.""" + for name, candidates in SYSTEMS.items(): + if system and name != system: + continue + for candidate in candidates: + if (root / candidate).is_file(): + return name, root / candidate + wanted = system or " or ".join(SYSTEMS) + raise DeclarationError( + f"no {wanted} input found; looked for " + + ", ".join(c for name, cs in SYSTEMS.items() for c in cs) + ) + + +def declared(system: str, selector: Path) -> list[Python]: + """The build system's Python versions, sorted, each listed once.""" + pythons = READERS[system](selector) + versions = [python.version for python in pythons] + if not pythons: + raise DeclarationError(f"{system} declares no Python versions") + if len(set(versions)) != len(versions): + raise DeclarationError( + f"{system} declares a Python version twice: {versions}" + ) + return sorted(pythons, key=lambda python: version_key(python.version)) + + +def declared_platforms(system: str, selector: Path) -> list[str]: + """The platforms the build system builds on, in PLATFORMS order.""" + return PLATFORM_READERS[system](selector) + + +def for_event(pythons: list[Python], event: str) -> list[Python]: + """ + All of them, or for a pull request the oldest and newest regular ones and + the newest free-threaded one. + """ + if event not in PULL_REQUEST_EVENTS: + return pythons + regular = [python for python in pythons if not python.version.endswith("t")] + threaded = [python for python in pythons if python.version.endswith("t")] + regular = regular or pythons + return list(dict.fromkeys(regular[:1] + regular[-1:] + threaded[-1:])) + + +def build_matrix(system: str, selector: Path, event: str) -> dict: + """ + Every declared platform Actions can build on, with every Python version the + event builds. + """ + pythons = for_event(declared(system, selector), event) + platforms = declared_platforms(system, selector) + waiting = [ + PLATFORMS[platform] + for platform in platforms + if platform not in BUILT_PLATFORMS + ] + if waiting: + print( + f'matrix: {", ".join(waiting)} declared but not built: ' + "no runners for it yet", + file=sys.stderr, + ) + built = [platform for platform in platforms if platform in BUILT_PLATFORMS] + if not built: + raise DeclarationError( + "the build files declare only platforms there are no runners " + "for yet: " + ", ".join(waiting) + ) + return { + "include": [ + { + "platform": platform, + "name": PLATFORMS[platform], + "python": python.version, + } + for platform in built + for python in pythons + ] + } + + +def main(argv: list[str] | None = None) -> int: + """Prints the matrix to build; writes it for GitHub Actions if asked.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--system", + default="", + choices=("", *SYSTEMS), + help="build system to read instead of the detected one", + ) + parser.add_argument( + "--event", default="", help="GitHub event name, such as pull_request" + ) + parser.add_argument( + "--github-output", + default="", + metavar="NAME", + help="also write NAME= to $GITHUB_OUTPUT", + ) + args = parser.parse_args(argv) + if args.github_output: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", args.github_output): + raise DeclarationError( + f"{args.github_output!r} is not an output name" + ) + if not os.environ.get("GITHUB_OUTPUT"): + raise DeclarationError( + "--github-output needs $GITHUB_OUTPUT, which Actions sets" + ) + system, selector = detect(ROOT, args.system) + matrix = build_matrix(system, selector, args.event) + cells = ", ".join( + f'{cell["name"]} {cell["python"]}' for cell in matrix["include"] + ) + print( + f"matrix: {system} from {selector.relative_to(ROOT)}: {cells}", + file=sys.stderr, + ) + text = json.dumps(matrix) + print(text) + if args.github_output: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"{args.github_output}={text}\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except DeclarationError as error: + print(f"matrix: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/test.py b/ci/test.py new file mode 100644 index 0000000000..6935de2354 --- /dev/null +++ b/ci/test.py @@ -0,0 +1,255 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests dpctl's installed package. + +Runs with one built package installed in a clean environment, as ci/hooks.py +sets up, from an empty working directory, so the tests import the installed +dpctl rather than the source tree. The test requirements come from +ci/config.yml. Writes a JUnit report to $GH_CI_RESULTS. + +The environment is scrubbed, so this sets what dpctl needs itself. It prints the +SYCL devices and fails without any, so a run that skipped every device test +cannot pass. With GH_CI_DEVICE=gpu a GPU device is required too. Without it the +OpenCL CPU runtime from the test environment is the device, as on GitHub's +runners. +""" + +import json +import os +import pathlib +import subprocess +import sys +import sysconfig +import tempfile + +PACKAGE = "dpctl" + +CPU_ICD = {"win32": "intelocl64.dll", "linux": "libintelocl.so"} + +DEVICES = """ +import json +import dpctl +print(json.dumps([ + [d.backend.name, d.device_type.name, d.name] for d in dpctl.get_devices() +])) +""" + +PROBE_SETUP = """ +import setuptools + +setuptools.setup( + name="probe", + ext_modules=[ + setuptools.Extension("probe_c", ["probe_c.c"], language="c"), + setuptools.Extension("probe_cpp", ["probe_cpp.cpp"], language="c++"), + ], +) +""" + +PROBE_SOURCE = """ +#include +static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, "NAME", 0, -1, 0}; +PyMODINIT_FUNC PyInit_NAME(void) { return PyModule_Create(&module); } +""" + +# These two tests build extensions with the Python environment's compiler. +NEED_COMPILER = ["test_c_headers", "test_cython_api"] + + +def running_python() -> str: + """This interpreter's version as artifact names spell it, such as 3.14t.""" + version = f"{sys.version_info.major}.{sys.version_info.minor}" + threaded = sysconfig.get_config_var("Py_GIL_DISABLED") + return version + ("t" if threaded else "") + + +def has_compilers(env: dict) -> bool: + """Whether setuptools can build a C and a C++ extension.""" + with tempfile.TemporaryDirectory() as scratch: + directory = pathlib.Path(scratch) + (directory / "setup.py").write_text(PROBE_SETUP, encoding="utf-8") + for language, suffix in (("c", "c"), ("c++", "cpp")): + source = PROBE_SOURCE.replace("NAME", f"probe_{suffix}") + (directory / f"probe_{suffix}.{suffix}").write_text( + source, encoding="utf-8" + ) + built = subprocess.run( + [sys.executable, "setup.py", "build_ext", "--inplace"], + cwd=directory, + env=env, + capture_output=True, + text=True, + check=False, + ) + if built.returncode: + print( + f"no usable compiler:\n{built.stdout[-1500:]}{built.stderr[-1500:]}" + ) + return not built.returncode + + +def compilers_installed() -> bool: + """Whether the environment has conda-forge's compiler packages.""" + meta = pathlib.Path(sys.prefix, "conda-meta") + return any(meta.glob("c-compiler-*.json")) or any( + meta.glob("cxx-compiler-*.json") + ) + + +def skipped_tests(compilers: bool) -> list: + """The tests this environment cannot run, said out loud.""" + skipped = [] + if os.environ.get("GH_CI_KIND") == "wheel": + print("deselecting test_syclinterface: a wheel keeps no symlinks") + skipped.append("test_syclinterface") + if not compilers: + print(f"deselecting {', '.join(NEED_COMPILER)}: they need a compiler") + skipped += NEED_COMPILER + return skipped + + +def cpu_runtime_icd() -> str | None: + """The OpenCL CPU runtime of the test environment, if it has one.""" + name = CPU_ICD.get(sys.platform, CPU_ICD["linux"]) + found = next(pathlib.Path(sys.prefix).rglob(name), None) + return str(found) if found else None + + +def merge_icd_vendors(env: dict, merged: pathlib.Path) -> None: + """Lets the environment's OpenCL loader see the system's GPU drivers too.""" + system = pathlib.Path("/etc/OpenCL/vendors") + own = pathlib.Path(sys.prefix, "etc", "OpenCL", "vendors") + if not (system.is_dir() and own.is_dir()): + return + for icd in [*own.glob("*.icd"), *system.glob("*.icd")]: + if not (merged / icd.name).exists(): + (merged / icd.name).symlink_to(icd) + env["OCL_ICD_VENDORS"] = str(merged) + + +def register_icd(icd: str) -> None: + """Registers the ICD in the registry, which Windows' loader reads.""" + import winreg + + try: + with winreg.CreateKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Khronos\OpenCL\Vendors", + 0, + winreg.KEY_SET_VALUE, + ) as vendors: + winreg.SetValueEx(vendors, icd, 0, winreg.REG_DWORD, 0) + except OSError as error: + print(f"could not register {icd}: {error}") + + +def sycl_env(gpu: bool, merged: pathlib.Path) -> dict: + """The environment dpctl runs in.""" + env = dict(os.environ, SYCL_CACHE_PERSISTENT="1") + if sys.platform == "win32": + # setuptools finds Visual C++ from these, and the scrubbed environment + # lacks them. + drive = env.get("SYSTEMDRIVE", "C:") + env.setdefault("PROGRAMFILES", drive + "\\Program Files") + env.setdefault("PROGRAMFILES(X86)", drive + "\\Program Files (x86)") + if gpu: + if sys.platform.startswith("linux"): + merge_icd_vendors(env, merged) + return env + # Linux and newer Windows loaders take the ICD from here. + icd = cpu_runtime_icd() + if icd: + env["OCL_ICD_FILENAMES"] = icd + if sys.platform == "win32": + register_icd(icd) + return env + + +def devices(env: dict) -> list: + """The [backend, type, name] of every SYCL device, in a fresh process.""" + listed = subprocess.run( + [sys.executable, "-c", DEVICES], + env=env, + capture_output=True, + text=True, + check=False, + ) + if listed.returncode: + print(listed.stderr) + return [] + return json.loads(listed.stdout) + + +def devices_ok(found: list, gpu: bool) -> bool: + """Prints the devices, and whether the ones the tests need are there.""" + for backend, kind, name in found: + print(f"SYCL device: {backend}:{kind} {name}") + if not found: + print("no SYCL device, so the tests would skip everything") + return False + if gpu and not any(kind == "gpu" for _, kind, _ in found): + print("GH_CI_DEVICE=gpu, but there is no GPU device") + return False + return True + + +def main() -> int: + """Runs the installed package's test suite.""" + sys.stdout.reconfigure(line_buffering=True) + expected = os.environ["GH_CI_PYTHON"] + if running_python() != expected: + print(f"running Python {running_python()}, expected {expected}") + return 1 + gpu = os.environ.get("GH_CI_DEVICE") == "gpu" + with tempfile.TemporaryDirectory() as merged: + env = sycl_env(gpu, pathlib.Path(merged)) + listed = subprocess.run( + [sys.executable, "-m", "dpctl", "-f"], env=env, check=False + ) + if listed.returncode: + print(f"python -m dpctl -f failed with exit {listed.returncode}") + return listed.returncode + if not devices_ok(devices(env), gpu): + return 1 + compilers = has_compilers(env) + if not compilers and compilers_installed(): + print("the environment's compilers cannot build an extension") + return 1 + # The suite name says which platform and Python ran. + cell = f"{os.environ.get('GH_CI_PLATFORM', 'linux')}.py{expected}" + report = pathlib.Path(os.environ["GH_CI_RESULTS"]) / f"{PACKAGE}.xml" + pytest = [ + sys.executable, + "-m", + "pytest", + "-v", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + "-o", + f"junit_suite_name={PACKAGE} {cell}", + "--pyargs", + PACKAGE, + ] + if skipped := skipped_tests(compilers): + pytest += ["-k", "not " + " and not ".join(skipped)] + return subprocess.run(pytest, env=env, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conda-recipe-cf/bld.bat b/conda-recipe-cf/bld.bat deleted file mode 100644 index 1c9e08701d..0000000000 --- a/conda-recipe-cf/bld.bat +++ /dev/null @@ -1,46 +0,0 @@ -REM A workaround for activate-dpcpp.bat issue to be addressed in 2021.4 -set "LIB=%BUILD_PREFIX%\Library\lib;%BUILD_PREFIX%\compiler\lib;%LIB%" -set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%" - -"%PYTHON%" setup.py clean --all - -REM useful for building in resources constrained VMs (public CI) -set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE" - -REM Try to detect SYCL resource/include dir from icpx -for /f "usebackq delims=" %%R in (`icpx --print-resource-dir 2^>NUL`) do ( - set "SYCL_RESOURCE_DIR=%%R" -) - -if defined SYCL_RESOURCE_DIR ( - if exist "%SYCL_RESOURCE_DIR%\include" ( - set "SYCL_INCLUDE_DIR_HINT=%SYCL_RESOURCE_DIR%" - ) -) - -set "CC=icx" -set "CXX=icx" - -set "CMAKE_GENERATOR=Ninja" -:: Make CMake verbose -set "VERBOSE=1" - -:: set CMAKE to use less threads to avoid OOM -set CMAKE_BUILD_PARALLEL_LEVEL=%CPU_COUNT% - -set "CMAKE_ARGS=%CMAKE_ARGS% -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=%PREFIX:\=/%/Library/include/level_zero" - -%PYTHON% -m build -w -n -x -if %ERRORLEVEL% neq 0 exit 1 - -:: wheel file was renamed -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m pip install %%f ^ - --no-build-isolation ^ - --no-deps ^ - --only-binary :all: ^ - --no-index ^ - --prefix %PREFIX% ^ - -vv - if errorlevel 1 exit 1 -) diff --git a/conda-recipe-cf/build.sh b/conda-recipe-cf/build.sh deleted file mode 100755 index 983ba5d76c..0000000000 --- a/conda-recipe-cf/build.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# This is necessary to help DPC++ find Intel libraries such as SVML, IRNG, etc in build prefix -export LIBRARY_PATH="$LIBRARY_PATH:${BUILD_PREFIX}/lib" - -# Intel LLVM must cooperate with compiler and sysroot from conda -echo "--gcc-toolchain=${BUILD_PREFIX} --sysroot=${BUILD_PREFIX}/${HOST}/sysroot -target ${HOST}" > icpx_for_conda.cfg -ICPXCFG="$(pwd)/icpx_for_conda.cfg" -export ICPXCFG -ICXCFG="$(pwd)/icpx_for_conda.cfg" -export ICXCFG - -if [ -e "_skbuild" ]; then - ${PYTHON} setup.py clean --all -fi - -export CC=icx -export CXX=icpx - -export CMAKE_GENERATOR=Ninja -# Make CMake verbose -export VERBOSE=1 - -# set CMAKE to use less threads to avoid OOM -export CMAKE_BUILD_PARALLEL_LEVEL=${CPU_COUNT} - -CMAKE_ARGS="${CMAKE_ARGS} -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=${PREFIX}/include/level_zero -DDPCTL_WITH_REDIST=ON" - -# -wnx flags mean: --wheel --no-isolation --skip-dependency-check -${PYTHON} -m build -w -n -x - -${PYTHON} -m pip install dist/dpctl*.whl \ - --no-build-isolation \ - --no-deps \ - --only-binary :all: \ - --no-index \ - --prefix "${PREFIX}" \ - -vv diff --git a/conda-recipe-cf/conda_build_config.yaml b/conda-recipe-cf/conda_build_config.yaml deleted file mode 100644 index 408d05b438..0000000000 --- a/conda-recipe-cf/conda_build_config.yaml +++ /dev/null @@ -1,18 +0,0 @@ -numpy: - - '2' -c_compiler: # [linux] - - gcc # [linux] -cxx_compiler: # [linux] - - gxx # [linux] -cxx_compiler_version: # [linux] - - '15' # [linux] -c_stdlib: # [linux] - - sysroot # [linux] -c_stdlib_version: # [linux] - - '2.28' # [linux] -c_stdlib: # [win] - - vs # [win] -cxx_compiler: # [win] - - vs2022 # [win] -c_compiler: # [win] - - vs2022 # [win] diff --git a/conda-recipe-cf/meta.yaml b/conda-recipe-cf/meta.yaml deleted file mode 100644 index 649d053129..0000000000 --- a/conda-recipe-cf/meta.yaml +++ /dev/null @@ -1,74 +0,0 @@ -{% set required_compiler_version = "2026.0.0" %} -package: - name: dpctl - version: {{ GIT_DESCRIBE_TAG }} - -source: - path: .. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports: - - level-zero - missing_dso_whitelist: - - $RPATH/DPCTLSyclInterface.dll # [win] - skip: true # [osx or py<39] - -requirements: - # TODO: keep in sync with /pyproject.toml - build: - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - {{ compiler('dpcpp') }} >={{ required_compiler_version }} - - ninja >=1.11.1 # [linux] - - cmake >=3.29.0 - host: - - python - - pip >=24.0 - - level-zero-devel >=1.16 - - pybind11 >=3.0.2 - - intel-sycl-rt - - intel-cmplr-lib-rt - # Ensure we are using latest version of setuptools, since we don't need - # editable environments for release. - - setuptools >=69 - - wheel >=0.43 - - python-build >=1.1 - - scikit-build >=0.17.0 - - cython >=3.1.0 # [py>=313] - - cython >=3.0.10 # [py<313] - - numpy - # WARNING: check with doc how to upgrade - - versioneer ==0.29 - # versioneer dependency - - tomli # [py<311] - run: - - python - - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} - - numpy - -test: - requires: - - pip - - pytest - - cython - - setuptools - commands: - - pip check - - python -c "import dpctl; print(dpctl.__version__)" - - python -m dpctl -f - -about: - home: https://github.com/IntelPython/dpctl.git - license: Apache-2.0 - license_file: LICENSE - summary: 'A lightweight Python wrapper for a subset of SYCL API.' - description: | - Data Parallel Control or dpctl is a Python library that allows users to control the execution placement of a compute kernel on an XPU. - -extra: - recipe-maintainers: - - ndgrigorian - - antonwolfy - - vlad-perevezentsev diff --git a/conda-recipe-cf/run_test.bat b/conda-recipe-cf/run_test.bat deleted file mode 100644 index 42816416d3..0000000000 --- a/conda-recipe-cf/run_test.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo on - -"%PYTHON%" -c "import dpctl; print(dpctl.__version__)" -if %ERRORLEVEL% neq 0 exit 1 - -"%PYTHON%" -m dpctl -f -if %ERRORLEVEL% neq 0 exit 1 - -python -m pytest -q -ra --disable-warnings --pyargs dpctl -vv -if %ERRORLEVEL% neq 0 exit 1 diff --git a/conda-recipe-cf/run_test.sh b/conda-recipe-cf/run_test.sh deleted file mode 100755 index de137e0caa..0000000000 --- a/conda-recipe-cf/run_test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -${PYTHON} -c "import dpctl; print(dpctl.__version__)" -${PYTHON} -m dpctl -f -${PYTHON} -m pytest -q -ra --disable-warnings --pyargs dpctl -vv diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index 2ff9e6cdcc..c697264145 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -4,15 +4,17 @@ set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%" "%PYTHON%" setup.py clean --all -REM Overriding IPO is useful for building in resources constrained VMs (public CI) -if DEFINED OVERRIDE_INTEL_IPO ( - set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE" +REM useful for building in resources constrained VMs (public CI) +set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE" + +REM Try to detect SYCL resource/include dir from icpx +for /f "usebackq delims=" %%R in (`icpx --print-resource-dir 2^>NUL`) do ( + set "SYCL_RESOURCE_DIR=%%R" ) -FOR %%V IN (17.0.0 17 18.0.0 18 19.0.0 19 20.0.0 20 21.0.0 21) DO @( - REM set DIR_HINT if directory exists - IF EXIST "%BUILD_PREFIX%\Library\lib\clang\%%V\" ( - SET "SYCL_INCLUDE_DIR_HINT=%BUILD_PREFIX%\Library\lib\clang\%%V" +if defined SYCL_RESOURCE_DIR ( + if exist "%SYCL_RESOURCE_DIR%\include" ( + set "SYCL_INCLUDE_DIR_HINT=%SYCL_RESOURCE_DIR%" ) ) @@ -23,18 +25,14 @@ set "CMAKE_GENERATOR=Ninja" :: Make CMake verbose set "VERBOSE=1" +:: one build job per CPU conda-build counts; set CPU_COUNT lower to save memory +set CMAKE_BUILD_PARALLEL_LEVEL=%CPU_COUNT% + set "CMAKE_ARGS=%CMAKE_ARGS% -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=%PREFIX:\=/%/Library/include/level_zero" %PYTHON% -m build -w -n -x if %ERRORLEVEL% neq 0 exit 1 -:: `pip install dist\numpy*.whl` does not work on windows, -:: so use a loop; there's only one wheel in dist/ anyway -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m wheel tags --remove --build %GIT_DESCRIBE_NUMBER% %%f - if %ERRORLEVEL% neq 0 exit 1 -) - :: wheel file was renamed for /f %%f in ('dir /b /S .\dist') do ( %PYTHON% -m pip install %%f ^ @@ -44,11 +42,5 @@ for /f %%f in ('dir /b /S .\dist') do ( --no-index ^ --prefix %PREFIX% ^ -vv - if %ERRORLEVEL% neq 0 exit 1 -) - -:: Copy wheel package -if NOT "%WHEELS_OUTPUT_FOLDER%"=="" ( - copy dist\dpctl*.whl %WHEELS_OUTPUT_FOLDER% - if %ERRORLEVEL% neq 0 exit 1 + if errorlevel 1 exit 1 ) diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index e892602ede..2178a963f3 100755 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -10,9 +10,6 @@ export ICPXCFG ICXCFG="$(pwd)/icpx_for_conda.cfg" export ICXCFG -read -r GLIBC_MAJOR GLIBC_MINOR <<<"$(conda list '^sysroot_linux-64$' \ - | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 2 | tr '\n' ' ')" - if [ -e "_skbuild" ]; then ${PYTHON} setup.py clean --all fi @@ -24,15 +21,14 @@ export CMAKE_GENERATOR=Ninja # Make CMake verbose export VERBOSE=1 +# one build job per CPU conda-build counts; set CPU_COUNT lower to save memory +export CMAKE_BUILD_PARALLEL_LEVEL=${CPU_COUNT} + CMAKE_ARGS="${CMAKE_ARGS} -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=${PREFIX}/include/level_zero -DDPCTL_WITH_REDIST=ON" # -wnx flags mean: --wheel --no-isolation --skip-dependency-check ${PYTHON} -m build -w -n -x -${PYTHON} -m wheel tags --remove --build "$GIT_DESCRIBE_NUMBER" \ - --platform-tag "manylinux_${GLIBC_MAJOR}_${GLIBC_MINOR}_x86_64" \ - dist/dpctl*.whl - ${PYTHON} -m pip install dist/dpctl*.whl \ --no-build-isolation \ --no-deps \ @@ -40,8 +36,3 @@ ${PYTHON} -m pip install dist/dpctl*.whl \ --no-index \ --prefix "${PREFIX}" \ -vv - -# Copy wheel package -if [[ -d "${WHEELS_OUTPUT_FOLDER}" ]]; then - cp dist/dpctl*.whl "${WHEELS_OUTPUT_FOLDER[@]}" -fi diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index c5df1a7f2e..e15e9c816e 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,11 +1,20 @@ +# Python versions for ci/matrix.py, one per line, without selectors +python: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14.* *_cp314" + - "3.14.* *_cp314t" + - "3.15.* *_cp315" numpy: - - '1.26' + - '2' c_compiler: # [linux] - gcc # [linux] cxx_compiler: # [linux] - gxx # [linux] cxx_compiler_version: # [linux] - - '14' # [linux] + - '15' # [linux] c_stdlib: # [linux] - sysroot # [linux] c_stdlib_version: # [linux] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 83a2818ed0..10310faeb0 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,87 +1,78 @@ {% set required_compiler_version = "2026.0.0" %} - -{% set pyproject = load_file_data('pyproject.toml') %} -{% set py_build_deps = pyproject.get('build-system', {}).get('requires', []) %} - package: - name: dpctl - version: {{ GIT_DESCRIBE_TAG }} + name: dpctl + version: {{ GIT_DESCRIBE_TAG }} source: - path: .. + path: .. build: - number: {{ GIT_DESCRIBE_NUMBER }} - script_env: - - WHEELS_OUTPUT_FOLDER - - OVERRIDE_INTEL_IPO # [win] - ignore_run_exports: - - level-zero - missing_dso_whitelist: - - $RPATH/DPCTLSyclInterface.dll # [win] + number: {{ GIT_DESCRIBE_NUMBER }} + ignore_run_exports: + - level-zero + missing_dso_whitelist: + - $RPATH/DPCTLSyclInterface.dll # [win] + skip: true # [osx] requirements: - # TODO: keep in sync with /pyproject.toml - build: - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - {{ compiler('dpcpp') }} >={{ required_compiler_version }} - host: - - python - - pip >=24.0 - - level-zero-devel >=1.16 - - pybind11 >=2.13 - - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} - # Ensure we are using latest version of setuptools, since we don't need - # editable environments for release. - - setuptools >=69 - {% for dep in py_build_deps %} - {% if dep.startswith('ninja') %} - - {{ dep.split(';')[0] }} # [not win] - {% elif dep.startswith('cmake') %} - - {{ dep }} - {% elif dep.startswith('build>=') %} - - {{ 'python-' ~ dep }} - {% else %} - - {{ dep|replace('_','-') }} - {% endif %} - {% endfor %} - # versioneer dependency - - tomli # [py<311] - - opencl-headers >=2025.06.13 # [win] - run: - - python - - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} - - numpy + # TODO: keep in sync with /pyproject.toml + build: + - {{ compiler('cxx') }} + - {{ stdlib('c') }} + - {{ compiler('dpcpp') }} >={{ required_compiler_version }} + - ninja >=1.11.1 # [linux] + - cmake >=3.29.0 + host: + - python + - pip >=24.0 + - level-zero-devel >=1.16 + - pybind11 >=3.0.2 + - intel-sycl-rt + - intel-cmplr-lib-rt + # Ensure we are using latest version of setuptools, since we don't need + # editable environments for release. + - setuptools >=69 + - wheel >=0.43 + - python-build >=1.1 + - scikit-build >=0.17.0,<1.0 + - cython >=3.1.0 # [py>=313] + - cython >=3.0.10 # [py<313] + - numpy + # WARNING: check with doc how to upgrade + - versioneer ==0.29 + # versioneer dependency + - tomli # [py<311] + run: + - python + - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} + - numpy test: - requires: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - cython - - setuptools - - pytest + requires: + - pip + commands: + - pip check + - python -c "import dpctl; print(dpctl.__version__)" + - python -m dpctl -f about: - home: https://github.com/IntelPython/dpctl.git - license: Apache-2.0 - license_file: LICENSE - summary: 'A lightweight Python wrapper for a subset of SYCL API.' - description: | - LEGAL NOTICE: Use of this software package is subject to the - software license agreement (as set forth above, in the license section of - the installed Conda package and/or the README file) and all notices, - disclaimers or license terms for third party or open source software - included in or with the software. -

- EULA: Apache-2.0 -

+ home: https://github.com/IntelPython/dpctl.git + license: Apache-2.0 + license_file: LICENSE + summary: 'A lightweight Python wrapper for a subset of SYCL API.' + description: | + LEGAL NOTICE: Use of this software package is subject to the + software license agreement (as set forth above, in the license section of + the installed Conda package and/or the README file) and all notices, + disclaimers or license terms for third party or open source software + included in or with the software. +

+ EULA: Apache-2.0 +

extra: - recipe-maintainers: - - ndgrigorian - - antonwolfy - - vlad-perevezentsev + recipe-maintainers: + - ndgrigorian + - antonwolfy + - vlad-perevezentsev diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat deleted file mode 100644 index 42816416d3..0000000000 --- a/conda-recipe/run_test.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo on - -"%PYTHON%" -c "import dpctl; print(dpctl.__version__)" -if %ERRORLEVEL% neq 0 exit 1 - -"%PYTHON%" -m dpctl -f -if %ERRORLEVEL% neq 0 exit 1 - -python -m pytest -q -ra --disable-warnings --pyargs dpctl -vv -if %ERRORLEVEL% neq 0 exit 1 diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh deleted file mode 100755 index de137e0caa..0000000000 --- a/conda-recipe/run_test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -${PYTHON} -c "import dpctl; print(dpctl.__version__)" -${PYTHON} -m dpctl -f -${PYTHON} -m pytest -q -ra --disable-warnings --pyargs dpctl -vv diff --git a/docs/_legacy/docfiles/user_guides/QuickStart.rst b/docs/_legacy/docfiles/user_guides/QuickStart.rst index 382528cc98..0daf02d77e 100644 --- a/docs/_legacy/docfiles/user_guides/QuickStart.rst +++ b/docs/_legacy/docfiles/user_guides/QuickStart.rst @@ -105,20 +105,18 @@ Build and Install Using Conda-Build You can use the conda-recipe included with the sources to build the dpctl package. The advantage of this approach is that all dependencies are pulled in -from oneAPI's intelpython conda channel that is installed as a part of oneAPI. +from the conda-forge channel. .. code-block:: bash - export ONEAPI_ROOT=/opt/intel/oneapi - conda build conda-recipe -c ${ONEAPI_ROOT}/conda_channel + conda build conda-recipe -c conda-forge --override-channels On Windows OS to cope with `long file names `_, use ``croot`` with a short folder path: .. code-block:: bat - set "ONEAPI_ROOT=C:\Program Files (x86)\Intel\oneAPI\" - conda build --croot=C:/tmp conda-recipe -c "%ONEAPI_ROOT%\conda_channel" + conda build --croot=C:/tmp conda-recipe -c conda-forge --override-channels After building the Conda package, install it by executing: diff --git a/docs/doc_sources/contributor_guides/building.rst b/docs/doc_sources/contributor_guides/building.rst index e492a42b4f..4dbece9aee 100644 --- a/docs/doc_sources/contributor_guides/building.rst +++ b/docs/doc_sources/contributor_guides/building.rst @@ -61,21 +61,19 @@ Build and Install Using Conda-Build ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can use the conda-recipe included with the sources to build the dpctl -package. The advantage of this approach is that all oneAPI library dependencies are -pulled in from oneAPI's local conda channel that is installed as a part of oneAPI. +package. The advantage of this approach is that all dependencies are pulled in +from the conda-forge channel. .. code-block:: bash - export ONEAPI_ROOT=/opt/intel/oneapi - conda build conda-recipe -c ${ONEAPI_ROOT}/conda_channel + conda build conda-recipe -c conda-forge --override-channels On Windows OS to cope with `long file names `_, use ``croot`` with a short folder path: .. code-block:: bat - set "ONEAPI_ROOT=C:\Program Files (x86)\Intel\oneAPI\" - conda build --croot=C:/tmp conda-recipe -c "%ONEAPI_ROOT%\conda_channel" + conda build --croot=C:/tmp conda-recipe -c conda-forge --override-channels After building the Conda package, install it by executing: diff --git a/pyproject.toml b/pyproject.toml index 833952dcce..058f08e8ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "wheel>=0.43", "build>=1.1", "setuptools>=63.0.0", - "scikit-build>=0.17.0", + "scikit-build>=0.17.0,<1.0", "ninja>=1.11.1; platform_system!='Windows'", "cmake>=3.29.0", "cython>=3.1.0", @@ -89,6 +89,34 @@ Repository = "https://github.com/IntelPython/dpctl.git" exclude = "dpctl/_version.py" line-length = 80 +[tool.cibuildwheel] +build-verbosity = 1 + +[tool.cibuildwheel.linux] +# Intel's icx, as the conda recipe builds with, from Intel's repository, and the Level Zero headers +before-all = [ + "rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB", + "printf '[oneAPI]\\nname=Intel oneAPI\\nbaseurl=https://yum.repos.intel.com/oneapi\\nenabled=1\\ngpgcheck=1\\nrepo_gpgcheck=1\\ngpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\\n' > /etc/yum.repos.d/oneAPI.repo", + "dnf install -y intel-oneapi-compiler-dpcpp-cpp-2026.2", + "git clone --depth 1 --branch v1.34.0 https://github.com/oneapi-src/level-zero /opt/level-zero" +] +# LDFLAGS gives libDPCTLSyclInterface the full RELRO the extensions link with +environment = { CC = "/opt/intel/oneapi/compiler/latest/bin/icx", CXX = "/opt/intel/oneapi/compiler/latest/bin/icpx", CMAKE_GENERATOR = "Ninja", CMAKE_ARGS = "-DDPCTL_LEVEL_ZERO_INCLUDE_DIR=/opt/level-zero/include -DDPCTL_WITH_REDIST=ON", LDFLAGS = "-Wl,-z,relro -Wl,-z,now" } +# The SYCL and icx runtime libraries come from wheels at run time, see [tool.gh-ci]; +# libsycl matches any soname, as wheel-requires allows any 2026 runtime +repair-wheel-command = "auditwheel repair --exclude 'libsycl.so.*' --exclude libintlc.so.5 --exclude libimf.so --exclude libsvml.so --exclude libirng.so -w {dest_dir} {wheel}" + +[tool.cibuildwheel.windows] +# icx from conda-forge, and the Level Zero headers +before-all = [ + "python {project}/ci/install_icx.py C:/icx", + "git clone --depth 1 --branch v1.34.0 https://github.com/oneapi-src/level-zero C:/level-zero" +] +environment = { CC = "C:/icx/Library/bin/icx.exe", CXX = "C:/icx/Library/bin/icx.exe", CMAKE_GENERATOR = "Ninja", CMAKE_ARGS = "-DDPCTL_LEVEL_ZERO_INCLUDE_DIR=C:/level-zero/include -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE", PATH = "C:/icx/Library/bin;C:/icx/Library/bin/compiler;$PATH", LIB = "C:/icx/Library/lib;$LIB", LINK = "/LIBPATH:C:/icx/Library/lib" } +# The SYCL and icx runtime libraries come from wheels at run time, see [tool.gh-ci]; +# sycl matches any version, as wheel-requires allows any 2026 runtime +repair-wheel-command = "delvewheel repair --no-dll DPCTLSyclInterface.dll;sycl*.dll;irc_msg.dll;libmmd.dll;svml_dispmd.dll;libircmd.dll;libirngmd.dll;ze_loader.dll;OpenCL.dll -w {dest_dir} {wheel}" + [tool.codespell] builtin = "clear,rare,informal,names" check-filenames = true @@ -119,6 +147,10 @@ source = [ [tool.cython-lint] max-line-length = 80 +[tool.gh-ci] +# What the wheels need at run time, which conda packages get from the recipe +wheel-requires = ["intel-sycl-rt >=2026.2,<2027", "intel-cmplr-lib-rt >=2026.2,<2027"] + [tool.isort] ensure_newline_before_comments = true force_grid_wrap = 0