From 377bd6032b9203404d720fc39153d14a1b7b595b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 30 Oct 2026 22:26:25 -0700 Subject: [PATCH 1/4] ci: use same-run CUDA 12 bindings for Core and metapackages Signed-off-by: Ralf W. Grosse-Kunstleve --- .github/workflows/build-wheel.yml | 116 ++++++++++++++--------- .github/workflows/ci.yml | 115 ++++------------------ .github/workflows/test-sdist-linux.yml | 33 ++++--- .github/workflows/test-sdist-windows.yml | 33 ++++--- .github/workflows/test-wheel-linux.yml | 41 ++++---- .github/workflows/test-wheel-windows.yml | 41 ++++---- ci/tools/env-vars | 12 ++- ci/tools/tests/test_bindings_config.py | 8 +- ci/tools/tests/test_cuda_python_setup.py | 65 +++++++++++++ cuda_core/docs/source/install.rst | 8 +- cuda_python/AGENTS.md | 18 ++-- cuda_python/docs/README.md | 6 +- cuda_python/pyproject.toml | 9 +- cuda_python/setup.py | 73 +++++++++++--- 14 files changed, 342 insertions(+), 236 deletions(-) create mode 100644 ci/tools/tests/test_cuda_python_setup.py diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 7042be13bea..5bfc3ddc732 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -46,6 +46,16 @@ jobs: BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} + BUILD_MAINTENANCE_BINDINGS: >- + ${{ !inputs.single-cuda-major && + (inputs.workplan == '' || + fromJSON(inputs.workplan).modules.bindings.needs_build || + fromJSON(inputs.workplan).modules.bindings.needs_test || + fromJSON(inputs.workplan).modules.core.needs_build || + fromJSON(inputs.workplan).modules.core.needs_test || + fromJSON(inputs.workplan).modules.python.needs_build || + fromJSON(inputs.workplan).modules.python.needs_test) }} BASELINE_RUN_ID: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.run_id || '' }} BASELINE_SHA: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.sha || '' }} strategy: @@ -88,14 +98,14 @@ jobs: run: echo "::add-matcher::.github/matchers/compiler.json" - name: Install rapidsai/sccache - if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true') }} + if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.BUILD_MAINTENANCE_BINDINGS == 'true') }} uses: ./.github/actions/install-sccache # The runner only exposes these values to actions, not to `run:` steps or # the cibuildwheel container, so re-export them for sccache's GHA backend. # xref: https://github.com/orgs/community/discussions/42856#discussioncomment-7678867 - name: Adding addtional GHA cache-related env vars - if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.BUILD_MAINTENANCE_BINDINGS == 'true' }} uses: actions/github-script@v9.0.0 with: script: | @@ -203,7 +213,7 @@ jobs: twine check --strict cuda_pathfinder/*.whl - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true' }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_MAINTENANCE_BINDINGS == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 @@ -419,6 +429,23 @@ jobs: run-id: ${{ env.BASELINE_RUN_ID }} # We only need/want a single pure python wheel, pick linux-64 index 0. + - name: Build and check maintenance cuda-python wheel + if: ${{ !inputs.single-cuda-major && (env.BUILD_PYTHON == 'true' || env.TEST_PYTHON == 'true') && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + run: | + pushd cuda_python + CUDA_PYTHON_BUILD_MAJOR="${MAINTENANCE_CUDA_MAJOR}" \ + pip wheel -v --no-deps --wheel-dir "dist/cu${MAINTENANCE_CUDA_MAJOR}" . + twine check --strict "dist/cu${MAINTENANCE_CUDA_MAJOR}"/*.whl + popd + + - name: Upload maintenance cuda-python wheel + if: ${{ !inputs.single-cuda-major && (env.BUILD_PYTHON == 'true' || env.TEST_PYTHON == 'true') && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: maintenance-python-wheel + path: cuda_python/dist/cu${{ env.MAINTENANCE_CUDA_MAJOR }}/*.whl + if-no-files-found: error + - name: Build and check cuda-python wheel if: ${{ env.BUILD_PYTHON == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} run: | @@ -533,7 +560,7 @@ jobs: # Note: This overwrites CUDA_PATH etc - name: Set up mini CTK - if: ${{ !inputs.single-cuda-major && (env.BUILD_CORE == 'true' || env.TEST_CORE == 'true' || env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true') }} + if: ${{ !inputs.single-cuda-major && (env.BUILD_CORE == 'true' || env.TEST_CORE == 'true' || env.BUILD_MAINTENANCE_BINDINGS == 'true') }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -559,7 +586,7 @@ jobs: if-no-files-found: error - name: Build maintenance cuda.bindings wheel - if: ${{ !inputs.single-cuda-major && (env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true') }} + if: ${{ env.BUILD_MAINTENANCE_BINDINGS == 'true' }} uses: pypa/cibuildwheel@b9e566a2b284570af229f6c2a33673410d0389a3 # v4.3.4 with: package-dir: ./${{ env.MAINTENANCE_BINDINGS_ROOT }}/ @@ -569,11 +596,11 @@ jobs: env: *bindings-wheel-env - name: Check maintenance cuda.bindings wheel - if: ${{ !inputs.single-cuda-major && (env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true') }} + if: ${{ env.BUILD_MAINTENANCE_BINDINGS == 'true' }} run: twine check --strict "${MAINTENANCE_BINDINGS_ARTIFACTS_DIR}"/*.whl - name: Upload maintenance cuda.bindings wheel - if: ${{ !inputs.single-cuda-major && (env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true') }} + if: ${{ env.BUILD_MAINTENANCE_BINDINGS == 'true' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ env.MAINTENANCE_BINDINGS_ARTIFACT_NAME }} @@ -599,48 +626,11 @@ jobs: path: ${{ env.MAINTENANCE_BINDINGS_CYTHON_TESTS_DIR }}/test_*${{ env.PY_EXT_SUFFIX }} if-no-files-found: error - - name: Look up prior-branch cuda.bindings run ID - id: prior-bindings - if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - if ! (command -v gh 2>&1 >/dev/null); then - # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt. - # gh is needed for lookup-run-id, which calls gh api. - mkdir -p -m 755 /etc/apt/keyrings \ - && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \ - && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ - && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ - && apt update \ - && apt install gh -y - fi - - OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) - OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]" - LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \ - --branch "${OLD_BRANCH}" \ - --artifact "${OLD_ARTIFACT_PATTERN}" \ - NVIDIA/cuda-python "CI") - echo "run-id=${LATEST_PRIOR_RUN_ID}" >> "${GITHUB_OUTPUT}" - echo "artifact-pattern=${OLD_ARTIFACT_PATTERN}" >> "${GITHUB_OUTPUT}" - - - name: Download prior-branch cuda.bindings build artifacts - if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }} - uses: $/.github/actions/download-artifact-retry - with: - pattern: ${{ steps.prior-bindings.outputs.artifact-pattern }} - merge-multiple: "true" - path: cuda_bindings_13/dist-prev - run-id: ${{ steps.prior-bindings.outputs.run-id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel + - name: Constrain maintenance cuda.core to the same-run cuda.bindings wheel if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) - bindings_wheels=(cuda_bindings_13/dist-prev/cuda_bindings-"${BUILD_PREV_CUDA_MAJOR}".*.whl) + bindings_wheels=("${MAINTENANCE_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-"${MAINTENANCE_CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" @@ -734,6 +724,40 @@ jobs: ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + - name: Build maintenance cuda.core Cython tests + if: ${{ !inputs.single-cuda-major && env.TEST_CORE == 'true' }} + run: | + # The current package's test extensions were built in this same source tree. + # Remove generated outputs so Cython and the compiler cannot reuse them. + find "${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}" -maxdepth 1 -type f \ + \( -name 'test_*.cpp' -o -name 'test_*.so' -o -name 'test_*.pyd' \) -delete + if [[ -d "${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}/build" ]]; then + find "${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}/build" -type f -delete + fi + pip uninstall -y cuda-bindings cuda-core + pip install ${{ env.MAINTENANCE_BINDINGS_ARTIFACTS_DIR }}/*.whl + if ${{ env.BUILD_CORE == 'true' }}; then + core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${MAINTENANCE_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit) + else + core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit) + fi + if [[ -z "${core_wheel}" ]]; then + echo "No cuda.core wheel found" >&2 + exit 1 + fi + pip install "${core_wheel}" --group ./cuda_core/pyproject.toml:test + pushd ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} + bash build_tests.sh + popd + + - name: Upload maintenance cuda.core Cython tests + if: ${{ !inputs.single-cuda-major && env.TEST_CORE == 'true' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-cu${{ env.MAINTENANCE_CUDA_MAJOR }}-tests + path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}/test_*${{ env.PY_EXT_SUFFIX }} + if-no-files-found: error + - name: Merge cuda.core wheels if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }} run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8934cc54c51..49a10f81ab8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -518,6 +518,10 @@ jobs: # - host-platform value # - uses: (test-sdist-linux.yml vs test-sdist-windows.yml) test-sdist-linux: + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.ci-vars.outputs.BINDINGS_CONFIG).package_roots }} needs: - ci-vars - should-skip @@ -535,11 +539,15 @@ jobs: uses: ./.github/workflows/test-sdist-linux.yml with: host-platform: linux-64 - cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + bindings-package: ${{ toJSON(matrix.package) }} workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.ci-vars.outputs.BINDINGS_CONFIG).package_roots }} needs: - ci-vars - should-skip @@ -558,7 +566,7 @@ jobs: uses: ./.github/workflows/test-sdist-windows.yml with: host-platform: win-64 - cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + bindings-package: ${{ toJSON(matrix.package) }} workplan: ${{ needs.detect-changes.outputs.workplan }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see @@ -587,9 +595,10 @@ jobs: secrets: inherit uses: ./.github/workflows/test-wheel-linux.yml with: - build-type: pull-request + build-type: ${{ startsWith(github.ref_name, 'pull-request/') && 'pull-request' || (github.event_name == 'schedule' && 'nightly' || 'branch') }} host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + bindings-config: ${{ needs.ci-vars.outputs.BINDINGS_CONFIG }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} workplan: ${{ needs.detect-changes.outputs.workplan }} @@ -616,9 +625,10 @@ jobs: secrets: inherit uses: ./.github/workflows/test-wheel-linux.yml with: - build-type: pull-request + build-type: ${{ startsWith(github.ref_name, 'pull-request/') && 'pull-request' || (github.event_name == 'schedule' && 'nightly' || 'branch') }} host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + bindings-config: ${{ needs.ci-vars.outputs.BINDINGS_CONFIG }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} workplan: ${{ needs.detect-changes.outputs.workplan }} @@ -645,90 +655,11 @@ jobs: secrets: inherit uses: ./.github/workflows/test-wheel-windows.yml with: - build-type: pull-request + build-type: ${{ startsWith(github.ref_name, 'pull-request/') && 'pull-request' || (github.event_name == 'schedule' && 'nightly' || 'branch') }} host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - workplan: ${{ needs.detect-changes.outputs.workplan }} - - test-maintenance-bindings-linux-64: - name: Maintenance bindings (linux-64) - if: ${{ github.repository_owner == 'nvidia' && - !fromJSON(needs.should-skip.outputs.skip) && - !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).modules.bindings.needs_test && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} - permissions: - actions: read - contents: read - needs: - - ci-vars - - should-skip - - detect-changes - - build-linux-64 - secrets: inherit - uses: ./.github/workflows/test-wheel-linux.yml - with: - build-type: ${{ startsWith(github.ref_name, 'pull-request/') && 'pull-request' || (github.event_name == 'schedule' && 'nightly' || 'branch') }} - host-platform: linux-64 - build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} bindings-config: ${{ needs.ci-vars.outputs.BINDINGS_CONFIG }} - maintenance-bindings: true - nruns: ${{ (github.event_name == 'schedule' && 5) || 1 }} - workplan: ${{ needs.detect-changes.outputs.workplan }} - - test-maintenance-bindings-linux-aarch64: - name: Maintenance bindings (linux-aarch64) - if: ${{ github.repository_owner == 'nvidia' && - !fromJSON(needs.should-skip.outputs.skip) && - !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).modules.bindings.needs_test && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }} - permissions: - actions: read - contents: read - needs: - - ci-vars - - should-skip - - detect-changes - - build-linux-64 - - build-linux-aarch64 - secrets: inherit - uses: ./.github/workflows/test-wheel-linux.yml - with: - build-type: ${{ startsWith(github.ref_name, 'pull-request/') && 'pull-request' || (github.event_name == 'schedule' && 'nightly' || 'branch') }} - host-platform: linux-aarch64 - build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} - bindings-config: ${{ needs.ci-vars.outputs.BINDINGS_CONFIG }} - maintenance-bindings: true - nruns: ${{ (github.event_name == 'schedule' && 5) || 1 }} - workplan: ${{ needs.detect-changes.outputs.workplan }} - - test-maintenance-bindings-windows: - name: Maintenance bindings (win-64) - if: ${{ github.repository_owner == 'nvidia' && - !fromJSON(needs.should-skip.outputs.skip) && - !fromJSON(needs.should-skip.outputs.doc-only) && - fromJSON(needs.detect-changes.outputs.workplan).modules.bindings.needs_test && - fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows }} - permissions: - actions: read - contents: read - needs: - - ci-vars - - should-skip - - detect-changes - - build-linux-64 - - build-windows - secrets: inherit - uses: ./.github/workflows/test-wheel-windows.yml - with: - build-type: ${{ startsWith(github.ref_name, 'pull-request/') && 'pull-request' || (github.event_name == 'schedule' && 'nightly' || 'branch') }} - host-platform: win-64 - build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} - bindings-config: ${{ needs.ci-vars.outputs.BINDINGS_CONFIG }} - maintenance-bindings: true - nruns: ${{ (github.event_name == 'schedule' && 5) || 1 }} + nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} workplan: ${{ needs.detect-changes.outputs.workplan }} doc: @@ -765,9 +696,6 @@ jobs: - test-linux-64 - test-linux-aarch64 - test-windows - - test-maintenance-bindings-linux-64 - - test-maintenance-bindings-linux-aarch64 - - test-maintenance-bindings-windows - api-check-core-vs-release - api-check-core-vs-base - doc @@ -795,7 +723,6 @@ jobs: windows_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows || false }}" windows_arm64_supported="${{ needs.ci-vars.outputs.WINDOWS_ARM64_SUPPORTED }}" build_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests || false }}" - maintenance_bindings_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).modules.bindings.needs_test || false }}" run_core_api_check="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks || false }}" is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" @@ -846,16 +773,6 @@ jobs: check_result "test-linux-aarch64" "$linux_expected" check_result "test-windows" "$windows_expected" - maintenance_linux_expected="skipped" - maintenance_windows_expected="skipped" - if [[ "$maintenance_bindings_selected" == "true" ]]; then - maintenance_linux_expected="$linux_expected" - maintenance_windows_expected="$windows_expected" - fi - check_result "test-maintenance-bindings-linux-64" "$maintenance_linux_expected" - check_result "test-maintenance-bindings-linux-aarch64" "$maintenance_linux_expected" - check_result "test-maintenance-bindings-windows" "$maintenance_windows_expected" - # API compatibility checks run for cuda_core source changes and for # conservative full runs when reusable base artifacts are unavailable. expected="skipped" diff --git a/.github/workflows/test-sdist-linux.yml b/.github/workflows/test-sdist-linux.yml index 542ba2ebc93..89c65fd10df 100644 --- a/.github/workflows/test-sdist-linux.yml +++ b/.github/workflows/test-sdist-linux.yml @@ -8,7 +8,8 @@ on: host-platform: required: true type: string - cuda-version: + bindings-package: + description: Normalized bindings root and toolkit for this sdist build. required: true type: string workplan: @@ -30,7 +31,9 @@ jobs: name: Test sdist builds if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} env: - BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BINDINGS_ROOT: ${{ fromJSON(inputs.bindings-package).package_root }} + CUDA_PYTHON_BUILD_MAJOR: ${{ fromJSON(inputs.bindings-package).cuda_major }} + BUILD_PATHFINDER: ${{ fromJSON(inputs.bindings-package).release_status == 'current' && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build) }} BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} @@ -60,8 +63,14 @@ jobs: with: python-version: "3.12" - - name: Install build tools - run: python -m pip install build + - name: Install build and CI tools + run: python -m pip install build -e ./ci + + - name: Prepare local maintenance SCM milestone + if: ${{ fromJSON(inputs.bindings-package).release_status == 'maintenance' }} + run: | + python -m ci.tools.bindings_scm_version \ + --package-root "${BINDINGS_ROOT}" --sha "${{ github.sha }}" # Pure Python packages -- no CTK needed. - name: Build cuda.pathfinder sdist and wheel-from-sdist @@ -111,7 +120,7 @@ jobs: continue-on-error: false with: host-platform: ${{ inputs.host-platform }} - cuda-version: ${{ inputs.cuda-version }} + cuda-version: ${{ fromJSON(inputs.bindings-package).toolkit_version }} # cuda_bindings_13/setup.py parses CUDA headers at import time, so CUDA_PATH # (set by fetch_ctk) must be available for both sdist and wheel builds. @@ -123,22 +132,22 @@ jobs: export CXX="sccache c++" export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-bindings.txt" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_bindings_13/ - pip wheel --no-deps --wheel-dir cuda_bindings_13/dist cuda_bindings_13/dist/*.tar.gz + python -m build --sdist "${BINDINGS_ROOT}/" + pip wheel --no-deps --wheel-dir "${BINDINGS_ROOT}/dist" "${BINDINGS_ROOT}"/dist/*.tar.gz - name: Download cuda.bindings wheel if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} uses: ./.github/actions/download-artifact-retry with: - name: cuda-bindings-python312-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }} - path: cuda_bindings_13/dist + name: ${{ fromJSON(inputs.bindings-package).release_status == 'maintenance' && 'maintenance-bindings' || 'cuda-bindings' }}-python312-cuda${{ fromJSON(inputs.bindings-package).toolkit_version }}-${{ inputs.host-platform }}-${{ github.sha }} + path: ${{ env.BINDINGS_ROOT }}/dist - name: Constrain cuda.core to the local cuda.bindings wheel if: ${{ env.BUILD_CORE == 'true' }} run: | - CUDA_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + CUDA_MAJOR="$(echo '${{ fromJSON(inputs.bindings-package).toolkit_version }}' | cut -d. -f1)" pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) - bindings_wheels=(cuda_bindings_13/dist/cuda_bindings-"${CUDA_MAJOR}".*.whl) + bindings_wheels=("${BINDINGS_ROOT}"/dist/cuda_bindings-"${CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" @@ -158,7 +167,7 @@ jobs: if: ${{ env.BUILD_CORE == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) - export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + export CUDA_CORE_BUILD_MAJOR="$(echo '${{ fromJSON(inputs.bindings-package).toolkit_version }}' | cut -d. -f1)" export CC="sccache cc" export CXX="sccache c++" export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt" diff --git a/.github/workflows/test-sdist-windows.yml b/.github/workflows/test-sdist-windows.yml index 0c0563126d1..7ede002a609 100644 --- a/.github/workflows/test-sdist-windows.yml +++ b/.github/workflows/test-sdist-windows.yml @@ -14,7 +14,8 @@ on: host-platform: required: true type: string - cuda-version: + bindings-package: + description: Normalized bindings root and toolkit for this sdist build. required: true type: string workplan: @@ -36,7 +37,9 @@ jobs: name: Test sdist builds if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} env: - BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BINDINGS_ROOT: ${{ fromJSON(inputs.bindings-package).package_root }} + CUDA_PYTHON_BUILD_MAJOR: ${{ fromJSON(inputs.bindings-package).cuda_major }} + BUILD_PATHFINDER: ${{ fromJSON(inputs.bindings-package).release_status == 'current' && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build) }} BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} @@ -73,8 +76,14 @@ jobs: if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - - name: Install build tools - run: python -m pip install build + - name: Install build and CI tools + run: python -m pip install build -e ./ci + + - name: Prepare local maintenance SCM milestone + if: ${{ fromJSON(inputs.bindings-package).release_status == 'maintenance' }} + run: | + python -m ci.tools.bindings_scm_version \ + --package-root "${BINDINGS_ROOT}" --sha "${{ github.sha }}" # Pure Python packages -- no CTK needed. - name: Build cuda.pathfinder sdist and wheel-from-sdist @@ -115,7 +124,7 @@ jobs: continue-on-error: false with: host-platform: ${{ inputs.host-platform }} - cuda-version: ${{ inputs.cuda-version }} + cuda-version: ${{ fromJSON(inputs.bindings-package).toolkit_version }} # cuda_bindings_13/setup.py parses CUDA headers at import time, so CUDA_PATH # (set by fetch_ctk) must be available for both sdist and wheel builds. @@ -127,22 +136,22 @@ jobs: export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-bindings.txt")" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" - python -m build --sdist cuda_bindings_13/ - pip wheel --no-deps --wheel-dir cuda_bindings_13/dist cuda_bindings_13/dist/*.tar.gz + python -m build --sdist "${BINDINGS_ROOT}/" + pip wheel --no-deps --wheel-dir "${BINDINGS_ROOT}/dist" "${BINDINGS_ROOT}"/dist/*.tar.gz - name: Download cuda.bindings wheel if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} uses: ./.github/actions/download-artifact-retry with: - name: cuda-bindings-python312-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }} - path: cuda_bindings_13/dist + name: ${{ fromJSON(inputs.bindings-package).release_status == 'maintenance' && 'maintenance-bindings' || 'cuda-bindings' }}-python312-cuda${{ fromJSON(inputs.bindings-package).toolkit_version }}-${{ inputs.host-platform }}-${{ github.sha }} + path: ${{ env.BINDINGS_ROOT }}/dist - name: Constrain cuda.core to the local cuda.bindings wheel if: ${{ env.BUILD_CORE == 'true' }} run: | - CUDA_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + CUDA_MAJOR="$(echo '${{ fromJSON(inputs.bindings-package).toolkit_version }}' | cut -d. -f1)" pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) - bindings_wheels=(cuda_bindings_13/dist/cuda_bindings-"${CUDA_MAJOR}".*.whl) + bindings_wheels=("${BINDINGS_ROOT}"/dist/cuda_bindings-"${CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" @@ -162,7 +171,7 @@ jobs: if: ${{ env.BUILD_CORE == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) - export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" + export CUDA_CORE_BUILD_MAJOR="$(echo '${{ fromJSON(inputs.bindings-package).toolkit_version }}' | cut -d. -f1)" export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" python -m build --sdist cuda_core/ diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 8287a41b8d2..85b29e1a792 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -17,13 +17,9 @@ on: type: string required: true bindings-config: - description: Normalized bindings registry for maintenance-only tests. + description: Normalized bindings registry; empty retains legacy artifact sources. type: string default: '' - maintenance-bindings: - description: Test this run's maintenance bindings wheel and Cython extensions. - type: boolean - default: false matrix_filter: type: string default: "." @@ -64,7 +60,6 @@ jobs: env: BUILD_TYPE: ${{ inputs.build-type }} BINDINGS_CONFIG: ${{ inputs.bindings-config }} - BINDINGS_ONLY: ${{ inputs.maintenance-bindings }} ARCH: ${{ (inputs.host-platform == 'linux-64' && 'amd64') || (inputs.host-platform == 'linux-aarch64' && 'arm64') }} outputs: @@ -93,11 +88,18 @@ jobs: # Read base matrix from YAML file for the specific architecture TEST_MATRIX=$(yq -o json ".linux[\"${MATRIX_TYPE}\"] | map(select(.ARCH == \"${ARCH}\"))" ci/test-matrix.yml) - if [[ "$BINDINGS_ONLY" == "true" ]]; then + if [[ -n "$BINDINGS_CONFIG" ]]; then TEST_MATRIX=$(jq -c --argjson config "$BINDINGS_CONFIG" ' - ($config.package_roots[] | select(.release_status == "maintenance")) as $package - | map(select(.CUDA_VER == $package.toolkit_version) - | . + {BINDINGS_ROOT: $package.package_root}) + ($config.package_roots | map({key: .ctk_target, value: .}) | from_entries) as $packages + | map(($packages[(.CUDA_VER | split(".")[:2] | join("."))]) as $package + | . + { + BINDINGS_ROOT: ($package.package_root // "cuda_bindings_13"), + BINDINGS_BUILD_VERSION: ($package.toolkit_version // ""), + BINDINGS_SOURCE: (if $package == null then "floor" else "main" end), + BINDINGS_ARTIFACT_PREFIX: (if $package.release_status == "maintenance" then "maintenance-bindings" else "cuda-bindings" end), + CORE_CYTHON_SUFFIX: (if $package.release_status == "maintenance" then "-cu\($package.cuda_major)-tests" else "-tests" end), + PYTHON_ARTIFACT_NAME: (if $package.release_status == "maintenance" then "maintenance-python-wheel" else "cuda-python-wheel" end) + }) ' <<< "$TEST_MATRIX") fi @@ -117,10 +119,10 @@ jobs: test: env: - TEST_PATHFINDER: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test) }} - TEST_BINDINGS: ${{ inputs.maintenance-bindings || inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} - TEST_CORE: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test) }} - TEST_PYTHON: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test) }} + TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }}${{ matrix.FLAVOR && format(', {0}', matrix.FLAVOR) || '' }}${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 needs: compute-matrix @@ -191,9 +193,12 @@ jobs: - name: Set environment variables env: - BUILD_CUDA_VER: ${{ inputs.maintenance-bindings && matrix.CUDA_VER || inputs.build-ctk-ver }} + BUILD_CUDA_VER: ${{ matrix.BINDINGS_BUILD_VERSION || inputs.build-ctk-ver }} CUDA_BINDINGS_ROOT: ${{ matrix.BINDINGS_ROOT || 'cuda_bindings_13' }} - BINDINGS_ARTIFACT_PREFIX: ${{ inputs.maintenance-bindings && 'maintenance-bindings' || 'cuda-bindings' }} + BINDINGS_SOURCE: ${{ matrix.BINDINGS_SOURCE || '' }} + BINDINGS_ARTIFACT_PREFIX: ${{ matrix.BINDINGS_ARTIFACT_PREFIX || 'cuda-bindings' }} + CORE_CYTHON_SUFFIX: ${{ matrix.CORE_CYTHON_SUFFIX || '-tests' }} + CUDA_PYTHON_ARTIFACT_NAME: ${{ matrix.PYTHON_ARTIFACT_NAME || 'cuda-python-wheel' }} CUDA_VER: ${{ matrix.CUDA_VER }} HOST_PLATFORM: ${{ inputs.host-platform }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} @@ -221,7 +226,7 @@ jobs: if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} uses: $/.github/actions/download-artifact-retry with: - name: cuda-python-wheel + name: ${{ env.CUDA_PYTHON_ARTIFACT_NAME || 'cuda-python-wheel' }} path: . run-id: ${{ inputs.run-id || github.run_id }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -345,7 +350,7 @@ jobs: if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: $/.github/actions/download-artifact-retry with: - name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests + name: ${{ env.CUDA_CORE_CYTHON_TEST_ARTIFACT_NAME || format('{0}-tests', env.CUDA_CORE_ARTIFACT_NAME) }} path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} run-id: ${{ inputs.run-id || github.run_id }} github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 79234697ef3..6c18f171e80 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -17,13 +17,9 @@ on: type: string required: true bindings-config: - description: Normalized bindings registry for maintenance-only tests. + description: Normalized bindings registry; empty retains legacy artifact sources. type: string default: '' - maintenance-bindings: - description: Test this run's maintenance bindings wheel and Cython extensions. - type: boolean - default: false matrix_filter: type: string default: "." @@ -63,7 +59,6 @@ jobs: env: BUILD_TYPE: ${{ inputs.build-type }} BINDINGS_CONFIG: ${{ inputs.bindings-config }} - BINDINGS_ONLY: ${{ inputs.maintenance-bindings }} ARCH: ${{ (inputs.host-platform == 'win-64' && 'amd64') }} outputs: MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }} @@ -89,11 +84,18 @@ jobs: # Read base matrix from YAML file for the specific architecture TEST_MATRIX=$(yq -o json ".windows[\"${MATRIX_TYPE}\"] | map(select(.ARCH == \"${ARCH}\"))" ci/test-matrix.yml) - if [[ "$BINDINGS_ONLY" == "true" ]]; then + if [[ -n "$BINDINGS_CONFIG" ]]; then TEST_MATRIX=$(jq -c --argjson config "$BINDINGS_CONFIG" ' - ($config.package_roots[] | select(.release_status == "maintenance")) as $package - | map(select(.CUDA_VER == $package.toolkit_version) - | . + {BINDINGS_ROOT: $package.package_root}) + ($config.package_roots | map({key: .ctk_target, value: .}) | from_entries) as $packages + | map(($packages[(.CUDA_VER | split(".")[:2] | join("."))]) as $package + | . + { + BINDINGS_ROOT: ($package.package_root // "cuda_bindings_13"), + BINDINGS_BUILD_VERSION: ($package.toolkit_version // ""), + BINDINGS_SOURCE: (if $package == null then "floor" else "main" end), + BINDINGS_ARTIFACT_PREFIX: (if $package.release_status == "maintenance" then "maintenance-bindings" else "cuda-bindings" end), + CORE_CYTHON_SUFFIX: (if $package.release_status == "maintenance" then "-cu\($package.cuda_major)-tests" else "-tests" end), + PYTHON_ARTIFACT_NAME: (if $package.release_status == "maintenance" then "maintenance-python-wheel" else "cuda-python-wheel" end) + }) ' <<< "$TEST_MATRIX") fi @@ -107,10 +109,10 @@ jobs: test: env: - TEST_PATHFINDER: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test) }} - TEST_BINDINGS: ${{ inputs.maintenance-bindings || inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} - TEST_CORE: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test) }} - TEST_PYTHON: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test) }} + TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }} (${{ matrix.DRIVER_MODE }})${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 # The build stage could fail but we want the CI to keep moving. @@ -175,9 +177,12 @@ jobs: - name: Set environment variables env: - BUILD_CUDA_VER: ${{ inputs.maintenance-bindings && matrix.CUDA_VER || inputs.build-ctk-ver }} + BUILD_CUDA_VER: ${{ matrix.BINDINGS_BUILD_VERSION || inputs.build-ctk-ver }} CUDA_BINDINGS_ROOT: ${{ matrix.BINDINGS_ROOT || 'cuda_bindings_13' }} - BINDINGS_ARTIFACT_PREFIX: ${{ inputs.maintenance-bindings && 'maintenance-bindings' || 'cuda-bindings' }} + BINDINGS_SOURCE: ${{ matrix.BINDINGS_SOURCE || '' }} + BINDINGS_ARTIFACT_PREFIX: ${{ matrix.BINDINGS_ARTIFACT_PREFIX || 'cuda-bindings' }} + CORE_CYTHON_SUFFIX: ${{ matrix.CORE_CYTHON_SUFFIX || '-tests' }} + CUDA_PYTHON_ARTIFACT_NAME: ${{ matrix.PYTHON_ARTIFACT_NAME || 'cuda-python-wheel' }} CUDA_VER: ${{ matrix.CUDA_VER }} HOST_PLATFORM: ${{ inputs.host-platform }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} @@ -207,7 +212,7 @@ jobs: if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} uses: $/.github/actions/download-artifact-retry with: - name: cuda-python-wheel + name: ${{ env.CUDA_PYTHON_ARTIFACT_NAME || 'cuda-python-wheel' }} path: . run-id: ${{ inputs.run-id || github.run_id }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -323,7 +328,7 @@ jobs: if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: $/.github/actions/download-artifact-retry with: - name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests + name: ${{ env.CUDA_CORE_CYTHON_TEST_ARTIFACT_NAME || format('{0}-tests', env.CUDA_CORE_ARTIFACT_NAME) }} path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }} run-id: ${{ inputs.run-id || github.run_id }} github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci/tools/env-vars b/ci/tools/env-vars index 66b401e8887..6d3e0df75d7 100755 --- a/ci/tools/env-vars +++ b/ci/tools/env-vars @@ -64,6 +64,7 @@ if [[ "${1}" == "build" ]]; then { echo "MAINTENANCE_BINDINGS_ROOT=${maintenance_root}" echo "MAINTENANCE_CUDA_VERSION=${maintenance_version}" + echo "MAINTENANCE_CUDA_MAJOR=${maintenance_version%%.*}" echo "MAINTENANCE_BINDINGS_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/${maintenance_root}/dist")" echo "MAINTENANCE_BINDINGS_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/${maintenance_root}/tests/cython")" echo "MAINTENANCE_BINDINGS_ARTIFACT_NAME=maintenance-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${maintenance_version}-${HOST_PLATFORM}-${SHA}" @@ -100,7 +101,14 @@ elif [[ "${1}" == "test" ]]; then BUILD_PREV_CUDA_VER="$(yq -r '.cuda.prev_build.version' ci/versions.yml)" BUILD_PREV_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${BUILD_PREV_CUDA_VER})" - if [[ ${BUILD_CUDA_MAJOR} != ${TEST_CUDA_MAJOR} ]]; then + if [[ -n "${BINDINGS_SOURCE:-}" ]]; then + SKIP_CUDA_BINDINGS_TEST=1 + SKIP_CYTHON_TEST=1 + if [[ "$BINDINGS_SOURCE" == main ]]; then + SKIP_CUDA_BINDINGS_TEST="${SKIP_BINDINGS_TEST_OVERRIDE:-0}" + SKIP_CYTHON_TEST=0 + fi + elif [[ ${BUILD_CUDA_MAJOR} != ${TEST_CUDA_MAJOR} ]]; then SKIP_CUDA_BINDINGS_TEST=1 SKIP_CYTHON_TEST=1 if [[ ${BUILD_PREV_CUDA_MINOR} != ${TEST_CUDA_MINOR} ]]; then @@ -154,4 +162,6 @@ fi echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${SHA}" echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/${CUDA_BINDINGS_ROOT}/dist")" echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/${CUDA_BINDINGS_ROOT}/tests/cython")" + echo "CUDA_CORE_CYTHON_TEST_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${SHA}${CORE_CYTHON_SUFFIX:--tests}" + echo "CUDA_PYTHON_ARTIFACT_NAME=${CUDA_PYTHON_ARTIFACT_NAME:-cuda-python-wheel}" } >> $GITHUB_ENV diff --git a/ci/tools/tests/test_bindings_config.py b/ci/tools/tests/test_bindings_config.py index e21adc79110..17608a3334c 100644 --- a/ci/tools/tests/test_bindings_config.py +++ b/ci/tools/tests/test_bindings_config.py @@ -129,9 +129,13 @@ def test_cuda12_source_changes_select_bindings_consumers(): @pytest.mark.agent_authored(model="gpt-6") def test_cuda12_milestone_does_not_change_current_package_versions(tmp_path, monkeypatch): current = load_config(ROOT / "ci" / "versions.yml").package_for_release_status("current") - for package_root, filename in (("cuda_bindings_13", "pyproject.toml"), ("cuda_python", "setup.py")): + for package_root, filename in ( + ("cuda_bindings_13", "pyproject.toml"), + ("cuda_python", "setup.py"), + ("cuda_python", "pyproject.toml"), + ): destination = tmp_path / package_root / filename - destination.parent.mkdir() + destination.parent.mkdir(exist_ok=True) shutil.copyfile(ROOT / package_root / filename, destination) def git(*args): diff --git a/ci/tools/tests/test_cuda_python_setup.py b/ci/tools/tests/test_cuda_python_setup.py new file mode 100644 index 00000000000..1faa3a4b792 --- /dev/null +++ b/ci/tools/tests/test_cuda_python_setup.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import runpy +import shutil +from pathlib import Path +from unittest.mock import Mock + +import pytest +import setuptools +import setuptools_scm +from packaging.requirements import Requirement +from packaging.version import Version + + +@pytest.fixture +def standalone_source(tmp_path, monkeypatch): + package = Path(__file__).resolve().parents[3] / "cuda_python" + for name in ("setup.py", "pyproject.toml"): + shutil.copyfile(package / name, tmp_path / name) + monkeypatch.chdir(tmp_path) + return tmp_path + + +@pytest.mark.parametrize(("requested_major", "expected_major"), ((None, 13), ("12", 12))) +@pytest.mark.agent_authored(model="gpt-6.1-sol") +def test_source_build_selects_bindings_and_major_specific_dependencies( + standalone_source, monkeypatch, requested_major, expected_major +): + if requested_major is None: + monkeypatch.delenv("CUDA_PYTHON_BUILD_MAJOR", raising=False) + else: + monkeypatch.setenv("CUDA_PYTHON_BUILD_MAJOR", requested_major) + setup = Mock() + monkeypatch.setattr(setuptools, "setup", setup) + monkeypatch.setattr( + setuptools_scm, + "get_version", + lambda **options: options["git_describe_command"][-1].removeprefix("v").replace("*", "0"), + ) + + runpy.run_path(str(standalone_source / "setup.py")) + + metadata = setup.call_args.kwargs + assert Version(metadata["version"]).major == expected_major + dependencies = {req.name: req for req in map(Requirement, metadata["install_requires"])} + expected = {"cuda-bindings", "cuda-core", "cuda-pathfinder"} if expected_major >= 13 else {"cuda-bindings"} + assert dependencies.keys() == expected + assert metadata["version"] in dependencies["cuda-bindings"].specifier + + +@pytest.mark.agent_authored(model="gpt-6.1-sol") +def test_maintenance_sdist_keeps_its_dependencies_without_a_build_selector(standalone_source, monkeypatch): + (standalone_source / "PKG-INFO").write_text( + "Metadata-Version: 2.4\nName: cuda-python\nVersion: 12.9.11\n", encoding="utf-8" + ) + monkeypatch.delenv("CUDA_PYTHON_BUILD_MAJOR", raising=False) + setup = Mock() + monkeypatch.setattr(setuptools, "setup", setup) + + runpy.run_path(str(standalone_source / "setup.py")) + + metadata = setup.call_args.kwargs + assert Version(metadata["version"]).major == 12 + assert {Requirement(value).name for value in metadata["install_requires"]} == {"cuda-bindings"} diff --git a/cuda_core/docs/source/install.rst b/cuda_core/docs/source/install.rst index e2d74063140..299c9ebb7be 100644 --- a/cuda_core/docs/source/install.rst +++ b/cuda_core/docs/source/install.rst @@ -101,10 +101,10 @@ obtain the most recent build, use the following commands: $ # Install the downloaded wheel: $ pip install cuda-core-python312-linux-64-*/cuda_core*.whl[cu13] -Replace ``python312`` with your Python version (e.g. ``python310``, ``python311``, -``python313``, ``python314``, ``python314t``). For aarch64, replace ``linux-64`` -with ``linux-aarch64``; for Windows, use ``win-64``. Replace ``cu13`` with -``cu12`` for CUDA 12.x environments. +Replace ``python312`` with your Python version (e.g. ``python311``, +``python313``, ``python314``, ``python314t``, ``python315``, ``python315t``). +For aarch64, replace ``linux-64`` with ``linux-aarch64``; for Windows, use +``win-64``. Replace ``cu13`` with ``cu12`` for CUDA 12.x environments. .. _cuda-core-source-install: diff --git a/cuda_python/AGENTS.md b/cuda_python/AGENTS.md index e4038dc5824..e86d0af29dd 100644 --- a/cuda_python/AGENTS.md +++ b/cuda_python/AGENTS.md @@ -5,7 +5,7 @@ monorepo. - `cuda_python` is primarily packaging and documentation glue. - It does not host substantial runtime APIs like `cuda_core`, - `cuda_bindings_13`, or `cuda_pathfinder`. + `cuda_bindings_12`, `cuda_bindings_13`, or `cuda_pathfinder`. ## Main files to edit @@ -20,14 +20,20 @@ monorepo. component packages rather than here. - Be careful when changing dependency/version logic in `setup.py`; preserve compatibility between metapackage versioning and subpackage constraints. +- Git source builds default to CUDA 13; set `CUDA_PYTHON_BUILD_MAJOR=12` to + select CUDA 12. Selectors live in `pyproject.toml`. Standalone sdists retain + their fixed version, which also determines their dependency major. Check + default CUDA 13, explicit CUDA 12, and CUDA 12 sdist rebuilds when changing + this shared packaging logic. ## Release coupling -- `setup.py` pins `cuda-core` to a minor series (`cuda-core~=X.Y.0`). Every - `cuda-core` release must bump that pin, and a `cuda-python` release should - follow right after, so that `pip install cuda-python` resolves to the new - `cuda-core`. The `cuda-core` release checklist (`.github/RELEASE-core.md`) - has this step. +- The current-major metapackage in `setup.py` pins `cuda-core` to a minor + series (`cuda-core~=X.Y.0`). The CUDA 12 metapackage depends only on bindings. + Every `cuda-core` release must bump that pin, and a `cuda-python` release + for the current major should follow right after, so that + `pip install cuda-python` resolves to the new `cuda-core`. The `cuda-core` + release checklist (`.github/RELEASE-core.md`) has this step. - Users should pin `cuda-python` alone. A separate `cuda-core` pin next to it can make the install unresolvable after a `cuda-core` release. - If you update docs structure, ensure `docs/build_all_docs.sh` still collects diff --git a/cuda_python/docs/README.md b/cuda_python/docs/README.md index 4cf18ee74d0..76b46c7d061 100644 --- a/cuda_python/docs/README.md +++ b/cuda_python/docs/README.md @@ -4,7 +4,11 @@ 2. Build the docs with `./build_docs.sh`. 3. The html artifacts should be available under both `./build/html/latest` and `./build/html/`. -Alternatively, we can build all the docs at once by running [`./build_all_docs.sh`](./build_all_docs.sh). +[`./build_all_docs.sh`](./build_all_docs.sh) builds the metapackage, current +CUDA 13 bindings, Core, and Pathfinder documentation together. It does not +build the [CUDA 12 maintenance documentation](../../cuda_bindings_12/docs/README.md). +Build that separately in its own environment: both bindings roots install the +same `cuda-bindings` distribution and `cuda.bindings` namespace. When building the docs, some (but not all) of the urls can be rendered/examined locally by setting the environment variable `CUDA_PYTHON_DOMAIN` as follows: diff --git a/cuda_python/pyproject.toml b/cuda_python/pyproject.toml index 5b0a55dc99f..d9391e513a1 100644 --- a/cuda_python/pyproject.toml +++ b/cuda_python/pyproject.toml @@ -42,6 +42,9 @@ documentation = "https://nvidia.github.io/cuda-python/" repository = "https://github.com/NVIDIA/cuda-python/" issues = "https://github.com/NVIDIA/cuda-python/issues/" -# The [tool.setuptools_scm] section is handled in setup.py since we need to -# dynamically set the dependency to cuda_bindings based on the dynamically -# determinded version +# setup.py selects one bindings line and derives the matching dependency +# from its SCM version. Keep these selectors aligned with the bindings roots; +# check-bindings-config verifies them against ci/versions.yml. +[tool.cuda-python.scm-describe-match-by-major] +"12" = "v12.9.*" +"13" = "v13.4.*" diff --git a/cuda_python/setup.py b/cuda_python/setup.py index 2223375aa8a..bf8a990f672 100644 --- a/cuda_python/setup.py +++ b/cuda_python/setup.py @@ -2,20 +2,58 @@ # # SPDX-License-Identifier: Apache-2.0 -import ast +import os +import subprocess +import tomllib +from pathlib import Path from packaging.version import Version from setuptools import setup from setuptools_scm import get_version -version = get_version( - root="..", - relative_to=__file__, - git_describe_command=["git", "describe", "--dirty", "--tags", "--long", "--match", "v13.*"], -) +# The selector table ships with standalone sdists, which have neither the +# CI registry nor sibling bindings packages. The metadata hook checks drift. +with Path(__file__).with_name("pyproject.toml").open("rb") as stream: + scm_selectors = tomllib.load(stream)["tool"]["cuda-python"]["scm-describe-match-by-major"] + +# CI selects either major explicitly. Direct tagged builds follow the tag; +# untagged checkouts and commits tagged for both majors default to CUDA 13. +build_major = os.environ.get("CUDA_PYTHON_BUILD_MAJOR") +if build_major is None: + build_major = "13" + source_root = Path(__file__).resolve().parent.parent + if (source_root / ".git").exists(): + tags = subprocess.check_output( + ["git", "tag", "--points-at", "HEAD", "--list", "v12.*", "v13.*"], # noqa: S607 + cwd=source_root, + text=True, + ).splitlines() + if any(tag.startswith("v12.") for tag in tags) and not any(tag.startswith("v13.") for tag in tags): + build_major = "12" +if build_major not in {"12", "13"}: + raise ValueError(f"CUDA_PYTHON_BUILD_MAJOR must be 12 or 13, got {build_major!r}") + +version_options = { + "root": "..", + "relative_to": __file__, + "dist_name": "cuda-python", + # Keep metapackage tag selection identical to its bindings source line. + "git_describe_command": [ + "git", + "describe", + "--dirty", + "--tags", + "--long", + "--match", + scm_selectors[build_major], + ], +} +version = get_version(**version_options) -base_version = Version(version).base_version + +resolved_version = Version(version) +base_version = resolved_version.base_version if base_version == version: @@ -25,16 +63,23 @@ # Pre-release version matcher = "==" +install_requires = [f"cuda-bindings{matcher}{version}"] +# An sdist recovers its fixed version without Git; its dependency set must +# follow that version even when the checkout selector defaults to CUDA 13. +if resolved_version.major >= 13: + install_requires.extend( + [ + # Bump this with every cuda-core release and release the current-major + # cuda-python right after it; see .github/RELEASE-core.md. + "cuda-core~=1.2.0", + "cuda-pathfinder~=1.1", + ] + ) + setup( version=version, - install_requires=[ - f"cuda-bindings{matcher}{version}", - # Bump this with every cuda-core release and release cuda-python right - # after it; see .github/RELEASE-core.md and cuda_python/AGENTS.md. - "cuda-core~=1.2.0", - "cuda-pathfinder~=1.1", - ], + install_requires=install_requires, extras_require={ "all": [f"cuda-bindings[all]{matcher}{version}"], }, From b74179c348809ea19f9507be31b67517dd3df108 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 3 Nov 2026 21:47:40 -0800 Subject: [PATCH 2/4] Address migration review: simplify configuration and tooling Signed-off-by: Ralf W. Grosse-Kunstleve --- cuda_python/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cuda_python/pyproject.toml b/cuda_python/pyproject.toml index d9391e513a1..1210baaae80 100644 --- a/cuda_python/pyproject.toml +++ b/cuda_python/pyproject.toml @@ -46,5 +46,5 @@ issues = "https://github.com/NVIDIA/cuda-python/issues/" # from its SCM version. Keep these selectors aligned with the bindings roots; # check-bindings-config verifies them against ci/versions.yml. [tool.cuda-python.scm-describe-match-by-major] -"12" = "v12.9.*" -"13" = "v13.4.*" +"12" = "v12.*" +"13" = "v13.*" From e692d7208609ade591f00dcbd2c798f72f74acdb Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 3 Nov 2026 21:51:15 -0800 Subject: [PATCH 3/4] Fix tagged metapackage builds and preserve toolkit patch coverage Signed-off-by: Ralf W. Grosse-Kunstleve --- ci/tools/tests/test_cuda_python_setup.py | 108 ++++++++++++----------- cuda_python/AGENTS.md | 5 +- 2 files changed, 60 insertions(+), 53 deletions(-) diff --git a/ci/tools/tests/test_cuda_python_setup.py b/ci/tools/tests/test_cuda_python_setup.py index 1faa3a4b792..54b247e17a0 100644 --- a/ci/tools/tests/test_cuda_python_setup.py +++ b/ci/tools/tests/test_cuda_python_setup.py @@ -1,65 +1,71 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import runpy +import os import shutil +import subprocess +import sys +import tarfile +from email import message_from_bytes from pathlib import Path -from unittest.mock import Mock +from zipfile import ZipFile import pytest -import setuptools -import setuptools_scm from packaging.requirements import Requirement -from packaging.version import Version -@pytest.fixture -def standalone_source(tmp_path, monkeypatch): - package = Path(__file__).resolve().parents[3] / "cuda_python" - for name in ("setup.py", "pyproject.toml"): - shutil.copyfile(package / name, tmp_path / name) - monkeypatch.chdir(tmp_path) - return tmp_path +@pytest.mark.agent_authored(model="gpt-6") +def test_tagged_maintenance_sdist_keeps_version_and_dependencies_without_selector(tmp_path): + source = Path(__file__).resolve().parents[3] / "cuda_python" + checkout = tmp_path / "checkout" + package = checkout / "cuda_python" + package.mkdir(parents=True) + for filename in ("setup.py", "pyproject.toml", "DESCRIPTION.rst", "LICENSE"): + shutil.copyfile(source / filename, package / filename) + env = os.environ.copy() + env.pop("CUDA_PYTHON_BUILD_MAJOR", None) + for args in (("init", "--quiet"), ("add", "."), ("commit", "--quiet", "-m", "Release"), ("tag", "v12.9.11")): + subprocess.run( # noqa: S603 + [ # noqa: S607 + "git", + "-c", + "user.name=CI test", + "-c", + "user.email=ci-test@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "tag.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + *args, + ], + cwd=checkout, + env=env, + check=True, + capture_output=True, + ) -@pytest.mark.parametrize(("requested_major", "expected_major"), ((None, 13), ("12", 12))) -@pytest.mark.agent_authored(model="gpt-6.1-sol") -def test_source_build_selects_bindings_and_major_specific_dependencies( - standalone_source, monkeypatch, requested_major, expected_major -): - if requested_major is None: - monkeypatch.delenv("CUDA_PYTHON_BUILD_MAJOR", raising=False) - else: - monkeypatch.setenv("CUDA_PYTHON_BUILD_MAJOR", requested_major) - setup = Mock() - monkeypatch.setattr(setuptools, "setup", setup) - monkeypatch.setattr( - setuptools_scm, - "get_version", - lambda **options: options["git_describe_command"][-1].removeprefix("v").replace("*", "0"), + subprocess.run( + [sys.executable, "-c", "from setuptools.build_meta import build_sdist; build_sdist('dist')"], + cwd=package, + env=env, + check=True, ) - - runpy.run_path(str(standalone_source / "setup.py")) - - metadata = setup.call_args.kwargs - assert Version(metadata["version"]).major == expected_major - dependencies = {req.name: req for req in map(Requirement, metadata["install_requires"])} - expected = {"cuda-bindings", "cuda-core", "cuda-pathfinder"} if expected_major >= 13 else {"cuda-bindings"} - assert dependencies.keys() == expected - assert metadata["version"] in dependencies["cuda-bindings"].specifier - - -@pytest.mark.agent_authored(model="gpt-6.1-sol") -def test_maintenance_sdist_keeps_its_dependencies_without_a_build_selector(standalone_source, monkeypatch): - (standalone_source / "PKG-INFO").write_text( - "Metadata-Version: 2.4\nName: cuda-python\nVersion: 12.9.11\n", encoding="utf-8" + unpacked = tmp_path / "unpacked" + with tarfile.open(next((package / "dist").glob("*.tar.gz"))) as archive: + archive.extractall(unpacked, filter="data") + standalone = next(unpacked.iterdir()) + subprocess.run( + [sys.executable, "-c", "from setuptools.build_meta import build_wheel; build_wheel('dist')"], + cwd=standalone, + env=env, + check=True, ) - monkeypatch.delenv("CUDA_PYTHON_BUILD_MAJOR", raising=False) - setup = Mock() - monkeypatch.setattr(setuptools, "setup", setup) - - runpy.run_path(str(standalone_source / "setup.py")) - - metadata = setup.call_args.kwargs - assert Version(metadata["version"]).major == 12 - assert {Requirement(value).name for value in metadata["install_requires"]} == {"cuda-bindings"} + with ZipFile(next((standalone / "dist").glob("*.whl"))) as wheel: + metadata = message_from_bytes( + wheel.read(next(name for name in wheel.namelist() if name.endswith(".dist-info/METADATA"))) + ) + assert metadata["Version"] == "12.9.11" + assert {Requirement(value).name for value in metadata.get_all("Requires-Dist", [])} == {"cuda-bindings"} diff --git a/cuda_python/AGENTS.md b/cuda_python/AGENTS.md index e86d0af29dd..dda1068bf5e 100644 --- a/cuda_python/AGENTS.md +++ b/cuda_python/AGENTS.md @@ -20,8 +20,9 @@ monorepo. component packages rather than here. - Be careful when changing dependency/version logic in `setup.py`; preserve compatibility between metapackage versioning and subpackage constraints. -- Git source builds default to CUDA 13; set `CUDA_PYTHON_BUILD_MAJOR=12` to - select CUDA 12. Selectors live in `pyproject.toml`. Standalone sdists retain +- Git source builds use CUDA 12 at an unambiguous CUDA 12 tag and otherwise + default to CUDA 13. Set `CUDA_PYTHON_BUILD_MAJOR` to select either major + explicitly. Selectors live in `pyproject.toml`. Standalone sdists retain their fixed version, which also determines their dependency major. Check default CUDA 13, explicit CUDA 12, and CUDA 12 sdist rebuilds when changing this shared packaging logic. From cdebef2ca01fb02093bf96b90ee161008e3fb809 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 6 Nov 2026 09:46:32 -0800 Subject: [PATCH 4/4] Use retained CUDA 12 milestone for source distributions Signed-off-by: Ralf W. Grosse-Kunstleve --- .github/workflows/test-sdist-linux.yml | 10 ++-------- .github/workflows/test-sdist-windows.yml | 10 ++-------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-sdist-linux.yml b/.github/workflows/test-sdist-linux.yml index 89c65fd10df..dd481a77ab8 100644 --- a/.github/workflows/test-sdist-linux.yml +++ b/.github/workflows/test-sdist-linux.yml @@ -63,14 +63,8 @@ jobs: with: python-version: "3.12" - - name: Install build and CI tools - run: python -m pip install build -e ./ci - - - name: Prepare local maintenance SCM milestone - if: ${{ fromJSON(inputs.bindings-package).release_status == 'maintenance' }} - run: | - python -m ci.tools.bindings_scm_version \ - --package-root "${BINDINGS_ROOT}" --sha "${{ github.sha }}" + - name: Install build tools + run: python -m pip install build # Pure Python packages -- no CTK needed. - name: Build cuda.pathfinder sdist and wheel-from-sdist diff --git a/.github/workflows/test-sdist-windows.yml b/.github/workflows/test-sdist-windows.yml index 7ede002a609..d15b747612b 100644 --- a/.github/workflows/test-sdist-windows.yml +++ b/.github/workflows/test-sdist-windows.yml @@ -76,14 +76,8 @@ jobs: if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - - name: Install build and CI tools - run: python -m pip install build -e ./ci - - - name: Prepare local maintenance SCM milestone - if: ${{ fromJSON(inputs.bindings-package).release_status == 'maintenance' }} - run: | - python -m ci.tools.bindings_scm_version \ - --package-root "${BINDINGS_ROOT}" --sha "${{ github.sha }}" + - name: Install build tools + run: python -m pip install build # Pure Python packages -- no CTK needed. - name: Build cuda.pathfinder sdist and wheel-from-sdist