Build and test conda packages and wheels

.github/workflows/intel-ci.yml

@@ -0,0 +1,192 @@
+# Builds and tests mkl_random's conda packages and wheels, see ci/README.md
+name: Intel CI
+
+on:
+  push:
+    branches: [master]
+    tags: ['*']
+  pull_request:
+
+permissions:
+  contents: read
+
+# A push or tag run gets a group of its own, so a queued master push is never dropped.
+concurrency:
+  group: intel-ci-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  matrix:
+    name: Platforms and Python versions
+    runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }}
+    timeout-minutes: 10
+    outputs:
+      matrix: ${{ steps.declared.outputs.matrix }}
+      wheel: ${{ steps.declared.outputs.wheel }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.14"
+
+      - name: Read the declared platforms and Python versions
+        id: declared
+        run: |
+          python ci/matrix.py --event "$GITHUB_EVENT_NAME" --github-output matrix
+          python ci/matrix.py --system wheel --event "$GITHUB_EVENT_NAME" --github-output wheel
+
+  # macOS runs on Intel runners, as MKL has no Apple silicon build.
+  build:
+    name: ${{ matrix.name }} Python ${{ matrix.python }}
+    needs: matrix
+    runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }}
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          # The recipe takes its version from git describe.
+          fetch-depth: 0
+
+      # Miniforge 26.9.4-0 fails setup-miniconda's conda init on Windows runners; 26.7.2-0 works.
+      - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0
+        with:
+          miniforge-version: 26.7.2-0
+          activate-environment: ""
+          conda-remove-defaults: "true"
+
+      - name: Install conda-build
+        run: conda install -y -q -n base --override-channels -c conda-forge conda-build
+
+      - name: Build and run the recipe tests
+        env:
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: python ci/build.py --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output"
+
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: mkl_random ${{ matrix.name }} Python ${{ matrix.python }}
+          path: ${{ runner.temp }}/output/*.conda
+          if-no-files-found: error
+          # Tags keep them 30 days, pull requests and pushes to master 3.
+          retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }}
+
+  test:
+    name: Test ${{ matrix.name }} Python ${{ matrix.python }}
+    needs: [matrix, build]
+    # Test every package that built, even when another build failed
+    if: ${{ !cancelled() && needs.matrix.result == 'success' }}
+    runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: mkl_random ${{ matrix.name }} Python ${{ matrix.python }}
+          path: ${{ runner.temp }}/artifacts
+
+      - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0
+        with:
+          miniforge-version: 26.7.2-0
+          activate-environment: ""
+          conda-remove-defaults: "true"
+
+      - name: Install the package and run ci/test.py
+        env:
+          PYTHON_VERSION: ${{ matrix.python }}
+          EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }}
+          COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: >-
+          python ci/hooks.py --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts"
+          --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT"
+
+  build-wheel:
+    name: ${{ matrix.name }} Wheels Python ${{ matrix.python }}
+    needs: matrix
+    runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }}
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.14"
+
+      - name: Install cibuildwheel
+        run: python -m pip install --no-input cibuildwheel==4.3.5
+
+      - name: Build the wheel
+        env:
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: python ci/build.py --system wheel --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output"
+
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: mkl_random ${{ matrix.name }} Wheels Python ${{ matrix.python }}
+          path: ${{ runner.temp }}/output/*.whl
+          if-no-files-found: error
+          retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }}
+
+  test-wheel:
+    name: Test ${{ matrix.name }} Wheels Python ${{ matrix.python }}
+    needs: [matrix, build-wheel]
+    # Test every package that built, even when another build failed
+    if: ${{ !cancelled() && needs.matrix.result == 'success' }}
+    runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: mkl_random ${{ matrix.name }} Wheels Python ${{ matrix.python }}
+          path: ${{ runner.temp }}/artifacts
+
+      - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0
+        with:
+          miniforge-version: 26.7.2-0
+          activate-environment: ""
+          conda-remove-defaults: "true"
+
+      - name: Install the wheel and run ci/test.py
+        env:
+          PYTHON_VERSION: ${{ matrix.python }}
+          EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }}
+          COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: >-
+          python ci/hooks.py --kind wheel --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts"
+          --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT"

README.md

@@ -1,7 +1,6 @@
 ## `mkl_random` -- a NumPy-based Python interface to Intel® oneAPI Math Kernel Library (OneMKL) Random Number Generation functionality
-[![Conda package](https://github.com/IntelPython/mkl_random/actions/workflows/conda-package.yml/badge.svg)](https://github.com/IntelPython/mkl_random/actions/workflows/conda-package.yml)
 [![Editable build using pip and pre-release NumPy](https://github.com/IntelPython/mkl_random/actions/workflows/build_pip.yml/badge.svg)](https://github.com/IntelPython/mkl_random/actions/workflows/build_pip.yml)
-[![Conda package using conda-forge](https://github.com/IntelPython/mkl_random/actions/workflows/conda-package-cf.yml/badge.svg)](https://github.com/IntelPython/mkl_random/actions/workflows/conda-package-cf.yml)
+[![Intel CI](https://github.com/IntelPython/mkl_random/actions/workflows/intel-ci.yml/badge.svg)](https://github.com/IntelPython/mkl_random/actions/workflows/intel-ci.yml)
 [![Coverity Scan Build Status](https://scan.coverity.com/projects/33198/badge.svg)](https://scan.coverity.com/projects/intelpython-mkl_random)
 [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/IntelPython/mkl_random/badge)](https://securityscorecards.dev/viewer/?uri=github.com/IntelPython/mkl_random)
 

ci/README.md

@@ -0,0 +1,32 @@
+# How Intel CI tests mkl_random
+
+`.github/workflows/intel-ci.yml` builds mkl_random's conda packages and wheels in GitHub Actions and tests each one. It needs this directory, the recipe, and `pyproject.toml`.
+
+| File | Role |
+|---|---|
+| `../conda-recipe/` | The conda recipe. `conda_build_config.yaml` lists the Python versions. The recipe builds on Linux, Windows and Intel macOS unless a platform-only skip in `meta.yaml`, such as `skip: true  # [win]`, rules one out. |
+| `../pyproject.toml` | What the wheels are built from. Its Python classifiers list the wheels' Python versions, and its Operating System classifiers their platforms. `[tool.cibuildwheel]` holds the wheel build settings, and `[tool.gh-ci]` holds the run-time requirements the wheels need on top of `[project]`'s, which the conda packages get from the recipe: `pin-compatible` requires at least the `mkl-devel` version the build used, below its next major. |
+| `../.github/workflows/intel-ci.yml` | One job reads both matrices. One job per platform and Python version builds and uploads `mkl_random <Platform> Python <version>`, and one builds `mkl_random <Platform> Wheels Python <version>`. Then one job per package tests it. |
+| `matrix.py` | Reads the platforms and Python versions from the recipe, or with `--system wheel` from `pyproject.toml`. A pull request gets the oldest and the newest regular versions and the newest free-threaded one. |
+| `build.py` | Builds one Python version: with conda-build, which runs the recipe's own tests, or with `--system wheel` with cibuildwheel, in a manylinux container on Linux. The ELF files in a wheel keep only their `$ORIGIN` RPATH and RUNPATH entries, and must have full RELRO. |
+| `hooks.py` | The workflow's test jobs: installs one package into a clean environment with the requirements in `config.yml`, and runs `test.py`. A conda package's environment comes from conda-forge; a wheel goes into a venv made from conda-forge's Python, with everything else from PyPI. |
+| `config.yml` | Names the workflow that builds the packages, its artifact names, and what the tests and benchmarks need. |
+| `test.py` | Runs the installed package's test suite with pytest. |
+| `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package. |
+
+The scripts need nothing beyond the Python standard library and the tools each one's docstring names. macOS packages build and test on Intel runners, as MKL has no Apple silicon build.
+
+**A pull request.** Actions builds the oldest and the newest regular Python versions and the newest free-threaded one on Linux, Windows and macOS, as a conda package and, where `pyproject.toml` declares the platform, as a wheel, and tests each package.
+
+**A push to `master` or a tag.** Actions builds and tests every declared version.
+
+To support another Python, add it to `conda-recipe/conda_build_config.yaml` for the conda packages and as a classifier in `pyproject.toml` for the wheels; nothing else lists versions. Try it locally from a conda base environment with conda-build, or with cibuildwheel for wheels:
+
+```bash
+python ci/matrix.py                           # every platform and Python version
+python ci/matrix.py --event pull_request      # what a pull request builds
+python ci/build.py --python VERSION --output /tmp/out     # VERSION: one matrix.py lists
+python ci/hooks.py --python VERSION --artifacts /tmp/out --work /tmp/hooks
+python ci/build.py --system wheel --python VERSION --output /tmp/wheels
+python ci/hooks.py --kind wheel --python VERSION --artifacts /tmp/wheels --work /tmp/wheel-hooks
+```

ci/benchmark.py

@@ -0,0 +1,84 @@
+"""
+Benchmarks the installed package.
+
+Runs the asv suite in benchmarks/ against the package installed in this
+environment, from an empty working directory, without a build of its own, and
+records the results under the commit being tested, in $GH_CI_RESULTS. A pull
+request samples each benchmark once, enough to show that every one still runs;
+a tag measures them properly. A benchmark that fails fails the run; a slower one
+does not, as nothing compares the timings with a baseline yet.
+"""
+
+import json
+import os
+import pathlib
+import subprocess
+import sys
+
+# asv's own dependency, which reads the comments in its configuration.
+import json5
+
+ASV_CONFIG = pathlib.Path("benchmarks", "asv.conf.json")
+
+
+def asv_config(source: pathlib.Path, results: pathlib.Path) -> pathlib.Path:
+    """Writes this run's asv configuration into the working directory.
+
+    It is the repository's own, with absolute paths, since asv reads them from
+    where the configuration is, and with results under GH_CI_RESULTS. It lists
+    one branch, HEAD: the checkout is the commit under test, detached, and asv
+    cannot resolve the repository's branches there.
+    """
+    path = source / ASV_CONFIG
+    config = json5.loads(path.read_text(encoding="utf-8"))
+    work = pathlib.Path.cwd()
+    config.update(
+        repo=str((path.parent / config.get("repo", "..")).resolve()),
+        benchmark_dir=str(
+            (path.parent / config.get("benchmark_dir", "benchmarks")).resolve()
+        ),
+        branches=["HEAD"],
+        results_dir=str(results / "asv"),
+        html_dir=str(work / "html"),
+        env_dir=str(work / "env"),
+    )
+    written = work / "asv.conf.json"
+    written.write_text(json.dumps(config, indent=2), encoding="utf-8")
+    return written
+
+
+def main() -> int:
+    """Runs the asv suite against the installed package."""
+    config = asv_config(
+        pathlib.Path(os.environ["GH_CI_SOURCE"]),
+        pathlib.Path(os.environ["GH_CI_RESULTS"]),
+    )
+    asv = [sys.executable, "-m", "asv"]
+    # asv keeps its machine description in ~/.asv-machine.json; this one stays
+    # with the run rather than with the machine's user.
+    home = pathlib.Path.cwd() / "home"
+    home.mkdir(exist_ok=True)
+    env = dict(os.environ, HOME=str(home))
+    described = subprocess.run(
+        asv + ["machine", "--yes", "--config", str(config)], env=env
+    )
+    if described.returncode:
+        return described.returncode
+    run = [
+        "run",
+        "--config",
+        str(config),
+        f"--environment=existing:{sys.executable}",
+        f"--set-commit-hash={os.environ['GH_CI_COMMIT']}",
+        "--no-pull",
+        "--show-stderr",
+        "--launch-method=spawn",
+    ]
+    if os.environ.get("GH_CI_EVENT") != "tag":
+        run.append("--quick")
+    # asv exits 2 when a benchmark failed.
+    return subprocess.run(asv + run, env=env).returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())