From 45a9f96fde2f0c5250f2e50d309c132f3a31f7a3 Mon Sep 17 00:00:00 2001 From: Neil Lawrence Date: Fri, 13 Nov 2026 19:50:31 +0000 Subject: [PATCH 1/2] CIP-0001 Phase 2: deprecate MPI and remove typemap. Refuse silently wrong mpi_comm without back-constraints, warn on use, let mpi4py infer buffer types, and add an optional 2-rank CI parity job. --- .coveragerc | 4 +- .github/workflows/tests.yml | 26 ++++++++++ README.md | 13 ++++- cip/cip0001.md | 8 +-- deepgp/layers/enclayer.py | 11 ++-- deepgp/models/model.py | 16 ++++++ deepgp/testing/mpi_parity.py | 86 +++++++++++++++++++++++++++++++ deepgp/testing/test_mpi_policy.py | 55 ++++++++++++++++++++ deepgp/util/parallel.py | 56 +++++++++----------- 9 files changed, 229 insertions(+), 46 deletions(-) create mode 100644 deepgp/testing/mpi_parity.py create mode 100644 deepgp/testing/test_mpi_policy.py diff --git a/.coveragerc b/.coveragerc index 3a0a96a..6cec78c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,8 @@ # Coverage config for PyDeepGP # # Measured surface is the maintained serial DeepGP path. The omitted modules -# are known gaps (MPI, MRD, prediction-only SVI ratio, Theano encoder) and are -# tracked as follow-up work rather than counted against the CI gate. +# are known gaps (MPI, MRD, prediction-only SVI ratio) and are tracked as +# follow-up work rather than counted against the CI gate (CIP-0001). [run] branch = True source = deepgp diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c2c570d..0d98dce 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,3 +45,29 @@ jobs: - name: Upload coverage XML if: always() run: python -m coverage xml -o coverage.xml + + # CIP-0001 R11: optional MPI parity while mpi4py support still exists. + mpi: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install MPI and Python deps + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libopenmpi-dev openmpi-bin + python -m pip install --upgrade pip setuptools wheel + python -m pip install "numpy>=1.7" "scipy>=0.12" "GPy>=1.15" matplotlib h5py tables mpi4py + python -m pip install -e . --no-deps + + - name: MPI parity (2 ranks vs serial) + env: + OMPI_MCA_btl_vader_single_copy_mechanism: none + OMPI_MCA_rmaps_base_oversubscribe: 'true' + run: mpiexec -n 2 python -m deepgp.testing.mpi_parity diff --git a/README.md b/README.md index b40eec4..9a67c09 100644 --- a/README.md +++ b/README.md @@ -35,4 +35,15 @@ tox Coverage is gated on the maintained serial DeepGP path, including the NumPy encoder (see `.coveragerc`). MPI parallel inference, MRD, and prediction-only -SVI helpers remain omitted until those paths are revived (CIP-0001 Phases 2–4). +SVI helpers remain omitted until those paths are revived or removed +(CIP-0001 Phases 2–4). + +MPI (deprecated) +---------------- + +Passing ``mpi_comm`` to ``DeepGP`` emits a ``FutureWarning`` (CIP-0001). Use +serial models for now; chunked psi-statistics (Phase 3) will replace the +single-machine role of MPI. ``mpi_comm`` with ``back_constraint=False`` raises +``ValueError`` (that configuration produced silently wrong results). + +An optional GitHub Actions job runs a 2-rank parity check while MPI remains. diff --git a/cip/cip0001.md b/cip/cip0001.md index ef7921d..2014e9e 100644 --- a/cip/cip0001.md +++ b/cip/cip0001.md @@ -2,7 +2,7 @@ author: Neil Lawrence created: "2026-11-13" id: "0001" -last_updated: "2026-11-13" # Phase 1 implementation started +last_updated: "2026-11-13" status: In Progress compressed: false related_requirements: [] @@ -27,7 +27,7 @@ title: Replace Theano and MPI with modern NumPy - [x] Proposed - Initial idea documented (open for Discussion feedback) - [x] Accepted - Phase 1 started on branch `cip0001-numpy-encoder` (regularisation: keep no-op + FutureWarning) -- [x] In Progress - Phase 1 (NumPy encoder) under way +- [x] In Progress - Phase 2 (MPI correctness / deprecation) under way - [ ] Implemented - Work complete, awaiting verification - [ ] Closed - Verified and complete - [ ] Rejected - Will not be implemented @@ -255,8 +255,8 @@ pluggable reducer in R14, so the MPI-specific code shrinks to that one place. - [x] Prototype: chunked/threaded psi benchmark (2-core) - [ ] Discussion opened - [x] Accept (partial): regularisation kept as no-op with `FutureWarning` (R2) -- [x] Phase 1: NumPy encoder (R1–R7) on `cip0001-numpy-encoder` — BC reference tests + encoder unit tests green -- [ ] Phase 2: MPI correctness and deprecation (R8–R11) — note: `np.int`→`np.int64` typemap fix included with Phase 1 +- [x] Phase 1: NumPy encoder (R1–R7) — merged (PR #37) +- [x] Phase 2: MPI correctness and deprecation (R8–R11) on `cip0001-mpi-deprecate` — typemap removed; unsupported configs raise; FutureWarning; optional MPI CI job - [ ] Phase 3: chunked psi-statistics (R12 in GPy, R13–R16) - [ ] Many-core benchmarks recorded - [ ] Phase 4: MPI removed (R17–R24) or reduced to the reducer hook diff --git a/deepgp/layers/enclayer.py b/deepgp/layers/enclayer.py index 251a9b6..3e943a3 100644 --- a/deepgp/layers/enclayer.py +++ b/deepgp/layers/enclayer.py @@ -90,9 +90,8 @@ def update_gradients(self): self.X.mean.gradient += X_grad def _gather_gradients(self): - from deepgp.util.parallel import numpy_to_MPI_typemap - t = numpy_to_MPI_typemap(self.encoder.gradient.dtype) - self.mpi_comm.Reduce([self.encoder.gradient.copy(), t], [self.encoder.gradient, t], root=self.mpi_root) - - t = numpy_to_MPI_typemap(self.Y_var.gradient.dtype) - self.mpi_comm.Reduce([self.Y_var.gradient.copy(), t], [self.Y_var.gradient, t], root=self.mpi_root) + # mpi4py infers MPI datatypes from NumPy buffers (CIP-0001 R8). + enc_grad = self.encoder.gradient + self.mpi_comm.Reduce(enc_grad.copy(), enc_grad, root=self.mpi_root) + y_grad = self.Y_var.gradient + self.mpi_comm.Reduce(y_grad.copy(), y_grad, root=self.mpi_root) diff --git a/deepgp/models/model.py b/deepgp/models/model.py index 811a94d..6a9743b 100644 --- a/deepgp/models/model.py +++ b/deepgp/models/model.py @@ -3,12 +3,17 @@ import sys +import warnings import numpy as np from scipy.linalg import LinAlgError from GPy import Model, likelihoods from GPy.core.parameterization.variational import VariationalPosterior, NormalPosterior from ..layers import ObservedLayer, ObservedMRDLayer, HiddenLayer, TopHiddenLayer +_MPI_DEPRECATION = ( + "MPI support is deprecated; use chunked psi-statistics (see CIP-0001)." +) + class DeepGP(Model): """ Deep Gaussian process model. @@ -31,6 +36,17 @@ def __init__(self, nDims, Y, X=None, num_inducing=10, # Back-forward compatibility with old interface if encoder_dims is None and 'MLP_dims' in kwargs: encoder_dims = kwargs['MLP_dims'] + + # CIP-0001 Phase 2: deprecate MPI; refuse silently-wrong configurations. + if mpi_comm is not None: + warnings.warn(_MPI_DEPRECATION, FutureWarning, stacklevel=2) + if not back_constraint: + raise ValueError( + "mpi_comm is not supported with back_constraint=False: " + "per-data-point variational parameters are not synchronised " + "under the root-broadcast design, so results would be wrong. " + "Use back_constraint=True, or omit mpi_comm (see CIP-0001)." + ) self.mpi_comm, self.mpi_root = mpi_comm, mpi_root self.obs_data = obs_data diff --git a/deepgp/testing/mpi_parity.py b/deepgp/testing/mpi_parity.py new file mode 100644 index 0000000..588e694 --- /dev/null +++ b/deepgp/testing/mpi_parity.py @@ -0,0 +1,86 @@ +# Copyright (c) 2015-2016, the authors (see AUTHORS.txt). +# Licensed under the BSD 3-clause license (see LICENSE.txt) + +""" +CIP-0001 R11: compare 2-rank MPI objective with a serial back-constrained model. + +Run under MPI:: + + mpiexec -n 2 python -m deepgp.testing.mpi_parity + +Exits 0 when the MPI log-likelihood on root matches the serial reference +within a relative tolerance; otherwise exits 1. +""" + +from __future__ import print_function + +import sys +import warnings + +import numpy as np +import GPy + +import deepgp + + +def _build_model(Y, X, mpi_comm=None): + kernels = [GPy.kern.RBF(3, ARD=True), GPy.kern.RBF(2, ARD=True)] + with warnings.catch_warnings(): + warnings.simplefilter('ignore', FutureWarning) + return deepgp.DeepGP( + [2, 3, 2], + Y, + X=X, + kernels=kernels, + num_inducing=4, + back_constraint=True, + mpi_comm=mpi_comm, + mpi_root=0, + ) + + +def main(): + from mpi4py import MPI + + comm = MPI.COMM_WORLD + rank = comm.rank + size = comm.size + if size < 2: + if rank == 0: + print('mpi_parity requires at least 2 ranks', file=sys.stderr) + return 2 + + rng = np.random.RandomState(42) + N, D, Q = 20, 2, 2 + Y_all = rng.randn(N, D) + X_all = rng.randn(N, Q) + + # Contiguous row partition across ranks (same total data as serial). + counts = [N // size + (1 if r < N % size else 0) for r in range(size)] + starts = np.cumsum([0] + counts[:-1]) + sl = slice(int(starts[rank]), int(starts[rank] + counts[rank])) + Y_local = np.ascontiguousarray(Y_all[sl]) + X_local = np.ascontiguousarray(X_all[sl]) + + serial_ll = None + if rank == 0: + m_serial = _build_model(Y_all, X_all, mpi_comm=None) + serial_ll = float(m_serial.log_likelihood()) + serial_ll = comm.bcast(serial_ll, root=0) + + m_mpi = _build_model(Y_local, X_local, mpi_comm=comm) + # After internal Allreduce of psi statistics, root holds the global objective. + mpi_obj = float(m_mpi.log_likelihood()) if rank == 0 else None + if rank == 0: + rel = abs(mpi_obj - serial_ll) / max(1.0, abs(serial_ll)) + print('serial_ll={:.10g} mpi_ll={:.10g} rel_diff={:.3e}'.format( + serial_ll, mpi_obj, rel)) + if not np.isfinite(mpi_obj) or rel > 1e-5: + print('MPI parity check failed', file=sys.stderr) + return 1 + print('MPI parity check passed') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/deepgp/testing/test_mpi_policy.py b/deepgp/testing/test_mpi_policy.py new file mode 100644 index 0000000..5de0fa2 --- /dev/null +++ b/deepgp/testing/test_mpi_policy.py @@ -0,0 +1,55 @@ +# Copyright (c) 2015-2016, the authors (see AUTHORS.txt). +# Licensed under the BSD 3-clause license (see LICENSE.txt) + +"""CIP-0001 Phase 2: MPI deprecation warning and unsupported configurations.""" + +import unittest +import warnings + +import numpy as np +import GPy + +import deepgp +from deepgp.models.model import _MPI_DEPRECATION + + +def _small_dims_and_data(): + rng = np.random.RandomState(0) + Y = rng.randn(8, 2) + X = rng.randn(8, 1) + kernels = [GPy.kern.RBF(2, ARD=True), GPy.kern.RBF(1, ARD=True)] + return [2, 2, 1], Y, X, kernels + + +class MPIPolicyTest(unittest.TestCase): + def test_mpi_comm_without_back_constraint_raises(self): + nDims, Y, X, kernels = _small_dims_and_data() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', FutureWarning) + with self.assertRaises(ValueError) as ctx: + deepgp.DeepGP( + nDims, Y, X=X, kernels=kernels, num_inducing=3, + back_constraint=False, mpi_comm=object(), + ) + self.assertIn('back_constraint=False', str(ctx.exception)) + + def test_mpi_comm_emits_future_warning(self): + nDims, Y, X, kernels = _small_dims_and_data() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + try: + deepgp.DeepGP( + nDims, Y, X=X, kernels=kernels, num_inducing=3, + back_constraint=True, mpi_comm=object(), + ) + except Exception: + # Fake communicator is enough to exercise the policy check; + # construction may fail later when real MPI ops are needed. + pass + future = [w for w in caught if issubclass(w.category, FutureWarning)] + self.assertTrue(future, 'expected FutureWarning for mpi_comm') + self.assertIn(_MPI_DEPRECATION, str(future[0].message)) + + +if __name__ == '__main__': + unittest.main() diff --git a/deepgp/util/parallel.py b/deepgp/util/parallel.py index f71a304..bd60062 100644 --- a/deepgp/util/parallel.py +++ b/deepgp/util/parallel.py @@ -4,60 +4,50 @@ import numpy as np + def get_id_within_node(comm=None): from mpi4py import MPI - if comm is None: comm = MPI.COMM_WORLD + if comm is None: + comm = MPI.COMM_WORLD rank = comm.rank - nodename = MPI.Get_processor_name() + nodename = MPI.Get_processor_name() nodelist = comm.allgather(nodename) - return len([i for i in nodelist[:rank] if i==nodename]) + return len([i for i in nodelist[:rank] if i == nodename]) -def numpy_to_MPI_typemap(np_type): - from mpi4py import MPI - typemap = { - np.dtype(np.float64) : MPI.DOUBLE, - np.dtype(np.float32) : MPI.FLOAT, - np.dtype(np.int64) : MPI.INT, - np.dtype(np.int8) : MPI.CHAR, - np.dtype(np.uint8) : MPI.UNSIGNED_CHAR, - np.dtype(np.int32) : MPI.INT, - np.dtype(np.uint32) : MPI.UNSIGNED_INT, - } - return typemap[np_type] - def allReduceArrays(arrL, comm): + """Allreduce a list of NumPy scalars/arrays. mpi4py infers MPI datatypes.""" res = [] for arr in arrL: if np.isscalar(arr): - arr_out = np.empty((1,),dtype=arr.dtype) - t = numpy_to_MPI_typemap(arr.dtype) - comm.Allreduce([arr, t], [arr_out, t]) - res.append(arr_out[0]) + send = np.asarray(arr).reshape(1) + arr_out = np.empty_like(send) + comm.Allreduce(send, arr_out) + res.append(arr_out.reshape(())) else: - arr_out = arr.copy() - t = numpy_to_MPI_typemap(arr.dtype) - comm.Allreduce([arr, t], [arr_out, t]) + arr_out = np.empty_like(arr) + comm.Allreduce(arr, arr_out) res.append(arr_out) return res + def reduceArrays(arrL, comm, root=0): + """Reduce a list of NumPy scalars/arrays to ``root``. Non-root entries are None.""" res = [] for arr in arrL: if np.isscalar(arr): - arr_out = np.empty((1,),dtype=arr.dtype) if comm.rank==root else None - t = numpy_to_MPI_typemap(arr.dtype) - comm.Reduce([arr, t], [arr_out, t], root=root) - res.append(arr_out[0] if comm.rank==root else None) + send = np.asarray(arr).reshape(1) + arr_out = np.empty_like(send) if comm.rank == root else None + comm.Reduce(send, arr_out, root=root) + res.append(arr_out.reshape(()) if comm.rank == root else None) else: - arr_out = arr.copy() if comm.rank==root else None - t = numpy_to_MPI_typemap(arr.dtype) - comm.Reduce([arr, t], [arr_out, t], root=root) - res.append(arr_out if comm.rank==root else None) + arr_out = np.empty_like(arr) if comm.rank == root else None + comm.Reduce(arr, arr_out, root=root) + res.append(arr_out if comm.rank == root else None) return res + def broadcastArrays(arrL, comm, root=0): + """Broadcast a list of NumPy arrays from ``root`` (in-place).""" for arr in arrL: - t = numpy_to_MPI_typemap(arr.dtype) comm.Bcast(arr, root=root) - From 10c45478d7ac89a5abad1347079729b4f84ff775 Mon Sep 17 00:00:00 2001 From: Neil Lawrence Date: Fri, 13 Nov 2026 19:54:54 +0000 Subject: [PATCH 2/2] CIP-0001: fix MPI parity test to sync parameters. Serial vs partitioned MPI LLs only match when they share parameters; the previous check compared independent random initialisations. --- deepgp/testing/mpi_parity.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/deepgp/testing/mpi_parity.py b/deepgp/testing/mpi_parity.py index 588e694..f9e62d7 100644 --- a/deepgp/testing/mpi_parity.py +++ b/deepgp/testing/mpi_parity.py @@ -8,8 +8,9 @@ mpiexec -n 2 python -m deepgp.testing.mpi_parity -Exits 0 when the MPI log-likelihood on root matches the serial reference -within a relative tolerance; otherwise exits 1. +Builds a partitioned MPI model, copies its parameters onto a serial model with +the full dataset (so the comparison is not confounded by independent random +initialisation), then checks that log-likelihoods agree. """ from __future__ import print_function @@ -62,20 +63,23 @@ def main(): Y_local = np.ascontiguousarray(Y_all[sl]) X_local = np.ascontiguousarray(X_all[sl]) - serial_ll = None + # Independent seeds per rank; root values are broadcast inside DeepGP. + np.random.seed(1000 + rank) + m_mpi = _build_model(Y_local, X_local, mpi_comm=comm) + mpi_ll = float(m_mpi.log_likelihood()) if rank == 0 else None + if rank == 0: + np.random.seed(7) m_serial = _build_model(Y_all, X_all, mpi_comm=None) + # Same parameters as the MPI model (full data, serial inference). + m_serial.param_array[:] = m_mpi.param_array + m_serial.trigger_update() serial_ll = float(m_serial.log_likelihood()) - serial_ll = comm.bcast(serial_ll, root=0) - m_mpi = _build_model(Y_local, X_local, mpi_comm=comm) - # After internal Allreduce of psi statistics, root holds the global objective. - mpi_obj = float(m_mpi.log_likelihood()) if rank == 0 else None - if rank == 0: - rel = abs(mpi_obj - serial_ll) / max(1.0, abs(serial_ll)) + rel = abs(mpi_ll - serial_ll) / max(1.0, abs(serial_ll)) print('serial_ll={:.10g} mpi_ll={:.10g} rel_diff={:.3e}'.format( - serial_ll, mpi_obj, rel)) - if not np.isfinite(mpi_obj) or rel > 1e-5: + serial_ll, mpi_ll, rel)) + if not np.isfinite(mpi_ll) or not np.isfinite(serial_ll) or rel > 1e-5: print('MPI parity check failed', file=sys.stderr) return 1 print('MPI parity check passed')