diff --git a/.coveragerc b/.coveragerc index 6cec78c..afaa1a8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,14 +1,12 @@ # Coverage config for PyDeepGP # -# Measured surface is the maintained serial DeepGP path. The omitted modules -# are known gaps (MPI, MRD, prediction-only SVI ratio) and are tracked as -# follow-up work rather than counted against the CI gate (CIP-0001). +# Measured surface is the maintained serial DeepGP path, including MRD +# (CIP-0002). Remaining omits are MPI / prediction-only SVI helpers (CIP-0001). [run] branch = True source = deepgp omit = */deepgp/testing/* - */deepgp/layers/mrd.py */deepgp/inference/vardtc_parallel.py */deepgp/inference/inferenceX.py */deepgp/inference/svi_ratio.py diff --git a/README.md b/README.md index 3150164..f4f4a72 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Currently implemented models are * Deep GPs * Variational Auto-encoded Deep GPs (NumPy MLP back-constraint; see CIP-0001) +* Multi-view / MRD Deep GPs (shared latent layer over observation views; see CIP-0002) Testing ------- @@ -41,10 +42,29 @@ tox ``` Coverage is gated on the maintained serial DeepGP path, including the NumPy -encoder (see `.coveragerc`). MPI parallel inference, MRD, and prediction-only -SVI helpers remain omitted until those paths are revived or removed +encoder and MRD (see `.coveragerc`). MPI parallel inference and prediction-only +SVI helpers remain omitted until those paths are removed or revived (CIP-0001 Phases 2–4). +Multi-view / MRD (CIP-0002) +--------------------------- + +Pass a list of observation matrices and a matching list of output dimensions. +The observation-layer kernels must be a **list of kernels** (one per view):: + + m = deepgp.DeepGP( + [[D1, D2], Q_shared, Q_top], + Y=[Y1, Y2], + kernels=[[kern_v1, kern_v2], kern_top], + num_inducing=40, + back_constraint=False, + ) + +With ``back_constraint=True``, ``encoder_dims[0]`` is a list of per-view MLP +hidden widths (e.g. ``[[[4], [4]], [3]]``); higher indices feed the usual +``EncoderLayer`` stack for layers above the MRD leaf. See +``examples/example_mrd.py``. + MPI (deprecated) ---------------- diff --git a/cip/cip0002.md b/cip/cip0002.md index f60baad..5611752 100644 --- a/cip/cip0002.md +++ b/cip/cip0002.md @@ -3,7 +3,7 @@ author: Neil Lawrence created: "2026-11-13" id: "0002" last_updated: "2026-11-13" -status: Proposed +status: In Progress compressed: false related_requirements: [] related_cips: @@ -23,8 +23,8 @@ title: Revive Manifold Relevance Determination (MRD) for multi-view DeepGPs ## Status - [x] Proposed - Initial idea documented (open for review) -- [ ] Accepted - Approved, ready to start work -- [ ] In Progress - Actively being implemented +- [x] Accepted - Approved; Phase C (back-constraint) included in first implementation PR +- [x] In Progress - Actively being implemented - [ ] Implemented - Work complete, awaiting verification - [ ] Closed - Verified and complete - [ ] Rejected - Will not be implemented @@ -187,9 +187,9 @@ in comments stay out of scope unless tests prove they already work. - Remove `mrd.py` from `.coveragerc` omit; ensure CI coverage does not regress the new tests. -3. **Phase C — Back-constraint + encoder (optional in first PR)** +3. **Phase C — Back-constraint + encoder (in first implementation PR)** - Fix MLP import (D4); add one back-constrained MRD test. - - Skip or xfail if encoder dims make the smoke test too heavy. + - Keep encoder dims small for CI smoke. 4. **Phase D — Align with CIP-0001 R20** - Remove or no-op MPI branches in `mrd.py` consistent with Phase 4 policy. @@ -231,11 +231,11 @@ frontmatter. ## Implementation Status -- [ ] Phase A: bound dtype + init fix + unit tests -- [ ] Phase B: docs, coverage omit removal, high-five example -- [ ] Phase C: back-constraint encoder path (optional first PR) +- [x] Phase A: bound dtype + init fix + unit tests +- [x] Phase B: docs, coverage omit removal, `examples/example_mrd.py` +- [x] Phase C: back-constraint encoder path (included in first implementation PR) - [ ] Phase D: MPI cleanup aligned with CIP-0001 R20 -- [ ] CIP review → Accepted → backlog tasks +- [x] CIP review → Accepted (2026-11-13); implementation on `cip0002-mrd-revival` ## References diff --git a/deepgp/layers/mrd.py b/deepgp/layers/mrd.py index 96b3882..8b9ea89 100644 --- a/deepgp/layers/mrd.py +++ b/deepgp/layers/mrd.py @@ -21,8 +21,8 @@ def __init__(self, Y, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_var self.scale = 1. if back_constraint: - from .mlp import MLP from copy import deepcopy + from deepgp.encoder.mlp import MLP self.encoder = MLP([dim_down, int((dim_down+dim_up)*2./3.), int((dim_down+dim_up)/3.), dim_up] if MLP_dims is None else [dim_down]+deepcopy(MLP_dims)+[dim_up]) X = self.encoder.predict(Y.mean.values if isinstance(Y, VariationalPosterior) else Y) X_variance = 0.0001*np.ones(X.shape) @@ -31,7 +31,10 @@ def __init__(self, Y, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_var self.back_constraint = False if Z is None: - Z = np.random.rand(num_inducing, dim_up)*2-1. #np.random.permutation(X.copy())[:num_inducing] + # Match Layer.__init__: random inducing inputs in [-1, 1]. + Z = np.random.rand(num_inducing, dim_up) * 2 - 1 + if X is None: + raise ValueError("MRDView requires X (or back_constraint=True to encode it from Y)") assert Z.shape[1] == X.shape[1] if likelihood is None: likelihood = likelihoods.Gaussian(variance=Y.var()*0.01) @@ -111,6 +114,11 @@ def __init__(self, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_varian Y = self.Y if self.layer_lower is None else self.layer_lower.X assert isinstance(dim_down, list) or isinstance(dim_down, tuple) assert isinstance(kernel, list) and len(kernel)==len(dim_down), "The number of kernels has to be equal to the number of input modalities!" + if MLP_dims is not None: + assert isinstance(MLP_dims, (list, tuple)) and len(MLP_dims) == len(dim_down), ( + "MLP_dims must be a list with one entry per view (each entry a list of " + "hidden widths, or None for the default encoder width)." + ) super(MRDLayer, self).__init__(name=name) self.mpi_comm, self.mpi_root = mpi_comm, mpi_root @@ -190,16 +198,30 @@ def update_encoder_gradients(self): v.update_encoder_gradients() def parameters_changed(self): - self._aggregate_qX() - self.X.mean.gradient[:] = 0 - self.X.variance.gradient[:] = 0 - for v in self.views: - v.update_gradients() - self.X.mean.gradient += v.Xv.mean.gradient - self.X.variance.gradient += v.Xv.variance.gradient - - self._log_marginal_likelihood = np.sum([v._log_marginal_likelihood for v in self.views]) - self.update_qX_gradients() + # Writing into self.X during aggregation notifies observers; guard re-entry. + if getattr(self, '_in_parameters_changed', False): + return + self._in_parameters_changed = True + try: + if self.back_constraint: + # Refresh each view's Xv from its encoder before aggregating q(X). + for v in self.views: + v.parameters_changed() + self._aggregate_qX() + self.X.mean.gradient[:] = 0 + self.X.variance.gradient[:] = 0 + for v in self.views: + v.update_gradients() + self.X.mean.gradient += v.Xv.mean.gradient + self.X.variance.gradient += v.Xv.variance.gradient + + # Keep shape (1,) like SparseGP layers so DeepGP.parameters_changed can sum. + self._log_marginal_likelihood = np.sum( + [np.asarray(v._log_marginal_likelihood) for v in self.views], axis=0 + ) + self.update_qX_gradients() + finally: + self._in_parameters_changed = False class ObservedMRDLayer(MRDLayer): @@ -226,11 +248,13 @@ def __init__(self, dim_down, dim_up, Y, X=None, X_variance=None, Z=None, num_ind def _init_X(self, Ylist, input_dim, init='PCA'): if Ylist is None: Ylist = self.Ylist - if init in "PCA_concat": + # Equality checks (legacy code used `init in "PCA_concat"`, which is + # substring/character membership and mis-routes several spellings). + if init in ('PCA', 'PCA_concat'): print('# Initializing latent space with: PCA_concat') X, fracs = initialize_latent('PCA', input_dim, np.hstack(Ylist)) fracs = [fracs]*len(Ylist) - elif init in "PCA_single": + elif init == 'PCA_single': print('# Initializing latent space with: PCA_single') X = np.zeros((Ylist[0].shape[0], input_dim)) fracs = [] @@ -238,7 +262,7 @@ def _init_X(self, Ylist, input_dim, init='PCA'): x,frcs = initialize_latent('PCA', len(qs), Y) X[:, qs] = x fracs.append(frcs) - else: # init == 'random': + else: # 'random', 'rand', or unknown → random print('# Initializing latent space with: random') X = np.random.randn(Ylist[0].shape[0], input_dim) fracs = X.var(0) diff --git a/deepgp/models/model.py b/deepgp/models/model.py index 7d5b1c4..f631299 100644 --- a/deepgp/models/model.py +++ b/deepgp/models/model.py @@ -59,6 +59,7 @@ def __init__(self, nDims, Y, X=None, num_inducing=10, # Manifold relevance determination mrd_flag = isinstance(Y,(list,tuple)) + self._mrd = mrd_flag if mrd_flag: assert isinstance(nDims[0], (list,tuple)), "The data dimension for outputs has to be a list or tuple for MRD!" else: @@ -105,8 +106,9 @@ def __init__(self, nDims, Y, X=None, num_inducing=10, for i in range(self.nLayers): if i==0: if mrd_flag:#isinstance(nDims[0], (list,tuple)): - # MRD - #raise NotImplementedError() + # MRD: encoder_dims[0] is a list of per-view MLP hidden sizes + # (see CIP-0002). Higher indices feed EncoderLayer as usual. + mrd_mlp_dims = encoder_dims[0] if encoder_dims is not None else None self.layers.append(ObservedMRDLayer(nDims[0], nDims[1], Y, X=Xs[i], likelihood=likelihood, @@ -114,6 +116,7 @@ def __init__(self, nDims, Y, X=None, num_inducing=10, init=inits[i], kernel=kernels[i] if kernels is not None else None, back_constraint=back_constraint, + MLP_dims=mrd_mlp_dims, mpi_comm=mpi_comm, mpi_root=mpi_root)) else: @@ -194,8 +197,11 @@ def __init__(self, nDims, Y, X=None, num_inducing=10, mpi_root=mpi_root, name='enclayer_'+str(i)) for i in range(self.nLayers-1,0,-1)] else: + # For MRD, layer 0 owns per-view encoders inside ObservedMRDLayer; + # EncoderLayer starts at layer 1 (CIP-0002 Phase C). + enc_start = 1 if mrd_flag else 0 self.enclayers = [ EncoderLayer(self.layers[i], direction=direction, encoder_dims=encoder_dims[i] if encoder_dims is not None else None, mpi_comm = mpi_comm, mpi_root=mpi_root, - name='enclayer_'+str(i)) for i in range(self.nLayers)] + name='enclayer_'+str(i)) for i in range(enc_start, self.nLayers)] self.link_parameters(*(self.enclayers+self.layers)) else: self.link_parameters(*self.layers) @@ -271,8 +277,14 @@ def log_likelihood(self): def parameters_changed(self): if not self.auto_update: [l.update_layer() for l in self.layers] - self._log_marginal_likelihood = np.sum([l._log_marginal_likelihood for l in self.layers]) + # Coerce to a Python float so mixed scalar / shape-(1,) layer bounds + # (historical MRD vs SparseGP) cannot raise under NumPy (CIP-0002 D1). + self._log_marginal_likelihood = float(np.sum( + [np.asarray(l._log_marginal_likelihood).sum() for l in self.layers] + )) if self.back_constraint: + if self._mrd and hasattr(self.layers[0], 'update_encoder_gradients'): + self.layers[0].update_encoder_gradients() [l.update_gradients() for l in self.enclayers[::-1]] if self.mpi_comm is not None: [l._gather_gradients() for l in self.enclayers] diff --git a/deepgp/testing/test_mrd.py b/deepgp/testing/test_mrd.py new file mode 100644 index 0000000..99f9a83 --- /dev/null +++ b/deepgp/testing/test_mrd.py @@ -0,0 +1,127 @@ +# Copyright (c) 2015-2016, the authors (see AUTHORS.txt). +# Licensed under the BSD 3-clause license (see LICENSE.txt) + +"""CIP-0002: multi-view / MRD DeepGP construction and optimisation tests.""" + +import unittest + +import numpy as np +import GPy + +import deepgp +from deepgp.layers.mrd import ObservedMRDLayer + + +def _correlated_views(n=24, q=3, d=6, seed=0): + rng = np.random.RandomState(seed) + X = rng.randn(n, q) + Y1 = X @ rng.randn(q, d) + 0.05 * rng.randn(n, d) + Y2 = X @ rng.randn(q, d) + 0.05 * rng.randn(n, d) + return Y1, Y2 + + +def _mrd_kernels(q_shared, q_top=None): + k_obs = [ + GPy.kern.RBF(q_shared, ARD=True) + GPy.kern.Bias(q_shared), + GPy.kern.RBF(q_shared, ARD=True) + GPy.kern.Bias(q_shared), + ] + if q_top is None: + return k_obs + return [k_obs, GPy.kern.RBF(q_top, ARD=True) + GPy.kern.Bias(q_top)] + + +class MRDConstructTest(unittest.TestCase): + def test_two_view_constructs_finite_bound(self): + Y1, Y2 = _correlated_views() + q1, q2 = 4, 3 + m = deepgp.DeepGP( + [[Y1.shape[1], Y2.shape[1]], q1, q2], + [Y1, Y2], + kernels=_mrd_kernels(q1, q2), + num_inducing=8, + back_constraint=False, + ) + ll = float(m.log_likelihood()) + self.assertTrue(np.isfinite(ll)) + self.assertEqual(np.asarray(m.layers[0]._log_marginal_likelihood).shape, (1,)) + + def test_optimize_smoke(self): + Y1, Y2 = _correlated_views(seed=1) + q1, q2 = 4, 3 + m = deepgp.DeepGP( + [[Y1.shape[1], Y2.shape[1]], q1, q2], + [Y1, Y2], + kernels=_mrd_kernels(q1, q2), + num_inducing=8, + back_constraint=False, + ) + before = float(m.log_likelihood()) + m.optimize(max_iters=5, messages=False) + after = float(m.log_likelihood()) + self.assertTrue(np.isfinite(after)) + self.assertGreaterEqual(after, before - 1.0) + + def test_kernel_list_required(self): + Y1, Y2 = _correlated_views(n=12, d=4, seed=2) + q1 = 3 + with self.assertRaises(AssertionError): + deepgp.DeepGP( + [[Y1.shape[1], Y2.shape[1]], q1], + [Y1, Y2], + kernels=[GPy.kern.RBF(q1, ARD=True)], # not a per-view list + num_inducing=5, + back_constraint=False, + ) + + +class MRDInitTest(unittest.TestCase): + def test_pca_concat_and_single(self): + Y1, Y2 = _correlated_views(n=20, d=5, seed=3) + layer = ObservedMRDLayer.__new__(ObservedMRDLayer) + Xc, _ = ObservedMRDLayer._init_X(layer, [Y1, Y2], 4, init='PCA_concat') + self.assertEqual(Xc.shape, (20, 4)) + Xs, _ = ObservedMRDLayer._init_X(layer, [Y1, Y2], 4, init='PCA_single') + self.assertEqual(Xs.shape, (20, 4)) + Xr, _ = ObservedMRDLayer._init_X(layer, [Y1, Y2], 4, init='random') + self.assertEqual(Xr.shape, (20, 4)) + # Legacy alias + Xp, _ = ObservedMRDLayer._init_X(layer, [Y1, Y2], 4, init='PCA') + self.assertEqual(Xp.shape, (20, 4)) + + +class MRDBackConstraintTest(unittest.TestCase): + def test_back_constraint_constructs_and_optimizes(self): + Y1, Y2 = _correlated_views(n=18, d=5, seed=4) + q1, q2 = 3, 2 + # encoder_dims[0]: per-view MLP hidden widths; encoder_dims[1]: top layer + m = deepgp.DeepGP( + [[Y1.shape[1], Y2.shape[1]], q1, q2], + [Y1, Y2], + kernels=_mrd_kernels(q1, q2), + num_inducing=6, + back_constraint=True, + encoder_dims=[[[4], [4]], [3]], + ) + self.assertTrue(m._mrd) + self.assertTrue(hasattr(m.layers[0].views[0], 'encoder')) + ll = float(m.log_likelihood()) + self.assertTrue(np.isfinite(ll)) + m.optimize(max_iters=3, messages=False) + self.assertTrue(np.isfinite(float(m.log_likelihood()))) + + def test_mlp_dims_length_checked(self): + Y1, Y2 = _correlated_views(n=10, d=4, seed=5) + q1 = 3 + with self.assertRaises(AssertionError): + deepgp.DeepGP( + [[Y1.shape[1], Y2.shape[1]], q1], + [Y1, Y2], + kernels=_mrd_kernels(q1), + num_inducing=4, + back_constraint=True, + encoder_dims=[[[3]]], # only one view entry for two views + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/examples/example_mrd.py b/examples/example_mrd.py new file mode 100644 index 0000000..ef4ef24 --- /dev/null +++ b/examples/example_mrd.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +"""Multi-view DeepGP (MRD) demo — CIP-0002. + +Two correlated observation views share a parent latent layer. This is the +architecture used for the Damianou & Lawrence (2013) high-five experiment; +see also ``examples/damianou2013/highfive_shared.py`` when that suite is +present. +""" + +from __future__ import annotations + +import numpy as np +import GPy +import deepgp + +np.random.seed(42) + +# Synthetic shared-latent data (replace with two MOCAP subjects for high-five) +n, q_true, d = 60, 3, 20 +X_true = np.random.randn(n, q_true) +Y1 = X_true @ np.random.randn(q_true, d) + 0.05 * np.random.randn(n, d) +Y2 = X_true @ np.random.randn(q_true, d) + 0.05 * np.random.randn(n, d) +Y1 = (Y1 - Y1.mean(0)) / (Y1.std() + 1e-12) +Y2 = (Y2 - Y2.mean(0)) / (Y2.std() + 1e-12) + +Q_shared, Q_top = 5, 3 +num_inducing = 25 + +kern_views = [ + GPy.kern.RBF(Q_shared, ARD=True) + GPy.kern.Bias(Q_shared), + GPy.kern.RBF(Q_shared, ARD=True) + GPy.kern.Bias(Q_shared), +] +kern_top = GPy.kern.RBF(Q_top, ARD=True) + GPy.kern.Bias(Q_top) + +m = deepgp.DeepGP( + [[Y1.shape[1], Y2.shape[1]], Q_shared, Q_top], + [Y1, Y2], + kernels=[kern_views, kern_top], + num_inducing=num_inducing, + back_constraint=False, +) + +from GPy.core.parameterization.variational import NormalPosterior, VariationalPosterior + + +def _out_var(y): + if isinstance(y, (NormalPosterior, VariationalPosterior)): + return float(np.asarray(y.mean).var()) + return float(np.asarray(y).var()) + + +for layer in m.layers: + if hasattr(layer, "views"): + for view in layer.views: + view.Gaussian_noise.variance = _out_var(view.Y) * 0.01 + view.Gaussian_noise.variance.fix() + else: + layer.Gaussian_noise.variance = _out_var(layer.Y) * 0.01 + layer.Gaussian_noise.variance.fix() + +m.optimize(max_iters=200, messages=True) +for layer in m.layers: + if hasattr(layer, "views"): + for view in layer.views: + view.Gaussian_noise.variance.unfix() + else: + layer.Gaussian_noise.variance.unfix() +m.optimize(max_iters=500, messages=True) + +print(f"MRD DeepGP variational bound: {float(m.log_likelihood()):.3f}") +for i, view in enumerate(m.layers[0].views): + ls = view.kern.rbf.lengthscale.values if hasattr(view.kern, "rbf") else view.kern.lengthscale.values + print(f"View {i} ARD (1/lengthscale): {1.0 / np.maximum(ls, 1e-12)}")