Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions .coveragerc
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
# Coverage config for PyDeepGP
#
# Measured surface is the maintained serial DeepGP path. The omitted modules
# are known gaps (MPI, MRD, prediction-only SVI ratio) and are tracked as
# follow-up work rather than counted against the CI gate (CIP-0001).
# Measured surface is the maintained serial DeepGP path, including MRD
# (CIP-0002). Remaining omits are MPI / prediction-only SVI helpers (CIP-0001).
[run]
branch = True
source = deepgp
omit =
*/deepgp/testing/*
*/deepgp/layers/mrd.py
*/deepgp/inference/vardtc_parallel.py
*/deepgp/inference/inferenceX.py
*/deepgp/inference/svi_ratio.py
Expand Down
24 changes: 22 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ Currently implemented models are

* Deep GPs
* Variational Auto-encoded Deep GPs (NumPy MLP back-constraint; see CIP-0001)
* Multi-view / MRD Deep GPs (shared latent layer over observation views; see CIP-0002)

Testing
-------
Expand Down Expand Up @@ -41,10 +42,29 @@ tox
```

Coverage is gated on the maintained serial DeepGP path, including the NumPy
encoder (see `.coveragerc`). MPI parallel inference, MRD, and prediction-only
SVI helpers remain omitted until those paths are revived or removed
encoder and MRD (see `.coveragerc`). MPI parallel inference and prediction-only
SVI helpers remain omitted until those paths are removed or revived
(CIP-0001 Phases 2–4).

Multi-view / MRD (CIP-0002)
---------------------------

Pass a list of observation matrices and a matching list of output dimensions.
The observation-layer kernels must be a **list of kernels** (one per view)::

m = deepgp.DeepGP(
[[D1, D2], Q_shared, Q_top],
Y=[Y1, Y2],
kernels=[[kern_v1, kern_v2], kern_top],
num_inducing=40,
back_constraint=False,
)

With ``back_constraint=True``, ``encoder_dims[0]`` is a list of per-view MLP
hidden widths (e.g. ``[[[4], [4]], [3]]``); higher indices feed the usual
``EncoderLayer`` stack for layers above the MRD leaf. See
``examples/example_mrd.py``.

MPI (deprecated)
----------------

Expand Down
18 changes: 9 additions & 9 deletions cip/cip0002.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ author: Neil Lawrence
created: "2026-11-13"
id: "0002"
last_updated: "2026-11-13"
status: Proposed
status: In Progress
compressed: false
related_requirements: []
related_cips:
Expand All @@ -23,8 +23,8 @@ title: Revive Manifold Relevance Determination (MRD) for multi-view DeepGPs
## Status

- [x] Proposed - Initial idea documented (open for review)
- [ ] Accepted - Approved, ready to start work
- [ ] In Progress - Actively being implemented
- [x] Accepted - Approved; Phase C (back-constraint) included in first implementation PR
- [x] In Progress - Actively being implemented
- [ ] Implemented - Work complete, awaiting verification
- [ ] Closed - Verified and complete
- [ ] Rejected - Will not be implemented
Expand Down Expand Up @@ -187,9 +187,9 @@ in comments stay out of scope unless tests prove they already work.
- Remove `mrd.py` from `.coveragerc` omit; ensure CI coverage does not
regress the new tests.

3. **Phase C — Back-constraint + encoder (optional in first PR)**
3. **Phase C — Back-constraint + encoder (in first implementation PR)**
- Fix MLP import (D4); add one back-constrained MRD test.
- Skip or xfail if encoder dims make the smoke test too heavy.
- Keep encoder dims small for CI smoke.

4. **Phase D — Align with CIP-0001 R20**
- Remove or no-op MPI branches in `mrd.py` consistent with Phase 4 policy.
Expand Down Expand Up @@ -231,11 +231,11 @@ frontmatter.

## Implementation Status

- [ ] Phase A: bound dtype + init fix + unit tests
- [ ] Phase B: docs, coverage omit removal, high-five example
- [ ] Phase C: back-constraint encoder path (optional first PR)
- [x] Phase A: bound dtype + init fix + unit tests
- [x] Phase B: docs, coverage omit removal, `examples/example_mrd.py`
- [x] Phase C: back-constraint encoder path (included in first implementation PR)
- [ ] Phase D: MPI cleanup aligned with CIP-0001 R20
- [ ] CIP review → Accepted → backlog tasks
- [x] CIP review → Accepted (2026-11-13); implementation on `cip0002-mrd-revival`

## References

Expand Down
54 changes: 39 additions & 15 deletions deepgp/layers/mrd.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ def __init__(self, Y, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_var
self.scale = 1.

if back_constraint:
from .mlp import MLP
from copy import deepcopy
from deepgp.encoder.mlp import MLP
self.encoder = MLP([dim_down, int((dim_down+dim_up)*2./3.), int((dim_down+dim_up)/3.), dim_up] if MLP_dims is None else [dim_down]+deepcopy(MLP_dims)+[dim_up])
X = self.encoder.predict(Y.mean.values if isinstance(Y, VariationalPosterior) else Y)
X_variance = 0.0001*np.ones(X.shape)
Expand All @@ -31,7 +31,10 @@ def __init__(self, Y, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_var
self.back_constraint = False

if Z is None:
Z = np.random.rand(num_inducing, dim_up)*2-1. #np.random.permutation(X.copy())[:num_inducing]
# Match Layer.__init__: random inducing inputs in [-1, 1].
Z = np.random.rand(num_inducing, dim_up) * 2 - 1
if X is None:
raise ValueError("MRDView requires X (or back_constraint=True to encode it from Y)")
assert Z.shape[1] == X.shape[1]

if likelihood is None: likelihood = likelihoods.Gaussian(variance=Y.var()*0.01)
Expand Down Expand Up @@ -111,6 +114,11 @@ def __init__(self, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_varian
Y = self.Y if self.layer_lower is None else self.layer_lower.X
assert isinstance(dim_down, list) or isinstance(dim_down, tuple)
assert isinstance(kernel, list) and len(kernel)==len(dim_down), "The number of kernels has to be equal to the number of input modalities!"
if MLP_dims is not None:
assert isinstance(MLP_dims, (list, tuple)) and len(MLP_dims) == len(dim_down), (
"MLP_dims must be a list with one entry per view (each entry a list of "
"hidden widths, or None for the default encoder width)."
)
super(MRDLayer, self).__init__(name=name)
self.mpi_comm, self.mpi_root = mpi_comm, mpi_root

Expand Down Expand Up @@ -190,16 +198,30 @@ def update_encoder_gradients(self):
v.update_encoder_gradients()

def parameters_changed(self):
self._aggregate_qX()
self.X.mean.gradient[:] = 0
self.X.variance.gradient[:] = 0
for v in self.views:
v.update_gradients()
self.X.mean.gradient += v.Xv.mean.gradient
self.X.variance.gradient += v.Xv.variance.gradient

self._log_marginal_likelihood = np.sum([v._log_marginal_likelihood for v in self.views])
self.update_qX_gradients()
# Writing into self.X during aggregation notifies observers; guard re-entry.
if getattr(self, '_in_parameters_changed', False):
return
self._in_parameters_changed = True
try:
if self.back_constraint:
# Refresh each view's Xv from its encoder before aggregating q(X).
for v in self.views:
v.parameters_changed()
self._aggregate_qX()
self.X.mean.gradient[:] = 0
self.X.variance.gradient[:] = 0
for v in self.views:
v.update_gradients()
self.X.mean.gradient += v.Xv.mean.gradient
self.X.variance.gradient += v.Xv.variance.gradient

# Keep shape (1,) like SparseGP layers so DeepGP.parameters_changed can sum.
self._log_marginal_likelihood = np.sum(
[np.asarray(v._log_marginal_likelihood) for v in self.views], axis=0
)
self.update_qX_gradients()
finally:
self._in_parameters_changed = False

class ObservedMRDLayer(MRDLayer):

Expand All @@ -226,19 +248,21 @@ def __init__(self, dim_down, dim_up, Y, X=None, X_variance=None, Z=None, num_ind
def _init_X(self, Ylist, input_dim, init='PCA'):
if Ylist is None:
Ylist = self.Ylist
if init in "PCA_concat":
# Equality checks (legacy code used `init in "PCA_concat"`, which is
# substring/character membership and mis-routes several spellings).
if init in ('PCA', 'PCA_concat'):
print('# Initializing latent space with: PCA_concat')
X, fracs = initialize_latent('PCA', input_dim, np.hstack(Ylist))
fracs = [fracs]*len(Ylist)
elif init in "PCA_single":
elif init == 'PCA_single':
print('# Initializing latent space with: PCA_single')
X = np.zeros((Ylist[0].shape[0], input_dim))
fracs = []
for qs, Y in zip(np.array_split(np.arange(input_dim), len(Ylist)), Ylist):
x,frcs = initialize_latent('PCA', len(qs), Y)
X[:, qs] = x
fracs.append(frcs)
else: # init == 'random':
else: # 'random', 'rand', or unknown → random
print('# Initializing latent space with: random')
X = np.random.randn(Ylist[0].shape[0], input_dim)
fracs = X.var(0)
Expand Down
20 changes: 16 additions & 4 deletions deepgp/models/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def __init__(self, nDims, Y, X=None, num_inducing=10,

# Manifold relevance determination
mrd_flag = isinstance(Y,(list,tuple))
self._mrd = mrd_flag
if mrd_flag:
assert isinstance(nDims[0], (list,tuple)), "The data dimension for outputs has to be a list or tuple for MRD!"
else:
Expand Down Expand Up @@ -105,15 +106,17 @@ def __init__(self, nDims, Y, X=None, num_inducing=10,
for i in range(self.nLayers):
if i==0:
if mrd_flag:#isinstance(nDims[0], (list,tuple)):
# MRD
#raise NotImplementedError()
# MRD: encoder_dims[0] is a list of per-view MLP hidden sizes
# (see CIP-0002). Higher indices feed EncoderLayer as usual.
mrd_mlp_dims = encoder_dims[0] if encoder_dims is not None else None
self.layers.append(ObservedMRDLayer(nDims[0],
nDims[1], Y,
X=Xs[i], likelihood=likelihood,
num_inducing=num_inducing[i],
init=inits[i],
kernel=kernels[i] if kernels is not None else None,
back_constraint=back_constraint,
MLP_dims=mrd_mlp_dims,
mpi_comm=mpi_comm,
mpi_root=mpi_root))
else:
Expand Down Expand Up @@ -194,8 +197,11 @@ def __init__(self, nDims, Y, X=None, num_inducing=10,
mpi_root=mpi_root,
name='enclayer_'+str(i)) for i in range(self.nLayers-1,0,-1)]
else:
# For MRD, layer 0 owns per-view encoders inside ObservedMRDLayer;
# EncoderLayer starts at layer 1 (CIP-0002 Phase C).
enc_start = 1 if mrd_flag else 0
self.enclayers = [ EncoderLayer(self.layers[i], direction=direction, encoder_dims=encoder_dims[i] if encoder_dims is not None else None, mpi_comm = mpi_comm, mpi_root=mpi_root,
name='enclayer_'+str(i)) for i in range(self.nLayers)]
name='enclayer_'+str(i)) for i in range(enc_start, self.nLayers)]
self.link_parameters(*(self.enclayers+self.layers))
else:
self.link_parameters(*self.layers)
Expand Down Expand Up @@ -271,8 +277,14 @@ def log_likelihood(self):

def parameters_changed(self):
if not self.auto_update: [l.update_layer() for l in self.layers]
self._log_marginal_likelihood = np.sum([l._log_marginal_likelihood for l in self.layers])
# Coerce to a Python float so mixed scalar / shape-(1,) layer bounds
# (historical MRD vs SparseGP) cannot raise under NumPy (CIP-0002 D1).
self._log_marginal_likelihood = float(np.sum(
[np.asarray(l._log_marginal_likelihood).sum() for l in self.layers]
))
if self.back_constraint:
if self._mrd and hasattr(self.layers[0], 'update_encoder_gradients'):
self.layers[0].update_encoder_gradients()
[l.update_gradients() for l in self.enclayers[::-1]]
if self.mpi_comm is not None:
[l._gather_gradients() for l in self.enclayers]
Expand Down
127 changes: 127 additions & 0 deletions deepgp/testing/test_mrd.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
# Copyright (c) 2015-2016, the authors (see AUTHORS.txt).
# Licensed under the BSD 3-clause license (see LICENSE.txt)

"""CIP-0002: multi-view / MRD DeepGP construction and optimisation tests."""

import unittest

import numpy as np
import GPy

import deepgp
from deepgp.layers.mrd import ObservedMRDLayer


def _correlated_views(n=24, q=3, d=6, seed=0):
rng = np.random.RandomState(seed)
X = rng.randn(n, q)
Y1 = X @ rng.randn(q, d) + 0.05 * rng.randn(n, d)
Y2 = X @ rng.randn(q, d) + 0.05 * rng.randn(n, d)
return Y1, Y2


def _mrd_kernels(q_shared, q_top=None):
k_obs = [
GPy.kern.RBF(q_shared, ARD=True) + GPy.kern.Bias(q_shared),
GPy.kern.RBF(q_shared, ARD=True) + GPy.kern.Bias(q_shared),
]
if q_top is None:
return k_obs
return [k_obs, GPy.kern.RBF(q_top, ARD=True) + GPy.kern.Bias(q_top)]


class MRDConstructTest(unittest.TestCase):
def test_two_view_constructs_finite_bound(self):
Y1, Y2 = _correlated_views()
q1, q2 = 4, 3
m = deepgp.DeepGP(
[[Y1.shape[1], Y2.shape[1]], q1, q2],
[Y1, Y2],
kernels=_mrd_kernels(q1, q2),
num_inducing=8,
back_constraint=False,
)
ll = float(m.log_likelihood())
self.assertTrue(np.isfinite(ll))
self.assertEqual(np.asarray(m.layers[0]._log_marginal_likelihood).shape, (1,))

def test_optimize_smoke(self):
Y1, Y2 = _correlated_views(seed=1)
q1, q2 = 4, 3
m = deepgp.DeepGP(
[[Y1.shape[1], Y2.shape[1]], q1, q2],
[Y1, Y2],
kernels=_mrd_kernels(q1, q2),
num_inducing=8,
back_constraint=False,
)
before = float(m.log_likelihood())
m.optimize(max_iters=5, messages=False)
after = float(m.log_likelihood())
self.assertTrue(np.isfinite(after))
self.assertGreaterEqual(after, before - 1.0)

def test_kernel_list_required(self):
Y1, Y2 = _correlated_views(n=12, d=4, seed=2)
q1 = 3
with self.assertRaises(AssertionError):
deepgp.DeepGP(
[[Y1.shape[1], Y2.shape[1]], q1],
[Y1, Y2],
kernels=[GPy.kern.RBF(q1, ARD=True)], # not a per-view list
num_inducing=5,
back_constraint=False,
)


class MRDInitTest(unittest.TestCase):
def test_pca_concat_and_single(self):
Y1, Y2 = _correlated_views(n=20, d=5, seed=3)
layer = ObservedMRDLayer.__new__(ObservedMRDLayer)
Xc, _ = ObservedMRDLayer._init_X(layer, [Y1, Y2], 4, init='PCA_concat')
self.assertEqual(Xc.shape, (20, 4))
Xs, _ = ObservedMRDLayer._init_X(layer, [Y1, Y2], 4, init='PCA_single')
self.assertEqual(Xs.shape, (20, 4))
Xr, _ = ObservedMRDLayer._init_X(layer, [Y1, Y2], 4, init='random')
self.assertEqual(Xr.shape, (20, 4))
# Legacy alias
Xp, _ = ObservedMRDLayer._init_X(layer, [Y1, Y2], 4, init='PCA')
self.assertEqual(Xp.shape, (20, 4))


class MRDBackConstraintTest(unittest.TestCase):
def test_back_constraint_constructs_and_optimizes(self):
Y1, Y2 = _correlated_views(n=18, d=5, seed=4)
q1, q2 = 3, 2
# encoder_dims[0]: per-view MLP hidden widths; encoder_dims[1]: top layer
m = deepgp.DeepGP(
[[Y1.shape[1], Y2.shape[1]], q1, q2],
[Y1, Y2],
kernels=_mrd_kernels(q1, q2),
num_inducing=6,
back_constraint=True,
encoder_dims=[[[4], [4]], [3]],
)
self.assertTrue(m._mrd)
self.assertTrue(hasattr(m.layers[0].views[0], 'encoder'))
ll = float(m.log_likelihood())
self.assertTrue(np.isfinite(ll))
m.optimize(max_iters=3, messages=False)
self.assertTrue(np.isfinite(float(m.log_likelihood())))

def test_mlp_dims_length_checked(self):
Y1, Y2 = _correlated_views(n=10, d=4, seed=5)
q1 = 3
with self.assertRaises(AssertionError):
deepgp.DeepGP(
[[Y1.shape[1], Y2.shape[1]], q1],
[Y1, Y2],
kernels=_mrd_kernels(q1),
num_inducing=4,
back_constraint=True,
encoder_dims=[[[3]]], # only one view entry for two views
)


if __name__ == '__main__':
unittest.main()
Loading
Loading