Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .coveragerc
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ branch = True
source = deepgp
omit =
*/deepgp/testing/*
*/deepgp/encoder/*
*/deepgp/layers/enclayer.py
*/deepgp/layers/mrd.py
*/deepgp/inference/vardtc_parallel.py
*/deepgp/inference/inferenceX.py
Expand Down
10 changes: 4 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ The Python Implementation of Deep Gaussian Processes
Currently implemented models are

* Deep GPs
* Variational Auto-encoded Deep GPs
* Variational Auto-encoded Deep GPs (NumPy MLP back-constraint; see CIP-0001)

Testing
-------
Expand All @@ -33,8 +33,6 @@ Or with [tox](https://tox.readthedocs.io/):
tox
```

Back-constraint encoder tests are skipped unless Theano is installed.

Coverage is gated on the maintained serial DeepGP path (see `.coveragerc`).
MPI parallel inference, MRD, Theano encoder, and prediction-only SVI helpers
are omitted from the CI percentage until those paths are revived.
Coverage is gated on the maintained serial DeepGP path, including the NumPy
encoder (see `.coveragerc`). MPI parallel inference, MRD, and prediction-only
SVI helpers remain omitted until those paths are revived (CIP-0001 Phases 2–4).
14 changes: 7 additions & 7 deletions cip/cip0001.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
author: Neil Lawrence
created: "2026-11-13"
id: "0001"
last_updated: "2026-11-13"
status: Proposed
last_updated: "2026-11-13" # Phase 1 implementation started
status: In Progress
compressed: false
related_requirements: []
related_cips: []
Expand All @@ -26,8 +26,8 @@ title: Replace Theano and MPI with modern NumPy
## Status

- [x] Proposed - Initial idea documented (open for Discussion feedback)
- [ ] Accepted - Approved, ready to start work
- [ ] In Progress - Actively being implemented
- [x] Accepted - Phase 1 started on branch `cip0001-numpy-encoder` (regularisation: keep no-op + FutureWarning)
- [x] In Progress - Phase 1 (NumPy encoder) under way
- [ ] Implemented - Work complete, awaiting verification
- [ ] Closed - Verified and complete
- [ ] Rejected - Will not be implemented
Expand Down Expand Up @@ -254,9 +254,9 @@ pluggable reducer in R14, so the MPI-specific code shrinks to that one place.
- [x] Prototype: NumPy encoder passes all 8 `test_model_basic` tests (incl. BC references)
- [x] Prototype: chunked/threaded psi benchmark (2-core)
- [ ] Discussion opened
- [ ] Accept: regularisation decision; MPI deprecation and removal timeline
- [ ] Phase 1: NumPy encoder (R1–R7)
- [ ] Phase 2: MPI correctness and deprecation (R8–R11)
- [x] Accept (partial): regularisation kept as no-op with `FutureWarning` (R2)
- [x] Phase 1: NumPy encoder (R1–R7) on `cip0001-numpy-encoder` — BC reference tests + encoder unit tests green
- [ ] Phase 2: MPI correctness and deprecation (R8–R11) — note: `np.int`→`np.int64` typemap fix included with Phase 1
- [ ] Phase 3: chunked psi-statistics (R12 in GPy, R13–R16)
- [ ] Many-core benchmarks recorded
- [ ] Phase 4: MPI removed (R17–R24) or reduced to the reducer hook
Expand Down
223 changes: 112 additions & 111 deletions deepgp/encoder/mlp.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,167 +2,168 @@
# Licensed under the BSD 3-clause license (see LICENSE.txt)

"""
The multilayer perceptron implementation according to deeplearning.net
Multilayer perceptron encoder in plain NumPy (drop-in replacement for the
former Theano implementation; same classes, parameters and gradients).

Behaviour notes preserved from the Theano version:
- gradients are accumulated between ``prepare()`` calls;
- when the gradient is supplied externally (``update_gradient``), the L1/L2
``regularization`` term is not added (the Theano graph dropped ``add_cost``
on that branch).
"""

from GPy.core import Model, Parameterized, Param
import numpy as np
import theano
from theano import tensor, shared
theano.config.exception_verbosity='high'


def _softplus(x):
return np.logaddexp(0.0, x)


def _sigmoid(x):
return 0.5 * (1.0 + np.tanh(0.5 * x))


class Layer(Parameterized):

def __init__(self, dim_up, dim_down, activation, regularization=None, reg_weight=0, W=None, b=None, name='layer'):
super(Layer,self).__init__(name=name)

super(Layer, self).__init__(name=name)
self.dim_down = dim_down
self.dim_up = dim_up
self.layer_forward = None # the link to its lower layer
self.layer_backward = None # the link to its upper layer
self.layer_forward = None
self.layer_backward = None
self.activation = None if activation is None else activation.lower()
if self.activation not in (None, 'tanh', 'softplus'):
raise ValueError('Unsupported activation function: %r' % activation)
self.regularization = regularization
self.reg_weight = reg_weight

if W is None:
W = np.random.rand(dim_down,dim_up)*2-1
W *= np.sqrt(6./(dim_up+dim_down))
W = np.random.rand(dim_down, dim_up) * 2 - 1
W *= np.sqrt(6. / (dim_up + dim_down))
if b is None:
b = np.zeros((dim_down,))
self.W = Param('W', W)
self.b = Param('b', b)
self.link_parameters(self.W, self.b)

self.W_theano = shared(self.W.values.astype(theano.config.floatX),name=name+'_W')
self.W_grad_theano = shared(self.W.gradient.astype(theano.config.floatX),name=name+'_W_grad')
self.b_theano = shared(self.b.values.astype(theano.config.floatX),name=name+'_b')
self.b_grad_theano = shared(self.b.gradient.astype(theano.config.floatX),name=name+'_b_grad')

self._W_acc = np.zeros_like(self.W.values)
self._b_acc = np.zeros_like(self.b.values)

def link_layers(self, layer_backward, layer_forward):
self.layer_forward = layer_forward # the link to its lower layer
self.layer_backward = layer_backward # the link to its upper layer
self.layer_forward = layer_forward
self.layer_backward = layer_backward

def _prepare(self):
self.W_theano.set_value(self.W.values.astype(theano.config.floatX))
self.W_grad_theano.set_value(np.zeros_like(self.W.gradient, dtype=theano.config.floatX))
self.b_theano.set_value(self.b.values.astype(theano.config.floatX))
self.b_grad_theano.set_value(np.zeros_like(self.b.gradient, dtype=theano.config.floatX))
self._W_acc[:] = 0.
self._b_acc[:] = 0.
if self.layer_forward is not None:
self.layer_forward._prepare()

def _update_gradient(self):
self.W.gradient[:] = self.W_grad_theano.get_value()
self.b.gradient[:] = self.b_grad_theano.get_value()
self.W.gradient[:] = self._W_acc
self.b.gradient[:] = self._b_acc
if self.layer_forward is not None:
self.layer_forward._update_gradient()

def _build_hidden_layers(self, input, add_cost, Y, updates, external_grad=None):

lin_output = tensor.dot(input, self.W_theano.T)+self.b_theano[None,:]
if self.activation=='tanh':
output = tensor.tanh(lin_output)
elif self.activation=='softplus':
output = tensor.nnet.softplus(lin_output)
elif self.activation is None:
output = lin_output
else:
raise 'Unsupported activation function!'

if self.regularization == 'L1':
add_cost = add_cost -self.reg_weight*tensor.abs(self.W_theano).sum()
elif self.regularization == 'L2':
add_cost = add_cost -self.reg_weight*(self.W_theano**2).sum()

# Compute the cost function
if self.layer_forward is None:
if external_grad is None:
cost = -((output-Y)**2).sum()/self.sigma2_theano[0]+add_cost
else:
cost = (external_grad*output).sum()
Y_out = output
else:
cost, Y_out = self.layer_forward._build_hidden_layers(output, add_cost, Y, updates, external_grad=external_grad)

# Update parameter gradients
W_grad = tensor.grad(cost, self.W_theano)
b_grad = tensor.grad(cost, self.b_theano)
updates.extend([(self.W_grad_theano,self.W_grad_theano+W_grad), (self.b_grad_theano,self.b_grad_theano+b_grad)])

return cost, Y_out

def build_theano_functions(self, external_grad=False):

Y = tensor.matrix('Y')
X = tensor.matrix('X')
exGrad = tensor.matrix('exGrad') if external_grad else None
updates = []

cost, Y_out = self._build_hidden_layers(X, 0, Y, updates, external_grad=exGrad)

self._predict = theano.function([X], Y_out, allow_input_downcast=True,name='predict')
if external_grad:
X_grad = tensor.grad(cost, X)
self._comp_grad = theano.function([X, exGrad], X_grad, updates=updates, allow_input_downcast=True,name='comp_grad')

def _forward(self, X):
lin = X.dot(self.W.values.T) + self.b.values[None, :]
if self.activation == 'tanh':
out = np.tanh(lin)
elif self.activation == 'softplus':
out = _softplus(lin)
else:
self._comp_cost = theano.function([X,Y], cost, updates=updates, allow_input_downcast=True,name='comp_cost')
out = lin
return lin, out

def _dact(self, lin, out):
if self.activation == 'tanh':
return 1. - out ** 2
if self.activation == 'softplus':
return _sigmoid(lin)
return None

def forward_all(self, X):
cache, layer, h = [], self, X
while layer is not None:
lin, out = layer._forward(h)
cache.append((layer, h, lin, out))
h, layer = out, layer.layer_forward
return h, cache

def backward_all(self, cache, dout):
for layer, h_in, lin, out in reversed(cache):
d = layer._dact(lin, out)
dlin = dout if d is None else dout * d
layer._W_acc += dlin.T.dot(h_in)
layer._b_acc += dlin.sum(0)
dout = dlin.dot(layer.W.values)
return dout


class MLP(Parameterized):
"""Multi-Layer Perceptron Class"""

@staticmethod
def clone(mlp):
mlp_new = MLP(mlp.nUnits, activation=mlp.layers[0].activation, X_center=mlp.X_center,
regularization=mlp.regularization, reg_weight=mlp.reg_weight, positive_obs=mlp.positive_obs, name=mlp.name)
mlp_new = MLP(mlp.nUnits, activation=mlp.layers[0].activation, X_center=mlp.X_center,
regularization=mlp.regularization, reg_weight=mlp.reg_weight, positive_obs=mlp.positive_obs, name=mlp.name)
mlp_new.param_array[:] = mlp.param_array
mlp_new._trigger_params_changed()
return mlp_new

def __init__(self, nUnits, activation='tanh', X_center=None, regularization=None, reg_weight=0, positive_obs=False, name='mlp'):
super(MLP, self).__init__(name=name)

self.Y = None
self.X = None
self.nLayers = len(nUnits)-1
self.nLayers = len(nUnits) - 1
self.nUnits = nUnits
self.X_center = X_center
self.regularization = regularization
self.reg_weight = reg_weight
self.positive_obs = positive_obs

# Layers order from input to output
self.layers = [Layer(nUnits[i],nUnits[i+1], activation=activation if i!=self.nLayers-1 else ('softplus' if positive_obs else None), regularization=regularization, reg_weight=reg_weight, name='layer_'+str(i+1)) for i in range(self.nLayers)]
self._warned_unused_regularization = False
act = None if activation is None else activation.lower()
if act not in (None, 'tanh', 'softplus'):
raise ValueError('Unsupported activation function: %r' % activation)
if regularization is not None:
import warnings
warnings.warn(
"MLP regularization=%r is accepted for API compatibility but has "
"no effect when gradients are supplied via update_gradient "
"(historical Theano behaviour; see CIP-0001)." % (regularization,),
FutureWarning,
stacklevel=2,
)
self._warned_unused_regularization = True
self.layers = [Layer(nUnits[i], nUnits[i + 1], activation=act if i != self.nLayers - 1 else ('softplus' if positive_obs else None),
regularization=regularization, reg_weight=reg_weight, name='layer_' + str(i + 1)) for i in range(self.nLayers)]
for i in range(self.nLayers):
self.layers[i].link_layers(self.layers[i-1] if i>0 else None, self.layers[i+1] if i<self.nLayers-1 else None)
self.layers[i].link_layers(self.layers[i - 1] if i > 0 else None, self.layers[i + 1] if i < self.nLayers - 1 else None)
self.link_parameters(*self.layers)
self.theano_init = False


def update_gradient(self, X, dL):
if not self.theano_init:
self.layers[0].build_theano_functions(external_grad=True)
self.theano_init = True
X = X if self.X_center is None else X-self.X_center
if len(X.shape)==1: X = X[None,:]; dL = dL[None,:]
X_grad = self.layers[0]._comp_grad(X, dL)
# External-gradient path: L1/L2 regularization remains a no-op (CIP-0001 R2).
X = X if self.X_center is None else X - self.X_center
if len(X.shape) == 1:
X = X[None, :]
dL = dL[None, :]
_, cache = self.layers[0].forward_all(X)
X_grad = self.layers[0].backward_all(cache, dL)
self.layers[0]._update_gradient()
return X_grad

def prepare(self):
"""
Load the model parameters (MLP weights) and clear the gradients.
This function should be called before updating the gradients.
"""
"""Clear accumulated gradients (call before updating gradients)."""
self.layers[0]._prepare()

def predict(self, X):
X = X if self.X_center is None else X-self.X_center
if not self.theano_init:
self.layers[0].build_theano_functions(external_grad=True)
self.theano_init = True

X = X if self.X_center is None else X - self.X_center
self.prepare()
if len(X.shape)==1: X = X[None,:]
return self.layers[0]._predict(X)

if len(X.shape) == 1:
X = X[None, :]
return self.layers[0].forward_all(X)[0]


class MLP_model(Model):
"""
The Multi-layer Perceptron (MLP) model with L2 cost function.
Expand All @@ -176,11 +177,11 @@ def __init__(self, inputs, outputs, nUnits, activation='tanh', X_center=None, re

def parameters_changed(self):
Y_pred = self.mlp.predict(self.X)
self.cost = np.sum((self.Y - Y_pred)**2)
dL_dYpred = -(2*Y_pred -2*self.Y)
self.cost = np.sum((self.Y - Y_pred) ** 2)
dL_dYpred = -(2 * Y_pred - 2 * self.Y)
self.mlp.update_gradient(self.X, dL_dYpred)
def predict(self,x):

def predict(self, x):
return self.mlp.predict(x)

def log_likelihood(self):
Expand Down
Loading
Loading