diff --git a/.coveragerc b/.coveragerc index 867e14e..3a0a96a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -8,8 +8,6 @@ branch = True source = deepgp omit = */deepgp/testing/* - */deepgp/encoder/* - */deepgp/layers/enclayer.py */deepgp/layers/mrd.py */deepgp/inference/vardtc_parallel.py */deepgp/inference/inferenceX.py diff --git a/README.md b/README.md index 750a9d2..b40eec4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The Python Implementation of Deep Gaussian Processes Currently implemented models are * Deep GPs -* Variational Auto-encoded Deep GPs +* Variational Auto-encoded Deep GPs (NumPy MLP back-constraint; see CIP-0001) Testing ------- @@ -33,8 +33,6 @@ Or with [tox](https://tox.readthedocs.io/): tox ``` -Back-constraint encoder tests are skipped unless Theano is installed. - -Coverage is gated on the maintained serial DeepGP path (see `.coveragerc`). -MPI parallel inference, MRD, Theano encoder, and prediction-only SVI helpers -are omitted from the CI percentage until those paths are revived. +Coverage is gated on the maintained serial DeepGP path, including the NumPy +encoder (see `.coveragerc`). MPI parallel inference, MRD, and prediction-only +SVI helpers remain omitted until those paths are revived (CIP-0001 Phases 2–4). diff --git a/cip/cip0001.md b/cip/cip0001.md index 686e6e9..ef7921d 100644 --- a/cip/cip0001.md +++ b/cip/cip0001.md @@ -2,8 +2,8 @@ author: Neil Lawrence created: "2026-11-13" id: "0001" -last_updated: "2026-11-13" -status: Proposed +last_updated: "2026-11-13" # Phase 1 implementation started +status: In Progress compressed: false related_requirements: [] related_cips: [] @@ -26,8 +26,8 @@ title: Replace Theano and MPI with modern NumPy ## Status - [x] Proposed - Initial idea documented (open for Discussion feedback) -- [ ] Accepted - Approved, ready to start work -- [ ] In Progress - Actively being implemented +- [x] Accepted - Phase 1 started on branch `cip0001-numpy-encoder` (regularisation: keep no-op + FutureWarning) +- [x] In Progress - Phase 1 (NumPy encoder) under way - [ ] Implemented - Work complete, awaiting verification - [ ] Closed - Verified and complete - [ ] Rejected - Will not be implemented @@ -254,9 +254,9 @@ pluggable reducer in R14, so the MPI-specific code shrinks to that one place. - [x] Prototype: NumPy encoder passes all 8 `test_model_basic` tests (incl. BC references) - [x] Prototype: chunked/threaded psi benchmark (2-core) - [ ] Discussion opened -- [ ] Accept: regularisation decision; MPI deprecation and removal timeline -- [ ] Phase 1: NumPy encoder (R1–R7) -- [ ] Phase 2: MPI correctness and deprecation (R8–R11) +- [x] Accept (partial): regularisation kept as no-op with `FutureWarning` (R2) +- [x] Phase 1: NumPy encoder (R1–R7) on `cip0001-numpy-encoder` — BC reference tests + encoder unit tests green +- [ ] Phase 2: MPI correctness and deprecation (R8–R11) — note: `np.int`→`np.int64` typemap fix included with Phase 1 - [ ] Phase 3: chunked psi-statistics (R12 in GPy, R13–R16) - [ ] Many-core benchmarks recorded - [ ] Phase 4: MPI removed (R17–R24) or reduced to the reducer hook diff --git a/deepgp/encoder/mlp.py b/deepgp/encoder/mlp.py index 3fbbb68..41ee31f 100644 --- a/deepgp/encoder/mlp.py +++ b/deepgp/encoder/mlp.py @@ -2,167 +2,168 @@ # Licensed under the BSD 3-clause license (see LICENSE.txt) """ -The multilayer perceptron implementation according to deeplearning.net +Multilayer perceptron encoder in plain NumPy (drop-in replacement for the +former Theano implementation; same classes, parameters and gradients). + +Behaviour notes preserved from the Theano version: +- gradients are accumulated between ``prepare()`` calls; +- when the gradient is supplied externally (``update_gradient``), the L1/L2 + ``regularization`` term is not added (the Theano graph dropped ``add_cost`` + on that branch). """ from GPy.core import Model, Parameterized, Param import numpy as np -import theano -from theano import tensor, shared -theano.config.exception_verbosity='high' + + +def _softplus(x): + return np.logaddexp(0.0, x) + + +def _sigmoid(x): + return 0.5 * (1.0 + np.tanh(0.5 * x)) + class Layer(Parameterized): - + def __init__(self, dim_up, dim_down, activation, regularization=None, reg_weight=0, W=None, b=None, name='layer'): - super(Layer,self).__init__(name=name) - + super(Layer, self).__init__(name=name) self.dim_down = dim_down self.dim_up = dim_up - self.layer_forward = None # the link to its lower layer - self.layer_backward = None # the link to its upper layer + self.layer_forward = None + self.layer_backward = None self.activation = None if activation is None else activation.lower() + if self.activation not in (None, 'tanh', 'softplus'): + raise ValueError('Unsupported activation function: %r' % activation) self.regularization = regularization self.reg_weight = reg_weight - + if W is None: - W = np.random.rand(dim_down,dim_up)*2-1 - W *= np.sqrt(6./(dim_up+dim_down)) + W = np.random.rand(dim_down, dim_up) * 2 - 1 + W *= np.sqrt(6. / (dim_up + dim_down)) if b is None: b = np.zeros((dim_down,)) self.W = Param('W', W) self.b = Param('b', b) self.link_parameters(self.W, self.b) - - self.W_theano = shared(self.W.values.astype(theano.config.floatX),name=name+'_W') - self.W_grad_theano = shared(self.W.gradient.astype(theano.config.floatX),name=name+'_W_grad') - self.b_theano = shared(self.b.values.astype(theano.config.floatX),name=name+'_b') - self.b_grad_theano = shared(self.b.gradient.astype(theano.config.floatX),name=name+'_b_grad') - + self._W_acc = np.zeros_like(self.W.values) + self._b_acc = np.zeros_like(self.b.values) + def link_layers(self, layer_backward, layer_forward): - self.layer_forward = layer_forward # the link to its lower layer - self.layer_backward = layer_backward # the link to its upper layer + self.layer_forward = layer_forward + self.layer_backward = layer_backward def _prepare(self): - self.W_theano.set_value(self.W.values.astype(theano.config.floatX)) - self.W_grad_theano.set_value(np.zeros_like(self.W.gradient, dtype=theano.config.floatX)) - self.b_theano.set_value(self.b.values.astype(theano.config.floatX)) - self.b_grad_theano.set_value(np.zeros_like(self.b.gradient, dtype=theano.config.floatX)) + self._W_acc[:] = 0. + self._b_acc[:] = 0. if self.layer_forward is not None: self.layer_forward._prepare() - + def _update_gradient(self): - self.W.gradient[:] = self.W_grad_theano.get_value() - self.b.gradient[:] = self.b_grad_theano.get_value() + self.W.gradient[:] = self._W_acc + self.b.gradient[:] = self._b_acc if self.layer_forward is not None: self.layer_forward._update_gradient() - - def _build_hidden_layers(self, input, add_cost, Y, updates, external_grad=None): - - lin_output = tensor.dot(input, self.W_theano.T)+self.b_theano[None,:] - if self.activation=='tanh': - output = tensor.tanh(lin_output) - elif self.activation=='softplus': - output = tensor.nnet.softplus(lin_output) - elif self.activation is None: - output = lin_output - else: - raise 'Unsupported activation function!' - - if self.regularization == 'L1': - add_cost = add_cost -self.reg_weight*tensor.abs(self.W_theano).sum() - elif self.regularization == 'L2': - add_cost = add_cost -self.reg_weight*(self.W_theano**2).sum() - - # Compute the cost function - if self.layer_forward is None: - if external_grad is None: - cost = -((output-Y)**2).sum()/self.sigma2_theano[0]+add_cost - else: - cost = (external_grad*output).sum() - Y_out = output - else: - cost, Y_out = self.layer_forward._build_hidden_layers(output, add_cost, Y, updates, external_grad=external_grad) - - # Update parameter gradients - W_grad = tensor.grad(cost, self.W_theano) - b_grad = tensor.grad(cost, self.b_theano) - updates.extend([(self.W_grad_theano,self.W_grad_theano+W_grad), (self.b_grad_theano,self.b_grad_theano+b_grad)]) - - return cost, Y_out - - def build_theano_functions(self, external_grad=False): - - Y = tensor.matrix('Y') - X = tensor.matrix('X') - exGrad = tensor.matrix('exGrad') if external_grad else None - updates = [] - - cost, Y_out = self._build_hidden_layers(X, 0, Y, updates, external_grad=exGrad) - - self._predict = theano.function([X], Y_out, allow_input_downcast=True,name='predict') - if external_grad: - X_grad = tensor.grad(cost, X) - self._comp_grad = theano.function([X, exGrad], X_grad, updates=updates, allow_input_downcast=True,name='comp_grad') + + def _forward(self, X): + lin = X.dot(self.W.values.T) + self.b.values[None, :] + if self.activation == 'tanh': + out = np.tanh(lin) + elif self.activation == 'softplus': + out = _softplus(lin) else: - self._comp_cost = theano.function([X,Y], cost, updates=updates, allow_input_downcast=True,name='comp_cost') + out = lin + return lin, out + + def _dact(self, lin, out): + if self.activation == 'tanh': + return 1. - out ** 2 + if self.activation == 'softplus': + return _sigmoid(lin) + return None + + def forward_all(self, X): + cache, layer, h = [], self, X + while layer is not None: + lin, out = layer._forward(h) + cache.append((layer, h, lin, out)) + h, layer = out, layer.layer_forward + return h, cache + + def backward_all(self, cache, dout): + for layer, h_in, lin, out in reversed(cache): + d = layer._dact(lin, out) + dlin = dout if d is None else dout * d + layer._W_acc += dlin.T.dot(h_in) + layer._b_acc += dlin.sum(0) + dout = dlin.dot(layer.W.values) + return dout + class MLP(Parameterized): """Multi-Layer Perceptron Class""" - + @staticmethod def clone(mlp): - mlp_new = MLP(mlp.nUnits, activation=mlp.layers[0].activation, X_center=mlp.X_center, - regularization=mlp.regularization, reg_weight=mlp.reg_weight, positive_obs=mlp.positive_obs, name=mlp.name) + mlp_new = MLP(mlp.nUnits, activation=mlp.layers[0].activation, X_center=mlp.X_center, + regularization=mlp.regularization, reg_weight=mlp.reg_weight, positive_obs=mlp.positive_obs, name=mlp.name) mlp_new.param_array[:] = mlp.param_array mlp_new._trigger_params_changed() return mlp_new def __init__(self, nUnits, activation='tanh', X_center=None, regularization=None, reg_weight=0, positive_obs=False, name='mlp'): super(MLP, self).__init__(name=name) - self.Y = None self.X = None - self.nLayers = len(nUnits)-1 + self.nLayers = len(nUnits) - 1 self.nUnits = nUnits self.X_center = X_center self.regularization = regularization self.reg_weight = reg_weight self.positive_obs = positive_obs - - # Layers order from input to output - self.layers = [Layer(nUnits[i],nUnits[i+1], activation=activation if i!=self.nLayers-1 else ('softplus' if positive_obs else None), regularization=regularization, reg_weight=reg_weight, name='layer_'+str(i+1)) for i in range(self.nLayers)] + self._warned_unused_regularization = False + act = None if activation is None else activation.lower() + if act not in (None, 'tanh', 'softplus'): + raise ValueError('Unsupported activation function: %r' % activation) + if regularization is not None: + import warnings + warnings.warn( + "MLP regularization=%r is accepted for API compatibility but has " + "no effect when gradients are supplied via update_gradient " + "(historical Theano behaviour; see CIP-0001)." % (regularization,), + FutureWarning, + stacklevel=2, + ) + self._warned_unused_regularization = True + self.layers = [Layer(nUnits[i], nUnits[i + 1], activation=act if i != self.nLayers - 1 else ('softplus' if positive_obs else None), + regularization=regularization, reg_weight=reg_weight, name='layer_' + str(i + 1)) for i in range(self.nLayers)] for i in range(self.nLayers): - self.layers[i].link_layers(self.layers[i-1] if i>0 else None, self.layers[i+1] if i 0 else None, self.layers[i + 1] if i < self.nLayers - 1 else None) self.link_parameters(*self.layers) - self.theano_init = False - + def update_gradient(self, X, dL): - if not self.theano_init: - self.layers[0].build_theano_functions(external_grad=True) - self.theano_init = True - X = X if self.X_center is None else X-self.X_center - if len(X.shape)==1: X = X[None,:]; dL = dL[None,:] - X_grad = self.layers[0]._comp_grad(X, dL) + # External-gradient path: L1/L2 regularization remains a no-op (CIP-0001 R2). + X = X if self.X_center is None else X - self.X_center + if len(X.shape) == 1: + X = X[None, :] + dL = dL[None, :] + _, cache = self.layers[0].forward_all(X) + X_grad = self.layers[0].backward_all(cache, dL) self.layers[0]._update_gradient() return X_grad - + def prepare(self): - """ - Load the model parameters (MLP weights) and clear the gradients. - This function should be called before updating the gradients. - """ + """Clear accumulated gradients (call before updating gradients).""" self.layers[0]._prepare() - + def predict(self, X): - X = X if self.X_center is None else X-self.X_center - if not self.theano_init: - self.layers[0].build_theano_functions(external_grad=True) - self.theano_init = True - + X = X if self.X_center is None else X - self.X_center self.prepare() - if len(X.shape)==1: X = X[None,:] - return self.layers[0]._predict(X) - + if len(X.shape) == 1: + X = X[None, :] + return self.layers[0].forward_all(X)[0] + + class MLP_model(Model): """ The Multi-layer Perceptron (MLP) model with L2 cost function. @@ -176,11 +177,11 @@ def __init__(self, inputs, outputs, nUnits, activation='tanh', X_center=None, re def parameters_changed(self): Y_pred = self.mlp.predict(self.X) - self.cost = np.sum((self.Y - Y_pred)**2) - dL_dYpred = -(2*Y_pred -2*self.Y) + self.cost = np.sum((self.Y - Y_pred) ** 2) + dL_dYpred = -(2 * Y_pred - 2 * self.Y) self.mlp.update_gradient(self.X, dL_dYpred) - - def predict(self,x): + + def predict(self, x): return self.mlp.predict(x) def log_likelihood(self): diff --git a/deepgp/testing/test_encoder.py b/deepgp/testing/test_encoder.py new file mode 100644 index 0000000..697f39f --- /dev/null +++ b/deepgp/testing/test_encoder.py @@ -0,0 +1,82 @@ +# Copyright (c) 2015-2016, the authors (see AUTHORS.txt). +# Licensed under the BSD 3-clause license (see LICENSE.txt) + +"""Unit tests for the NumPy MLP encoder (CIP-0001 Phase 1).""" + +import unittest +import warnings + +import numpy as np + +from deepgp.encoder.mlp import MLP, MLP_model + + +class MLPModelGradientTest(unittest.TestCase): + def _toy(self, activation='tanh', positive_obs=False, seed=0): + rng = np.random.RandomState(seed) + X = rng.randn(12, 3) + Y = rng.randn(12, 2) + if positive_obs: + Y = np.abs(Y) + 0.1 + return MLP_model( + X, Y, [3, 5, 2], + activation=activation, + positive_obs=positive_obs, + ) + + def test_checkgrad_tanh(self): + self.assertTrue(self._toy('tanh', seed=1).checkgrad()) + + def test_checkgrad_softplus(self): + self.assertTrue(self._toy('softplus', seed=2).checkgrad()) + + def test_checkgrad_positive_obs(self): + self.assertTrue(self._toy('tanh', positive_obs=True, seed=3).checkgrad()) + + def test_input_gradient_finite_difference(self): + rng = np.random.RandomState(4) + X = rng.randn(8, 3) + dL = rng.randn(8, 2) + mlp = MLP([3, 4, 2], activation='tanh') + mlp.prepare() + g_analytic = mlp.update_gradient(X, dL) + + eps = 1e-6 + g_num = np.zeros_like(X) + for i in range(X.shape[0]): + for j in range(X.shape[1]): + Xp, Xm = X.copy(), X.copy() + Xp[i, j] += eps + Xm[i, j] -= eps + # Cost proxy: sum(pred * dL); gradient w.r.t. X matches update_gradient contract + fp = np.sum(mlp.predict(Xp) * dL) + fm = np.sum(mlp.predict(Xm) * dL) + g_num[i, j] = (fp - fm) / (2 * eps) + self.assertTrue(np.allclose(g_analytic, g_num, rtol=1e-5, atol=1e-6)) + + def test_clone_round_trip(self): + mlp = MLP([3, 4, 2], activation='tanh', X_center=np.array([0.1, -0.2, 0.0])) + mlp.param_array[:] = np.linspace(0.1, 1.0, mlp.param_array.size) + mlp._trigger_params_changed() + clone = MLP.clone(mlp) + X = np.random.RandomState(5).randn(5, 3) + self.assertTrue(np.allclose(mlp.predict(X), clone.predict(X))) + self.assertTrue(np.allclose(mlp.param_array, clone.param_array)) + + def test_unsupported_activation_raises(self): + with self.assertRaises(ValueError): + MLP([3, 2], activation='relu') + + def test_regularization_warns_and_is_noop(self): + X = np.random.RandomState(6).randn(6, 3) + Y = np.random.RandomState(7).randn(6, 2) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + model = MLP_model( + X, Y, [3, 4, 2], + activation='tanh', + regularization='L2', + reg_weight=1.0, + ) + self.assertTrue(any(issubclass(w.category, FutureWarning) for w in caught)) + self.assertTrue(model.checkgrad()) diff --git a/deepgp/testing/test_model_basic.py b/deepgp/testing/test_model_basic.py index 8bbf08d..ab55146 100644 --- a/deepgp/testing/test_model_basic.py +++ b/deepgp/testing/test_model_basic.py @@ -11,12 +11,6 @@ base_path = os.path.dirname(__file__) -try: - import theano # noqa: F401 - HAS_THEANO = True -except ImportError: - HAS_THEANO = False - def _scalar_from_hdf5(node): """Read a stored likelihood scalar from tables/h5py across NumPy versions.""" @@ -98,10 +92,9 @@ def test_checkLogLikelihood(self): self.assertTrue(np.allclose(float(self.model._log_marginal_likelihood), self.L, rtol=1e-05, atol=1e-08), str(float(self.model._log_marginal_likelihood))+', '+str(self.L)) -@unittest.skipUnless(HAS_THEANO, "back-constraint encoder requires theano") class DGP_BC_UnsupervisedTest(unittest.TestCase): """ - Normal Deep GPs with back-constraint + Deep GPs with NumPy MLP back-constraint (unsupervised). """ datafile = 'random_10_3.txt' modelfile = 'dgp_bc_unsupervised.h5' @@ -135,10 +128,9 @@ def test_gradient(self): def test_checkLogLikelihood(self): self.assertTrue(np.allclose(float(self.model._log_marginal_likelihood), self.L, rtol=1e-05, atol=1e-08), str(float(self.model._log_marginal_likelihood))+', '+str(self.L)) -@unittest.skipUnless(HAS_THEANO, "back-constraint encoder requires theano") class DGP_BC_SupervisedTest(unittest.TestCase): """ - Normal Deep GPs with back-constraint + Deep GPs with NumPy MLP back-constraint (supervised). """ outputfile = 'random_10_3.txt' inputfile = 'random_10_2.txt' diff --git a/deepgp/util/parallel.py b/deepgp/util/parallel.py index d659be6..f71a304 100644 --- a/deepgp/util/parallel.py +++ b/deepgp/util/parallel.py @@ -17,7 +17,7 @@ def numpy_to_MPI_typemap(np_type): typemap = { np.dtype(np.float64) : MPI.DOUBLE, np.dtype(np.float32) : MPI.FLOAT, - np.dtype(np.int) : MPI.INT, + np.dtype(np.int64) : MPI.INT, np.dtype(np.int8) : MPI.CHAR, np.dtype(np.uint8) : MPI.UNSIGNED_CHAR, np.dtype(np.int32) : MPI.INT,