diff --git a/README.md b/README.md index f4f4a72..18d49b1 100644 --- a/README.md +++ b/README.md @@ -86,3 +86,10 @@ linear psi-statistics (CIP-0001 Phase 3):: Defaults (``psi_chunk_size=None``, ``psi_threads=1``) preserve previous behaviour. Threading runs inside GPy's pure psi helpers; PyDeepGP itself stays serial at the kernel API. + +Examples +-------- + +* ``examples/`` — short supervised / unsupervised demos (oil, CMU walk/jog). +* ``examples/damianou2013/`` — recreations of the AISTATS 2013 experiments + (toy hierarchy, warped regression, USPS depth/ARD, high-five / MRD). diff --git a/cip/cip0002.md b/cip/cip0002.md index 5611752..a35a331 100644 --- a/cip/cip0002.md +++ b/cip/cip0002.md @@ -232,7 +232,8 @@ frontmatter. ## Implementation Status - [x] Phase A: bound dtype + init fix + unit tests -- [x] Phase B: docs, coverage omit removal, `examples/example_mrd.py` +- [x] Phase B: docs, coverage omit removal, `examples/example_mrd.py`, + `examples/damianou2013/highfive_shared.py` runnable - [x] Phase C: back-constraint encoder path (included in first implementation PR) - [ ] Phase D: MPI cleanup aligned with CIP-0001 R20 - [x] CIP review → Accepted (2026-11-13); implementation on `cip0002-mrd-revival` diff --git a/examples/damianou2013/README.md b/examples/damianou2013/README.md new file mode 100644 index 0000000..a461aa3 --- /dev/null +++ b/examples/damianou2013/README.md @@ -0,0 +1,73 @@ +# Damianou & Lawrence (AISTATS 2013) — PyDeepGP recreations + +Scripts that recreate the **feasible** experiments from +[Damianou & Lawrence, *Deep Gaussian Processes*](https://proceedings.mlr.press/v31/damianou13a.html) +(AISTATS 2013) on the maintained `DeepGP` path. + +These are **modern recreations**, not bit-identical paper scripts: optimiser +schedules, inducing-point counts, and initialisation differ from the 2013 +Matlab stack. Use `--quick` for a short smoke run. + +## Mapping + +| Paper section | Script | Status | +| --- | --- | --- | +| §4.1 Toy hierarchy (fig. 2) | `toy_hierarchy.py` | Runnable | +| §4.1 Warped regression (fig. 3) | `toy_warped_regression.py` | Runnable | +| §4.3 USPS {0,1,6} depth / ARD / 1-NN (figs. 6–8) | `usps_digits.py` | Runnable (downloads ESL `zip.train`) | +| §4.2 CMU high-five shared views (figs. 4–5) | `highfive_shared.py` | Runnable via CIP-0002 MRD (synthetic or pods stand-in) | + +## Setup + +From the repository root (Python ≥ 3.11, GPy ≥ 1.15): + +```bash +python -m pip install -e . +# optional: matplotlib for figures; already a project dependency +``` + +USPS data are cached under `examples/damianou2013/data/zip.train.gz` on first run +(ElemStatLearn `zip.train`). + +## Quick smoke tests + +```bash +cd examples/damianou2013 +python toy_hierarchy.py --quick --no-plot +python toy_warped_regression.py --quick --no-plot +python usps_digits.py --quick --no-plot --depths 1 2 +python highfive_shared.py --quick --no-plot --synthetic +``` + +Save figures without interactive display: + +```bash +python toy_hierarchy.py --quick --no-plot --outdir /tmp/dgp2013 +``` + +## Paper-scale runs + +Omit `--quick` (slower; especially USPS depth 1–5): + +```bash +python toy_warped_regression.py --repeats 10 --outdir /tmp/dgp2013 +python usps_digits.py --depths 1 2 3 4 5 --outdir /tmp/dgp2013 +``` + +## High-five / MRD + +The high-five experiment uses **conditionally independent observation views** +with a shared parent latent layer (`ObservedMRDLayer`, CIP-0002). +`highfive_shared.py` fits that model; use `--synthetic` when MOCAP data are +unavailable. See also ``examples/example_mrd.py``. + +## Citation + +``` +@inproceedings{damianou2013deep, + title={Deep {G}aussian {P}rocesses}, + author={Damianou, Andreas and Lawrence, Neil}, + booktitle={AISTATS}, + year={2013} +} +``` diff --git a/examples/damianou2013/_common.py b/examples/damianou2013/_common.py new file mode 100644 index 0000000..13bd45c --- /dev/null +++ b/examples/damianou2013/_common.py @@ -0,0 +1,183 @@ +"""Shared helpers for Damianou & Lawrence (AISTATS 2013) recreation scripts.""" + +from __future__ import annotations + +import argparse +import os +from typing import List, Optional, Tuple + +import numpy as np + +import GPy +import deepgp + + +def add_common_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + parser.add_argument( + "--quick", + action="store_true", + help="Fewer iterations / smaller models for a smoke test (not paper-faithful).", + ) + parser.add_argument( + "--seed", + type=int, + default=42, + help="RNG seed (default: 42).", + ) + parser.add_argument( + "--messages", + action="store_true", + help="Show optimiser progress messages.", + ) + parser.add_argument( + "--no-plot", + action="store_true", + help="Skip matplotlib figures.", + ) + parser.add_argument( + "--outdir", + type=str, + default=None, + help="If set, save figures under this directory.", + ) + return parser + + +def ensure_outdir(outdir: Optional[str]) -> Optional[str]: + if outdir is None: + return None + os.makedirs(outdir, exist_ok=True) + return outdir + + +def save_or_show(fig, outdir: Optional[str], name: str, no_plot: bool) -> None: + import matplotlib.pyplot as plt + + if outdir is not None: + path = os.path.join(outdir, name) + fig.savefig(path, bbox_inches="tight", dpi=120) + print(f"Wrote {path}") + if no_plot: + plt.close(fig) + else: + plt.show() + + +def rbf_bias(input_dim: int, ard: bool = True) -> GPy.kern.Kern: + return GPy.kern.RBF(input_dim, ARD=ard) + GPy.kern.Bias(input_dim) + + +def _output_variance(y) -> float: + """Variance of observed data or of a variational posterior mean.""" + from GPy.core.parameterization.variational import NormalPosterior, VariationalPosterior + + if isinstance(y, (NormalPosterior, VariationalPosterior)): + return float(np.asarray(y.mean).var()) + return float(np.asarray(y).var()) + + +def init_noise(model: deepgp.DeepGP, fraction: float = 0.01, fix: bool = False) -> None: + for layer in model.layers: + if hasattr(layer, "views"): + for view in layer.views: + view.Gaussian_noise.variance = _output_variance(view.Y) * fraction + if fix: + view.Gaussian_noise.variance.fix() + else: + layer.Gaussian_noise.variance = _output_variance(layer.Y) * fraction + if fix: + layer.Gaussian_noise.variance.fix() + + +def unfix_noise(model: deepgp.DeepGP) -> None: + for layer in model.layers: + if hasattr(layer, "views"): + for view in layer.views: + view.Gaussian_noise.variance.unfix() + else: + layer.Gaussian_noise.variance.unfix() + + +def optimize_staged( + model: deepgp.DeepGP, + iters_fixed: int, + iters_free: int, + messages: bool = False, +) -> None: + """Fix noise briefly, then free it — matches the shipped supervised examples.""" + init_noise(model, fix=True) + if iters_fixed > 0: + model.optimize(max_iters=iters_fixed, messages=messages) + unfix_noise(model) + if iters_free > 0: + model.optimize(max_iters=iters_free, messages=messages) + + +def lengthscales(kern: GPy.kern.Kern) -> np.ndarray: + """Return ARD lengthscales from an RBF(+Bias) kernel.""" + if hasattr(kern, "rbf"): + return np.asarray(kern.rbf.lengthscale).ravel() + if hasattr(kern, "lengthscale"): + return np.asarray(kern.lengthscale).ravel() + # Additive kernels: prefer the first RBF-like part + if hasattr(kern, "parts"): + for part in kern.parts: + if hasattr(part, "lengthscale"): + return np.asarray(part.lengthscale).ravel() + raise AttributeError("Could not find lengthscale on kernel") + + +def ard_weights(kern: GPy.kern.Kern) -> np.ndarray: + """Inverse lengthscale (larger ⇒ more relevant), as used in the paper plots.""" + ls = lengthscales(kern) + return 1.0 / np.maximum(ls, 1e-12) + + +def print_ard(model: deepgp.DeepGP) -> None: + for i, layer in enumerate(model.layers): + try: + w = ard_weights(layer.kern) + except AttributeError: + continue + print(f"Layer {i} ARD (1/lengthscale): {np.array2string(w, precision=3)}") + + +def nearest_neighbour_error(X: np.ndarray, labels: np.ndarray) -> Tuple[int, float]: + """Leave-one-out 1-NN error count and rate in Euclidean feature space.""" + X = np.asarray(X, dtype=float) + labels = np.asarray(labels).ravel() + n = X.shape[0] + # Squared distances via (x-y)^2 = x^2 + y^2 - 2xy + sq = np.sum(X * X, axis=1, keepdims=True) + d2 = sq + sq.T - 2.0 * X @ X.T + np.fill_diagonal(d2, np.inf) + nn = np.argmin(d2, axis=1) + mistakes = int(np.sum(labels[nn] != labels)) + return mistakes, mistakes / float(n) + + +def dominant_dims(weights: np.ndarray, k: int = 2) -> List[int]: + order = np.argsort(-weights) + return [int(i) for i in order[:k]] + + +def mse(pred: np.ndarray, target: np.ndarray) -> float: + return float(np.mean((np.asarray(pred) - np.asarray(target)) ** 2)) + + +def rmse(pred: np.ndarray, target: np.ndarray) -> float: + return float(np.sqrt(mse(pred, target))) + + +def sample_gp_prior( + X: np.ndarray, + kernel: GPy.kern.Kern, + output_dim: int, + noise: float = 1e-6, +) -> np.ndarray: + """Draw ``output_dim`` independent prior samples at inputs ``X``.""" + K = kernel.K(X) + K = K + noise * np.eye(X.shape[0]) + L = np.linalg.cholesky(K) + Z = np.random.randn(X.shape[0], output_dim) + return L @ Z diff --git a/examples/damianou2013/data/.gitignore b/examples/damianou2013/data/.gitignore new file mode 100644 index 0000000..d19067e --- /dev/null +++ b/examples/damianou2013/data/.gitignore @@ -0,0 +1,2 @@ +# Cached ElemStatLearn USPS split (downloaded by usps_digits.py) +zip.train.gz diff --git a/examples/damianou2013/data/.gitkeep b/examples/damianou2013/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/examples/damianou2013/highfive_shared.py b/examples/damianou2013/highfive_shared.py new file mode 100644 index 0000000..afd26a9 --- /dev/null +++ b/examples/damianou2013/highfive_shared.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +"""Section 4.2 (figs. 4–5): CMU high-five — shared / MRD hierarchy. + +Paper: Damianou & Lawrence, Deep Gaussian Processes, AISTATS 2013. + +Two observation views share a parent latent layer (conditionally independent +given the shared features). With CIP-0002, this uses the maintained +``ObservedMRDLayer`` path. + +Real CMU high-five MOCAP (two subjects, ~78 frames × 62 dims) is used when +``pods`` can supply a suitable sequence; otherwise the script falls back to +synthetic correlated views so the shared-hierarchy fit still runs. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +import numpy as np + +import deepgp + +from _common import ( + add_common_args, + ard_weights, + ensure_outdir, + init_noise, + print_ard, + rbf_bias, + save_or_show, + unfix_noise, +) + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + add_common_args(p) + p.add_argument( + "--synthetic", + action="store_true", + help="Force synthetic correlated views (skip pods MOCAP).", + ) + return p + + +def load_two_views(seed: int, synthetic: bool): + """Return (Y1, Y2, source_label).""" + if not synthetic: + try: + import pods + + if hasattr(pods.datasets, "cmu_mocap_35_walk_jog"): + data = pods.datasets.cmu_mocap_35_walk_jog() + Y = np.asarray(data["Y"], dtype=float) + n = min(80, Y.shape[0]) + Y = Y[:n] + mid = Y.shape[1] // 2 + print( + "Note: using walk/jog feature split as a two-view stand-in; " + "paper §4.2 used the CMU high-five pair of subjects." + ) + return Y[:, :mid], Y[:, mid:], "pods:cmu_mocap_35_walk_jog" + except Exception as exc: + print(f"pods MOCAP unavailable ({exc}); using synthetic views.") + + rng = np.random.RandomState(seed) + n, q, d = 60, 3, 20 + X = rng.randn(n, q) + Y1 = X @ rng.randn(q, d) + 0.05 * rng.randn(n, d) + Y2 = X @ rng.randn(q, d) + 0.05 * rng.randn(n, d) + return Y1, Y2, "synthetic" + + +def fit_mrd(Y1, Y2, quick: bool, messages: bool) -> deepgp.DeepGP: + q1, q2 = (4, 3) if quick else (6, 4) + num_inducing = 15 if quick else 30 + iters_fixed = 40 if quick else 200 + iters_free = 80 if quick else 800 + kernels = [ + [rbf_bias(q1, ard=True), rbf_bias(q1, ard=True)], + rbf_bias(q2, ard=True), + ] + m = deepgp.DeepGP( + [[Y1.shape[1], Y2.shape[1]], q1, q2], + [Y1, Y2], + kernels=kernels, + num_inducing=num_inducing, + back_constraint=False, + ) + init_noise(m, fix=True) + m.optimize(max_iters=iters_fixed, messages=messages) + unfix_noise(m) + m.optimize(max_iters=iters_free, messages=messages) + return m + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + outdir = ensure_outdir(args.outdir) + + Y1, Y2, source = load_two_views(args.seed, synthetic=args.synthetic or args.quick) + Y1 = (Y1 - Y1.mean(0)) / (Y1.std() + 1e-12) + Y2 = (Y2 - Y2.mean(0)) / (Y2.std() + 1e-12) + print(f"Views: {Y1.shape}, {Y2.shape} (source={source})") + + m = fit_mrd(Y1, Y2, quick=args.quick, messages=args.messages) + print(f"MRD DeepGP variational bound: {float(m.log_likelihood()):.3f}") + for i, view in enumerate(m.layers[0].views): + w = ard_weights(view.kern) + print(f"View {i} ARD (1/lengthscale): {np.array2string(w, precision=3)}") + print_ard(m) + + if not args.no_plot or outdir: + import matplotlib.pyplot as plt + + fig, axes = plt.subplots(1, 3, figsize=(10, 3.2)) + for i, view in enumerate(m.layers[0].views): + w = ard_weights(view.kern) + axes[i].bar(np.arange(1, len(w) + 1), w) + axes[i].set_title(f"view {i} ARD") + axes[i].set_xlabel("shared dim") + w_top = ard_weights(m.layers[-1].kern) + axes[2].bar(np.arange(1, len(w_top) + 1), w_top) + axes[2].set_title("top layer ARD") + axes[2].set_xlabel("dimension") + fig.suptitle("Damianou & Lawrence 2013 — high-five / shared MRD (fig. 4 style)") + fig.tight_layout() + save_or_show(fig, outdir, "highfive_ard.png", args.no_plot) + + return 0 + + +if __name__ == "__main__": + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + raise SystemExit(main()) diff --git a/examples/damianou2013/toy_hierarchy.py b/examples/damianou2013/toy_hierarchy.py new file mode 100644 index 0000000..ab5548f --- /dev/null +++ b/examples/damianou2013/toy_hierarchy.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +"""Section 4.1 (fig. 2): unsupervised recovery of a stacked-GP hierarchy. + +Paper: Damianou & Lawrence, Deep Gaussian Processes, AISTATS 2013. + +Generate a three-level stack (top latent → 2D intermediate → 10D observations), +fit a DeepGP with overcomplete ARD widths, and contrast against stacked PCA. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +import numpy as np + +import GPy +import deepgp + +from _common import ( + add_common_args, + ard_weights, + dominant_dims, + ensure_outdir, + optimize_staged, + print_ard, + rbf_bias, + sample_gp_prior, + save_or_show, +) + + +def _pca_project(Y: np.ndarray, q: int) -> np.ndarray: + try: + from sklearn.decomposition import PCA + + return PCA(n_components=q).fit_transform(Y) + except Exception: + Yc = Y - Y.mean(0) + U, S, _ = np.linalg.svd(Yc, full_matrices=False) + q = min(q, U.shape[1]) + return U[:, :q] * S[:q] + + +def stacked_pca(Y: np.ndarray, dims: list[int]) -> list[np.ndarray]: + """Greedy PCA stack returning latent matrices from bottom to top.""" + spaces = [] + cur = Y + for q in dims: + cur = _pca_project(cur, q) + spaces.append(cur) + return spaces + + +def generate_hierarchy(n: int, seed: int) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Return (Z_top, X_mid, Y_obs) for a 1→2→10 stacked GP.""" + rng = np.random.RandomState(seed) + np.random.seed(seed) + + Z = np.linspace(-1.0, 1.0, n)[:, None] + kern_mid = GPy.kern.RBF(1, lengthscale=0.4, variance=1.0) + X = sample_gp_prior(Z, kern_mid, output_dim=2) + X += 0.02 * rng.randn(*X.shape) + kern_obs = GPy.kern.RBF(2, lengthscale=0.8, variance=1.0, ARD=False) + Y = sample_gp_prior(X, kern_obs, output_dim=10) + Y += 0.05 * rng.randn(*Y.shape) + return Z, X, Y + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + add_common_args(p) + p.add_argument("--n", type=int, default=80, help="Number of observations.") + p.add_argument("--num-inducing", type=int, default=25) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + outdir = ensure_outdir(args.outdir) + n = 40 if args.quick else args.n + q_mid, q_top = (4, 3) if args.quick else (5, 4) + iters_fixed = 50 if args.quick else 200 + iters_free = 150 if args.quick else 1500 + num_inducing = 15 if args.quick else args.num_inducing + + Z, X_true, Y = generate_hierarchy(n, args.seed) + print(f"Generated hierarchy: Y={Y.shape}, true mid={X_true.shape}, true top={Z.shape}") + + kernels = [rbf_bias(q_mid, ard=True), rbf_bias(q_top, ard=True)] + m = deepgp.DeepGP( + [Y.shape[1], q_mid, q_top], + Y, + kernels=kernels, + num_inducing=num_inducing, + back_constraint=False, + ) + optimize_staged(m, iters_fixed, iters_free, messages=args.messages) + print(f"DeepGP variational bound: {float(m.log_likelihood()):.3f}") + print_ard(m) + + pca_spaces = stacked_pca(Y, [q_mid, q_top]) + + if not args.no_plot or outdir: + import matplotlib.pyplot as plt + + fig, axes = plt.subplots(1, 3, figsize=(10, 3.2)) + t = np.arange(n) + axes[0].plot(t, Z[:, 0], "k-", label="top") + axes[0].plot(t, X_true[:, 0], "C0-", label="mid0") + axes[0].plot(t, X_true[:, 1], "C1-", label="mid1") + axes[0].set_title("Ground truth") + axes[0].legend(fontsize=8) + + for ax, layer_idx, title in ( + (axes[1], 0, "DeepGP layer 0 (near data)"), + (axes[2], 1, "DeepGP layer 1 (top)"), + ): + Xmean = m.layers[layer_idx].X.mean + dims = dominant_dims(ard_weights(m.layers[layer_idx].kern), k=min(2, Xmean.shape[1])) + for d in dims: + ax.plot(t, Xmean[:, d], label=f"dim {d}") + ax.set_title(title) + ax.legend(fontsize=8) + + fig.suptitle("Damianou & Lawrence 2013 — toy hierarchy (fig. 2 style)") + fig.tight_layout() + save_or_show(fig, outdir, "toy_hierarchy.png", args.no_plot) + + fig2, ax = plt.subplots(figsize=(5, 3)) + ax.plot(t, pca_spaces[0][:, 0], label="PCA mid 0") + if pca_spaces[0].shape[1] > 1: + ax.plot(t, pca_spaces[0][:, 1], label="PCA mid 1") + ax.plot(t, pca_spaces[1][:, 0], label="PCA top 0") + ax.set_title("Stacked PCA (greedy)") + ax.legend(fontsize=8) + fig2.tight_layout() + save_or_show(fig2, outdir, "toy_hierarchy_pca.png", args.no_plot) + + return 0 + + +if __name__ == "__main__": + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + raise SystemExit(main()) diff --git a/examples/damianou2013/toy_warped_regression.py b/examples/damianou2013/toy_warped_regression.py new file mode 100644 index 0000000..0ef8a97 --- /dev/null +++ b/examples/damianou2013/toy_warped_regression.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +"""Section 4.1 (fig. 3): warped toy regression — DeepGP vs sparse GP. + +Paper: Damianou & Lawrence, Deep Gaussian Processes, AISTATS 2013. + +Stack Linear+RBF → RBF to make a non-stationary 10D regression problem from a +1D input. Train on 25 points; compare MSE of a standard sparse GP against a +DeepGP that is one hidden layer deeper than the generative process. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +import numpy as np + +import GPy +import deepgp + +from _common import ( + add_common_args, + ensure_outdir, + mse, + optimize_staged, + rbf_bias, + sample_gp_prior, + save_or_show, +) + + +def generate_warped( + n: int = 120, + seed: int = 0, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Return (X, H_hidden, Y) for the paper's warped process.""" + rng = np.random.RandomState(seed) + np.random.seed(seed) + X = np.linspace(-1.0, 1.0, n)[:, None] + kern_warp = GPy.kern.Linear(1) + GPy.kern.RBF(1, lengthscale=0.3, variance=1.0) + H = sample_gp_prior(X, kern_warp, output_dim=1) + H += 0.01 * rng.randn(*H.shape) + kern_obs = GPy.kern.RBF(1, lengthscale=0.25, variance=1.0) + Y = sample_gp_prior(H, kern_obs, output_dim=10) + Y += 0.05 * rng.randn(*Y.shape) + return X, H, Y + + +def train_test_split( + X: np.ndarray, + Y: np.ndarray, + n_train: int, + seed: int, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + rng = np.random.RandomState(seed) + idx = rng.permutation(X.shape[0]) + tr, te = np.sort(idx[:n_train]), np.sort(idx[n_train:]) + return X[tr], Y[tr], X[te], Y[te] + + +def fit_deepgp( + Xtr: np.ndarray, + Ytr: np.ndarray, + q1: int, + q2: int, + num_inducing: int, + iters_fixed: int, + iters_free: int, + messages: bool, +) -> deepgp.DeepGP: + # Two hidden layers between X and Y (paper: one more than the generative depth) + dims = [Ytr.shape[1], q1, q2, Xtr.shape[1]] + kernels = [ + rbf_bias(q1, ard=True), + rbf_bias(q2, ard=True), + rbf_bias(Xtr.shape[1], ard=False), + ] + m = deepgp.DeepGP( + dims, + Ytr, + X=Xtr, + kernels=kernels, + num_inducing=num_inducing, + back_constraint=False, + ) + optimize_staged(m, iters_fixed, iters_free, messages=messages) + return m + + +def fit_gp( + Xtr: np.ndarray, + Ytr: np.ndarray, + num_inducing: int, + messages: bool, +) -> GPy.core.Model: + kern = GPy.kern.RBF(Xtr.shape[1]) + GPy.kern.Bias(Xtr.shape[1]) + m = GPy.models.SparseGPRegression(Xtr, Ytr, kernel=kern, num_inducing=num_inducing) + m.Gaussian_noise.variance = Ytr.var() * 0.01 + m.Gaussian_noise.variance.fix() + m.optimize(max_iters=100, messages=messages) + m.Gaussian_noise.variance.unfix() + m.optimize(max_iters=400, messages=messages) + return m + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + add_common_args(p) + p.add_argument("--repeats", type=int, default=10, help="Paper uses 10 repeats.") + p.add_argument("--n-train", type=int, default=25) + p.add_argument("--num-inducing", type=int, default=20) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + outdir = ensure_outdir(args.outdir) + + repeats = 2 if args.quick else args.repeats + n_train = 20 if args.quick else args.n_train + q1, q2 = (3, 2) if args.quick else (4, 3) + iters_fixed = 40 if args.quick else 200 + iters_free = 120 if args.quick else 1200 + num_inducing = 12 if args.quick else args.num_inducing + + gp_mses = [] + dgp_mses = [] + + for r in range(repeats): + seed = args.seed + r + X, H, Y = generate_warped(n=120, seed=seed) + Xtr, Ytr, Xte, Yte = train_test_split(X, Y, n_train=n_train, seed=seed + 1000) + + m_gp = fit_gp(Xtr, Ytr, num_inducing=num_inducing, messages=False) + m_dgp = fit_deepgp( + Xtr, + Ytr, + q1=q1, + q2=q2, + num_inducing=num_inducing, + iters_fixed=iters_fixed, + iters_free=iters_free, + messages=args.messages, + ) + + mse_gp = mse(m_gp.predict(Xte)[0], Yte) + mse_dgp = mse(m_dgp.predict(Xte)[0], Yte) + gp_mses.append(mse_gp) + dgp_mses.append(mse_dgp) + print(f"Repeat {r + 1}/{repeats}: MSE GP={mse_gp:.4f} DeepGP={mse_dgp:.4f}") + + print( + f"Mean MSE (±std): GP {np.mean(gp_mses):.4f}±{np.std(gp_mses):.4f} " + f"DeepGP {np.mean(dgp_mses):.4f}±{np.std(dgp_mses):.4f}" + ) + + if not args.no_plot or outdir: + import matplotlib.pyplot as plt + + # Illustrate one generative draw + X, H, Y = generate_warped(n=120, seed=args.seed) + fig, axes = plt.subplots(2, 1, figsize=(7, 4), sharex=True) + axes[0].plot(X[:, 0], H[:, 0], "k-") + axes[0].set_ylabel("hidden warp") + axes[0].set_title("Warped process (one draw)") + axes[1].plot(X[:, 0], Y[:, 0], label="Y[:,0]") + axes[1].plot(X[:, 0], Y[:, 1], label="Y[:,1]") + axes[1].set_xlabel("X") + axes[1].set_ylabel("outputs") + axes[1].legend(fontsize=8) + fig.tight_layout() + save_or_show(fig, outdir, "toy_warped_process.png", args.no_plot) + + fig2, ax = plt.subplots(figsize=(5, 3.5)) + xs = np.arange(1, repeats + 1) + ax.plot(xs, gp_mses, "o-", label="GP") + ax.plot(xs, dgp_mses, "s-", label="DeepGP") + ax.set_xlabel("experiment #") + ax.set_ylabel("MSE") + ax.set_title("Damianou & Lawrence 2013 — fig. 3(b) style") + ax.legend() + fig2.tight_layout() + save_or_show(fig2, outdir, "toy_warped_mse.png", args.no_plot) + + return 0 + + +if __name__ == "__main__": + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + raise SystemExit(main()) diff --git a/examples/damianou2013/usps_digits.py b/examples/damianou2013/usps_digits.py new file mode 100644 index 0000000..aab1dd3 --- /dev/null +++ b/examples/damianou2013/usps_digits.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python +"""Section 4.3 (figs. 6–8): USPS digits {0,1,6} — depth, ARD, 1-NN in latent space. + +Paper: Damianou & Lawrence, Deep Gaussian Processes, AISTATS 2013. + +Downloads the ESL ``zip.train`` USPS split (16×16), keeps 50 examples of each of +digits {0,1,6}, and trains unsupervised DeepGPs of increasing depth. Reports the +variational bound and leave-one-out 1-NN error in the top latent features. +""" + +from __future__ import annotations + +import argparse +import gzip +import os +import sys +import urllib.request +from typing import List, Sequence, Tuple + +import numpy as np + +import deepgp + +from _common import ( + add_common_args, + ard_weights, + dominant_dims, + ensure_outdir, + nearest_neighbour_error, + optimize_staged, + print_ard, + rbf_bias, + save_or_show, +) + +ESL_ZIP_TRAIN = "https://web.stanford.edu/~hastie/ElemStatLearn/datasets/zip.train.gz" +DIGITS = (0, 1, 6) + + +def _cache_path(cache_dir: str) -> str: + return os.path.join(cache_dir, "zip.train.gz") + + +def download_usps(cache_dir: str) -> str: + os.makedirs(cache_dir, exist_ok=True) + path = _cache_path(cache_dir) + if os.path.isfile(path) and os.path.getsize(path) > 0: + return path + print(f"Downloading USPS zip.train → {path}") + req = urllib.request.Request(ESL_ZIP_TRAIN, headers={"User-Agent": "PyDeepGP-examples"}) + with urllib.request.urlopen(req, timeout=120) as resp, open(path, "wb") as out: + out.write(resp.read()) + return path + + +def load_usps_matrix(path: str) -> Tuple[np.ndarray, np.ndarray]: + """Return (Y [N,256], labels [N]) with pixels in roughly [-1, 1].""" + rows = [] + with gzip.open(path, "rt") as f: + for line in f: + parts = line.strip().split() + if not parts: + continue + rows.append([float(x) for x in parts]) + data = np.asarray(rows, dtype=float) + labels = data[:, 0].astype(int) + Y = data[:, 1:] + return Y, labels + + +def subsample_digits( + Y: np.ndarray, + labels: np.ndarray, + digits: Sequence[int] = DIGITS, + per_class: int = 50, + seed: int = 42, +) -> Tuple[np.ndarray, np.ndarray]: + rng = np.random.RandomState(seed) + pieces_Y = [] + pieces_L = [] + for d in digits: + idx = np.where(labels == d)[0] + if idx.size < per_class: + raise RuntimeError(f"Digit {d}: need {per_class} examples, found {idx.size}") + choose = rng.choice(idx, size=per_class, replace=False) + pieces_Y.append(Y[choose]) + pieces_L.append(np.full(per_class, d, dtype=int)) + return np.vstack(pieces_Y), np.concatenate(pieces_L) + + +def depth_dimensions(depth: int, data_dim: int, quick: bool) -> List[int]: + """Latent widths from data up to the top layer (length = depth + 1 entries for DeepGP). + + ``DeepGP(nDims, Y)`` expects ``nDims = [Y_dim, H1, H2, ..., H_depth]``. + """ + if quick: + # Narrower / shallower budgets for smoke tests + widths = {1: [8], 2: [8, 5], 3: [8, 6, 4]} + else: + # Overcomplete relative to the true manifold; ARD prunes + widths = { + 1: [12], + 2: [12, 8], + 3: [12, 10, 6], + 4: [12, 10, 8, 5], + 5: [12, 10, 8, 6, 4], + } + if depth not in widths: + raise ValueError(f"Unsupported depth {depth}; choose from {sorted(widths)}") + return [data_dim] + widths[depth] + + +def fit_depth( + Y: np.ndarray, + depth: int, + num_inducing: int, + iters_fixed: int, + iters_free: int, + messages: bool, + quick: bool, +) -> deepgp.DeepGP: + dims = depth_dimensions(depth, Y.shape[1], quick=quick) + kernels = [rbf_bias(q, ard=True) for q in dims[1:]] + m = deepgp.DeepGP( + dims, + Y, + kernels=kernels, + num_inducing=num_inducing, + back_constraint=False, + ) + optimize_staged(m, iters_fixed, iters_free, messages=messages) + return m + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + add_common_args(p) + p.add_argument( + "--depths", + type=int, + nargs="+", + default=None, + help="Hierarchy depths to fit (default: 1..5, or 1..2 with --quick).", + ) + p.add_argument("--per-class", type=int, default=50) + p.add_argument("--num-inducing", type=int, default=40) + p.add_argument( + "--cache-dir", + type=str, + default=os.path.join(os.path.dirname(__file__), "data"), + help="Directory for zip.train.gz cache.", + ) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + outdir = ensure_outdir(args.outdir) + + depths = args.depths + if depths is None: + depths = [1, 2] if args.quick else [1, 2, 3, 4, 5] + + per_class = 20 if args.quick else args.per_class + num_inducing = 20 if args.quick else args.num_inducing + iters_fixed = 40 if args.quick else 200 + iters_free = 100 if args.quick else 2000 + + path = download_usps(args.cache_dir) + Y_all, labels_all = load_usps_matrix(path) + Y, labels = subsample_digits(Y_all, labels_all, per_class=per_class, seed=args.seed) + # Zero-mean unit-ish scaling helps optimisation + Y = Y - Y.mean(0) + Y = Y / (Y.std() + 1e-12) + print(f"USPS subset: Y={Y.shape}, classes={sorted(set(labels.tolist()))}") + + results = [] + models = {} + for depth in depths: + print(f"\n=== depth {depth} ===") + m = fit_depth( + Y, + depth=depth, + num_inducing=num_inducing, + iters_fixed=iters_fixed, + iters_free=iters_free, + messages=args.messages, + quick=args.quick, + ) + bound = float(m.log_likelihood()) + # Top latent features = last layer's X + Xtop = np.asarray(m.layers[-1].X.mean) + mistakes, rate = nearest_neighbour_error(Xtop, labels) + print(f"bound={bound:.3f} 1-NN mistakes={mistakes}/{Y.shape[0]} ({rate:.3%})") + print_ard(m) + results.append((depth, bound, mistakes, rate)) + models[depth] = m + + print("\nSummary") + print("depth bound 1-NN errors") + for depth, bound, mistakes, rate in results: + print(f"{depth:5d} {bound:9.3f} {mistakes:3d} ({rate:.3%})") + + deepest = max(depths) + m = models[deepest] + if not args.no_plot or outdir: + import matplotlib.pyplot as plt + + Xtop = np.asarray(m.layers[-1].X.mean) + w = ard_weights(m.layers[-1].kern) + dims = dominant_dims(w, k=2) + colors = ["C0", "C1", "C2"] + markers = ["o", "s", "^"] + + fig, ax = plt.subplots(figsize=(4.5, 4.5)) + for i, digit in enumerate(DIGITS): + mask = labels == digit + ax.scatter( + Xtop[mask, dims[0]], + Xtop[mask, dims[1]], + c=colors[i], + marker=markers[i], + s=28, + label=str(digit), + ) + ax.set_xlabel(f"top dim {dims[0]}") + ax.set_ylabel(f"top dim {dims[1]}") + ax.set_title(f"USPS top latent (depth {deepest})") + ax.legend() + fig.tight_layout() + save_or_show(fig, outdir, "usps_top_latent.png", args.no_plot) + + fig2, axes = plt.subplots(1, len(m.layers), figsize=(3.2 * len(m.layers), 3)) + if len(m.layers) == 1: + axes = [axes] + for i, layer in enumerate(m.layers): + w = ard_weights(layer.kern) + axes[i].bar(np.arange(1, len(w) + 1), w) + axes[i].set_title(f"layer {i} ARD") + axes[i].set_xlabel("dimension") + fig2.suptitle("ARD weights (1/lengthscale)") + fig2.tight_layout() + save_or_show(fig2, outdir, "usps_ard.png", args.no_plot) + + fig3, ax = plt.subplots(figsize=(5, 3.5)) + ax.plot([r[0] for r in results], [r[1] for r in results], "o-", label="variational bound") + ax.set_xlabel("depth") + ax.set_ylabel("bound") + ax2 = ax.twinx() + ax2.plot([r[0] for r in results], [r[2] for r in results], "s--", color="C1", label="1-NN mistakes") + ax2.set_ylabel("1-NN mistakes") + ax.set_title("Depth selection via bound / 1-NN") + fig3.tight_layout() + save_or_show(fig3, outdir, "usps_depth.png", args.no_plot) + + return 0 + + +if __name__ == "__main__": + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + raise SystemExit(main()) diff --git a/examples/example_mrd.py b/examples/example_mrd.py index ef4ef24..01c3ab0 100644 --- a/examples/example_mrd.py +++ b/examples/example_mrd.py @@ -3,8 +3,7 @@ Two correlated observation views share a parent latent layer. This is the architecture used for the Damianou & Lawrence (2013) high-five experiment; -see also ``examples/damianou2013/highfive_shared.py`` when that suite is -present. +see also ``examples/damianou2013/highfive_shared.py``. """ from __future__ import annotations