Gaussian Processes: Regression and Classification

Module: Machine Learning | Difficulty: Advanced

GP Prior

GP Posterior

Marginal Likelihood

Common Kernels

import numpy as np
from scipy.optimize import minimize

class GaussianProcess:
    def __init__(self, kernel='rbf', noise=0.1):
        self.kernel = kernel; self.noise = noise
    def _k(self, X1, X2, l=1.0):
        sq = np.sum(X1**2,1).reshape(-1,1) + np.sum(X2**2,1) - 2*X1@X2.T
        return np.exp(-sq/(2*l**2))
    def fit(self, X, y):
        self.X_train = X; self.y_train = y
        K = self._k(X, X) + self.noise*np.eye(len(y))
        self.L = np.linalg.cholesky(K)
    def predict(self, X):
        K_s = self._k(X, self.X_train)
        v = np.linalg.solve(self.L, K_s.T)
        mu = K_s @ np.linalg.solve(self.L.T, np.linalg.solve(self.L, self.y_train))
        var = self._k(X, X) - np.sum(v**2, axis=1)
        return mu, var

Research Insight: GPs provide uncertainty estimates for free, but scale as in training. Sparse GP methods (SVGP, FITC) reduce this to by using inducing points.