Chain Rule and Implicit Differentiation

import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_grad(x):
    s = sigmoid(x)
    return s * (1 - s)

# Forward pass
x = np.array([2.0])
W1 = np.array([[0.5]])
b1 = np.array([0.1])
W2 = np.array([[0.8]])
b2 = np.array([0.2])
y_true = np.array([1.0])

z1 = W1 @ x + b1
a1 = sigmoid(z1)
z2 = W2 @ a1 + b2
a2 = sigmoid(z2)
loss = 0.5 * (a2 - y_true) ** 2

# Backward pass (chain rule)
dL_da2 = a2 - y_true
da2_dz2 = sigmoid_grad(z2)
dL_dz2 = dL_da2 * da2_dz2
dL_dW2 = dL_dz2 @ a1.T
dL_da1 = W2.T @ dL_dz2
da1_dz1 = sigmoid_grad(z1)
dL_dz1 = dL_da1 * da1_dz1
dL_dW1 = dL_dz1 @ x.T

print(f"Loss: {loss[0]:.4f}")
print(f"dL/dW2: {dL_dW2[0][0]:.4f}")
print(f"dL/dW1: {dL_dW1[0][0]:.4f}")

import numpy as np

# ============================================
# Manual Chain Rule Implementation
# ============================================

def chain_rule_example():
    """Differentiate f(x) = sin(x^2) using the chain rule."""
    x = 1.5

    # Outer: sin(u), Inner: u = x^2
    u = x ** 2
    f = np.sin(u)

    # Derivatives
    df_du = np.cos(u)       # derivative of sin
    du_dx = 2 * x            # derivative of x^2
    df_dx = df_du * du_dx   # chain rule: multiply

    print(f"f({x}) = sin({x}^2) = {f:.4f}")
    print(f"f'({x}) = {df_dx:.4f}")

chain_rule_example()

# ============================================
# Numerical Gradient Verification
# ============================================

def numerical_gradient(f, x, h=1e-7):
    """Central difference approximation."""
    return (f(x + h) - f(x - h)) / (2 * h)

def analytical_chain_rule(x):
    """Derivative of sin(x^2) using chain rule."""
    return np.cos(x ** 2) * 2 * x

x_test = 1.5
numerical = numerical_gradient(lambda x: np.sin(x**2), x_test)
analytical = analytical_chain_rule(x_test)
print(f"Numerical:  {numerical:.6f}")
print(f"Analytical: {analytical:.6f}")

# ============================================
# Deep Learning: Manual Backward Pass
# ============================================

def manual_backprop():
    """Full backward pass for a 3-layer network."""
    np.random.seed(42)

    # Forward
    x = np.random.randn(4, 1)
    W1 = np.random.randn(8, 4) * 0.5
    b1 = np.zeros((8, 1))
    W2 = np.random.randn(4, 8) * 0.5
    b2 = np.zeros((4, 1))
    W3 = np.random.randn(1, 4) * 0.5
    b3 = np.zeros((1, 1))

    def sigmoid(z):
        return 1 / (1 + np.exp(-np.clip(z, -500, 500)))

    # Forward
    z1 = W1 @ x + b1
    a1 = sigmoid(z1)
    z2 = W2 @ a1 + b2
    a2 = sigmoid(z2)
    z3 = W3 @ a2 + b3
    a3 = sigmoid(z3)

    y_true = np.array([[1.0]])
    loss = 0.5 * (a3 - y_true) ** 2

    # Backward (chain rule layer by layer)
    dL_da3 = a3 - y_true
    da3_dz3 = a3 * (1 - a3)
    dL_dz3 = dL_da3 * da3_dz3

    dL_dW3 = dL_dz3 @ a2.T
    dL_db3 = dL_dz3
    dL_da2 = W3.T @ dL_dz3

    da2_dz2 = a2 * (1 - a2)
    dL_dz2 = dL_da2 * da2_dz2
    dL_dW2 = dL_dz2 @ a1.T
    dL_db2 = dL_dz2
    dL_da1 = W2.T @ dL_dz2

    da1_dz1 = a1 * (1 - a1)
    dL_dz1 = dL_da1 * da1_dz1
    dL_dW1 = dL_dz1 @ x.T
    dL_db1 = dL_dz1

    print(f"Loss: {loss[0][0]:.6f}")
    print(f"dL/dW1 shape: {dL_dW1.shape}")
    print(f"dL/dW2 shape: {dL_dW2.shape}")
    print(f"dL/dW3 shape: {dL_dW3.shape}")

manual_backprop()

# ============================================
# PyTorch Autograd (Same Computation)
# ============================================

try:
    import torch

    x_t = torch.tensor([1.5], requires_grad=True)
    f_t = torch.sin(x_t ** 2)
    f_t.backward()
    print(f"PyTorch grad: {x_t.grad.item():.6f}")
except ImportError:
    print("PyTorch not available")