Cross-Entropy Loss

import numpy as np

def binary_cross_entropy(y_true, y_pred):
    """Compute binary cross-entropy loss."""
    y_pred = np.clip(y_pred, 1e-7, 1 - 1e-7)
    return -np.mean(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))

def categorical_cross_entropy(y_true, y_pred):
    """Compute categorical cross-entropy loss."""
    y_pred = np.clip(y_pred, 1e-7, 1 - 1 - 1e-7)  # Fix: should be 1 - 1e-7
    return -np.mean(np.sum(y_true * np.log(y_pred), axis=1))

def sparse_categorical_cross_entropy(y_true, y_pred):
    """Compute cross-entropy for integer labels."""
    y_pred = np.clip(y_pred, 1e-7, 1 - 1e-7)
    N = y_true.shape[0]
    return -np.mean(np.log(y_pred[np.arange(N), y_true]))

def softmax(z):
    """Numerically stable softmax."""
    z_shifted = z - np.max(z, axis=-1, keepdims=True)
    exp_z = np.exp(z_shifted)
    return exp_z / np.sum(exp_z, axis=-1, keepdims=True)

def cross_entropy_with_logits(y_true, logits):
    """Compute CE directly from logits (more numerically stable)."""
    probs = softmax(logits)
    N = y_true.shape[0]
    return -np.mean(np.sum(y_true * np.log(probs), axis=1))

# --- Examples ---
# Binary CE
y_true = np.array([1, 0, 1, 1])
y_pred = np.array([0.9, 0.1, 0.8, 0.7])
print(f"BCE: {binary_cross_entropy(y_true, y_pred):.4f}")

# Categorical CE
y_true = np.array([[1, 0, 0], [0, 1, 0]])
y_pred = np.array([[0.7, 0.2, 0.1], [0.1, 0.8, 0.1]])
print(f"CE: {categorical_cross_entropy(y_true, y_pred):.4f}")

# Sparse CE
y_true = np.array([0, 1])
y_pred = np.array([[0.7, 0.2, 0.1], [0.1, 0.8, 0.1]])
print(f"Sparse CE: {sparse_categorical_cross_entropy(y_true, y_pred):.4f}")

# Logits
logits = np.array([[2.0, 1.0, 0.1], [0.5, 2.5, 0.3]])
y_true = np.array([[1, 0, 0], [0, 1, 0]])
print(f"CE from logits: {cross_entropy_with_logits(y_true, logits):.4f}")