Entropy

import numpy as np
from collections import Counter

def entropy(probs):
    """Compute Shannon entropy in bits from a probability distribution."""
    probs = np.array(probs)
    probs = probs[probs > 0]  # Avoid log(0)
    return -np.sum(probs * np.log2(probs))

def entropy_from_samples(samples):
    """Estimate entropy from a list of samples."""
    counter = Counter(samples)
    total = len(samples)
    probs = np.array([count / total for count in counter.values()])
    return entropy(probs)

def joint_entropy(joint_probs):
    """Compute joint entropy from a 2D joint probability matrix."""
    flat = joint_probs.flatten()
    return entropy(flat)

def conditional_entropy(joint_probs):
    """Compute H(Y|X) from a joint probability matrix."""
    h_y_given_x = 0.0
    for i in range(joint_probs.shape[0]):
        p_x = joint_probs[i].sum()
        if p_x > 0:
            h_y_given_x += p_x * entropy(joint_probs[i] / p_x)
    return h_y_given_x

# --- Examples ---
# Fair coin
print(f"Fair coin: {entropy([0.5, 0.5]):.3f} bits")          # 1.000

# Biased coin
print(f"Biased (0.9,0.1): {entropy([0.9, 0.1]):.3f} bits")  # 0.469

# Fair die
print(f"Fair die: {entropy([1/6]*6):.3f} bits")              # 2.585

# Loaded die
loaded = [1/2, 1/4, 1/8, 1/16, 1/32, 1/32]
print(f"Loaded die: {entropy(loaded):.3f} bits")              # 1.938

# Joint entropy example
joint = np.array([[0.4, 0.1], [0.1, 0.4]])
print(f"Joint entropy: {joint_entropy(joint):.3f} bits")      # 1.722
print(f"Conditional H(Y|X): {conditional_entropy(joint):.3f}")  # 0.722

# Entropy from samples
np.random.seed(42)
samples = np.random.choice(['A', 'B', 'C', 'D'], size=10000, p=[0.5, 0.25, 0.125, 0.125])
print(f"Estimated entropy: {entropy_from_samples(samples):.3f}")  # ~1.75