Meta-Learning: Learning to Learn

Module: Machine Learning | Difficulty: Advanced

MAML

Prototypical Networks

Meta-Learning Theory

Task Distribution

import torch
import torch.nn as nn

class MAML:
    def __init__(self, model, inner_lr=0.01, outer_lr=0.001):
        self.model = model; self.inner_lr = inner_lr
        self.outer_optimizer = torch.optim.Adam(model.parameters(), lr=outer_lr)
    def inner_loop(self, support_x, support_y):
        fast_weights = list(self.model.parameters())
        for _ in range(5):
            loss = nn.functional.cross_entropy(self.model.functional_forward(support_x, fast_weights), support_y)
            grads = torch.autograd.grad(loss, fast_weights)
            fast_weights = [w - self.inner_lr * g for w, g in zip(fast_weights, grads)]
        return fast_weights
    def outer_loop(self, tasks):
        meta_loss = 0
        for task in tasks:
            fast_w = self.inner_loop(task.support_x, task.support_y)
            query_loss = nn.functional.cross_entropy(self.model.functional_forward(task.query_x, fast_w), task.query_y)
            meta_loss += query_loss
        self.outer_optimizer.zero_grad()
        (meta_loss / len(tasks)).backward()
        self.outer_optimizer.step()

Research Insight: MAML's inner loop acts as a learned optimizer. The key insight is that the initialization learned by MAML makes the loss landscape smooth, enabling rapid adaptation with few gradient steps.