Lifelong Learning for Visual Recognition Systems

Module: Computer Vision | Difficulty: Premium

Catastrophic Forgetting

Elastic Weight Consolidation (EWC)

where is the Fisher information diagonal.

Experience Replay

PackNet (Progressive Architecture)

import torch
import torch.nn as nn
import torch.nn.functional as F

class EWC:
    def __init__(self, model, importance=1000):
        self.model = model
        self.importance = importance
        self.fisher = {}
        self.old_params = {}

    def compute_fisher(self, dataloader):
        self.model.eval()
        for name, param in self.model.named_parameters():
            self.fisher[name] = torch.zeros_like(param.data)
            self.old_params[name] = param.data.clone()
        for data, target in dataloader:
            self.model.zero_grad()
            output = self.model(data)
            loss = F.cross_entropy(output, target)
            loss.backward()
            for name, param in self.model.named_parameters():
                self.fisher[name] += param.grad.data ** 2

    def penalty(self, model):
        loss = 0
        for name, param in model.named_parameters():
            if name in self.fisher:
                loss += (self.fisher[name] *
                         (param - self.old_params[name]) ** 2).sum()
        return self.importance * loss

class ExperienceReplay:
    def __init__(self, buffer_size=2000):
        self.buffer_size = buffer_size
        self.buffer_x = None
        self.buffer_y = None

    def add_to_buffer(self, x, y):
        if self.buffer_x is None:
            self.buffer_x = x[:self.buffer_size]
            self.buffer_y = y[:self.buffer_size]
        else:
            self.buffer_x = torch.cat([
                self.buffer_x[-self.buffer_size + len(x):], x])
            self.buffer_y = torch.cat([
                self.buffer_y[-self.buffer_size + len(y):], y])

    def sample(self, batch_size):
        idx = torch.randperm(len(self.buffer_x))[:batch_size]
        return self.buffer_x[idx], self.buffer_y[idx]

class LearningToPrompt(nn.Module):
    def __init__(self, backbone, prompt_length=10, num_tasks=10):
        super().__init__()
        self.backbone = backbone
        self.prompts = nn.ParameterList([
            nn.Parameter(torch.randn(prompt_length, backbone.embed_dim))
            for _ in range(num_tasks)])
        self.task_id_head = nn.Linear(backbone.embed_dim, num_tasks)

    def forward(self, x, task_id=None):
        features = self.backbone(x, return_features=True)
        if task_id is not None:
            prompt = self.prompts[task_id]
            features = torch.cat([
                prompt.unsqueeze(0).expand(x.shape[0], -1, -1),
                features], dim=1)
        return self.backbone.classifier(features.mean(dim=1))

Research Insight: Continual learning for vision has shifted from regularization-based methods (EWC) to prompt-based approaches (L2P) that learn task-specific prompts for frozen foundation models. The key insight is that large pretrained models contain sufficient knowledge for all tasks; the challenge is accessing the right knowledge at the right time. L2P uses a prompt pool and a small key network to select appropriate prompts, achieving near-zero forgetting without any exemplar memory. This approach leverages the fact that foundation models have already learned a rich feature space that can be adapted to new tasks.