Vision-Language Models

Module: Computer Vision | Difficulty: Advanced

CLIP

Contrastive Objective

Zero-Shot Classification

Visual QA

import torch
import torch.nn as nn

class CLIP(nn.Module):
    def __init__(self, image_encoder, text_encoder, embed_dim=512):
        super().__init__()
        self.image_encoder = image_encoder
        self.text_encoder = text_encoder
        self.image_proj = nn.Linear(image_encoder.out_features, embed_dim)
        self.text_proj = nn.Linear(text_encoder.out_features, embed_dim)
        self.temperature = nn.Parameter(torch.ones([]) * 0.07)
    
    def forward(self, images, texts):
        img_feats = self.image_proj(self.image_encoder(images))
        txt_feats = self.text_proj(self.text_encoder(texts))
        img_feats = nn.functional.normalize(img_feats, dim=1)
        txt_feats = nn.functional.normalize(txt_feats, dim=1)
        logits = img_feats @ txt_feats.T / self.temperature
        return logits

Key Takeaways

• CLIP learns visual concepts from natural language supervision
• Zero-shot classification requires no task-specific training
• Vision-language pre-training transfers across diverse vision tasks