VQA Models and Multimodal Reasoning

Module: Computer Vision | Difficulty: Premium

VQA Formulation

Soft Attention

VQA Accuracy

import torch
import torch.nn as nn
import torch.nn.functional as F

class TopDownAttention(nn.Module):
    def __init__(self, v_dim, q_dim, hidden_dim):
        super().__init__()
        self.v_proj = nn.Linear(v_dim, hidden_dim)
        self.q_proj = nn.Linear(q_dim, hidden_dim)
        self.out = nn.Linear(hidden_dim, 1)

    def forward(self, v, q):
        v_proj = self.v_proj(v)
        q_proj = self.q_proj(q)
        attention = self.out(torch.tanh(v_proj + q_proj.unsqueeze(1)))
        attention = F.softmax(attention.squeeze(-1), dim=1)
        return attention, (attention.unsqueeze(-1) * v).sum(dim=1)

class VQAModel(nn.Module):
    def __init__(self, vocab_size, embed_dim=300,
                 v_dim=2048, q_dim=1024, num_classes=3129):
        super().__init__()
        self.q_embed = nn.Embedding(vocab_size, embed_dim)
        self.q_lstm = nn.LSTM(embed_dim, q_dim // 2,
                              batch_first=True, bidirectional=True)
        self.attention = TopDownAttention(v_dim, q_dim, 512)
        self.classifier = nn.Sequential(
            nn.Linear(v_dim + q_dim, 1024),
            nn.ReLU(inplace=True), nn.Dropout(0.5),
            nn.Linear(1024, num_classes),
        )

    def forward(self, features, question):
        q_emb = self.q_embed(question)
        _, (h, _) = self.q_lstm(q_emb)
        q = torch.cat([h[0], h[1]], dim=1)
        alpha, v = self.attention(features, q)
        combined = torch.cat([v, q], dim=1)
        return self.classifier(combined)

Research Insight: Modern VQA has shifted from attention-based fusion to large multimodal models (LMMs) that unify vision and language via instruction tuning. Models like PaLI-X and GPT-4V demonstrate that scaling both data and parameters enables strong compositional reasoning without explicit structural priors. The challenge of compositional generalization (e.g., understanding "red cube on blue sphere" vs "blue cube on red sphere") remains an active research area, with datasets like CLEVR probing systematic generalization.