Multimodal Medical Data Fusion

Module: Healthcare AI | Difficulty: Advanced

Early Fusion

Late Fusion

Cross-Attention Fusion

Attention-Weighted Fusion

Fusion Strategy Comparison

import torch
import torch.nn as nn

class CrossModalAttention(nn.Module):
    def __init__(self, dim_img=512, dim_clin=128, dim_genomic=64):
        super().__init__()
        self.proj_img = nn.Linear(dim_img, 256)
        self.proj_clin = nn.Linear(dim_clin, 256)
        self.proj_genomic = nn.Linear(dim_genomic, 256)
        self.attn = nn.MultiheadAttention(256, num_heads=8)
        self.fusion_gate = nn.Sequential(
            nn.Linear(256 * 3, 256), nn.Sigmoid())
        self.output = nn.Linear(256, 1)

    def forward(self, img_feat, clin_feat, genomic_feat):
        q = self.proj_img(img_feat).unsqueeze(0)
        k = torch.cat([
            self.proj_clin(clin_feat).unsqueeze(0),
            self.proj_genomic(genomic_feat).unsqueeze(0)
        ], dim=0)
        v = k
        attn_out, _ = self.attn(q, k, v)
        combined = torch.cat([
            attn_out.squeeze(0), clin_feat, genomic_feat], dim=-1)
        gate = self.fusion_gate(combined)
        fused = gate * attn_out.squeeze(0)
        return self.output(fused)

class MultimodalClassifier(nn.Module):
    def __init__(self, num_classes=2):
        super().__init__()
        self.img_encoder = nn.Sequential(
            nn.Conv2d(3, 64, 3, padding=1), nn.ReLU(),
            nn.AdaptiveAvgPool2d(1))
        self.clinical_encoder = nn.Sequential(
            nn.Linear(50, 128), nn.ReLU(), nn.Linear(128, 128))
        self.cross_attn = CrossModalAttention()
        self.classifier = nn.Linear(256, num_classes)

    def forward(self, img, clinical):
        img_feat = self.img_encoder(img).flatten(1)
        clin_feat = self.clinical_encoder(clinical)
        fused = self.cross_attn(img_feat, clin_feat, clin_feat)
        return self.classifier(fused)

model = MultimodalClassifier()
img = torch.randn(1, 3, 224, 224)
clinical = torch.randn(1, 50)
output = model(img, clinical)
print(f'Output logits: {output.shape}')

Research Insight: Multimodal fusion with cross-attention significantly outperforms single-modality approaches, but the improvement depends critically on modality alignment. Contrastive learning objectives applied to medical data can learn modality-invariant representations that improve fusion even when one modality is missing at inference time.