Transformer Architecture: Self-Attention and Beyond
Module: Natural Language Processing | Difficulty: Advanced
Scaled Dot-Product Attention
Multi-Head Attention
Positional Encoding
Layer Normalization
import torch
import torch.nn as nn
import math
class TransformerBlock(nn.Module):
def __init__(self, d_model, nhead, dim_ff, dropout=0.1):
super().__init__()
self.attn = nn.MultiheadAttention(d_model, nhead, batch_first=True)
self.ff = nn.Sequential(
nn.Linear(d_model, dim_ff), nn.ReLU(), nn.Linear(dim_ff, d_model))
self.norm1 = nn.LayerNorm(d_model)
self.norm2 = nn.LayerNorm(d_model)
self.dropout = nn.Dropout(dropout)
def forward(self, x, mask=None):
attn_out, _ = self.attn(x, x, x, attn_mask=mask)
x = self.norm1(x + self.dropout(attn_out))
x = self.norm2(x + self.dropout(self.ff(x)))
return x
Research Insight: The transformer's key innovation is replacing recurrence with self-attention, enabling parallel computation across all positions. However, the quadratic complexity limits the context length, motivating efficient attention variants.