Video Generation

Module: Computer Vision | Difficulty: Advanced

Video Prediction

Temporal Consistency Loss

Video Diffusion

FVD (Fréchet Video Distance)

import torch
import torch.nn as nn

class VideoPredNet(nn.Module):
    def __init__(self, num_frames=16):
        super().__init__()
        self.temporal = nn.LSTM(256, 256, num_layers=2, batch_first=True)
        self.spatial = nn.Sequential(
            nn.Conv2d(256, 128, 3, padding=1), nn.ReLU(True),
            nn.ConvTranspose2d(128, 64, 4, 2, 1), nn.ReLU(True),
            nn.ConvTranspose2d(64, 3, 4, 2, 1), nn.Tanh(),
        )
    
    def forward(self, features):
        # features: (B, T, C)
        temporal_out, _ = self.temporal(features)
        B, T, C = temporal_out.shape
        frames = []
        for t in range(T):
            feat_map = temporal_out[:, t].view(B, C, 1, 1).expand(-1, -1, 64, 64)
            frames.append(self.spatial(feat_map))
        return torch.stack(frames, dim=1)

Key Takeaways

• Video generation requires maintaining temporal consistency
• Text-to-video combines language understanding with visual synthesis
• FVD evaluates both visual quality and temporal coherence