NeRF and Neural Implicit Representations for 3D

Module: Computer Vision | Difficulty: Premium

Volume Rendering

Positional Encoding

NeRF Loss

import torch
import torch.nn as nn

class NeRF(nn.Module):
    def __init__(self, pos_enc_dim=10, dir_enc_dim=4, hidden=256):
        super().__init__()
        pos_dim = 3 + 3 * 2 * pos_enc_dim
        dir_dim = 3 + 3 * 2 * dir_enc_dim
        self.pos_encoder = nn.Sequential(
            nn.Linear(pos_dim, hidden), nn.ReLU(inplace=True),
            nn.Linear(hidden, hidden), nn.ReLU(inplace=True),
            nn.Linear(hidden, hidden), nn.ReLU(inplace=True),
            nn.Linear(hidden, hidden), nn.ReLU(inplace=True),
        )
        self.sigma_head = nn.Linear(hidden, 1)
        self.feature_head = nn.Linear(hidden, hidden)
        self.color_head = nn.Sequential(
            nn.Linear(hidden + dir_dim, hidden // 2),
            nn.ReLU(inplace=True),
            nn.Linear(hidden // 2, 3), nn.Sigmoid(),
        )

    def forward(self, positions, directions):
        pos_enc = self.encode_position(positions)
        dir_enc = self.encode_direction(directions)
        h = self.pos_encoder(pos_enc)
        sigma = self.sigma_head(h)
        features = self.feature_head(h)
        color = self.color_head(
            torch.cat([features, dir_enc], dim=-1))
        return color, sigma

    def encode_position(self, x):
        encodings = [x]
        for i in range(10):
            encodings.append(torch.sin(2 ** i * 3.14159 * x))
            encodings.append(torch.cos(2 ** i * 3.14159 * x))
        return torch.cat(encodings, dim=-1)

    def encode_direction(self, x):
        encodings = [x]
        for i in range(4):
            encodings.append(torch.sin(2 ** i * 3.14159 * x))
            encodings.append(torch.cos(2 ** i * 3.14159 * x))
        return torch.cat(encodings, dim=-1)

def volume_rendering(sigma, color, distances):
    delta = distances[:, 1:] - distances[:, :-1]
    alpha = 1.0 - torch.exp(-sigma[:, :-1] * delta)
    T = torch.cumprod(
        torch.cat([torch.ones_like(alpha[:, :1]),
                    1.0 - alpha + 1e-10], dim=1), dim=1)
    weights = T[:, :-1] * alpha
    rgb = (weights.unsqueeze(-1) * color[:, :-1]).sum(dim=1)
    depth = (weights * distances[:, :-1]).sum(dim=1)
    return rgb, depth

Research Insight: Neural Radiance Fields have evolved from slow optimization-based methods (original NeRF: 30s per image) to real-time approaches (3D Gaussian Splatting: 20ms per image). The key breakthrough of 3DGS was replacing implicit neural representations with an explicit Gaussian representation that can be rasterized efficiently. Instant-NGP introduced multi-resolution hash encoding for fast training. The combination of neural rendering with SLAM (NeRF-SLAM) enables real-time 3D reconstruction from monocular video. The frontier is photorealistic avatars for telepresence, requiring real-time rendering of dynamic humans with facial expressions.