Point Cloud Processing and 3D Deep Learning

Module: Computer Vision | Difficulty: Premium

PointNet Set Abstraction

where is the centroid of a local region.

Farthest Point Sampling (FPS)

Chamfer Distance

Earth Mover's Distance

import torch
import torch.nn as nn
import torch.nn.functional as F

class PointNet(nn.Module):
    def __init__(self, num_classes=40):
        super().__init__()
        self.mlp1 = nn.Sequential(
            nn.Conv1d(3, 64, 1),
            nn.BatchNorm1d(64), nn.ReLU(inplace=True),
            nn.Conv1d(64, 64, 1),
            nn.BatchNorm1d(64), nn.ReLU(inplace=True),
        )
        self.mlp2 = nn.Sequential(
            nn.Conv1d(64, 128, 1),
            nn.BatchNorm1d(128), nn.ReLU(inplace=True),
            nn.Conv1d(128, 1024, 1),
            nn.BatchNorm1d(1024), nn.ReLU(inplace=True),
        )
        self.head = nn.Sequential(
            nn.Linear(1024, 512),
            nn.BatchNorm1d(512), nn.ReLU(inplace=True),
            nn.Dropout(0.5),
            nn.Linear(512, 256),
            nn.BatchNorm1d(256), nn.ReLU(inplace=True),
            nn.Dropout(0.5),
            nn.Linear(256, num_classes),
        )

    def forward(self, x):
        B, N, _ = x.shape
        x = x.permute(0, 2, 1)
        local_features = self.mlp1(x)
        global_features = self.mlp2(local_features).max(dim=2)[0]
        return self.head(global_features)

class SetAbstraction(nn.Module):
    def __init__(self, npoint, nsample, in_channel, mlp_channels):
        super().__init__()
        self.npoint = npoint
        self.nsample = nsample
        layers = []
        last_c = in_channel
        for out_c in mlp_channels:
            layers.extend([
                nn.Conv2d(last_c, out_c, 1),
                nn.BatchNorm2d(out_c),
                nn.ReLU(inplace=True)])
            last_c = out_c
        self.mlp = nn.Sequential(*layers)

    def forward(self, xyz, points):
        B, N, C = xyz.shape
        if self.npoint is not None:
            idx = self.farthest_point_sample(xyz, self.npoint)
            new_xyz = torch.gather(
                xyz, 1, idx.unsqueeze(-1).expand(-1, -1, C))
        else:
            new_xyz = xyz.mean(dim=1, keepdim=True)
        return new_xyz, self.mlp(
            points.unsqueeze(-1)).squeeze(-1)

Research Insight: 3D deep learning has evolved from PointNet's simple MLP approach to sophisticated transformer architectures for point clouds (Point Transformer V2, Point-BERT). The key challenge is the irregular, unordered nature of point data, which prevents direct application of convolutional architectures. Point-BERT and Point-MAE apply masked autoencoding to discretized point clouds, achieving breakthroughs in 3D representation learning. These methods pretrain on large 3D datasets and fine-tune for downstream tasks, mirroring the success of masked language modeling in NLP.