Semantic, Instance, and Panoptic Segmentation

Module: Computer Vision | Difficulty: Premium

Semantic Segmentation Output

For each pixel , predict class :

Panoptic Quality

Atrous Spatial Pyramid Pooling (ASPP)

Dice Loss

import torch
import torch.nn as nn

class DoubleConv(nn.Module):
    def __init__(self, in_ch, out_ch):
        super().__init__()
        self.block = nn.Sequential(
            nn.Conv2d(in_ch, out_ch, 3, padding=1, bias=False),
            nn.BatchNorm2d(out_ch),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_ch, out_ch, 3, padding=1, bias=False),
            nn.BatchNorm2d(out_ch),
            nn.ReLU(inplace=True),
        )
    def forward(self, x):
        return self.block(x)

class UNet(nn.Module):
    def __init__(self, in_ch=3, num_classes=21):
        super().__init__()
        self.enc1 = DoubleConv(in_ch, 64)
        self.enc2 = DoubleConv(64, 128)
        self.enc3 = DoubleConv(128, 256)
        self.pool = nn.MaxPool2d(2)
        self.up3 = nn.ConvTranspose2d(256, 128, 2, stride=2)
        self.up2 = nn.ConvTranspose2d(128, 64, 2, stride=2)
        self.dec3 = DoubleConv(256, 256)
        self.dec2 = DoubleConv(128, 128)
        self.out = nn.Conv2d(64, num_classes, 1)

    def forward(self, x):
        e1 = self.enc1(x)
        e2 = self.enc2(self.pool(e1))
        e3 = self.enc3(self.pool(e2))
        d3 = self.dec3(torch.cat([self.up3(e3), e2], dim=1))
        d2 = self.dec2(torch.cat([self.up2(d3), e1], dim=1))
        return self.out(d2)

Research Insight: Panoptic segmentation unifies semantic and instance segmentation by assigning each pixel a semantic label and instance ID. Mask2Former demonstrated that a single transformer architecture can handle all three segmentation tasks (semantic, instance, panoptic) by reformulating them as mask classification. The panoptic quality metric elegantly balances detection and segmentation quality, making it the standard evaluation measure for unified segmentation systems.