Semantic Segmentation for Drones
Semantic segmentation assigns a class label to every pixel in an image—transforming drone footage into detailed maps showing roads, buildings, vegetation, and water. This tutorial covers the architectures enabling pixel-perfect scene understanding.
The Segmentation Pipeline
Sponsored
Unlike object detection (bounding boxes), segmentation provides precise pixel-level classification for comprehensive scene analysis.
Architecture Diagram
**Real-world analogy:** Semantic segmentation is like a city planner coloring a map—every inch of land gets classified as residential, commercial, park, or road. Nothing is left unclassified.
U-Net Architecture
U-Net's symmetric encoder-decoder with skip connections is the gold standard for drone image segmentation.
python
class ConvBlock:
"""Double convolution block for U-Net."""
def __init__(self, in_channels, out_channels):
self.weights1 = np.random.randn(out_channels, in_channels, 3, 3) * np.sqrt(2.0 / (in_channels * 9))
self.bias1 = np.zeros(out_channels)
self.weights2 = np.random.randn(out_channels, out_channels, 3, 3) * np.sqrt(2.0 / (out_channels * 9))
self.bias2 = np.zeros(out_channels)
def conv2d(self, x, weights, bias, stride=1, padding=1):
"""2D convolution."""
batch, in_ch, h, w = x.shape
out_ch = weights.shape[0]
k = weights.shape[2]
# Padding
x_padded = np.pad(x, ((0,0), (0,0), (padding,padding), (padding,padding)), mode='reflect')
out_h = (h + 2*padding - k) // stride + 1
out_w = (w + 2*padding - k) // stride + 1
output = np.zeros((batch, out_ch, out_h, out_w))
for b in range(batch):
for oc in range(out_ch):
for i in range(out_h):
for j in range(out_w):
h_start = i * stride
h_end = h_start + k
w_start = j * stride
w_end = w_start + k
output[b, oc, i, j] = np.sum(x_padded[b, :, h_start:h_end, w_start:w_end] * weights[oc]) + bias[oc]
return output
def relu(self, x):
return np.maximum(0, x)
def forward(self, x):
x = self.conv2d(x, self.weights1, self.bias1)
x = self.relu(x)
x = self.conv2d(x, self.weights2, self.bias2)
x = self.relu(x)
return x
class UNet:
"""U-Net architecture for semantic segmentation."""
def __init__(self, in_channels=3, num_classes=6):
# Encoder
self.enc1 = ConvBlock(in_channels, 64)
self.enc2 = ConvBlock(64, 128)
self.enc3 = ConvBlock(128, 256)
self.enc4 = ConvBlock(256, 512)
# Bottleneck
self.bottleneck = ConvBlock(512, 1024)
# Decoder
self.dec4 = ConvBlock(1024 + 512, 512)
self.dec3 = ConvBlock(512 + 256, 256)
self.dec2 = ConvBlock(256 + 128, 128)
self.dec1 = ConvBlock(128 + 64, 64)
# Output
self.output_conv = np.random.randn(num_classes, 64, 1, 1) * 0.01
self.output_bias = np.zeros(num_classes)
def max_pool(self, x, pool_size=2):
"""Max pooling."""
batch, ch, h, w = x.shape
out_h = h // pool_size
out_w = w // pool_size
output = np.zeros((batch, ch, out_h, out_w))
for b in range(batch):
for c in range(ch):
for i in range(out_h):
for j in range(out_w):
region = x[b, c, i*pool_size:(i+1)*pool_size,
j*pool_size:(j+1)*pool_size]
output[b, c, i, j] = np.max(region)
return output
def upsample(self, x, target_size):
"""Bilinear upsampling."""
batch, ch, h, w = x.shape
output = np.zeros((batch, ch, target_size, target_size))
for b in range(batch):
for c in range(ch):
for i in range(target_size):
for j in range(target_size):
src_i = i * h / target_size
src_j = j * w / target_size
i0, j0 = int(src_i), int(src_j)
i1, j1 = min(i0+1, h-1), min(j0+1, w-1)
di, dj = src_i - i0, src_j - j0
output[b, c, i, j] = (
x[b, c, i0, j0] * (1-di) * (1-dj) +
x[b, c, i1, j0] * di * (1-dj) +
x[b, c, i0, j1] * (1-di) * dj +
x[b, c, i1, j1] * di * dj
)
return output
def forward(self, x):
"""Forward pass through U-Net."""
# Encoder
e1 = self.enc1.forward(x)
e2 = self.enc2.forward(self.max_pool(e1))
e3 = self.enc3.forward(self.max_pool(e2))
e4 = self.enc4.forward(self.max_pool(e3))
# Bottleneck
b = self.bottleneck.forward(self.max_pool(e4))
# Decoder with skip connections
d4 = self.upsample(b, e4.shape[2])
d4 = np.concatenate([d4, e4], axis=1)
d4 = self.dec4.forward(d4)
d3 = self.upsample(d4, e3.shape[2])
d3 = np.concatenate([d3, e3], axis=1)
d3 = self.dec3.forward(d3)
d2 = self.upsample(d3, e2.shape[2])
d2 = np.concatenate([d2, e2], axis=1)
d2 = self.dec2.forward(d2)
d1 = self.upsample(d2, e1.shape[2])
d1 = np.concatenate([d1, e1], axis=1)
d1 = self.dec1.forward(d1)
# Output
out = np.zeros((x.shape[0], self.output_conv.shape[0], x.shape[2], x.shape[3]))
for b in range(x.shape[0]):
for c in range(self.output_conv.shape[0]):
out[b, c] = np.sum(d1[b] * self.output_conv[c].squeeze(), axis=0) + self.output_bias[c]
# Softmax
out_exp = np.exp(out - np.max(out, axis=1, keepdims=True))
out = out_exp / np.sum(out_exp, axis=1, keepdims=True)
return out
# Example: Run U-Net inference
np.random.seed(42)
model = UNet(in_channels=3, num_classes=6)
# Simulate drone image batch
batch = np.random.randn(2, 3, 64, 64) # Smaller for demo
output = model.forward(batch)
print(f"Input shape: {batch.shape}")
print(f"Output shape: {output.shape}")
print(f"Classes: Building, Road, Vegetation, Water, Vehicle, Person")
# Per-pixel predictions
predictions = np.argmax(output, axis=1)
for i in range(2):
unique, counts = np.unique(predictions[i], return_counts=True)
print(f"\nImage {i+1} class distribution:")
for cls, count in zip(unique, counts):
pct = count / predictions[i].size * 100
print(f" Class {cls}: {count} pixels ({pct:.1f}%)")
Loss Functions for Segmentation
Segmentation models use specialized loss functions to handle class imbalance and boundary precision.
python
class SegmentationLosses:
"""Loss functions for semantic segmentation."""
@staticmethod
def cross_entropy_loss(predictions, targets):
"""Standard cross-entropy loss."""
batch, classes, h, w = predictions.shape
eps = 1e-7
# Clip predictions
predictions = np.clip(predictions, eps, 1 - eps)
# One-hot encode targets
targets_onehot = np.zeros_like(predictions)
for b in range(batch):
for c in range(classes):
targets_onehot[b, c] = (targets[b] == c).astype(float)
# Compute loss
loss = -np.sum(targets_onehot * np.log(predictions)) / batch
return loss
@staticmethod
def dice_loss(predictions, targets, smooth=1.0):
"""Dice loss for handling class imbalance."""
batch, classes, h, w = predictions.shape
dice = 0
for c in range(classes):
pred_flat = predictions[:, c].flatten()
target_flat = (targets == c).astype(float).flatten()
intersection = np.sum(pred_flat * target_flat)
union = np.sum(pred_flat) + np.sum(target_flat)
dice += (2.0 * intersection + smooth) / (union + smooth)
return 1 - dice / (batch * classes)
@staticmethod
def focal_loss(predictions, targets, alpha=0.25, gamma=2.0):
"""Focal loss for addressing class imbalance."""
batch, classes, h, w = predictions.shape
eps = 1e-7
predictions = np.clip(predictions, eps, 1 - eps)
targets_onehot = np.zeros_like(predictions)
for b in range(batch):
for c in range(classes):
targets_onehot[b, c] = (targets[b] == c).astype(float)
cross_entropy = -targets_onehot * np.log(predictions)
p_t = targets_onehot * predictions + (1 - targets_onehot) * (1 - predictions)
focal_weight = alpha * (1 - p_t) ** gamma
loss = focal_weight * cross_entropy
return np.mean(loss)
@staticmethod
def boundary_loss(predictions, targets, sigma=1.0):
"""Boundary-aware loss for sharp edges."""
batch, classes, h, w = predictions.shape
# Compute distance transform of target boundaries
loss = 0
for b in range(batch):
for c in range(classes):
target_mask = (targets[b] == c).astype(float)
# Simple boundary detection
grad_x = np.gradient(target_mask, axis=1)
grad_y = np.gradient(target_mask, axis=0)
boundary = np.sqrt(grad_x**2 + grad_y**2)
# Weight loss by distance to boundary
loss += np.sum(predictions[b, c] * boundary)
return loss / (batch * classes)
# Example: Compare losses
np.random.seed(42)
predictions = np.random.rand(2, 6, 64, 64)
predictions = predictions / predictions.sum(axis=1, keepdims=True)
targets = np.random.randint(0, 6, (2, 64, 64))
losses = SegmentationLosses()
ce_loss = losses.cross_entropy_loss(predictions, targets)
dice = losses.dice_loss(predictions, targets)
focal = losses.focal_loss(predictions, targets)
print(f"Cross-Entropy Loss: {ce_loss:.4f}")
print(f"Dice Loss: {dice:.4f}")
print(f"Focal Loss: {focal:.4f}")
Evaluation Metrics
Advertisement
python
class SegmentationMetrics:
"""Evaluation metrics for segmentation models."""
def __init__(self, num_classes):
self.num_classes = num_classes
self.confusion_matrix = np.zeros((num_classes, num_classes))
def update(self, predictions, targets):
"""Update confusion matrix with batch."""
for pred, target in zip(predictions, targets):
for i in range(self.num_classes):
for j in range(self.num_classes):
self.confusion_matrix[i, j] += np.sum(
(pred == i) & (target == j)
)
def pixel_accuracy(self):
"""Overall pixel accuracy."""
correct = np.trace(self.confusion_matrix)
total = np.sum(self.confusion_matrix)
return correct / total if total > 0 else 0
def mean_accuracy(self):
"""Mean class accuracy."""
per_class = np.diag(self.confusion_matrix) / np.sum(
self.confusion_matrix, axis=1
)
return np.mean(per_class[~np.isnan(per_class)])
def mean_iou(self):
"""Mean Intersection over Union (mIoU)."""
iou = np.diag(self.confusion_matrix) / (
np.sum(self.confusion_matrix, axis=1) +
np.sum(self.confusion_matrix, axis=0) -
np.diag(self.confusion_matrix)
)
return np.mean(iou[~np.isnan(iou)])
def dice_coefficient(self):
"""Dice coefficient per class."""
dice = 2 * np.diag(self.confusion_matrix) / (
np.sum(self.confusion_matrix, axis=1) +
np.sum(self.confusion_matrix, axis=0)
)
return dice
def frequency_weighted_iou(self):
"""Frequency-weighted mIoU."""
freq = np.sum(self.confusion_matrix, axis=1) / np.sum(self.confusion_matrix)
iou = np.diag(self.confusion_matrix) / (
np.sum(self.confusion_matrix, axis=1) +
np.sum(self.confusion_matrix, axis=0) -
np.diag(self.confusion_matrix)
)
return np.nansum(freq * iou)
def get_report(self, class_names=None):
"""Generate full metrics report."""
if class_names is None:
class_names = [f"Class_{i}" for i in range(self.num_classes)]
report = {
'pixel_accuracy': self.pixel_accuracy(),
'mean_accuracy': self.mean_accuracy(),
'mIoU': self.mean_iou(),
'fwIoU': self.frequency_weighted_iou(),
'per_class_iou': {},
'per_class_dice': {}
}
iou = np.diag(self.confusion_matrix) / (
np.sum(self.confusion_matrix, axis=1) +
np.sum(self.confusion_matrix, axis=0) -
np.diag(self.confusion_matrix)
)
dice = self.dice_coefficient()
for i, name in enumerate(class_names):
report['per_class_iou'][name] = iou[i]
report['per_class_dice'][name] = dice[i]
return report
# Example evaluation
np.random.seed(42)
class_names = ['Building', 'Road', 'Vegetation', 'Water', 'Vehicle', 'Person']
metrics = SegmentationMetrics(num_classes=6)
# Simulate predictions vs ground truth
for _ in range(10):
pred = np.random.randint(0, 6, (32, 32))
target = np.random.randint(0, 6, (32, 32))
metrics.update(pred[np.newaxis], target[np.newaxis])
report = metrics.get_report(class_names)
print("Segmentation Evaluation Report")
print("=" * 40)
print(f"Pixel Accuracy: {report['pixel_accuracy']:.2%}")
print(f"Mean Accuracy: {report['mean_accuracy']:.2%}")
print(f"mIoU: {report['mIoU']:.2%}")
print(f"Frequency-weighted IoU: {report['fwIoU']:.2%}")
print("\nPer-class IoU:")
for name, iou in report['per_class_iou'].items():
print(f" {name}: {iou:.2%}")
Hands-On Project: Land-Use Classification System
Build a complete drone land-use segmentation system.
python
class LandUseSegmentation:
"""Complete land-use classification from drone imagery."""
def __init__(self, num_classes=8):
self.num_classes = num_classes
self.class_names = [
'Residential', 'Commercial', 'Industrial', 'Agricultural',
'Forest', 'Water', 'Road', 'Park'
]
self.class_colors = [
[139, 69, 19], # Brown - Residential
[255, 140, 0], # Orange - Commercial
[128, 128, 128], # Gray - Industrial
[255, 255, 0], # Yellow - Agricultural
[0, 128, 0], # Green - Forest
[0, 0, 255], # Blue - Water
[64, 64, 64], # Dark Gray - Road
[144, 238, 144] # Light Green - Park
]
def preprocess_image(self, image):
"""Preprocess drone image for segmentation."""
# Normalize
image = image.astype(np.float32) / 255.0
# Mean subtraction
mean = np.array([0.485, 0.456, 0.406])
std = np.array([0.229, 0.224, 0.225])
image = (image - mean) / std
# Add batch dimension
return image.transpose(2, 0, 1)[np.newaxis]
def predict_segmentation(self, image):
"""Predict segmentation map."""
batch = self.preprocess_image(image)
# Simulate model output (replace with actual model)
h, w = image.shape[:2]
output = np.random.rand(batch.shape[0], self.num_classes, h, w)
# Softmax
output = np.exp(output) / np.sum(output, axis=1, keepdims=True)
# Get predictions
pred_map = np.argmax(output[0], axis=0)
confidence = np.max(output[0], axis=0)
return pred_map, confidence
def apply_color_map(self, seg_map):
"""Convert segmentation map to color image."""
h, w = seg_map.shape
color_image = np.zeros((h, w, 3), dtype=np.uint8)
for cls in range(self.num_classes):
mask = seg_map == cls
color_image[mask] = self.class_colors[cls]
return color_image
def compute_statistics(self, seg_map, pixel_size=0.1):
"""Compute area statistics."""
total_pixels = seg_map.size
stats = {}
for cls in range(self.num_classes):
pixel_count = np.sum(seg_map == cls)
area_m2 = pixel_count * pixel_size * pixel_size
area_ha = area_m2 / 10000
percentage = pixel_count / total_pixels * 100
stats[self.class_names[cls]] = {
'pixels': int(pixel_count),
'area_m2': round(area_m2, 2),
'area_ha': round(area_ha, 4),
'percentage': round(percentage, 2)
}
return stats
def detect_changes(self, seg_map1, seg_map2):
"""Detect land-use changes between two maps."""
changes = {}
for cls in range(self.num_classes):
for cls2 in range(self.num_classes):
if cls != cls2:
mask = (seg_map1 == cls) & (seg_map2 == cls2)
count = np.sum(mask)
if count > 0:
change_key = f"{self.class_names[cls]} → {self.class_names[cls2]}"
changes[change_key] = int(count)
return dict(sorted(changes.items(), key=lambda x: x[1], reverse=True))
def generate_report(self, seg_map, pixel_size=0.1):
"""Generate comprehensive segmentation report."""
stats = self.compute_statistics(seg_map, pixel_size)
report = {
'total_area_m2': seg_map.size * pixel_size * pixel_size,
'classes_detected': sum(1 for s in stats.values() if s['pixels'] > 0),
'dominant_class': max(stats.items(), key=lambda x: x[1]['pixels'])[0],
'statistics': stats
}
return report
# Example: Complete land-use analysis
np.random.seed(42)
system = LandUseSegmentation()
# Simulate drone image
image = np.random.randint(0, 255, (256, 256, 3), dtype=np.uint8)
# Add some structure
image[50:100, 50:100] = [139, 69, 19] # Residential
image[120:180, 30:90] = [0, 128, 0] # Forest
image[150:200, 150:220] = [0, 0, 255] # Water
# Run segmentation
seg_map, confidence = system.predict_segmentation(image)
color_map = system.apply_color_map(seg_map)
# Generate report
report = system.generate_report(seg_map, pixel_size=0.05)
print("Land-Use Segmentation Report")
print("=" * 50)
print(f"Total Area: {report['total_area_m2']:.0f} m²")
print(f"Classes Detected: {report['classes_detected']}/{system.num_classes}")
print(f"Dominant Class: {report['dominant_class']}")
print("\nArea Distribution:")
for cls_name, stats in report['statistics'].items():
if stats['pixels'] > 0:
print(f" {cls_name}: {stats['percentage']:.1f}% "
f"({stats['area_ha']:.4f} ha)")
# Change detection example
print("\nSimulating change detection...")
old_seg = np.random.randint(0, 6, (128, 128))
new_seg = old_seg.copy()
new_seg[30:50, 30:50] = 5 # Simulate land-use change
changes = system.detect_changes(old_seg, new_seg)
print("Detected Changes:")
for change, count in list(changes.items())[:5]:
print(f" {change}: {count} pixels")
Key Takeaways
- Semantic segmentation provides pixel-level scene understanding
- U-Net with skip connections preserves spatial details
- Loss functions like Dice and Focal handle class imbalance
- mIoU is the standard metric for segmentation performance
- Land-use classification enables urban planning and environmental monitoring
Next, we'll explore image classification techniques for categorizing entire drone scenes.