Pose Estimation for Drones
Pose estimation identifies human body keypoints from drone footage—enabling activity recognition, crowd analysis, and search-and-rescue operations. This tutorial covers the architectures for accurate aerial human pose estimation.
Aerial Pose Estimation Pipeline
Drone pose estimation faces unique challenges: small subjects, extreme viewpoints, and motion blur.
**Real-world analogy:** Pose estimation from a drone is like watching people from a second-story window. You can see their overall posture and movement, but fine details like facial expressions are harder to distinguish. The challenge is reconstructing 3D body poses from these overhead 2D views.
Keypoint Detection
from dataclasses import dataclass
from typing import List, Tuple, Optional
@dataclass
class Keypoint:
"""Single body keypoint."""
x: float
y: float
confidence: float
keypoint_id: int
@dataclass
class Pose:
"""Complete human pose."""
keypoints: List[Keypoint]
bbox: Tuple[int, int, int, int]
person_id: Optional[int] = None
action: Optional[str] = None
class AerialPoseEstimator:
"""Pose estimation optimized for drone (aerial) views."""
# COCO keypoint connections for skeleton
SKELETON = [
(0, 1), (0, 2), (1, 3), (2, 4), # Head
(5, 6), # Shoulders
(5, 7), (7, 9), # Left arm
(6, 8), (8, 10), # Right arm
(5, 11), (6, 12), # Torso
(11, 12), # Hips
(11, 13), (13, 15), # Left leg
(12, 14), (14, 16), # Right leg
]
def __init__(self, input_size=(256, 256), num_keypoints=17):
self.input_size = input_size
self.num_keypoints = num_keypoints
# Simulated heatmap model weights
self.heatmap_weights = np.random.randn(
num_keypoints, 64, input_size[0]//4, input_size[1]//4
) * 0.01
def generate_heatmaps(self, person_region):
"""Generate keypoint heatmaps for a detected person."""
h, w = person_region.shape[:2]
heatmap_h, heatmap_w = h // 4, w // 4
heatmaps = np.zeros((self.num_keypoints, heatmap_h, heatmap_w))
# Simulate keypoint positions (top-down view adjustments)
keypoint_positions = [
(heatmap_h//2, heatmap_w//2), # Nose (center of head)
(heatmap_h//2-2, heatmap_w//2-2), # Left eye
(heatmap_h//2-2, heatmap_w//2+2), # Right eye
(heatmap_h//2-1, heatmap_w//2-4), # Left ear
(heatmap_h//2-1, heatmap_w//2+4), # Right ear
(heatmap_h//3, heatmap_w//3), # Left shoulder
(heatmap_h//3, 2*heatmap_w//3), # Right shoulder
(heatmap_h//2, heatmap_w//4), # Left elbow
(heatmap_h//2, 3*heatmap_w//4), # Right elbow
(2*heatmap_h//3, heatmap_w//5), # Left wrist
(2*heatmap_h//3, 4*heatmap_w//5), # Right wrist
(2*heatmap_h//3, heatmap_w//3), # Left hip
(2*heatmap_h//3, 2*heatmap_w//3), # Right hip
(5*heatmap_h//6, heatmap_w//4), # Left knee
(5*heatmap_h//6, 3*heatmap_w//4), # Right knee
(heatmap_h-1, heatmap_w//5), # Left ankle
(heatmap_h-1, 4*heatmap_w//5), # Right ankle
]
for kp_id, (ky, kx) in enumerate(keypoint_positions):
if ky < heatmap_h and kx < heatmap_w:
# Generate Gaussian heatmap
y, x = np.ogrid[:heatmap_h, :heatmap_w]
sigma = 2
heatmap = np.exp(-((x - kx)**2 + (y - ky)**2) / (2 * sigma**2))
heatmaps[kp_id] = heatmap
return heatmaps
def decode_heatmaps(self, heatmaps, threshold=0.3):
"""Decode heatmaps to keypoint coordinates."""
keypoints = []
for kp_id in range(self.num_keypoints):
heatmap = heatmaps[kp_id]
# Find maximum response
max_idx = np.unravel_index(np.argmax(heatmap), heatmap.shape)
max_val = heatmap[max_idx]
if max_val > threshold:
keypoints.append(Keypoint(
x=float(max_idx[1]),
y=float(max_idx[0]),
confidence=float(max_val),
keypoint_id=kp_id
))
else:
keypoints.append(Keypoint(
x=0.0, y=0.0,
confidence=0.0,
keypoint_id=kp_id
))
return keypoints
def adjust_for_aerial_view(self, keypoints, altitude=50):
"""Adjust keypoints for top-down aerial perspective."""
# Aerial view transformations
scale_factor = 100 / altitude # Higher altitude = smaller scale
adjusted = []
for kp in keypoints:
# Scale coordinates
new_x = kp.x * scale_factor
new_y = kp.y * scale_factor
# Adjust for top-down view (foreshortening)
# Vertical body parts appear compressed
if kp.keypoint_id in [13, 14, 15, 16]: # Knees and ankles
new_y = kp.y * 0.6 # Compress leg appearance
adjusted.append(Keypoint(
x=new_x,
y=new_y,
confidence=kp.confidence,
keypoint_id=kp.keypoint_id
))
return adjusted
def estimate_pose(self, person_image, altitude=50):
"""Complete pose estimation pipeline."""
# Generate heatmaps
heatmaps = self.generate_heatmaps(person_image)
# Decode to keypoints
keypoints = self.decode_heatmaps(heatmaps)
# Adjust for aerial view
keypoints = self.adjust_for_aerial_view(keypoints, altitude)
return keypoints
def compute_skeleton(self, keypoints, image_shape):
"""Compute skeleton lines from keypoints."""
skeleton = []
h, w = image_shape[:2]
for start_idx, end_idx in self.SKELETON:
kp_start = keypoints[start_idx]
kp_end = keypoints[end_idx]
if kp_start.confidence > 0.3 and kp_end.confidence > 0.3:
skeleton.append({
'start': (int(kp_start.x), int(kp_start.y)),
'end': (int(kp_end.x), int(kp_end.y)),
'confidence': (kp_start.confidence + kp_end.confidence) / 2
})
return skeleton
# Example: Aerial pose estimation
np.random.seed(42)
estimator = AerialPoseEstimator()
print("Aerial Pose Estimation")
print("=" * 50)
# Simulate person region from drone
person_region = np.random.randint(0, 255, (64, 64, 3), dtype=np.uint8)
# Estimate pose
keypoints = estimator.estimate_pose(person_region, altitude=30)
print("Detected Keypoints (top-down adjusted):")
kp_names = ['Nose', 'L-Eye', 'R-Eye', 'L-Ear', 'R-Ear',
'L-Shoulder', 'R-Shoulder', 'L-Elbow', 'R-Elbow',
'L-Wrist', 'R-Wrist', 'L-Hip', 'R-Hip',
'L-Knee', 'R-Knee', 'L-Ankle', 'R-Ankle']
for i, (kp, name) in enumerate(zip(keypoints, kp_names)):
if kp.confidence > 0.3:
print(f" {name}: ({kp.x:.1f}, {kp.y:.1f}) conf={kp.confidence:.2f}")
# Compute skeleton
skeleton = estimator.compute_skeleton(keypoints, (64, 64))
print(f"\nSkeleton segments: {len(skeleton)}")
Pose-Based Action Recognition
from collections import deque
class ActionRecognizer:
"""Recognize actions from pose sequences."""
ACTIONS = {
'walking': 'Person is walking',
'running': 'Person is running',
'standing': 'Person is standing still',
'sitting': 'Person is sitting',
'fallen': 'Person appears to have fallen',
'waving': 'Person is waving hand',
'carrying': 'Person is carrying object',
}
def __init__(self, sequence_length=10):
self.sequence_length = sequence_length
self.pose_history = deque(maxlen=sequence_length)
def update(self, keypoints):
"""Update with new pose and predict action."""
self.pose_history.append(keypoints)
if len(self.pose_history) < 3:
return 'unknown', 0.0
# Compute motion features
features = self._extract_motion_features()
# Simple rule-based classification
action, confidence = self._classify_action(features)
return action, confidence
def _extract_motion_features(self):
"""Extract motion features from pose sequence."""
poses = list(self.pose_history)
features = {}
# Average keypoint velocity
velocities = []
for i in range(1, len(poses)):
vel = []
for kp1, kp2 in zip(poses[i-1], poses[i]):
dx = kp2.x - kp1.x
dy = kp2.y - kp1.y
vel.append(np.sqrt(dx**2 + dy**2))
velocities.append(np.mean(vel))
features['avg_velocity'] = np.mean(velocities)
features['velocity_variance'] = np.var(velocities)
# Joint angle changes
if len(poses) >= 2:
# Elbow angle
angles = []
for pose in poses:
if (pose[7].confidence > 0.3 and pose[9].confidence > 0.3):
angle = self._compute_angle(
pose[5], pose[7], pose[9] # Shoulder, elbow, wrist
)
angles.append(angle)
if angles:
features['elbow_angle_mean'] = np.mean(angles)
features['elbow_angle_std'] = np.std(angles)
else:
features['elbow_angle_mean'] = 0
features['elbow_angle_std'] = 0
else:
features['elbow_angle_mean'] = 0
features['elbow_angle_std'] = 0
# Body center height (for sitting/standing detection)
center_heights = []
for pose in poses:
if pose[11].confidence > 0.3 and pose[12].confidence > 0.3: # Hips
center_y = (pose[11].y + pose[12].y) / 2
center_heights.append(center_y)
if center_heights:
features['avg_hip_height'] = np.mean(center_heights)
features['hip_height_variance'] = np.var(center_heights)
else:
features['avg_hip_height'] = 0
features['hip_height_variance'] = 0
return features
def _compute_angle(self, p1, p2, p3):
"""Compute angle between three points."""
v1 = np.array([p1.x - p2.x, p1.y - p2.y])
v2 = np.array([p3.x - p2.x, p3.y - p2.y])
cos_angle = np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2) + 1e-6)
angle = np.arccos(np.clip(cos_angle, -1, 1))
return np.degrees(angle)
def _classify_action(self, features):
"""Classify action based on motion features."""
action = 'unknown'
confidence = 0.0
# Running: high velocity, high variance
if features['avg_velocity'] > 5 and features['velocity_variance'] > 2:
action = 'running'
confidence = min(0.95, 0.6 + features['avg_velocity'] * 0.05)
# Walking: moderate velocity
elif features['avg_velocity'] > 2:
action = 'walking'
confidence = min(0.9, 0.5 + features['avg_velocity'] * 0.1)
# Standing: low velocity, low variance
elif features['avg_velocity'] < 1 and features['velocity_variance'] < 0.5:
action = 'standing'
confidence = 0.85
# Sitting: low velocity, specific hip height
elif features['avg_velocity'] < 1 and features['avg_hip_height'] > 30:
action = 'sitting'
confidence = 0.75
# Fallen: very low, specific pose
elif features['avg_hip_height'] > 40 and features['avg_velocity'] < 0.5:
action = 'fallen'
confidence = 0.8
# Waving: high elbow angle variance
elif features['elbow_angle_std'] > 20:
action = 'waving'
confidence = 0.7
return action, confidence
# Example: Action recognition from pose sequence
np.random.seed(42)
recognizer = ActionRecognizer(sequence_length=10)
print("Action Recognition from Pose Sequences")
print("=" * 50)
# Simulate walking sequence
for frame in range(15):
# Create keypoints with walking motion
keypoints = []
base_y = 30 + frame * 2
for kp_id in range(17):
x = 32 + np.sin(frame * 0.5 + kp_id * 0.3) * 5
y = base_y + np.cos(frame * 0.5 + kp_id * 0.2) * 3
conf = 0.7 + np.random.uniform(-0.1, 0.1)
keypoints.append(Keypoint(x=x, y=y, confidence=conf, keypoint_id=kp_id))
action, confidence = recognizer.update(keypoints)
if frame >= 3 and (frame + 1) % 3 == 0:
print(f"Frame {frame+1}: {action} ({confidence:.1%})")
print("\nAction recognition complete!")
Hands-On Project: Search and Rescue Pose Detector
Build a pose detection system for search and rescue operations.
from dataclasses import dataclass
from typing import List
@dataclass
class RescueAlert:
"""Alert for potential rescue situations."""
person_id: int
location: tuple
alert_type: str
severity: str
confidence: float
class SearchRescuePoseDetector:
"""Pose-based detection system for search and rescue."""
def __init__(self, grid_size=100):
self.grid_size = grid_size
self.person_poses = {}
self.alerts = []
# Emergency pose indicators
self.emergency_indicators = {
'lying_down': self._check_lying_down,
'waving_distress': self._check_distress_waving,
'immobile': self._check_immobile,
'isolated': self._check_isolated,
}
def _check_lying_down(self, keypoints):
"""Check if person appears to be lying down."""
if len(keypoints) < 17:
return False
# Check if head and feet are at similar height (top-down view)
head_y = np.mean([kp.y for kp in keypoints[:5] if kp.confidence > 0.3])
feet_y = np.mean([kp.y for kp in keypoints[15:17] if kp.confidence > 0.3])
if head_y > 0 and feet_y > 0:
# In top-down view, lying down means spread out
spread = abs(head_y - feet_y)
return spread < 15 # Small vertical spread = lying down
return False
def _check_distress_waving(self, keypoints):
"""Check for distress waving pattern."""
if len(keypoints) < 11:
return False
# Check wrist positions relative to shoulders
l_wrist = keypoints[9]
r_wrist = keypoints[10]
l_shoulder = keypoints[5]
r_shoulder = keypoints[6]
if all(kp.confidence > 0.3 for kp in [l_wrist, r_wrist, l_shoulder, r_shoulder]):
# Hands raised above shoulders
hands_above = (l_wrist.y < l_shoulder.y or r_wrist.y < r_shoulder.y)
return hands_above
return False
def _check_immobile(self, person_id, current_poses):
"""Check if person has been immobile for extended period."""
if person_id not in self.person_poses:
return False
history = self.person_poses[person_id]
if len(history) < 5:
return False
# Check position variance over recent frames
recent = history[-5:]
positions = [(p['position'][0], p['position'][1]) for p in recent]
variance = np.var(positions, axis=0)
return np.sum(variance) < 5 # Very little movement
def _check_isolated(self, person_id, all_persons):
"""Check if person is isolated from others."""
if person_id not in self.person_poses:
return False
my_pos = np.array(self.person_poses[person_id][-1]['position'])
for other_id, other_data in self.person_poses.items():
if other_id != person_id and other_data:
other_pos = np.array(other_data[-1]['position'])
distance = np.linalg.norm(my_pos - other_pos)
if distance < 200: # Within 200 pixels
return False
return True
def process_frame(self, frame_id, detected_poses):
"""Process frame with detected poses."""
frame_alerts = []
for person_id, pose_data in detected_poses.items():
keypoints = pose_data['keypoints']
position = pose_data['position']
# Update history
if person_id not in self.person_poses:
self.person_poses[person_id] = []
self.person_poses[person_id].append({
'frame': frame_id,
'position': position,
'keypoints': keypoints
})
# Check emergency indicators
for indicator_name, check_func in self.emergency_indicators.items():
if indicator_name == 'immobile':
is_emergency = check_func(person_id, self.person_poses)
elif indicator_name == 'isolated':
is_emergency = check_func(person_id, self.person_poses)
else:
is_emergency = check_func(keypoints)
if is_emergency:
alert = RescueAlert(
person_id=person_id,
location=position,
alert_type=indicator_name,
severity='high' if indicator_name in ['lying_down', 'immobile'] else 'medium',
confidence=0.8
)
frame_alerts.append(alert)
self.alerts.append(alert)
return frame_alerts
def generate_report(self):
"""Generate search and rescue report."""
report = {
'total_persons_tracked': len(self.person_poses),
'total_alerts': len(self.alerts),
'alerts_by_type': {},
'alerts_by_severity': {'high': 0, 'medium': 0},
'persons_of_concern': []
}
for alert in self.alerts:
report['alerts_by_type'][alert.alert_type] = \
report['alerts_by_type'].get(alert.alert_type, 0) + 1
report['alerts_by_severity'][alert.severity] += 1
# Find persons with multiple alerts
person_alerts = {}
for alert in self.alerts:
if alert.person_id not in person_alerts:
person_alerts[alert.person_id] = []
person_alerts[alert.person_id].append(alert)
for person_id, alerts in person_alerts.items():
if len(alerts) >= 2:
report['persons_of_concern'].append({
'person_id': person_id,
'alert_count': len(alerts),
'alert_types': list(set(a.alert_type for a in alerts))
})
return report
# Example: Search and rescue simulation
np.random.seed(42)
sar_system = SearchRescuePoseDetector()
print("Search and Rescue Pose Detection System")
print("=" * 55)
# Simulate detection frames
for frame in range(20):
detected_poses = {}
# Simulate 3 persons
for person_id in range(3):
# Create keypoints
keypoints = []
base_x = 100 + person_id * 200
base_y = 150 + (frame if person_id != 2 else 0) # Person 2 stays still
for kp_id in range(17):
x = base_x + np.random.uniform(-5, 5)
y = base_y + np.random.uniform(-5, 5)
conf = 0.7 + np.random.uniform(-0.1, 0.1)
# Person 2: lying down pose
if person_id == 2 and kp_id > 12:
y += 20 # Spread out
keypoints.append(Keypoint(x=x, y=y, confidence=conf, keypoint_id=kp_id))
detected_poses[person_id] = {
'keypoints': keypoints,
'position': (base_x, base_y)
}
# Process frame
alerts = sar_system.process_frame(frame, detected_poses)
if alerts:
print(f"\nFrame {frame+1} - ALERTS:")
for alert in alerts:
print(f" Person {alert.person_id}: {alert.alert_type} "
f"(severity: {alert.severity})")
# Generate report
report = sar_system.generate_report()
print(f"\nSearch and Rescue Report:")
print(f" Persons tracked: {report['total_persons_tracked']}")
print(f" Total alerts: {report['total_alerts']}")
print(f" Alerts by type: {report['alerts_by_type']}")
print(f" Persons of concern: {len(report['persons_of_concern'])}")
if report['persons_of_concern']:
print("\nPersons requiring attention:")
for person in report['persons_of_concern']:
print(f" Person {person['person_id']}: "
f"{person['alert_count']} alerts "
f"({', '.join(person['alert_types'])})")
print("\nSearch and rescue system operational!")
Key Takeaways
- Aerial pose estimation requires adapting models for top-down views
- Keypoint heatmaps predict body joint locations with confidence
- Skeleton assembly connects keypoints into meaningful body structures
- Action recognition uses temporal pose sequences for behavior analysis
- Rescue detection identifies emergency situations from pose patterns
Next, we'll explore gesture recognition for drone-human interaction.