Gesture Recognition for Drone Control
Gesture recognition enables intuitive drone control through hand waves, body postures, and visual signalsβeliminating the need for controllers in many scenarios. This tutorial covers the complete pipeline for visual gesture-based drone interaction.
Gesture Recognition Pipeline
From camera input to drone commands, gesture recognition transforms visual signals into actionable instructions.
**Real-world analogy:** Gesture recognition for drones is like teaching a dog commandsβexcept instead of voice, you use visual signals. The drone learns to associate specific hand shapes or movements with actions like "hover," "land," or "follow me."
## Hand Landmark Detection
```python
from dataclasses import dataclass
from typing import List, Tuple
@dataclass
class HandLandmark:
"""Single hand landmark point."""
x: float
y: float
z: float
confidence: float
landmark_id: int
@dataclass
class HandGesture:
"""Detected hand gesture."""
gesture_name: str
confidence: float
landmarks: List[HandLandmark]
hand_side: str # 'left' or 'right'
class HandGestureDetector:
"""Hand gesture detection from drone camera."""
# Hand landmark connections
HAND_CONNECTIONS = [
(0, 1), (1, 2), (2, 3), (3, 4), # Thumb
(0, 5), (5, 6), (6, 7), (7, 8), # Index
(0, 9), (9, 10), (10, 11), (11, 12), # Middle
(0, 13), (13, 14), (14, 15), (15, 16), # Ring
(0, 17), (17, 18), (18, 19), (19, 20), # Pinky
(5, 9), (9, 13), (13, 17), # Palm
]
def __init__(self):
self.gesture_templates = self._load_gesture_templates()
def _load_gesture_templates(self):
"""Load reference gesture templates."""
return {
'open_palm': {
'finger_angles': [160, 160, 160, 160, 160], # All fingers extended
'finger_lengths': [0.8, 0.9, 1.0, 0.9, 0.7],
},
'fist': {
'finger_angles': [30, 30, 30, 30, 30], # All fingers curled
'finger_lengths': [0.3, 0.3, 0.3, 0.3, 0.3],
},
'pointing_up': {
'finger_angles': [160, 30, 30, 30, 30], # Only index extended
'finger_lengths': [0.9, 0.3, 0.3, 0.3, 0.3],
},
'peace': {
'finger_angles': [160, 160, 30, 30, 30], # Index and middle
'finger_lengths': [0.9, 0.9, 0.3, 0.3, 0.3],
},
'thumbs_up': {
'finger_angles': [160, 30, 30, 30, 30], # Thumb extended
'finger_lengths': [0.9, 0.3, 0.3, 0.3, 0.3],
},
}
def detect_landmarks(self, hand_region):
"""Detect hand landmarks (simplified)."""
h, w = hand_region.shape[:2]
landmarks = []
# Simulated landmark positions (21 points)
base_positions = [
(0.5, 0.9), # 0: Wrist
(0.4, 0.75), # 1: Thumb CMC
(0.3, 0.6), # 2: Thumb MCP
(0.25, 0.45), # 3: Thumb IP
(0.2, 0.3), # 4: Thumb Tip
(0.4, 0.5), # 5: Index MCP
(0.35, 0.35), # 6: Index PIP
(0.3, 0.2), # 7: Index DIP
(0.25, 0.05), # 8: Index Tip
(0.5, 0.45), # 9: Middle MCP
(0.5, 0.3), # 10: Middle PIP
(0.5, 0.15), # 11: Middle DIP
(0.5, 0.0), # 12: Middle Tip
(0.6, 0.5), # 13: Ring MCP
(0.65, 0.35), # 14: Ring PIP
(0.7, 0.2), # 15: Ring DIP
(0.75, 0.1), # 16: Ring Tip
(0.7, 0.6), # 17: Pinky MCP
(0.75, 0.45), # 18: Pinky PIP
(0.8, 0.3), # 19: Pinky DIP
(0.85, 0.2), # 20: Pinky Tip
]
for i, (nx, ny) in enumerate(base_positions):
landmarks.append(HandLandmark(
x=nx * w + np.random.uniform(-2, 2),
y=ny * h + np.random.uniform(-2, 2),
z=np.random.uniform(-0.1, 0.1),
confidence=0.85 + np.random.uniform(-0.1, 0.1),
landmark_id=i
))
return landmarks
def compute_finger_angles(self, landmarks):
"""Compute angles for each finger."""
angles = []
# Finger tip indices: 4, 8, 12, 16, 20
# Finger pip indices: 3, 6, 10, 14, 18
finger_tips = [4, 8, 12, 16, 20]
finger_pips = [3, 6, 10, 14, 18]
for tip_idx, pip_idx in zip(finger_tips, finger_pips):
tip = landmarks[tip_idx]
pip = landmarks[pip_idx]
mcp = landmarks[pip_idx - 2]
# Compute angle
v1 = np.array([mcp.x - pip.x, mcp.y - pip.y])
v2 = np.array([tip.x - pip.x, tip.y - pip.y])
cos_angle = np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2) + 1e-6)
angle = np.degrees(np.arccos(np.clip(cos_angle, -1, 1)))
angles.append(angle)
return angles
def compute_finger_lengths(self, landmarks):
"""Compute relative finger lengths."""
wrist = landmarks[0]
tips = [4, 8, 12, 16, 20]
lengths = []
for tip_idx in tips:
tip = landmarks[tip_idx]
length = np.sqrt((tip.x - wrist.x)**2 + (tip.y - wrist.y)**2)
lengths.append(length)
# Normalize
max_len = max(lengths) if lengths else 1
return [l / max_len for l in lengths]
def classify_gesture(self, landmarks):
"""Classify hand gesture from landmarks."""
angles = self.compute_finger_angles(landmarks)
lengths = self.compute_finger_lengths(landmarks)
best_match = 'unknown'
best_score = 0
for gesture_name, template in self.gesture_templates.items():
# Compare angles
angle_diff = np.mean(np.abs(np.array(angles) - np.array(template['finger_angles'])))
length_diff = np.mean(np.abs(np.array(lengths) - np.array(template['finger_lengths'])))
# Combined similarity score
score = 1.0 / (1.0 + angle_diff * 0.01 + length_diff)
if score > best_score:
best_score = score
best_match = gesture_name
return best_match, best_score
def detect_gesture(self, hand_region):
"""Complete gesture detection pipeline."""
# Detect landmarks
landmarks = self.detect_landmarks(hand_region)
# Classify gesture
gesture_name, confidence = self.classify_gesture(landmarks)
return HandGesture(
gesture_name=gesture_name,
confidence=confidence,
landmarks=landmarks,
hand_side='right'
)
# Example: Hand gesture detection
np.random.seed(42)
detector = HandGestureDetector()
print("Hand Gesture Detection")
print("=" * 50)
# Simulate hand regions with different gestures
test_gestures = ['open_palm', 'fist', 'pointing_up', 'peace']
for gesture_type in test_gestures:
hand_region = np.random.randint(0, 255, (200, 200, 3), dtype=np.uint8)
result = detector.detect_gesture(hand_region)
print(f"\nGesture: {gesture_type}")
print(f" Detected: {result.gesture_name}")
print(f" Confidence: {result.confidence:.1%}")
print(f" Landmarks: {len(result.landmarks)} points")
Gesture-to-Command Mapping
Dynamic Gesture Recognition
Hands-On Project: Gesture-Controlled Drone Interface
Build a complete gesture control interface for drone operations.
Key Takeaways
- Hand landmarks provide precise finger position tracking
- Static gestures map directly to drone commands
- Dynamic gestures require temporal analysis for motion patterns
- Safety validation prevents dangerous or conflicting commands
- Gesture buffering ensures consistent command recognition
Next, we'll explore face detection and recognition for security drone applications.