Video Analysis for Drone Surveillance
Video analysis transforms sequential drone footage into temporal intelligence—recognizing actions, detecting anomalies, and understanding events as they unfold. This tutorial covers the architectures for comprehensive aerial video understanding.
Video Analysis Pipeline
From frame extraction to temporal reasoning, video analysis builds understanding across time.
**Real-world analogy:** Video analysis is like watching a movie instead of looking at a single photo. A photo shows what's happening now, but video shows the story—how things change, what actions occur, and what events unfold over time.
Action Recognition
from collections import deque
from dataclasses import dataclass
from typing import List, Dict
@dataclass
class ActionPrediction:
"""Action recognition prediction."""
action_name: str
confidence: float
start_frame: int
end_frame: int
class VideoActionRecognizer:
"""Recognize actions from drone video streams."""
ACTIONS = {
'walking': 0,
'running': 1,
'standing': 2,
'sitting': 3,
'fighting': 4,
'falling': 5,
'entering_vehicle': 6,
'exiting_vehicle': 7,
'carrying_object': 8,
'waving': 9,
}
def __init__(self, num_frames=16, img_size=(112, 112)):
self.num_frames = num_frames
self.img_size = img_size
# Simulated model weights
self.spatial_weights = np.random.randn(512, 2048) * 0.01
self.temporal_weights = np.random.randn(len(self.ACTIONS), 512) * 0.01
# Frame buffer
self.frame_buffer = deque(maxlen=num_frames)
def extract_spatial_features(self, frame):
"""Extract spatial features from a single frame (simulated CNN)."""
# In production, use ResNet3D, I3D, or similar
features = np.random.randn(2048)
return features / np.linalg.norm(features)
def extract_temporal_features(self, frame_sequence):
"""Extract temporal features from frame sequence."""
# Simulate optical flow or temporal convolution
if len(frame_sequence) < 2:
return np.zeros(1024)
# Compute frame differences
diffs = []
for i in range(1, len(frame_sequence)):
diff = np.mean(np.abs(
frame_sequence[i].astype(float) -
frame_sequence[i-1].astype(float)
))
diffs.append(diff)
# Temporal features
features = np.array(diffs + [np.mean(diffs), np.std(diffs)])
features = np.pad(features, (0, 1024 - len(features)))
return features[:1024]
def classify_action(self, spatial_features, temporal_features):
"""Classify action from combined features."""
# Combine features
combined = np.concatenate([spatial_features, temporal_features[:1024]])
# Simulated classification
logits = np.random.randn(len(self.ACTIONS))
probs = np.exp(logits) / np.sum(np.exp(logits))
action_idx = np.argmax(probs)
action_name = list(self.ACTIONS.keys())[action_idx]
confidence = probs[action_idx]
return action_name, confidence
def process_video_clip(self, frames):
"""Process a video clip for action recognition."""
if len(frames) < self.num_frames:
# Pad with last frame
while len(frames) < self.num_frames:
frames.append(frames[-1])
# Sample frames
indices = np.linspace(0, len(frames) - 1, self.num_frames, dtype=int)
sampled_frames = [frames[i] for i in indices]
# Extract spatial features from each frame
spatial_features = []
for frame in sampled_frames:
feat = self.extract_spatial_features(frame)
spatial_features.append(feat)
# Average spatial features
avg_spatial = np.mean(spatial_features, axis=0)
# Extract temporal features
temporal_features = self.extract_temporal_features(sampled_frames)
# Classify action
action, confidence = self.classify_action(avg_spatial, temporal_features)
return ActionPrediction(
action_name=action,
confidence=confidence,
start_frame=0,
end_frame=len(frames) - 1
)
def process_stream(self, frame):
"""Process streaming video frame."""
self.frame_buffer.append(frame)
if len(self.frame_buffer) >= self.num_frames:
prediction = self.process_video_clip(list(self.frame_buffer))
return prediction
return None
# Example: Action recognition from drone video
np.random.seed(42)
recognizer = VideoActionRecognizer(num_frames=16)
print("Video Action Recognition for Drones")
print("=" * 50)
# Simulate video stream
print("\nProcessing simulated video stream:")
for frame_idx in range(30):
# Create frame with simulated motion
frame = np.random.randint(0, 255, (240, 320, 3), dtype=np.uint8)
# Add motion pattern
if 10 <= frame_idx <= 20:
# Simulate walking motion
frame[100:150, 100:200] = [100, 150, 100]
prediction = recognizer.process_stream(frame)
if prediction and frame_idx >= 15:
print(f" Frame {frame_idx}: {prediction.action_name} "
f"({prediction.confidence:.1%})")
print("\nAction recognition complete!")
Anomaly Detection
from collections import deque
from dataclasses import dataclass
from typing import List, Dict, Tuple
@dataclass
class AnomalyAlert:
"""Anomaly detection alert."""
anomaly_type: str
severity: str # 'low', 'medium', 'high', 'critical'
confidence: float
timestamp: float
location: Tuple[int, int]
description: str
class VideoAnomalyDetector:
"""Detect anomalies in drone surveillance video."""
ANOMALY_TYPES = {
'loitering': 'Person staying in area too long',
'running': 'Unusual running behavior',
'crowd_formation': 'Unexpected crowd gathering',
'vehicle_stop': 'Vehicle stopping in unusual location',
'abandoned_object': 'Object left unattended',
'fight': 'Physical altercation detected',
'fall': 'Person falling down',
'trespassing': 'Unauthorized area entry',
}
def __init__(self, history_size=100):
self.history_size = history_size
# Motion history
self.motion_history = deque(maxlen=history_size)
self.person_tracks = {}
self.vehicle_tracks = {}
# Baseline statistics
self.baseline_motion = {'mean': 5.0, 'std': 2.0}
self.baseline_density = {'mean': 3.0, 'std': 1.5}
# Alert thresholds
self.thresholds = {
'motion': 3.0, # Standard deviations from mean
'density': 2.5,
'loiter_time': 300, # frames (~10 seconds at 30fps)
'crowd_size': 10,
}
def compute_motion_features(self, frame, prev_frame):
"""Compute motion features between frames."""
if prev_frame is None:
return {'magnitude': 0, 'direction': 0}
# Frame difference
diff = np.mean(np.abs(frame.astype(float) - prev_frame.astype(float)))
# Optical flow approximation
magnitude = diff
direction = np.random.uniform(0, 2 * np.pi)
return {
'magnitude': magnitude,
'direction': direction,
'frame_diff': diff
}
def compute_scene_density(self, detections):
"""Compute scene density features."""
num_persons = len([d for d in detections if d.get('type') == 'person'])
num_vehicles = len([d for d in detections if d.get('type') == 'vehicle'])
# Spatial distribution
if num_persons > 0:
person_positions = [d['position'] for d in detections
if d.get('type') == 'person']
positions = np.array(person_positions)
center = np.mean(positions, axis=0)
spread = np.mean(np.linalg.norm(positions - center, axis=1))
else:
center = np.array([0, 0])
spread = 0
return {
'num_persons': num_persons,
'num_vehicles': num_vehicles,
'center': center,
'spread': spread,
'density': num_persons / (spread + 1)
}
def detect_loitering(self, track_id, position, frame_count):
"""Detect loitering behavior."""
if track_id not in self.person_tracks:
self.person_tracks[track_id] = {
'positions': [],
'first_seen': frame_count,
'last_seen': frame_count
}
track = self.person_tracks[track_id]
track['positions'].append(position)
track['last_seen'] = frame_count
# Check loitering
time_in_area = frame_count - track['first_seen']
if time_in_area > self.thresholds['loiter_time']:
# Check if staying in same area
positions = np.array(track['positions'][-100:])
if len(positions) > 10:
spread = np.std(positions, axis=0)
if np.mean(spread) < 50: # Staying in small area
return True, time_in_area
return False, time_in_area
def detect_crowd(self, density_features):
"""Detect unexpected crowd formation."""
num_persons = density_features['num_persons']
spread = density_features['spread']
# Crowd: many people in small area
if (num_persons > self.thresholds['crowd_size'] and
spread < 100):
return True, num_persons
return False, num_persons
def detect_fight(self, motion_features, person_tracks):
"""Detect fighting behavior."""
# Check for rapid, erratic motion
if len(self.motion_history) < 5:
return False
recent_motion = list(self.motion_history)[-5:]
motion_variance = np.var([m['magnitude'] for m in recent_motion])
# High motion variance suggests erratic movement
if motion_variance > 100:
# Check for multiple people in close proximity
active_tracks = [t for t in person_tracks.values()
if t['last_seen'] > len(self.motion_history) - 10]
if len(active_tracks) >= 2:
positions = [t['positions'][-1] for t in active_tracks
if t['positions']]
if len(positions) >= 2:
min_distance = min(
np.linalg.norm(np.array(p1) - np.array(p2))
for i, p1 in enumerate(positions)
for p2 in positions[i+1:]
)
if min_distance < 50:
return True
return False
def process_frame(self, frame, detections, frame_count, prev_frame=None):
"""Process frame for anomaly detection."""
alerts = []
# Compute motion features
motion = self.compute_motion_features(frame, prev_frame)
self.motion_history.append(motion)
# Compute scene density
density = self.compute_scene_density(detections)
# Check for loitering
for det in detections:
if det.get('type') == 'person':
track_id = det.get('track_id', id(det))
is_loitering, duration = self.detect_loitering(
track_id, det['position'], frame_count
)
if is_loitering:
alerts.append(AnomalyAlert(
anomaly_type='loitering',
severity='medium',
confidence=0.8,
timestamp=frame_count / 30.0,
location=det['position'],
description=f"Person loitering for {duration/30:.1f} seconds"
))
# Check for crowd
is_crowd, crowd_size = self.detect_crowd(density)
if is_crowd:
alerts.append(AnomalyAlert(
anomaly_type='crowd_formation',
severity='high',
confidence=0.75,
timestamp=frame_count / 30.0,
location=tuple(density['center'].astype(int)),
description=f"Crowd of {crowd_size} people detected"
))
# Check for fighting
if self.detect_fight(motion, self.person_tracks):
alerts.append(AnomalyAlert(
anomaly_type='fight',
severity='critical',
confidence=0.7,
timestamp=frame_count / 30.0,
location=(0, 0),
description="Possible physical altercation detected"
))
# Check for unusual motion
if len(self.motion_history) > 10:
recent = list(self.motion_history)[-10:]
avg_motion = np.mean([m['magnitude'] for m in recent])
if avg_motion > self.baseline_motion['mean'] + \
self.thresholds['motion'] * self.baseline_motion['std']:
alerts.append(AnomalyAlert(
anomaly_type='unusual_motion',
severity='medium',
confidence=0.65,
timestamp=frame_count / 30.0,
location=(0, 0),
description=f"Unusual motion level: {avg_motion:.1f}"
))
return alerts
# Example: Anomaly detection
np.random.seed(42)
detector = VideoAnomalyDetector()
print("Video Anomaly Detection for Drone Surveillance")
print("=" * 55)
# Simulate video with anomalies
print("\nProcessing video stream:")
all_alerts = []
for frame_idx in range(50):
frame = np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8)
prev_frame = np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8) if frame_idx > 0 else None
# Simulate detections
detections = []
if frame_idx > 20:
# Add loitering person
detections.append({
'type': 'person',
'position': (300, 200),
'track_id': 'person_1'
})
if frame_idx > 30:
# Add crowd
for i in range(8):
detections.append({
'type': 'person',
'position': (250 + i * 10, 150 + i * 5),
'track_id': f'person_{i+2}'
})
alerts = detector.process_frame(frame, detections, frame_idx, prev_frame)
all_alerts.extend(alerts)
if alerts and frame_idx % 5 == 0:
print(f" Frame {frame_idx}:")
for alert in alerts:
print(f" {alert.severity.upper()}: {alert.anomaly_type}")
print(f" {alert.description}")
print(f"\nTotal alerts generated: {len(all_alerts)}")
print(f"Alert types: {set(a.anomaly_type for a in all_alerts)}")
Event Summarization
from dataclasses import dataclass
from typing import List, Dict
from collections import defaultdict
@dataclass
class VideoEvent:
"""Single video event."""
event_type: str
start_time: float
end_time: float
confidence: float
description: str
keyframes: List[int]
persons_involved: List[str]
class VideoEventSummarizer:
"""Summarize events from video analysis."""
def __init__(self):
self.events = []
self.current_event = None
def add_event(self, event_type, start_time, confidence, description,
keyframes=None, persons=None):
"""Add a new event."""
event = VideoEvent(
event_type=event_type,
start_time=start_time,
end_time=start_time,
confidence=confidence,
description=description,
keyframes=keyframes or [],
persons_involved=persons or []
)
self.events.append(event)
return event
def update_event(self, event_idx, end_time):
"""Update event end time."""
if event_idx < len(self.events):
self.events[event_idx].end_time = end_time
def merge_events(self, event_type, time_threshold=5.0):
"""Merge nearby events of same type."""
merged = []
i = 0
while i < len(self.events):
current = self.events[i]
# Look for nearby events of same type
j = i + 1
while (j < len(self.events) and
self.events[j].event_type == event_type and
self.events[j].start_time - current.end_time < time_threshold):
# Merge events
current.end_time = self.events[j].end_time
current.confidence = max(current.confidence,
self.events[j].confidence)
current.keyframes.extend(self.events[j].keyframes)
current.persons_involved.extend(
self.events[j].persons_involved
)
j += 1
merged.append(current)
i = j
return merged
def generate_summary(self, max_duration=60):
"""Generate video summary."""
summary = {
'total_events': len(self.events),
'event_types': defaultdict(int),
'highlights': [],
'timeline': [],
'persons_activity': defaultdict(int),
}
for event in self.events:
summary['event_types'][event.event_type] += 1
for person in event.persons_involved:
summary['persons_activity'][person] += 1
# Add to timeline
summary['timeline'].append({
'time': f"{event.start_time:.1f}s - {event.end_time:.1f}s",
'event': event.event_type,
'description': event.description
})
# High confidence events are highlights
if event.confidence > 0.8:
summary['highlights'].append({
'time': f"{event.start_time:.1f}s",
'event': event.event_type,
'confidence': event.confidence
})
return summary
def generate_narrative(self):
"""Generate natural language narrative of events."""
if not self.events:
return "No significant events detected in this video."
narrative_parts = []
# Sort events by time
sorted_events = sorted(self.events, key=lambda e: e.start_time)
for event in sorted_events:
duration = event.end_time - event.start_time
if duration > 1:
narrative_parts.append(
f"At {event.start_time:.1f}s, {event.description} "
f"(duration: {duration:.1f}s, confidence: {event.confidence:.0%})"
)
else:
narrative_parts.append(
f"At {event.start_time:.1f}s, {event.description}"
)
return "\n".join(narrative_parts)
# Example: Event summarization
np.random.seed(42)
summarizer = VideoEventSummarizer()
print("Video Event Summarization")
print("=" * 50)
# Simulate events
events_data = [
('person_detected', 5.0, 0.9, 'Person entering monitored area', [150, 200], ['Alice']),
('vehicle_detected', 15.0, 0.85, 'Vehicle approaching checkpoint', [450, 500], []),
('loitering', 25.0, 0.8, 'Person loitering near restricted zone', [750, 800, 850], ['Alice']),
('crowd_formation', 35.0, 0.75, 'Group of people gathering', [1050, 1100], ['Bob', 'Carol', 'David']),
('unusual_motion', 45.0, 0.7, 'Rapid movement detected', [1350, 1400], ['Eve']),
]
print("\nRecording events:")
for event_type, time, conf, desc, keyframes, persons in events_data:
summarizer.add_event(event_type, time, conf, desc, keyframes, persons)
print(f" {time:.1f}s: {desc}")
# Generate summary
summary = summarizer.generate_summary()
print(f"\nVideo Summary:")
print(f" Total events: {summary['total_events']}")
print(f" Event types: {dict(summary['event_types'])}")
print(f" Highlights: {len(summary['highlights'])}")
# Generate narrative
narrative = summarizer.generate_narrative()
print(f"\nNarrative:")
print(narrative)
Hands-On Project: Complete Video Surveillance System
Build a complete video surveillance system for drone monitoring.
from collections import defaultdict, deque
from datetime import datetime
class DroneVideoSurveillance:
"""Complete video surveillance system for drones."""
def __init__(self):
self.action_recognizer = VideoActionRecognizer()
self.anomaly_detector = VideoAnomalyDetector()
self.event_summarizer = VideoEventSummarizer()
# Surveillance state
self.frame_count = 0
self.monitored_zones = {}
self.alert_history = []
self persons_tracked = defaultdict(list)
# Configuration
self.alert_cooldown = 30 # frames between same alert type
self.last_alert_time = defaultdict(int)
def setup_zones(self, zones_config):
"""Setup monitoring zones."""
for zone_id, config in zones_config.items():
self.monitored_zones[zone_id] = {
'boundary': config['boundary'],
'max_persons': config.get('max_persons', 10),
'restricted': config.get('restricted', False),
'current_persons': 0
}
def process_frame(self, frame, detections, prev_frame=None):
"""Process a video frame."""
self.frame_count += 1
frame_results = {
'frame_id': self.frame_count,
'actions': [],
'alerts': [],
'zone_status': {}
}
# Update zone occupancy
self._update_zones(detections)
# Action recognition
action_pred = self.action_recognizer.process_stream(frame)
if action_pred:
frame_results['actions'].append({
'action': action_pred.action_name,
'confidence': action_pred.confidence
})
# Log suspicious actions
if action_pred.action_name in ['fighting', 'falling']:
if self._check_alert_cooldown('suspicious_action'):
alert = {
'type': 'suspicious_action',
'action': action_pred.action_name,
'confidence': action_pred.confidence,
'timestamp': self.frame_count / 30.0
}
frame_results['alerts'].append(alert)
self.alert_history.append(alert)
# Anomaly detection
anomalies = self.anomaly_detector.process_frame(
frame, detections, self.frame_count, prev_frame
)
for anomaly in anomalies:
if self._check_alert_cooldown(anomaly.anomaly_type):
alert = {
'type': anomaly.anomaly_type,
'severity': anomaly.severity,
'confidence': anomaly.confidence,
'timestamp': anomaly.timestamp,
'description': anomaly.description
}
frame_results['alerts'].append(alert)
self.alert_history.append(alert)
# Zone status
for zone_id, zone in self.monitored_zones.items():
frame_results['zone_status'][zone_id] = {
'persons': zone['current_persons'],
'restricted': zone['restricted'],
'alert': (zone['current_persons'] > zone['max_persons'] or
(zone['restricted'] and zone['current_persons'] > 0))
}
return frame_results
def _update_zones(self, detections):
"""Update zone occupancy from detections."""
# Reset counts
for zone in self.monitored_zones.values():
zone['current_persons'] = 0
# Count persons in each zone
for det in detections:
if det.get('type') == 'person':
pos = det['position']
for zone_id, zone in self.monitored_zones.items():
if self._point_in_zone(pos, zone['boundary']):
zone['current_persons'] += 1
# Track person
track_id = det.get('track_id', str(pos))
self.persons_tracked[track_id].append({
'frame': self.frame_count,
'position': pos
})
def _point_in_zone(self, point, boundary):
"""Check if point is inside zone boundary."""
# Simplified point-in-polygon check
x, y = point
n = len(boundary)
inside = False
j = n - 1
for i in range(n):
xi, yi = boundary[i]
xj, yj = boundary[j]
if ((yi > y) != (yj > y)) and (x < (xj - xi) * (y - yi) / (yj - yi) + xi):
inside = not inside
j = i
return inside
def _check_alert_cooldown(self, alert_type):
"""Check if alert is in cooldown period."""
if self.frame_count - self.last_alert_time[alert_type] > self.alert_cooldown:
self.last_alert_time[alert_type] = self.frame_count
return True
return False
def generate_surveillance_report(self):
"""Generate comprehensive surveillance report."""
report = {
'monitoring_duration': self.frame_count / 30.0,
'total_frames': self.frame_count,
'total_alerts': len(self.alert_history),
'alerts_by_type': defaultdict(int),
'alerts_by_severity': defaultdict(int),
'persons_detected': len(self.persons_tracked),
'zones_monitored': len(self.monitored_zones),
'zone_history': {},
}
for alert in self.alert_history:
report['alerts_by_type'][alert['type']] += 1
report['alerts_by_severity'][alert.get('severity', 'medium')] += 1
# Zone statistics
for zone_id in self.monitored_zones:
report['zone_history'][zone_id] = {
'max_occupancy': self.monitored_zones[zone_id]['current_persons'],
'is_restricted': self.monitored_zones[zone_id]['restricted']
}
return report
def get_person_trajectories(self, person_id):
"""Get trajectory for a specific person."""
if person_id in self.persons_tracked:
return self.persons_tracked[person_id]
return []
# Example: Complete surveillance system
np.random.seed(42)
surveillance = DroneVideoSurveillance()
print("Drone Video Surveillance System")
print("=" * 55)
# Setup zones
zones_config = {
'entrance': {
'boundary': [(0, 0), (200, 0), (200, 100), (0, 100)],
'max_persons': 5,
'restricted': False
},
'restricted_area': {
'boundary': [(300, 200), (500, 200), (500, 400), (300, 400)],
'max_persons': 2,
'restricted': True
},
'parking': {
'boundary': [(600, 100), (800, 100), (800, 300), (600, 300)],
'max_persons': 20,
'restricted': False
}
}
surveillance.setup_zones(zones_config)
print(f"Monitoring {len(zones_config)} zones")
# Process video frames
print("\nProcessing surveillance footage:")
all_alerts = []
for frame_idx in range(30):
frame = np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8)
prev_frame = np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8) if frame_idx > 0 else None
# Simulate detections
detections = []
if frame_idx > 5:
# Person in entrance
detections.append({
'type': 'person',
'position': (100, 50),
'track_id': 'person_1'
})
if frame_idx > 15:
# Person in restricted area
detections.append({
'type': 'person',
'position': (400, 300),
'track_id': 'person_2'
})
results = surveillance.process_frame(frame, detections, prev_frame)
all_alerts.extend(results['alerts'])
if results['alerts']:
print(f"\n Frame {frame_idx}:")
for alert in results['alerts']:
print(f" {alert['type'].upper()}: {alert.get('description', 'Alert')}")
# Generate report
report = surveillance.generate_surveillance_report()
print(f"\nSurveillance Report:")
print(f" Duration: {report['monitoring_duration']:.1f} seconds")
print(f" Total frames: {report['total_frames']}")
print(f" Total alerts: {report['total_alerts']}")
print(f" Persons tracked: {report['persons_detected']}")
print(f" Zones monitored: {report['zones_monitored']}")
print(f" Alerts by type: {dict(report['alerts_by_type'])}")
# Get person trajectory
trajectory = surveillance.get_person_trajectories('person_2')
if trajectory:
print(f"\nPerson 2 trajectory:")
print(f" Frames tracked: {len(trajectory)}")
print(f" Last position: {trajectory[-1]['position']}")
print("\nSurveillance system operational!")
Key Takeaways
- Action recognition identifies human activities in real-time
- Anomaly detection flags unusual behavior for security alerts
- Event summarization creates concise video overviews
- Zone monitoring tracks occupancy and restricted access
- Trajectory analysis follows individual movements over time
Course Complete
Congratulations on completing the AI Fundamentals for Drones course! You've learned:
- Machine learning and deep learning fundamentals
- Computer vision and image processing
- Object detection and tracking
- Semantic and instance segmentation
- Image classification with transfer learning
- Pose estimation and gesture recognition
- Face detection and recognition
- Video analysis and event understanding
These skills form the foundation for building intelligent drone systems that can perceive, understand, and interact with their environment autonomously.