Agent Error Recovery
Why This Matters
Error recovery is the foundation of reliable AI agents. Without proper error handling, a single failed API call can cascade into complete system failure. Production agents must handle transient errors, permanent failures, rate limits, and unexpected edge cases while maintaining user trust and system stability.
Real-World Analogy
Think of error recovery like a safety net system in a circus. The primary performer (your main tool) does the work. If they fall, there's a backup performer (fallback chain). If the whole act fails, the ringmaster (circuit breaker) stops the show temporarily rather than letting performers get hurt. The audience (users) always sees something, even if it's a simpler act.
Error Recovery Architecture
Retry Strategy Implementation
import asyncio
import time
import random
import logging
from dataclasses import dataclass, field
from typing import Any, Callable, Coroutine, Optional
from enum import Enum
logger = logging.getLogger(__name__)
class ErrorCategory(Enum):
TRANSIENT = "transient"
PERMANENT = "permanent"
RATE_LIMIT = "rate_limit"
TIMEOUT = "timeout"
VALIDATION = "validation"
@dataclass
class RetryConfig:
max_retries: int = 3
base_delay: float = 1.0
max_delay: float = 60.0
exponential_base: float = 2.0
jitter: bool = True
retryable_errors: list[type] = field(default_factory=list)
class RetryStrategy:
def __init__(self, config: RetryConfig = None):
self.config = config or RetryConfig()
self.attempt_history: list[dict] = []
def calculate_delay(self, attempt: int) -> float:
delay = self.config.base_delay * (self.config.exponential_base ** attempt)
delay = min(delay, self.config.max_delay)
if self.config.jitter:
delay = delay * (0.5 + random.random())
return delay
async def execute_with_retry(
self,
func: Callable[..., Coroutine],
*args,
**kwargs,
) -> Any:
last_exception = None
for attempt in range(self.config.max_retries + 1):
try:
result = await func(*args, **kwargs)
self.attempt_history.append({
"attempt": attempt,
"success": True,
"timestamp": time.time(),
})
return result
except Exception as e:
last_exception = e
self.attempt_history.append({
"attempt": attempt,
"success": False,
"error": str(e),
"timestamp": time.time(),
})
if attempt < self.config.max_retries:
delay = self.config.base_delay * (self.config.exponential_base ** attempt)
if self.config.jitter:
delay *= (0.5 + random.random())
delay = min(delay, self.config.max_delay)
logger.warning(f"Attempt {attempt} failed, retrying in {delay:.2f}s: {e}")
await asyncio.sleep(delay)
logger.error(f"All {self.config.max_retries} retry attempts failed")
raise last_exception
def get_success_rate(self) -> float:
if not self.attempt_history:
return 0.0
successes = sum(1 for h in self.attempt_history if h["success"])
return successes / len(self.attempt_history)
Circuit Breaker
import asyncio
import time
from dataclasses import dataclass
from typing import Any, Callable, Coroutine
from enum import Enum
import statistics
class CircuitState(Enum):
CLOSED = "closed"
OPEN = "open"
HALF_OPEN = "half_open"
@dataclass
class CircuitBreakerConfig:
failure_threshold: int = 5
recovery_timeout: float = 30.0
half_open_max_calls: int = 3
success_threshold: int = 2
window_size: int = 10
class CircuitBreaker:
def __init__(self, name: str, config: CircuitBreakerConfig = None):
self.name = name
self.config = config or CircuitBreakerConfig()
self.state = CircuitState.CLOSED
self.failure_count = 0
self.success_count = 0
self.last_failure_time: Optional[float] = None
self.call_history: list[dict] = []
self.half_open_calls = 0
async def execute(
self,
func: Callable[..., Coroutine],
*args,
**kwargs,
) -> Any:
if self.state == CircuitState.OPEN:
if self._should_attempt_reset():
self.state = CircuitState.HALF_OPEN
self.half_open_calls = 0
else:
raise CircuitOpenError(f"Circuit {self.name} is OPEN")
if self.state == CircuitState.HALF_OPEN:
if self.half_open_calls >= self.config.half_open_max_calls:
raise CircuitOpenError(f"Circuit {self.name} HALF_OPEN limit reached")
self.half_open_calls += 1
try:
result = await func(*args, **kwargs)
self._on_success()
return result
except Exception as e:
self._on_failure()
raise
def _on_success(self):
self.call_history.append({"success": True, "timestamp": time.time()})
if self.state == CircuitState.HALF_OPEN:
self.success_count += 1
if self.success_count >= self.config.success_threshold:
self.state = CircuitState.CLOSED
self.failure_count = 0
self.success_count = 0
elif self.state == CircuitState.CLOSED:
self.failure_count = max(0, self.failure_count - 1)
def _on_failure(self):
self.call_history.append({"success": False, "timestamp": time.time()})
self.failure_count += 1
self.last_failure_time = time.time()
if self.state == CircuitState.HALF_OPEN:
self.state = CircuitState.OPEN
elif self.state == CircuitState.CLOSED:
if self.failure_count >= self.config.failure_threshold:
self.state = CircuitState.OPEN
def _should_attempt_reset(self) -> bool:
if self.last_failure_time is None:
return True
return time.time() - self.last_failure_time >= self.config.recovery_timeout
def get_status(self) -> dict:
recent_calls = self.call_history[-self.config.window_size:]
successes = sum(1 for c in recent_calls if c["success"])
return {
"name": self.name,
"state": self.state.value,
"failure_count": self.failure_count,
"success_rate": successes / max(len(recent_calls), 1),
"total_calls": len(self.call_history),
}
class CircuitOpenError(Exception):
pass
Fallback Chain
from dataclasses import dataclass
from typing import Any, Callable, Coroutine
import asyncio
@dataclass
class FallbackStep:
name: str
handler: Callable[..., Coroutine]
error_types: list[type]
description: str = ""
class FallbackChain:
def __init__(self, steps: list[FallbackStep] = None):
self.steps = steps or []
self.execution_log: list[dict] = []
def add_step(self, step: FallbackStep):
self.steps.append(step)
async def execute(self, *args, **kwargs) -> Any:
for i, step in enumerate(self.steps):
try:
result = await step.handler(*args, **kwargs)
self.execution_log.append({
"step": step.name,
"success": True,
"step_index": i,
})
return result
except Exception as e:
self.execution_log.append({
"step": step.name,
"success": False,
"error": str(e),
"step_index": i,
})
if i == len(self.steps) - 1:
raise
continue
raise FallbackExhaustedError("All fallback steps exhausted")
def get_log(self) -> list[dict]:
return self.execution_log.copy()
class FallbackExhaustedError(Exception):
pass
Graceful Degradation
from dataclasses import dataclass
from typing import Any, Callable, Coroutine
import asyncio
@dataclass
class DegradationLevel:
name: str
handler: Callable[..., Coroutine]
quality_level: float
description: str = ""
class GracefulDegradation:
def __init__(self, levels: list[DegradationLevel] = None):
self.levels = sorted(levels or [], key=lambda l: l.quality_level, reverse=True)
self.current_level: int = 0
self.degradation_history: list[dict] = []
async def execute(self, *args, **kwargs) -> tuple[Any, float]:
for i, level in enumerate(self.levels):
try:
result = await level.handler(*args, **kwargs)
self.current_level = i
return result, level.quality_level
except Exception as e:
self.degradation_history.append({
"level": level.name,
"error": str(e),
})
continue
raise DegradationExhaustedError("All degradation levels exhausted")
def get_quality_level(self) -> float:
if self.current_level < len(self.levels):
return self.levels[self.current_level].quality_level
return 0.0
def get_status(self) -> dict:
return {
"current_level": self.current_level,
"quality_level": self.get_quality_level(),
"total_levels": len(self.levels),
"degradation_count": len(self.degradation_history),
}
class DegradationExhaustedError(Exception):
pass
Mathematical Foundation
Exponential Backoff Delay:
Where:
- — Delay at attempt n
- — Exponential base (typically 2)
- — Base delay
- — Random factor (0 to )
- — Maximum delay cap
Circuit Breaker Failure Rate:
Circuit opens when .
Retry Success Probability (with per attempt):
Expected Total Delay:
Fallback Chain Reliability:
Where is reliability of fallback step i.
Performance Considerations
| Strategy | Latency Impact | Cost Impact | Accuracy Impact |
|---|---|---|---|
| Exponential Backoff | +100-500ms per retry | +API calls | Maintained |
| Circuit Breaker | Fast fail (0ms) | -API calls | Partial availability |
| Fallback Chain | +50-200ms per fallback | +Alternative API | Reduced quality |
| Graceful Degradation | Minimal | -Compute | Reduced functionality |
Security Considerations
- Rate limiting: Implement client-side rate limits to prevent abuse
- Circuit breaker thresholds: Prevent cascade failures that could be weaponized
- Fallback authentication: Ensure fallback paths maintain security
- Error logging: Log sensitive errors securely, never expose credentials
- Retry budgets: Limit total retry attempts to prevent resource exhaustion
Interview Questions
1. What is the difference between transient and permanent errors?
Answer: Transient errors are temporary failures that may succeed on retry—network timeouts, rate limits, temporary service unavailability. Permanent errors are definitive failures that won't resolve with retry—invalid input, authentication failures, resource not found. Classification is critical: retrying permanent errors wastes resources and delays failure notification. Implement error classification using heuristics (error codes, message patterns) and domain knowledge. Some errors require special handling (rate limits need delay, not just retry).
2. How does exponential backoff work and why is it important?
Answer: Exponential backoff increases delay between retries exponentially: 1s, 2s, 4s, 8s... This prevents overwhelming a failing service with rapid retries (thundering herd). The formula: delay = base * 2^attempt. Add jitter (random factor) to prevent synchronized retries from multiple clients. Cap the maximum delay to avoid excessive waiting. Backoff is critical for: rate-limited APIs, recovering services, and distributed system stability. Without it, retries can worsen outages.
3. What is a circuit breaker and when should you use one?
Answer: A circuit breaker monitors failures and "opens" (blocks requests) when failures exceed a threshold, preventing cascading failures. States: CLOSED (normal), OPEN (blocking), HALF-OPEN (testing recovery). Use when: calling external services, database connections, or any unreliable dependency. Benefits: fast failure (no waiting for timeouts), prevents cascade, allows recovery time. Configure: failure threshold, recovery timeout, half-open probe count. Monitor circuit state for operational visibility.
4. How do you design a fallback chain for graceful degradation?
Answer: Order fallbacks by quality/cost: 1) Primary tool (highest quality), 2) Cheaper alternative, 3) Cached results, 4) Default/partial response, 5) Graceful failure message. Each fallback should: handle specific error types, degrade quality progressively, maintain core functionality. Test each fallback independently. Monitor which fallbacks are used to identify reliability issues. Consider: async fallbacks for latency, fallback composition, and user notification when degraded.
5. What metrics should you monitor for error recovery?
Answer: Key metrics: 1) Error rate — Total errors / total requests, 2) Retry rate — Retries / total requests, 3) Circuit state — Open/closed/half-open, 4) Fallback usage — Which fallbacks are triggered, 5) Recovery time — Time to restore service, 6) Mean time between failures (MTBF), 7) Mean time to recovery (MTTR), 8) Success rate by error type — Which errors are recoverable. Set alerts on high error rates, frequent circuit opens, or degraded fallback usage.
6. How do you handle partial failures in multi-step operations?
Answer: Use saga pattern: 1) Execute each step, 2) On failure, execute compensating actions for completed steps, 3) Track partial progress, 4) Support resume from checkpoint. Implementation: write-ahead logs for each step, compensating transactions, idempotent operations. For agent workflows: save state after each step, implement step-level rollback, provide partial results when full completion fails. Consider: eventual consistency, user notification of partial results, and retry from last successful step.
7. What is the thundering herd problem and how do you prevent it?
Answer: Thundering herd occurs when many clients retry simultaneously after a service recovers, overwhelming it again. Prevention: 1) Exponential backoff with jitter (randomize retry times), 2) Circuit breaker to prevent mass retries, 3) Rate limiting on retries, 4) Gradual recovery with probe requests, 5) Load shedding during recovery. Jitter is critical—without it, synchronized retries create periodic spikes. Implement adaptive backoff based on server feedback (e.g., retry-after headers).
8. How would you implement error recovery for LLM API calls?
Answer: LLM-specific strategies: 1) Handle rate limits with exponential backoff and retry-after headers, 2) Retry on transient network errors, 3) Don't retry on context length exceeded (permanent), 4) Implement token budget monitoring to prevent overspend, 5) Use fallback models when primary is unavailable, 6) Cache successful responses for repeated queries, 7) Degrade to simpler models under load, 8) Monitor latency and error patterns. Special case: if LLM returns malformed output, retry with different sampling parameters before falling back.
Common Pitfalls
| Pitfall | Solution |
|---|---|
| Retrying permanent errors | Classify errors; only retry transient failures |
| No jitter in backoff | Add random jitter to prevent thundering herd |
| Circuit breaker too sensitive | Tune thresholds based on normal error rates |
| Fallback chain too deep | Limit to 3-4 levels; each adds latency |
| No monitoring of recovery | Track error rates, circuit states, fallback usage |
| Partial failure without rollback | Implement saga pattern with compensating actions |
| Retry storms | Use circuit breakers and rate limiting |
| Silent failures | Always log errors and notify when degraded |
KnowledgeCheck
-
What is the primary purpose of exponential backoff?
- a) To increase retry frequency
- b) To prevent overwhelming failing services
- c) To reduce error handling code
- d) To simplify error logging
-
When should a circuit breaker transition from CLOSED to OPEN?
- a) On first error
- b) When failure threshold is exceeded
- c) After recovery timeout
- d) On every request
-
What is the benefit of adding jitter to retry delays?
- a) Faster retries
- b) Preventing synchronized retry storms
- c) Simpler implementation
- d) Lower memory usage
-
In a fallback chain, how should fallbacks be ordered?
- a) Random order
- b) By implementation date
- c) By quality/cost (best first)
- d) Alphabetical order
-
What is the thundering herd problem?
- a) Too many tools registered
- b) Simultaneous retries overwhelming a recovering service
- c) Memory exhaustion from error logs
- d) Circuit breaker state confusion
-
What is the purpose of the HALF-OPEN circuit breaker state?
- a) To block all requests
- b) To test if the service has recovered
- c) To reset failure counters
- d) To log all requests
Answers: 1-b, 2-b, 3-b, 4-c, 5-b, 6-b