Logging Best Practices - Log Levels, Handlers, Formatters

Introduction

Proper logging is essential for debugging and monitoring applications. This tutorial covers Python's logging module with log levels, handlers, formatters, and best practices.

Basic Logging Configuration

import logging
import sys

# Configure basic logging
logging.basicConfig(
    level=logging.DEBUG,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    handlers=[
        logging.StreamHandler(sys.stdout)
    ]
)

logger = logging.getLogger(__name__)

# Log messages at different levels
logger.debug("Debug message - for development")
logger.info("Info message - general information")
logger.warning("Warning message - something might be wrong")
logger.error("Error message - something failed")
logger.critical("Critical message - serious error")

# Create child logger
api_logger = logging.getLogger('myapp.api')
api_logger.info("API request received")

Advanced Configuration

import logging
from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler

# Create logger
logger = logging.getLogger('myapp')
logger.setLevel(logging.DEBUG)

# Console handler
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO)
console_formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
console_handler.setFormatter(console_formatter)

# File handler with rotation
file_handler = RotatingFileHandler(
    'app.log',
    maxBytes=10 * 1024 * 1024,  # 10MB
    backupCount=5
)
file_handler.setLevel(logging.DEBUG)
file_formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s'
)
file_handler.setFormatter(file_formatter)

# Add handlers to logger
logger.addHandler(console_handler)
logger.addHandler(file_handler)

# Time-based rotation
timed_handler = TimedRotatingFileHandler(
    'app.log',
    when='midnight',
    interval=1,
    backupCount=30
)

Structured Logging

import logging
import json
from datetime import datetime

class JSONFormatter(logging.Formatter):
    """Format logs as JSON."""
    
    def format(self, record):
        log_data = {
            'timestamp': datetime.utcnow().isoformat(),
            'level': record.levelname,
            'logger': record.name,
            'message': record.getMessage(),
            'module': record.module,
            'function': record.funcName,
            'line': record.lineno
        }
        
        if record.exc_info:
            log_data['exception'] = self.formatException(record.exc_info)
        
        if hasattr(record, 'extra'):
            log_data.update(record.extra)
        
        return json.dumps(log_data)

# Usage with extra fields
logger = logging.getLogger('myapp')
logger.info(
    "User logged in",
    extra={'user_id': 123, 'ip_address': '192.168.1.1'}
)

Context Variables

import logging
from contextvars import ContextVar

# Context variable for request ID
request_id_var: ContextVar[str] = ContextVar('request_id', default='')

class ContextFilter(logging.Filter):
    """Add context variables to log records."""
    
    def filter(self, record):
        record.request_id = request_id_var.get()
        record.user_id = user_id_var.get()
        return True

# Configure with context filter
logger = logging.getLogger('myapp')
logger.addFilter(ContextFilter())

# Usage in request handling
async def handle_request(request_id: str):
    request_id_var.set(request_id)
    logger.info("Processing request")

Practice Problems

1. Create a logging configuration for different environments
2. Implement log rotation with compression
3. Add structured logging with correlation IDs
4. Create a custom log handler for sending logs to external services
5. Implement log level filtering based on module