Why This Matters

Amazon EventBridge is the central nervous system for event-driven data architectures on AWS. Unlike traditional polling-based approaches that waste compute cycles checking for changes, EventBridge enables reactive architectures where services respond to events as they occur. For data engineers, this means pipelines that start processing the instant new data arrives, rather than waiting for the next scheduled batch window.

EventBridge processes over 650 billion events per month across AWS customers and integrates natively with 35+ AWS services plus hundreds of SaaS applications. It charges only per event published ($1.00 per million events), making it cost-effective for both high-volume streaming and low-frequency batch triggers.

Architecture Overview

Real-World Project Structure

eventbridge-data-pipeline/
├── infrastructure/
│   ├── cdk/
│   │   ├── app.py
│   │   ├── stacks/
│   │   │   ├── eventbridge_stack.py
│   │   │   ├── lambda_targets_stack.py
│   │   │   ├── api_destinations_stack.py
│   │   │   └── monitoring_stack.py
│   │   └── cdk.json
│   └── terraform/
│       ├── main.tf
│       ├── eventbridge.tf
│       ├── rules.tf
│       └── variables.tf
├── event_patterns/
│   ├── s3_event_pattern.json
│   ├── custom_event_pattern.json
│   ├── saas_partner_pattern.json
│   └── scheduled_rules.json
├── lambda_functions/
│   ├── process_s3_event/
│   │   ├── app.py
│   │   └── requirements.txt
│   ├── aggregate_daily/
│   │   ├── app.py
│   │   └── requirements.txt
│   └── notify_downstream/
│       ├── app.py
│       └── requirements.txt
├── schemas/
│   ├── order_event.json
│   ├── user_event.json
│   └── data_event.json
├── archives/
│   └── archive_config.json
├── tests/
│   ├── unit/
│   ├── integration/
│   └── test_event_patterns.py
├── monitoring/
│   ├── dashboards/
│   └── alarms/
└── README.md

Scheduling with EventBridge

EventBridge Scheduler enables cron and rate-based event generation for data engineering workflows.

Schedule Expression Formats

# Rate-based schedules - fixed intervals
rate(1 hour)           # Every hour
rate(5 minutes)        # Every 5 minutes
rate(1 day)            # Once per day
rate(30 minutes)       # Every 30 minutes

# Cron-based schedules - specific times (UTC)
cron(0 12 * * ? *)     # Daily at noon UTC
cron(0/15 * * * ? *)   # Every 15 minutes
cron(0 8 ? * MON-FRI *) # Weekdays at 8 AM
cron(0 1 1 * ? *)      # First of each month
cron(0 2 ? * SUN *)    # Every Sunday at 2 AM

Event-Driven ETL Pattern

import boto3
import json
import os

s3_client = boto3.client('s3')
glue_client = boto3.client('glue')

def lambda_handler(event, context):
    """
    EventBridge target: Trigger Glue ETL when new files land in S3.
    """
    try:
        detail = event['detail']
        bucket = detail['bucket']['name']
        key = detail['object']['key']

        # Validate file before processing
        if not key.endswith('.parquet'):
            print(f"Skipping non-Parquet file: {key}")
            return {'statusCode': 200, 'body': 'Skipped'}

        # Trigger Glue ETL job
        response = glue_client.start_job_run(
            JobName=os.environ['GLUE_JOB_NAME'],
            Arguments={
                '--source_bucket': bucket,
                '--source_key': key,
                '--target_path': f"s3://curated-data/processed/{key.split('/')[1]}/",
                '--execution_id': context.aws_request_id
            },
            Timeout=3600
        )

        print(f"Glue job started: {response['JobRunId']} for {bucket}/{key}")

        return {
            'statusCode': 200,
            'body': json.dumps({
                'jobRunId': response['JobRunId'],
                'source': f"{bucket}/{key}"
            })
        }

    except Exception as e:
        print(f"Error triggering Glue job: {str(e)}")
        raise

Event Pattern Filtering

{
  "source": ["aws.s3"],
  "detail-type": ["Object Created"],
  "detail": {
    "bucket": {
      "name": ["data-lake-bucket"]
    },
    "object": {
      "key": [{"prefix": "raw/"}],
      "size": [{"numeric": [">", 0]}]
    }
  }
}

Scheduled Aggregation Pattern

import boto3
import json
from datetime import datetime, timedelta

athena_client = boto3.client('athena')

def lambda_handler(event, context):
    """
    EventBridge scheduled rule: Aggregate yesterday's data daily.
    """
    try:
        yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')

        query = f"""
        INSERT OVERWRITE TABLE analytics.daily_summary
        SELECT
            date_trunc('day', event_time) as event_date,
            source,
            event_type,
            COUNT(*) as event_count,
            COUNT(DISTINCT user_id) as unique_users,
            SUM(amount) as total_amount
        FROM raw.events
        WHERE dt = '{yesterday}'
        GROUP BY 1, 2, 3
        """

        response = athena_client.start_query_execution(
            QueryString=query,
            ResultConfiguration={
                'OutputLocation': 's3://athena-results/eventbridge/'
            },
            WorkGroup='data-engineering'
        )

        print(f"Athena query started: {response['QueryExecutionId']}")

        return {
            'statusCode': 200,
            'body': json.dumps({
                'queryExecutionId': response['QueryExecutionId'],
                'date': yesterday
            })
        }

    except Exception as e:
        print(f"Error running aggregation: {str(e)}")
        raise

Performance Considerations

Security Considerations

• Resource-Based Policies: Use event bus resource policies to control who can publish and receive events.
• IAM Roles: Assign least-privilege IAM roles to each target (Lambda, Step Functions, etc.).
• Encryption: Enable encryption at rest for custom event buses using AWS KMS.
• VPC Endpoints: Use VPC endpoints for private connectivity without public internet exposure.
• CloudTrail: Log all EventBridge API calls for audit and compliance.
• Input Transformation: Sanitize and validate event data before passing to targets.
• Cross-Account: Use AssumeRole with explicit trust policies for cross-account event routing.
• API Destinations: Use OAuth or SigV4 authentication for external HTTP endpoints.

Interview Questions & Answers

Q1: What is Amazon EventBridge and how does it differ from SNS?

Answer: Amazon EventBridge is a serverless event bus that connects AWS services, SaaS applications, and custom applications through events. Unlike SNS which uses a topic-based pub/sub model, EventBridge uses an event bus pattern with rule-based JSON filtering. EventBridge offers built-in event filtering with pattern matching, 35+ AWS service integrations, SaaS partner integrations, schema discovery for custom events, and archive/replay capabilities. SNS is simpler for basic pub/sub but lacks the rich filtering, schema management, and archive features that EventBridge provides for complex data architectures.

Q2: Explain EventBridge event buses and when you would use custom event buses?

Answer: Event buses are pipelines that receive and route events. The default bus is AWS-managed and receives all AWS service events. Custom buses are user-created for application-specific events, providing isolation, separate permissions, and custom routing rules. Partner buses receive events from SaaS integrations. Use custom buses when you need isolation between different applications or environments, separate IAM policies and governance, custom event routing rules, and better cost tracking per application domain. Custom buses also enable cross-account event sharing with explicit trust policies.

Q3: How would you design an event-driven data pipeline using EventBridge?

Answer: A robust design includes: (1) Event sources: S3 for file events, IoT Core for sensor data, API Gateway for REST events, custom applications via PutEvents API. (2) Event Bridge: Single default bus for AWS events, custom buses per application domain. (3) Rules: Pattern-matching rules for different event types with precise filtering to reduce unnecessary invocations. (4) Targets: Lambda for simple processing, Step Functions for complex orchestration, SQS for buffering and decoupling. (5) Dead Letter Queues: Configure DLQs on every target for failed event handling. (6) Monitoring: CloudWatch metrics (Invocations, FailedInvocations, ThrottledRules) with alarms.

Q4: What are EventBridge schedule expressions and when would you use them?

Answer: Schedule expressions trigger events at specific times using rate or cron formats. Rate expressions use fixed intervals (rate(5 minutes), rate(1 hour)). Cron expressions specify exact times (cron(0 12 * * ? *) for daily at noon). Use cases: data ingestion from external APIs hourly, daily aggregation queries for reporting, weekly cleanup of temporary data, intraday monitoring checks every 15 minutes. The cron format uses 6 fields: minutes, hours, day-of-month, month, day-of-week, and year. Day-of-week uses 1-7 or SUN-SAT.

Q5: How does EventBridge handle event filtering?

Answer: EventBridge uses JSON pattern matching for content-based filtering. Filtering types include exact match ("source": ["aws.s3"]), prefix match ("source": [{"prefix": "aws."}]), anything-but ("source": [{"anything-but": "aws.ec2"}]), numeric comparisons ("size": [{"numeric": [">", 1000]}]), and dot-notation for nested fields. Filtering reduces unnecessary invocations and costs by ensuring targets only receive relevant events. For high-throughput scenarios, use prefix and numeric filters to minimize processing overhead. Combine multiple filter conditions with AND logic for precise routing.

Q6: Explain EventBridge archives and replay capabilities.

Answer: Archives capture all events (or filtered events) passing through an event bus for later replay. Configuration includes retention period (1-365 days), event selection patterns for selective archival, and archive state (enabled/disabled). Use cases: debugging failed event processing by replaying events after fixing the handler, backfilling historical data into new systems, testing new event handlers with production events, disaster recovery by replaying events to a new bus. Replay sends archived events to a specified event bus or ARN at up to 100,000 events per second.

Q7: How would you implement error handling in EventBridge?

Answer: Multi-layer error handling approach: (1) Dead Letter Queues (DLQ): Capture failed events for investigation - always configure on every target. (2) Retry Policies: Configure exponential backoff with max attempts (default 185 retries over ~24 hours). (3) Lambda Dead Letter Config: Lambda-specific DLQ handling with bisect-on-error for large payloads. (4) CloudWatch Alarms: Monitor FailedInvocations metric and set alarms for threshold breaches. (5) Event Archive: Capture all events for debugging and replay. (6) API Destination DLQs: For HTTP targets, configure separate DLQs for failed API calls.

Q8: What are the cost optimization strategies for EventBridge?

Answer: (1) Precise filtering: Use specific event patterns to reduce unnecessary rule matches and target invocations. (2) Event batching: Configure batch size on Lambda targets to process multiple events per invocation. (3) Rule consolidation: Combine related rules where possible to stay within limits and reduce overhead. (4) Archive selective events: Only archive events that need replay capability, not all events. (5) Schedule optimization: Use rate expressions instead of cron where possible for simpler schedules. (6) Custom bus per domain: Track costs per application and identify unused rules. (7) API destination rate limiting: Configure appropriate invocation rates to avoid throttling costs.

Common Pitfalls

Advanced: Cross-Account Event Routing

EventBridge supports cross-account event sharing, enabling centralized event buses that receive events from multiple AWS accounts:

import boto3
import json

events_client = boto3.client('events')

def setup_cross_account_events():
    """
    Configure cross-account event bus for multi-account data pipeline.
    """
    # Create custom event bus in central account
    bus = events_client.create_event_bus(
        Name='central-data-events'
    )

    # Add resource policy for cross-account access
    events_client.put_permission(
        EventBusName='central-data-events',
        Action='events:PutEvents',
        Principal='123456789012',  # Source account
        StatementId='allow-data-account'
    )

    # Create rule for cross-account events
    events_client.put_rule(
        Name='cross-account-data-event',
        EventBusName='central-data-events',
        EventPattern=json.dumps({
            'source': ['custom.data-pipeline'],
            'detail-type': ['data-ready'],
            'detail': {
                'account': ['123456789012']
            }
        }),
        State='ENABLED'
    )

    # Add Lambda target
    events_client.put_targets(
        Rule='cross-account-data-event',
        EventBusName='central-data-events',
        Targets=[{
            'Id': 'process-cross-account',
            'Arn': 'arn:aws:lambda:us-east-1:123456789012:function:process-event'
        }]
    )

Advanced: Schema Discovery and Registry

EventBridge Schema Discovery automatically detects event schemas from your custom events, enabling type-safe code generation:

import boto3

schemas_client = boto3.client('schemas')

def discover_and_register_schema():
    """
    Discover event schemas and generate code bindings.
    """
    # List discovered schemas
    schemas = schemas_client.list_schemas(
        RegistryName='discovered-schemas'
    )

    for schema in schemas['Schemas']:
        schema_name = schema['SchemaName']
        print(f"Discovered schema: {schema_name}")

        # Generate code binding for the schema
        try:
            schemas_client.get_code_binding_source(
                RegistryName='discovered-schemas',
                SchemaName=schema_name,
                Language='python'
            )
            print(f"Code binding available for {schema_name}")
        except Exception as e:
            print(f"No code binding: {str(e)}")

Advanced: Event Replay Configuration

import boto3
import json
from datetime import datetime, timedelta

events_client = boto3.client('events')

def setup_event_archive_and_replay():
    """
    Configure event archive for compliance and replay capability.
    """
    # Create archive for all events
    archive = events_client.create_archive(
        ArchiveName='data-pipeline-events',
        SourceArn='arn:aws:events:us-east-1:123456789012:event-bus/central-data-events',
        Description='Archive for data pipeline events',
        RetentionDays=365,
        EventPattern=json.dumps({
            'source': [{'prefix': 'custom.'}]
        })
    )

    print(f"Archive created: {archive['ArchiveArn']}")

    # Start replay after incident
    start_time = datetime.now() - timedelta(hours=24)
    end_time = datetime.now() - timedelta(hours=23)

    replay = events_client.start_replay(
        ReplayName='incident-replay-2025-01-15',
        Description='Replay events from incident window',
        EventSourceArn=archive['ArchiveArn'],
        EventStartTime=start_time,
        EventEndTime=end_time,
        Destination={
            'Arn': 'arn:aws:events:us-east-1:123456789012:event-bus/replay-events'
        }
    )

    print(f"Replay started: {replay['ReplayArn']}")

See Also

• AWS Step Functions
• AWS Lambda
• AWS SQS SNS
• AWS Glue