{
  "dashboard": {
    "title": "Airflow Overview",
    "panels": [
      {
        "title": "Task Success Rate",
        "type": "stat",
        "targets": [
          {
            "expr": "rate(airflow_task_success_total[5m]) / (rate(airflow_task_success_total[5m]) + rate(airflow_task_failure_total[5m]))",
            "legendFormat": "Success Rate"
          }
        ],
        "thresholds": [
          {"value": 0.95, "color": "green"},
          {"value": 0.9, "color": "yellow"},
          {"value": 0.8, "color": "red"}
        ]
      },
      {
        "title": "Task Duration",
        "type": "graph",
        "targets": [
          {
            "expr": "histogram_quantile(0.95, rate(airflow_task_duration_seconds_bucket[5m]))",
            "legendFormat": "P95 Duration"
          }
        ]
      },
      {
        "title": "Queue Depth",
        "type": "graph",
        "targets": [
          {
            "expr": "airflow_executor_queue_depth",
            "legendFormat": "Queued Tasks"
          }
        ]
      },
      {
        "title": "Scheduler Lag",
        "type": "stat",
        "targets": [
          {
            "expr": "airflow_scheduler_lag_seconds",
            "legendFormat": "Lag (seconds)"
          }
        ],
        "thresholds": [
          {"value": 60, "color": "green"},
          {"value": 300, "color": "yellow"},
          {"value": 600, "color": "red"}
        ]
      }
    ]
  }
}

# prometheus/alert_rules.yml
groups:
  - name: airflow_alerts
    rules:
      - alert: AirflowSchedulerLagHigh
        expr: airflow_scheduler_lag_seconds > 300
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Airflow scheduler lag is high"
          description: "Scheduler lag is {{ $value }} seconds"
      
      - alert: AirflowTaskFailureRateHigh
        expr: rate(airflow_task_failure_total[5m]) / rate(airflow_task_total[5m]) > 0.05
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "High task failure rate"
          description: "Task failure rate is {{ $value | humanizePercentage }}"
      
      - alert: AirflowQueueDepthHigh
        expr: airflow_executor_queue_depth > 100
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "High queue depth"
          description: "{{ $value }} tasks queued"
      
      - alert: AirflowDagRunStale
        expr: time() - airflow_dag_run_last_scheduling_decision > 3600
        for: 30m
        labels:
          severity: critical
        annotations:
          summary: "Stale DAG run detected"
          description: "DAG run has not been scheduled for {{ $value }} seconds"