python -m venv venv
source venv/bin/activate
pip install openai anthropic google-generativeai pandas
export OPENAI_API_KEY="sk-your-key"
export ANTHROPIC_API_KEY="sk-ant-your-key"
export GOOGLE_API_KEY="your-key"

# models/gpt4.py
from openai import OpenAI
import time

class GPT4Adapter:
    def __init__(self, model: str = "gpt-4-turbo-preview"):
        self.client = OpenAI()
        self.model = model

    def generate(self, prompt: str, tools: list = None) -> dict:
        start = time.time()
        kwargs = {"model": self.model, "messages": [{"role": "user", "content": prompt}], "temperature": 0.0}
        if tools:
            kwargs["tools"] = tools
        response = self.client.chat.completions.create(**kwargs)
        latency = (time.time() - start) * 1000
        return {
            "response": response.choices[0].message.content,
            "tool_calls": [tc.function.name for tc in (response.choices[0].message.tool_calls or [])],
            "latency_ms": latency,
            "tokens_input": response.usage.prompt_tokens,
            "tokens_output": response.usage.completion_tokens,
            "model": self.model,
        }

# models/claude.py
import anthropic
import time

class ClaudeAdapter:
    def __init__(self, model: str = "claude-3-opus-20240229"):
        self.client = anthropic.Anthropic()
        self.model = model

    def generate(self, prompt: str, tools: list = None) -> dict:
        start = time.time()
        kwargs = {"model": self.model, "max_tokens": 1024, "messages": [{"role": "user", "content": prompt}]}
        if tools:
            kwargs["tools"] = tools
        response = self.client.messages.create(**kwargs)
        latency = (time.time() - start) * 1000
        return {
            "response": response.content[0].text if response.content else "",
            "tool_calls": [tc.name for tc in response.tool_use] if hasattr(response, 'tool_use') else [],
            "latency_ms": latency,
            "tokens_input": response.usage.input_tokens,
            "tokens_output": response.usage.output_tokens,
            "model": self.model,
        }

# models/gemini.py
import google.generativeai as genai
import time

class GeminiAdapter:
    def __init__(self, model: str = "gemini-pro"):
        genai.configure()
        self.model = genai.GenerativeModel(model)

    def generate(self, prompt: str, tools: list = None) -> dict:
        start = time.time()
        response = self.model.generate_content(prompt)
        latency = (time.time() - start) * 1000
        return {
            "response": response.text,
            "tool_calls": [],
            "latency_ms": latency,
            "tokens_input": response.usage_metadata.prompt_token_count if response.usage_metadata else 0,
            "tokens_output": response.usage_metadata.candidates_token_count if response.usage_metadata else 0,
            "model": self.model.model_name,
        }

# tasks/agent_tasks.py
from typing import List, Dict

class AgentBenchmarkTasks:
    REASONING_TASKS = [
        {"id": "reason_1", "input": "If a train travels at 60 mph for 2.5 hours, then 80 mph for 1.5 hours, what is the total distance?", "expected": "270 miles", "category": "math"},
        {"id": "reason_2", "input": "What comes next: 2, 6, 12, 20, 30, ?", "expected": "42", "category": "pattern"},
        {"id": "reason_3", "input": "A farmer has 17 sheep. All but 9 die. How many are left?", "expected": "9", "category": "logic"},
    ]

    CODE_TASKS = [
        {"id": "code_1", "input": "Write a Python function to check if a string is a palindrome", "expected_contains": ["def", "return"], "category": "function"},
        {"id": "code_2", "input": "Write a Python function to find the factorial of a number", "expected_contains": ["def", "return"], "category": "function"},
        {"id": "code_3", "input": "Write a SQL query to find the second highest salary", "expected_contains": ["SELECT"], "category": "sql"},
    ]

    TOOL_TASKS = [
        {"id": "tool_1", "input": "Search the web for Python 3.12 features", "expected_tool": "web_search", "category": "search"},
        {"id": "tool_2", "input": "Calculate 15% tip on a $85 bill", "expected_tool": "calculator", "category": "math"},
        {"id": "tool_3", "input": "What's the weather in New York?", "expected_tool": "weather", "category": "lookup"},
    ]

    INSTRUCTION_TASKS = [
        {"id": "inst_1", "input": "List exactly 3 benefits of exercise", "expected_contains": ["1.", "2.", "3."], "category": "formatting"},
        {"id": "inst_2", "input": "Explain quantum computing in exactly 2 sentences", "expected_contains": ["."], "category": "constraint"},
    ]

    def get_all_tasks(self) -> List[Dict]:
        return (
            self.REASONING_TASKS +
            self.CODE_TASKS +
            self.TOOL_TASKS +
            self.INSTRUCTION_TASKS
        )

    def get_tasks_by_category(self, category: str) -> List[Dict]:
        return [t for t in self.get_all_tasks() if t["category"] == category]

# evaluation/metrics.py
from typing import Dict, List
import re

class BenchmarkMetrics:
    def evaluate_accuracy(self, response: str, expected: str) -> float:
        if not expected:
            return 1.0
        response_lower = response.lower().strip()
        expected_lower = expected.lower().strip()
        if expected_lower in response_lower:
            return 1.0
        response_numbers = re.findall(r'\d+\.?\d*', response)
        expected_numbers = re.findall(r'\d+\.?\d*', expected)
        if expected_numbers and response_numbers:
            if expected_numbers[0] in response_numbers:
                return 1.0
        return 0.0

    def evaluate_contains(self, response: str, expected_contains: List[str]) -> float:
        if not expected_contains:
            return 1.0
        matches = sum(1 for exp in expected_contains if exp.lower() in response.lower())
        return matches / len(expected_contains)

    def evaluate_tool_use(self, result: Dict, expected_tool: str) -> float:
        return 1.0 if expected_tool in result.get("tool_calls", []) else 0.0

    def calculate_cost(self, result: Dict, pricing: Dict) -> float:
        model = result.get("model", "")
        p = pricing.get(model, {"input": 0.01, "output": 0.03})
        return (result["tokens_input"] * p["input"] + result["tokens_output"] * p["output"]) / 1000

# reporting/reporter.py
from typing import Dict, List
import pandas as pd

class BenchmarkReporter:
    def generate_report(self, results: List[Dict]) -> str:
        df = pd.DataFrame(results)
        report = "# LLM Benchmark Report\n\n"
        report += "## Summary by Model\n"
        summary = df.groupby("model").agg({
            "accuracy": "mean",
            "latency_ms": "mean",
            "cost": "sum",
            "tokens_input": "mean",
        }).round(3)
        report += summary.to_string() + "\n\n"
        report += "## Results by Task Category\n"
        cat_summary = df.groupby(["model", "category"]).agg({"accuracy": "mean"}).round(3)
        report += cat_summary.to_string() + "\n\n"
        report += "## Recommendations\n"
        best_accuracy = df.groupby("model")["accuracy"].mean().idxmax()
        fastest = df.groupby("model")["latency_ms"].mean().idxmin()
        cheapest = df.groupby("model")["cost"].sum().idxmin()
        report += f"- Best Accuracy: {best_accuracy}\n"
        report += f"- Fastest: {fastest}\n"
        report += f"- Most Cost-Effective: {cheapest}\n"
        return report

    def compare_models(self, results: List[Dict]) -> pd.DataFrame:
        df = pd.DataFrame(results)
        return df.groupby("model").agg({
            "accuracy": ["mean", "std"],
            "latency_ms": ["mean", "std"],
            "cost": ["sum", "mean"],
        }).round(3)

# benchmark.py
from models.gpt4 import GPT4Adapter
from models.claude import ClaudeAdapter
from models.gemini import GeminiAdapter
from tasks.agent_tasks import AgentBenchmarkTasks
from evaluation.metrics import BenchmarkMetrics
from reporting.reporter import BenchmarkReporter
from typing import Dict, List

class AgentBenchmark:
    def __init__(self):
        self.models = {
            "gpt-4": GPT4Adapter(),
            "claude-3": ClaudeAdapter(),
            "gemini": GeminiAdapter(),
        }
        self.tasks = AgentBenchmarkTasks()
        self.metrics = BenchmarkMetrics()
        self.reporter = BenchmarkReporter()
        self.results: List[Dict] = []

    def run(self, categories: List[str] = None) -> List[Dict]:
        tasks = self.tasks.get_all_tasks()
        if categories:
            tasks = [t for t in tasks if t["category"] in categories]
        for model_name, model in self.models.items():
            for task in tasks:
                result = model.generate(task["input"])
                if "expected" in task:
                    accuracy = self.metrics.evaluate_accuracy(result["response"], task["expected"])
                elif "expected_contains" in task:
                    accuracy = self.metrics.evaluate_contains(result["response"], task["expected_contains"])
                elif "expected_tool" in task:
                    accuracy = self.metrics.evaluate_tool_use(result, task["expected_tool"])
                else:
                    accuracy = 1.0
                cost = self.metrics.calculate_cost(result, {
                    "gpt-4-turbo-preview": {"input": 0.01, "output": 0.03},
                    "claude-3-opus-20240229": {"input": 0.015, "output": 0.075},
                    "gemini-pro": {"input": 0.00025, "output": 0.0005},
                })
                self.results.append({
                    "model": model_name,
                    "task_id": task["id"],
                    "category": task["category"],
                    "accuracy": accuracy,
                    "latency_ms": result["latency_ms"],
                    "cost": cost,
                    "tokens_input": result["tokens_input"],
                    "tokens_output": result["tokens_output"],
                })
        return self.results

    def report(self) -> str:
        return self.reporter.generate_report(self.results)

import pytest
from benchmark import AgentBenchmark

def test_benchmark():
    bench = AgentBenchmark()
    results = bench.run(categories=["math"])
    assert len(results) > 0
    assert all("accuracy" in r for r in results)

# main.py
from benchmark import AgentBenchmark

def main():
    bench = AgentBenchmark()
    print("Running benchmark...\n")
    results = bench.run()
    print(bench.report())

if __name__ == "__main__":
    main()