Code Generation with LLMs

Large Language Models have transformed software development, enabling everything from intelligent autocomplete to autonomous program synthesis. This tutorial covers the architectures, training methods, and evaluation frameworks that make code generation possible.

Code-Specific Architectures

How Code LLMs Differ from General LLMs

Code LLMs require specialized handling due to the unique characteristics of programming languages:

State-of-the-Art Code LLMs

Training Code LLMs

Pretraining on Code

Code-Specific Training Strategies

training_strategies = {
    "fill_in_middle": {
        "description": "Train model to fill in middle of code given prefix and suffix",
        "objective": "P(middle | prefix, suffix)",
        "benefit": "Enables intelligent code completion"
    },
    "commit_message": {
        "description": "Train to generate commit messages from diffs",
        "objective": "P(message | diff)",
        "benefit": "Automated documentation"
    },
    "code_review": {
        "description": "Train to review code and suggest improvements",
        "objective": "P(review | code, context)",
        "benefit": "Automated code review"
    },
    "test_generation": {
        "description": "Train to generate test cases from code",
        "objective": "P(tests | function)",
        "benefit": "Automated testing"
    },
    "bug_detection": {
        "description": "Train to identify and fix bugs",
        "objective": "P(fix | buggy_code, error)",
        "benefit": "Automated debugging"
    }
}

Fine-Tuning for Code Tasks

Evaluation Benchmarks

Core Benchmarks

Benchmark Comparison

Evaluation Framework

class CodeLLMEvaluator:
    """Comprehensive evaluation framework for code LLMs."""
    
    def __init__(self, model, benchmarks=["humaneval", "mbpp"]):
        self.model = model
        self.benchmarks = benchmarks
    
    def evaluate(self, benchmark_name, n_samples=100):
        """Run evaluation on a benchmark."""
        dataset = load_dataset(benchmark_name)
        results = []
        
        for problem in dataset:
            # Generate solutions
            solutions = self.model.generate(
                prompt=problem["prompt"],
                n=n_samples,
                temperature=0.8
            )
            
            # Execute solutions
            pass_count = 0
            for solution in solutions:
                if self.execute_and_test(solution, problem["test_cases"]):
                    pass_count += 1
            
            # Calculate pass@k
            for k in [1, 5, 10, 100]:
                pass_at_k = self.calculate_pass_at_k(
                    n=n_samples, c=pass_count, k=k
                )
                results.append({
                    "problem": problem["id"],
                    "pass_at_k": pass_at_k,
                    "k": k
                })
        
        return self.aggregate_results(results)
    
    def execute_and_test(self, code, test_cases):
        """Safely execute code and run tests."""
        try:
            # Create isolated execution environment
            exec_globals = {}
            exec(code, exec_globals)
            
            # Run test cases
            for test in test_cases:
                result = eval(test, exec_globals)
                if not result:
                    return False
            return True
        except Exception:
            return False

Production Deployment

Code Assistant Architecture

class CodeAssistant:
    """Production code assistant with context management."""
    
    def __init__(self, model, context_window=8192):
        self.model = model
        self.context_window = context_window
        self.file_cache = {}
    
    def autocomplete(self, file_path, cursor_position, prefix, suffix):
        """Provide intelligent code completion."""
        # Gather context
        context = self.gather_context(file_path, cursor_position)
        
        # Format prompt with fill-in-middle
        prompt = f"<prefix>{prefix}<middle>{suffix}"
        
        # Generate completion
        completion = self.model.generate(
            prompt=prompt,
            max_tokens=256,
            stop_tokens=["\ndef ", "\nclass ", "\n# "]
        )
        
        return completion
    
    def gather_context(self, file_path, cursor_position):
        """Gather relevant context from the codebase."""
        context = {
            "current_file": self.get_file_content(file_path),
            "imports": self.get_imports(file_path),
            "related_files": self.get_related_files(file_path),
            "definitions": self.get_definitions(file_path)
        }
        
        # Truncate to fit context window
        return self.truncate_context(context)
    
    def generate_function(self, description, context):
        """Generate a complete function from description."""
        prompt = f"""Write a Python function that: {description}

Context from the codebase:
{context}

Requirements:
- Follow existing code style
- Include type hints
- Add docstring
- Handle edge cases

Function:"""
        
        return self.model.generate(prompt, max_tokens=512)

Multi-File Code Generation

class RepoLevelGenerator:
    """Generate code at repository level."""
    
    def __init__(self, repo_path, model):
        self.repo = Repository(repo_path)
        self.model = model
    
    def generate_feature(self, feature_description):
        """Generate a complete feature across multiple files."""
        # Analyze repository structure
        structure = self.repo.analyze()
        
        # Identify files to modify/create
        plan = self.plan_feature(feature_description, structure)
        
        # Generate changes for each file
        changes = []
        for file_change in plan:
            if file_change["action"] == "modify":
                new_content = self.modify_file(
                    file_change["path"],
                    file_change["instructions"]
                )
            else:
                new_content = self.create_file(
                    file_change["path"],
                    file_change["instructions"]
                )
            
            changes.append({
                "path": file_change["path"],
                "content": new_content,
                "action": file_change["action"]
            })
        
        return changes
    
    def modify_file(self, file_path, instructions):
        """Modify an existing file with new code."""
        current_content = self.repo.get_file(file_path)
        
        prompt = f"""Modify the following code according to these instructions:

Current code:
{current_content}

Instructions: {instructions}

Modified code:"""
        
        return self.model.generate(prompt, max_tokens=2048)

Advanced Techniques

Execution-Based Generation

class ExecutionBasedGenerator:
    """Generate code with execution feedback."""
    
    def __init__(self, model, executor):
        self.model = model
        self.executor = executor
    
    def generate_with_feedback(self, problem, max_attempts=5):
        """Generate code with iterative improvement."""
        for attempt in range(max_attempts):
            # Generate code
            code = self.model.generate(problem)
            
            # Execute and get feedback
            result = self.executor.execute(code)
            
            if result["success"]:
                return code, attempt + 1
            
            # Add error feedback to prompt
            problem = f"""{problem}

Previous attempt failed with error:
{result['error']}

Fix the error and try again:"""
        
        return None, max_attempts

Code Explanation and Documentation

Practice Exercises

1. Conceptual: Explain the difference between Pass@1 and Pass@k metrics. Why is Pass@k more appropriate for evaluating code generation?

2. Mathematical: If a code LLM generates 100 samples for a problem and 40 pass all tests, calculate Pass@1, Pass@5, and Pass@10.

3. Practical: Implement a simple code completion system using a pre-trained code LLM. Test it on 10 Python functions and measure completion accuracy.

4. Research: Compare the performance of Code Llama 7B and StarCoder 15B on the HumanEval benchmark. What are the trade-offs between model size and performance?

What to Learn Next