Open Source LLM Ecosystem

The open-source LLM ecosystem has rapidly expanded, offering powerful alternatives to proprietary models. This tutorial covers the major model families, tools, and strategies for choosing and deploying open-source LLMs.

Major Open Source Model Families

LLaMA Family (Meta)

from transformers import AutoModelForCausalLM, AutoTokenizer

# Loading LLaMA models
def load_llama(model_name, device_map="auto"):
    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        torch_dtype="auto",
        device_map=device_map,
        trust_remote_code=True
    )
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    return model, tokenizer

# Example usage
model, tokenizer = load_llama("meta-llama/Llama-3.1-8B")

Mistral Family

# Mixtral MoE Architecture
from transformers import MixtralForCausalLM

model = MixtralForCausalLM.from_pretrained(
    "mistralai/Mixtral-8x7B-Instruct-v0.1",
    torch_dtype="auto",
    device_map="auto"
)

Other Notable Families

Licensing Considerations

License Comparison

license_information = {
    "mit": {
        "commercial": True,
        "restrictions": [],
        "examples": ["phi-2", "gemma"]
    },
    "apache_2": {
        "commercial": True,
        "restrictions": ["patent grant"],
        "examples": ["falcon", "mistral-7b", "mixtral", "qwen"]
    },
    "llama_2_community": {
        "commercial": True,
        "restrictions": ["700M monthly active users limit", "acceptable use policy"],
        "examples": ["llama-2-7b", "llama-2-13b", "llama-2-70b"]
    },
    "llama_3_community": {
        "commercial": True,
        "restrictions": ["700M monthly active users limit", "acceptable use policy"],
        "examples": ["llama-3-8b", "llama-3-70b", "llama-3.1-405b"]
    }
}

HuggingFace Ecosystem

Transformers

The core library for model loading and inference:

from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    pipeline,
    BitsAndBytesConfig
)

# Load with quantization
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype="auto"
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3.1-8B",
    quantization_config=bnb_config,
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B")

# Quick inference
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
output = pipe("Explain quantum computing:", max_new_tokens=200)

PEFT (Parameter-Efficient Fine-Tuning)

from peft import (
    LoraConfig,
    get_peft_model,
    TaskType,
    prepare_model_for_kbit_training
)

# Prepare model for QLoRA
model = prepare_model_for_kbit_training(model)

# LoRA configuration
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,
    lora_alpha=32,
    lora_dropout=0.1,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"]
)

# Apply LoRA
peft_model = get_peft_model(model, lora_config)
peft_model.print_trainable_parameters()
# trainable params: 13,631,488 || all params: 8,048,558,080 || 0.17%

TRL (Transformer Reinforcement Learning)

from trl import SFTTrainer, PPOTrainer, DPOTrainer
from datasets import load_dataset

dataset = load_dataset("tatsu-lab/alpaca", split="train")

# Supervised Fine-Tuning
sft_trainer = SFTTrainer(
    model=peft_model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    max_seq_length=512,
    args=TrainingArguments(
        output_dir="./sft_output",
        num_train_epochs=3,
        per_device_train_batch_size=4,
        learning_rate=2e-4,
        fp16=True
    )
)

sft_trainer.train()

Datasets Library

from datasets import load_dataset, Dataset, DatasetDict

# Load existing datasets
alpaca = load_dataset("tatsu-lab/alpaca")
dolly = load_dataset("databricks/databricks-dolly-15k")

# Create custom dataset
custom_data = {
    "instruction": ["Summarize this text:", "Translate to French:"],
    "input": ["Long text here...", "Hello world"],
    "output": ["Summary here...", "Bonjour le monde"]
}
dataset = Dataset.from_dict(custom_data)

# Format for instruction tuning
def format_alpaca(example):
    if example["input"]:
        prompt = f"### Instruction:\n{example['instruction']}\n\n### Input:\n{example['input']}\n\n### Response:\n"
    else:
        prompt = f"### Instruction:\n{example['instruction']}\n\n### Response:\n"
    return {"text": prompt + example["output"]}

formatted = alpaca.map(format_alpaca)

Model Hubs and Quantized Models

Finding Models on HuggingFace

from huggingface_hub import HfApi

api = HfApi()

# Search for models
models = api.list_models(
    search="llama",
    sort="downloads",
    direction=-1,
    filter="text-generation"
)

# Find quantized versions
quantized = api.list_models(
    search="llama-3-8b-GPTQ",
    sort="downloads",
    direction=-1
)

Popular Quantized Model Sources

Open Source vs Proprietary

Performance Comparison

When to Choose Which

decision_framework = {
    "use_proprietary_when": [
        "Need best overall quality (GPT-4, Claude)",
        "Low-latency requirements without GPU infrastructure",
        "Rapid prototyping without deployment concerns",
        "Tasks requiring latest capabilities"
    ],
    "use_open_source_when": [
        "Data privacy requirements (healthcare, finance)",
        "High volume inference (cost sensitivity)",
        "Custom fine-tuning needed",
        "On-premise deployment required",
        "Full control over model behavior"
    ],
    "use_smallest_model_when": [
        "Edge deployment",
        "Latency-critical applications",
        "Limited compute budget",
        "Simple tasks (classification, extraction)"
    ]
}

Model Selection Guide

def select_model(
    task_type: str,
    quality_requirement: str,
    budget: str,
    privacy: bool
) -> str:
    if privacy or budget == "low":
        if quality_requirement == "high":
            return "meta-llama/Llama-3.1-70B or Qwen-2.5-72B"
        elif quality_requirement == "medium":
            return "mistralai/Mixtral-8x7B or Qwen-2.5-32B"
        else:
            return "meta-llama/Llama-3.1-8B or Qwen-2.5-7B"

    if quality_requirement == "highest":
        return "gpt-4o or claude-3.5-sonnet"

    if task_type == "code":
        return "codellama/CodeLlama-34b or deepseek-coder-33b"

    if task_type == "chat":
        return "meta-llama/Llama-3.1-8B-Instruct"

    return "meta-llama/Llama-3.1-8B"

Summary

Practice Exercises

1. Model Comparison: Compare LLaMA 3.1 8B, Mixtral 8x7B, and Qwen 2.5 72B on your specific task. Which provides the best quality-cost tradeoff?

2. License Analysis: Review licenses for 5 open-source models. Which ones can you use commercially in your application?

3. HuggingFace Pipeline: Build a complete inference pipeline using Transformers with quantization and batching.

4. PEFT Fine-tuning: Fine-tune LLaMA 3.1 8B with LoRA on a custom dataset. Compare performance with full fine-tuning.

5. Deployment Comparison: Deploy the same model using vLLM, TGI, and llama.cpp. Compare latency, throughput, and resource usage.

Previous: 22 - LLM Safety & Red Teaming <- | Next: 24 - Scaling Laws & Chinchilla ->