NLP Evaluation Metrics

from sklearn.metrics import (
    accuracy_score, precision_score, recall_score, f1_score,
    classification_report, confusion_matrix, roc_auc_score
)
import numpy as np

def comprehensive_classification_metrics(y_true, y_pred, y_prob=None, average="macro"):
    """Compute comprehensive classification metrics."""
    metrics = {
        "accuracy": accuracy_score(y_true, y_pred),
        "precision": precision_score(y_true, y_pred, average=average, zero_division=0),
        "recall": recall_score(y_true, y_pred, average=average, zero_division=0),
        "f1": f1_score(y_true, y_pred, average=average, zero_division=0),
    }

    if y_prob is not None:
        try:
            metrics["auc_roc"] = roc_auc_score(y_true, y_prob, multi_class="ovr")
        except ValueError:
            metrics["auc_roc"] = None

    # Per-class metrics
    report = classification_report(y_true, y_pred, output_dict=True, zero_division=0)
    metrics["per_class"] = report

    # Confusion matrix
    metrics["confusion_matrix"] = confusion_matrix(y_true, y_pred).tolist()

    return metrics

# Example
y_true = [0, 1, 1, 0, 2, 1, 0, 2]
y_pred = [0, 1, 0, 0, 2, 2, 1, 2]

results = comprehensive_classification_metrics(y_true, y_pred)
print(f"Accuracy: {results['accuracy']:.4f}")
print(f"F1 (macro): {results['f1']:.4f}")

import torch
import math

def compute_perplexity(model, tokenizer, text, max_length=512):
    """Compute perplexity of text using a language model."""
    encodings = tokenizer(text, return_tensors="pt", max_length=max_length, truncation=True)
    input_ids = encodings.input_ids.to(model.device)

    with torch.no_grad():
        outputs = model(input_ids, labels=input_ids)
        neg_log_likelihood = outputs.loss

    return math.exp(neg_log_likelihood.item())

def compute_dataset_perplexity(model, tokenizer, texts, max_length=512):
    """Compute average perplexity over a dataset."""
    perplexities = []
    for text in texts:
        ppl = compute_perplexity(model, tokenizer, text, max_length)
        perplexities.append(ppl)

    return {
        "mean_ppl": np.mean(perplexities),
        "std_ppl": np.std(perplexities),
        "median_ppl": np.median(perplexities),
        "min_ppl": min(perplexities),
        "max_ppl": max(perplexities),
    }

def error_analysis(y_true, y_pred, texts, labels):
    """Detailed error analysis for classification."""
    errors = []
    for i, (true, pred, text) in enumerate(zip(y_true, y_pred, texts)):
        if true != pred:
            errors.append({
                "index": i,
                "text": text,
                "true_label": labels[true],
                "predicted_label": labels[pred],
            })

    # Confusion pairs
    confusion_pairs = {}
    for error in errors:
        pair = (error["true_label"], error["predicted_label"])
        confusion_pairs[pair] = confusion_pairs.get(pair, 0) + 1

    # Sort by frequency
    sorted_pairs = sorted(confusion_pairs.items(), key=lambda x: -x[1])

    return {
        "total_errors": len(errors),
        "error_rate": len(errors) / len(y_true),
        "top_confusion_pairs": sorted_pairs[:10],
        "error_examples": errors[:20],
    }