Text Generation Evaluation

from nltk.translate.bleu_score import corpus_bleu, SmoothingFunction
import numpy as np

def compute_bleu(references, candidates, max_n=4):
    """
    Compute corpus-level BLEU score.

    Args:
        references: list of list of reference token sequences
        candidates: list of candidate token sequences
        max_n: maximum n-gram order
    """
    # Convert to format expected by NLTK
    refs = [[ref] for ref in references]

    # Individual n-gram precisions
    weights = tuple([1.0 / max_n] * max_n)

    smoothing = SmoothingFunction().method1

    bleu = corpus_bleu(
        refs,
        candidates,
        weights=weights,
        smoothing_function=smoothing
    )

    # Also compute individual n-gram scores
    individual = {}
    for n in range(1, max_n + 1):
        individual[f"bleu-{n}"] = corpus_bleu(
            refs, candidates,
            weights=tuple([1.0 if i == n-1 else 0.0 for i in range(max_n)]),
            smoothing_function=smoothing
        )

    return {"bleu": bleu, **individual}

# Example
references = [
    ["the", "cat", "sat", "on", "the", "mat"],
    ["there", "is", "a", "cat", "on", "the", "mat"],
]
candidate = ["the", "cat", "is", "on", "the", "mat"]

scores = compute_bleu(references, [candidate])
print(f"BLEU: {scores['bleu']:.4f}")

from rouge_score import rouge_scorer

scorer = rouge_scorer.RougeScorer(
    ['rouge1', 'rouge2', 'rougeL', 'rougeLsum'],
    use_stemmer=True
)

def evaluate_rouge(references, candidate):
    """Compute ROUGE scores against multiple references."""
    all_scores = {}

    for ref in references:
        scores = scorer.score(ref, candidate)
        for metric, score in scores.items():
            if metric not in all_scores:
                all_scores[metric] = {"precision": [], "recall": [], "fmeasure": []}
            all_scores[metric]["precision"].append(score.precision)
            all_scores[metric]["recall"].append(score.recall)
            all_scores[metric]["fmeasure"].append(score.fmeasure)

    # Average across references
    avg_scores = {}
    for metric in all_scores:
        avg_scores[metric] = {
            "precision": np.mean(all_scores[metric]["precision"]),
            "recall": np.mean(all_scores[metric]["recall"]),
            "fmeasure": np.mean(all_scores[metric]["fmeasure"]),
        }
    return avg_scores

# Example
reference = "The cat sat on the mat and looked out the window"
candidate = "The cat was sitting on the mat looking outside"

scores = evaluate_rouge([reference], candidate)
for metric, values in scores.items():
    print(f"{metric}: P={values['precision']:.3f} R={values['recall']:.3f} F={values['fmeasure']:.3f}")

from bert_score import score as bert_score
import torch

def evaluate_bertscore(references, candidates, lang="en", model_type="microsoft/deberta-xlarge-mnli"):
    """
    Compute BERTScore between candidates and references.
    """
    P, R, F1 = bert_score(
        candidates,
        references,
        lang=lang,
        model_type=model_type,
        verbose=True,
        rescale_with_baseline=True  # Recommended for better correlation
    )

    results = {
        "precision": P.mean().item(),
        "recall": R.mean().item(),
        "f1": F1.mean().item(),
        "precision_std": P.std().item(),
        "recall_std": R.std().item(),
        "f1_std": F1.std().item(),
    }
    return results

# Example
references = ["The cat sat on the mat", "A feline rested on the rug"]
candidates = ["The cat was sitting on the mat", "A cat lay on the carpet"]

scores = evaluate_bertscore(references, candidates)
print(f"BERTScore F1: {scores['f1']:.4f} (±{scores['f1_std']:.4f})")

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

def compute_perplexity(text, model_name="gpt2", max_length=512):
    """Compute perplexity of text using a language model."""
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForCausalLM.from_pretrained(model_name)
    model.eval()

    encodings = tokenizer(text, return_tensors="pt", max_length=max_length, truncation=True)
    input_ids = encodings.input_ids

    with torch.no_grad():
        outputs = model(input_ids, labels=input_ids)
        loss = outputs.loss

    return torch.exp(loss).item()

# Compare quality of two texts
text_good = "The weather is pleasant today with clear blue skies."
text_bad = "Weather is today good sky blue clear."

ppl_good = compute_perplexity(text_good)
ppl_bad = compute_perplexity(text_bad)

print(f"Good text perplexity: {ppl_good:.2f}")
print(f"Bad text perplexity: {ppl_bad:.2f}")

# Human evaluation framework
HUMAN_EVAL_TEMPLATE = {
    "dimensions": {
        "fluency": {
            "description": "Is the text grammatically correct and natural?",
            "scale": ["Incomprehensible", "Poor", "Acceptable", "Good", "Excellent"]
        },
        "relevance": {
            "description": "Does the output address the input appropriately?",
            "scale": ["Completely off", "Mostly off", "Partially relevant", "Mostly relevant", "Fully relevant"]
        },
        "coherence": {
            "description": "Is the text logically organized and consistent?",
            "scale": ["Incoherent", "Mostly disorganized", "Somewhat coherent", "Mostly coherent", "Fully coherent"]
        },
        "informativeness": {
            "description": "How much useful information does the output contain?",
            "scale": ["Empty", "Minimal", "Some", "Substantial", "Comprehensive"]
        }
    },
    "guidelines": """
    Rate each dimension on the 1-5 scale provided.
    Base your judgment on the overall quality of the generated text.
    Consider the context and intended use case.
    """
}

# Complete evaluation pipeline
class TextGenerationEvaluator:
    def __init__(self, metrics=None):
        self.metrics = metrics or ["bleu", "rouge", "bertscore", "perplexity"]

    def evaluate(self, references, candidates):
        results = {}

        if "bleu" in self.metrics:
            results["bleu"] = compute_bleu(references, candidates)

        if "rouge" in self.metrics:
            rouge_scores = [evaluate_rouge([ref], cand) for ref, cand in zip(references, candidates)]
            results["rouge"] = average_rouge_scores(rouge_scores)

        if "bertscore" in self.metrics:
            results["bertscore"] = evaluate_bertscore(references, candidates)

        return results

    def compare_models(self, refs, model_a_outputs, model_b_outputs):
        scores_a = self.evaluate(refs, model_a_outputs)
        scores_b = self.evaluate(refs, model_b_outputs)

        comparison = {}
        for metric in scores_a:
            comparison[metric] = {
                "model_a": scores_a[metric],
                "model_b": scores_b[metric],
                "difference": scores_b[metric] - scores_a[metric]
            }
        return comparison