Vision Model Deployment

Module: Computer Vision | Difficulty: Intermediate

Model Optimization

Quantization

INT8 Quantization

Knowledge Distillation

ONNX Export

import torch

def export_onnx(model, input_shape=(1, 3, 224, 224), path='model.onnx'):
    dummy = torch.randn(*input_shape)
    torch.onnx.export(
        model, dummy, path,
        opset_version=11,
        input_names=['input'],
        output_names=['output'],
        dynamic_axes={'input': {0: 'batch_size'}, 'output': {0: 'batch_size'}}
    )

def quantize_model(model, calibration_loader):
    model.eval()
    for images in calibration_loader:
        model(images)
    quantized = torch.quantization.quantize_dynamic(
        model, {torch.nn.Linear}, torch.qint8
    )
    return quantized

TensorRT Optimization

import tensorrt as trt

def build_engine(onnx_path, engine_path, fp16=True):
    logger = trt.Logger(trt.Logger.WARNING)
    builder = trt.Builder(logger)
    network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
    parser = trt.OnnxParser(network, logger)
    
    with open(onnx_path, 'rb') as f:
        parser.parse(f.read())
    
    config = builder.create_builder_config()
    if fp16:
        config.set_flag(trt.BuilderFlag.FP16)
    
    engine = builder.build_engine(network, config)
    with open(engine_path, 'wb') as f:
        f.write(engine.serialize())

Key Takeaways

• Quantization reduces model size with minimal accuracy loss
• ONNX provides cross-platform model interoperability
• TensorRT optimizes inference speed on NVIDIA hardware