Divergence of accuracy in mobilenetv2_12_int8.onnx

Bug Report

Which model does this pertain to?

mobilenetv2-12-int8.onnx

Describe the bug

I am trying to achieve the same performance in mobilenetv2_12_int8.onnx, using pytorch to read imagenet dataset, onnxruntime to read model and torchmetrics to calculate accuracy. However, the only model which I have a significant accuracy drop is mobilenetv2_12_int8.onnx, reaching 64.346 % (vs 68.30 reporting on table https://github.com/onnx/models/tree/main/vision/classification/mobilenet#model).

Reproduction instructions

System Information

OS Platform and Distribution (Linux Ubuntu 20.04.4 LTS):
ONNX version (1.13.1):
Backend/Runtime version (ONNX Runtime 1.14.1, PyTorch 2.0.0):

Provide a code snippet to reproduce your errors.

import os
import io
import tarfile
from PIL import Image
from tqdm import tqdm

import torch
from torchvision import transforms as T
import torchmetrics

import onnxruntime

_TORCH_DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu"


class ImagenetValDataset(torch.utils.data.Dataset):
    def __init__(self, img_dir, transform=None):
        images_path = os.path.join(img_dir, 'ILSVRC2012_img_val')
        try:
            self._tf = images_path + '.tar'
            with tarfile.open(self._tf) as tf:
                self._img_names = tf.getnames()
        except Exception as e:
            raise ValueError(f"{img_dir} have not 'ILSVRC2012_img_val.tar' "
                                "file or it is corrupted.") from e
        self._img_names = sorted(self._img_names)

        # Read labels
        self._labels = []
        with open(os.path.join(img_dir, 'imagenet_2012_validation_synset_labels.txt')) as f:
            while label := f.readline():
                self._labels.append(label.strip())
        self._label_names = sorted(set(self._labels))
        assert len(self._img_names) == len(self._labels), "Incomplete labels!"

        self.transform = transform

    def _get_image(self, name):
        image = self._tf.extractfile(name)
        image = io.BytesIO(image.read())
        image = Image.open(image).convert('RGB')
        return image

    def __len__(self):
        return len(self._img_names)

    def __getitem__(self, index):
        # Read tar file here to proper parallelization (just one time)
        if isinstance(self._tf, str):
            self._tf = tarfile.open(self._tf)

        # Read image from tar file
        image = self._get_image(self._img_names[index])

        # Apply transformation
        if self.transform is not None:
            image = self.transform(image)

        # Return image with his label
        label = self._labels[index]
        return image, self._label_names.index(label)

class OnnxInferencePipeline:
    def __init__(self, onnx_path):
        self._ort_session = onnxruntime.InferenceSession(onnx_path)

    @property
    def inputs(self):
        return self._ort_session.get_inputs()[0]

    @staticmethod
    def to_numpy(tensor):
        return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()

    def __call__(self, inputs: torch.Tensor):
        # Generate ort inputs
        ort_inputs = {self.inputs.name: self.to_numpy(inputs)}

        # Run inputs in graph
        ort_outputs = self._ort_session.run(None, ort_inputs)
        return torch.from_numpy(ort_outputs[0]).to(inputs.device)


def get_imagenet_dataset(data_path, batch_size=128, image_size=224, num_workers=0):
    transform = T.Compose([T.Resize(int(image_size * 1.1429)),
                           T.CenterCrop(image_size),
                           T.ToTensor(),
                           T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
    imagenet_data = ImagenetValDataset(data_path, transform=transform)
    return torch.utils.data.DataLoader(imagenet_data,
                                       batch_size=batch_size,
                                       shuffle=False,
                                       num_workers=num_workers)


def evaluate_model(model, dataset):
    print("Starting evaluation...")
    num_classes = len(dataset.dataset._label_names)
    accuracy = torchmetrics.Accuracy(task="multiclass", num_classes=num_classes)

    for images, gt_labels in (barprog := tqdm(dataset)):
        images = images.to(_TORCH_DEVICE)
        pred_labels = model(images).argmax(-1)
        acc_step = accuracy(pred_labels.cpu(), gt_labels)
        barprog.set_postfix({'acc': acc_step.item()})
    print(f"[INFO] Accuracy: {accuracy.compute()}")


if __name__ == "__main__":
    model_path = "mobilenetv2-12-int8.onnx"
    imagenet_path = "/imagenet/dataset"
    model = OnnxInferencePipeline(model_path)

    # Read dataset
    val_dataset = get_imagenet_dataset(imagenet_path, num_workers=8)

    # Process
    evaluate_model(model, val_dataset)

Notes

ImagenetValDataset needs the list of ordered-labels to work. If you need it, I could provide it.

Start by running the provided Python reproduction against mobilenetv2-12-int8.onnx and compare its result with the accuracy reported in the Mobilenet model table. Read the linked model page and verify the dataset labels and preprocessing used in the script. Done means the accuracy discrepancy has a verified explanation and the affected model or reported result is identified for correction.