No Performance Benefit from OnnxRuntime.GPU in ML.NET

Describe the bug
I have an Image classification model that was trained using Microsoft CustomVision and exported as an ONNX model. I am able to run inferencing using this model with an average inference time of around 45ms. My computer is equipped with an NVIDIA GPU and I have been trying to reduce the inference time.

My application is a .NET console application written in C#.

I tried utilizing the OnnxRuntime.GPU nuget package version 1.10 and followed in steps given on the link below to install the relevant CUDA Toolkit and Cudnn packages. (https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements). Despite this, I have not seem any performance improvement when using OnnxRuntime or OnnxRuntime.GPU. The average inference time is similar and varies between 45 to 60ms.

Urgency
I have been trying various options to improve inference performance but none of them seem to be working. Urgent support would be appreciated.

System information
Windows 10 Home 21H1, Dell Inspiron 5406, Core i7 1165G7, 16GB RAM with Nvidia MX330 2GB GPU
ONNX Runtime installed from Nuget
ONNX Runtime version: 1.10.0
Program is written in C#, .NET 5, Console App
Visual Studio 2019 v16.10.3
CUDA/CudNN version: CUDA Tooklit 11.4.3 , CudNN 8.2.2.26 for Cuda 11.4
GPU model and memory: Nvidia MX330 with 2GB Memory

To Reproduce
I use the following class to initiate an ONNX Scoring class:
`public class OnnxModelScorer
{

public class ImageInputData
{
    [ImageType(300, 300)]
    public Bitmap Image { get; set; }
}

public class ImagePrediction
{
        
    [ColumnName("model_output")]
    public float[] PredictedLabels;
}

PredictionEngine<ImageInputData, ImagePrediction> predictionEngine;
ModelMetadataPropertiesClass modelprops;
Dictionary<int, string> ModelLabels = new Dictionary<int, string>();

public void SetupPredictionEngine(string modelFolderPath, out string errors)
{
    errors = "";
    predictionEngine = null;
    try
    {
        var mlContext = new MLContext();

        modelprops = LoadProperties(modelFolderPath + "metadata_properties.json", out string error);

        var pipeline = mlContext.Transforms
                        .ResizeImages("image", modelprops.CustomVisionPreprocessTargetWidth, modelprops.CustomVisionPreprocessTargetHeight, nameof(ImageInputData.Image), ImageResizingEstimator.ResizingKind.Fill)
                        .Append(mlContext.Transforms.ExtractPixels("data", "image"))
                        .Append(mlContext.Transforms.ApplyOnnxModel("model_output", "data", modelFolderPath + @"model.onnx"));

        var data = mlContext.Data.LoadFromEnumerable(new List<ImageInputData>());
        var model = pipeline.Fit(data);

        predictionEngine = mlContext.Model.CreatePredictionEngine<ImageInputData, ImagePrediction>(model);

        string[] labels = File.ReadAllText(modelFolderPath + @"labels.txt").Split('\n');

        int i = 0;
        foreach (var label in labels)
        {
            ModelLabels.Add(i, label);
            i++;
        }
    }
    catch (Exception ex)
    {
        errors = "Model Loading Failed: " + ex.ToString();
    }
        
}

public PredictionResultClass GetModelPrediction(Bitmap sample, out string error)
{
    PredictionResultClass pr = new PredictionResultClass();
    error = "";
    if (predictionEngine != null)
    {
        var input = new ImageInputData { Image = sample };

        var prediction = predictionEngine.Predict(input);
        Dictionary<int, PredictionResultClass> predictionResults = new Dictionary<int, PredictionResultClass>();
        int indexofMaxProb = -1;
        float maxProbability = 0;
        for (int i = 0; i < prediction.PredictedLabels.Count(); i++)
        {
            predictionResults.Add(i,new PredictionResultClass() { Label = ModelLabels[i], probability = prediction.PredictedLabels[i] });

            if(prediction.PredictedLabels[i]>maxProbability)
            {
                maxProbability = prediction.PredictedLabels[i];
                indexofMaxProb = i;
            }
        }

        pr = predictionResults[indexofMaxProb];

    }
    else error = "Prediction Engine Not initialized";

    return pr;
}
public class PredictionResultClass
{
    public string Label = "";
    public float probability = 0;
}

public void ModelMassTest(string samplesfolder)
{
        
    string[] inputfiles = Directory.GetFiles(samplesfolder);
    List<double> analysistimes = new List<double>();
    foreach (var fl in inputfiles)
    {

        //Emgu.CV.Image<Emgu.CV.Structure.Bgr, byte> Img = new Emgu.CV.Image<Emgu.CV.Structure.Bgr, byte>(fl);
        // Img.ROI = JsonConvert.DeserializeObject<Rectangle>("\"450, 288, 420, 1478\"");
        // string savePath = @"C:\ImageMLProjects\Tresseme200Ml Soiling Experiment\Tresseme200MlImages\ROIApplied\Bad\" + Path.GetFileName(fl);
        // Img.Save(savePath);

        //Bitmap bitmap = Emgu.CV.BitmapExtension.ToBitmap(Img); // your source of a bitmap
        Bitmap bitmap = new Bitmap(fl);
        Stopwatch sw = new Stopwatch();
        sw.Start();
        var res =  GetModelPrediction(bitmap, out string error);

        sw.Stop();
        PrintResultsonConsole(res, Path.GetFileName(fl));




        Console.WriteLine($"Analysis Time(ms): {sw.ElapsedMilliseconds}");
        analysistimes.Add(sw.ElapsedMilliseconds);

    }

    if(analysistimes.Count()>0)
        Console.WriteLine($"Average Analysis Time(ms): {analysistimes.Average()}");
}


public static ModelMetadataPropertiesClass LoadProperties(string MetadatePropertiesFilepath, out string error)
{
    string propertiesText = File.ReadAllText(MetadatePropertiesFilepath);
    error = "";
    ModelMetadataPropertiesClass mtp = new ModelMetadataPropertiesClass();

    try
    {
        mtp = JsonConvert.DeserializeObject<ModelMetadataPropertiesClass>(propertiesText);
    }
    catch (Exception ex)
    {
        error = ex.ToString();
        mtp = null;
    }

    return mtp;
}
public class ModelMetadataPropertiesClass
{
    [JsonProperty("CustomVision.Metadata.AdditionalModelInfo")]
    public string CustomVisionMetadataAdditionalModelInfo { get; set; }

    [JsonProperty("CustomVision.Metadata.Version")]
    public string CustomVisionMetadataVersion { get; set; }

    [JsonProperty("CustomVision.Postprocess.Method")]
    public string CustomVisionPostprocessMethod { get; set; }

    [JsonProperty("CustomVision.Postprocess.Yolo.Biases")]
    public string CustomVisionPostprocessYoloBiases { get; set; }

    [JsonProperty("CustomVision.Postprocess.Yolo.NmsThreshold")]
    public string CustomVisionPostprocessYoloNmsThreshold { get; set; }

    [JsonProperty("CustomVision.Preprocess.CropHeight")]
    public string CustomVisionPreprocessCropHeight { get; set; }

    [JsonProperty("CustomVision.Preprocess.CropMethod")]
    public string CustomVisionPreprocessCropMethod { get; set; }

    [JsonProperty("CustomVision.Preprocess.CropWidth")]
    public string CustomVisionPreprocessCropWidth { get; set; }

    [JsonProperty("CustomVision.Preprocess.MaxDimension")]
    public string CustomVisionPreprocessMaxDimension { get; set; }

    [JsonProperty("CustomVision.Preprocess.MaxScale")]
    public string CustomVisionPreprocessMaxScale { get; set; }

    [JsonProperty("CustomVision.Preprocess.MinDimension")]
    public string CustomVisionPreprocessMinDimension { get; set; }

    [JsonProperty("CustomVision.Preprocess.MinScale")]
    public string CustomVisionPreprocessMinScale { get; set; }

    [JsonProperty("CustomVision.Preprocess.NormalizeMean")]
    public string CustomVisionPreprocessNormalizeMean { get; set; }

    [JsonProperty("CustomVision.Preprocess.NormalizeStd")]
    public string CustomVisionPreprocessNormalizeStd { get; set; }

    [JsonProperty("CustomVision.Preprocess.ResizeMethod")]
    public string CustomVisionPreprocessResizeMethod { get; set; }

    [JsonProperty("CustomVision.Preprocess.TargetHeight")]
    public int CustomVisionPreprocessTargetHeight { get; set; }

    [JsonProperty("CustomVision.Preprocess.TargetWidth")]
    public int CustomVisionPreprocessTargetWidth { get; set; }

    [JsonProperty("Image.BitmapPixelFormat")]
    public string ImageBitmapPixelFormat { get; set; }

    [JsonProperty("Image.ColorSpaceGamma")]
    public string ImageColorSpaceGamma { get; set; }

    [JsonProperty("Image.NominalPixelRange")]
    public string ImageNominalPixelRange { get; set; }
}


public static void PrintResultsonConsole( PredictionResultClass pr,string  filePath)
{
    var defaultForeground = Console.ForegroundColor;
    var labelColor = ConsoleColor.Magenta;
    var probColor = ConsoleColor.Blue;
    var exactLabel = ConsoleColor.Green;
    var failLabel = ConsoleColor.Red;

    Console.Write("ImagePath: ");
    Console.ForegroundColor = labelColor;
    Console.Write($"{Path.GetFileName(filePath)}");
    Console.ForegroundColor = defaultForeground;

    Console.ForegroundColor = defaultForeground;
    Console.Write(" predicted as ");
    Console.ForegroundColor = exactLabel;
    Console.Write($"{pr.Label}");

    Console.ForegroundColor = defaultForeground;
    Console.Write(" with probability ");
    Console.ForegroundColor = probColor;
    Console.Write(pr.probability);
    Console.ForegroundColor = defaultForeground;
    Console.WriteLine("");
}

}
`

To execute inferencing, I then initiate the modelScorer and consume it.
`static void Main(string[] args)
{
var onnxModelScorer = new OnnxModelScorer();

        onnxModelScorer.SetupPredictionEngine(@"..\..\..\OnnxModel\", out string error);
        onnxModelScorer.ModelMassTest(@"..\..\..\SampleImages\Bad\");
        ConsoleHelpers.ConsolePressAnyKey();
        onnxModelScorer.ModelMassTest(@"..\..\..\SampleImages\Good\");


        ConsoleHelpers.ConsolePressAnyKey();

        
ConsoleHelpers.ConsolePressAnyKey();

}
`

Expected behavior
When utilizing the Onnxruntime package, the average inferencing time is ~40ms, with Onnxruntime.GPU I expected it to be less than 10ms

Screenshots
NA

Additional context
This is a performance oriented question, on how well Onnxruntime.GPU allows .NET developers to exploit benefits of faster inferencing using Nvidia GPUs.

If having the full project with OnnxModel and sample images would help you investigate better, please access the following link and request access:
https://drive.google.com/drive/folders/1DqnUvTaU9xp2QLuV_X9jFCjkratckMYL?usp=sharing