microsoft / microsoft/onnxruntime
Why Float16 model takes twice as long as Float under GPU
Open
@hariharans29 is already working on this.
Since Oct 25, 2021.
ep:CUDA
- Dominant language
- C++
- Stars
- 21.9k
- Forks
- 4.2k
- Avg merge
- 4d 8h
- Merged PRs (30d)
- 179
Description
float16 model is generated by convert_float_to_float16() in onnxmltools.
Here is inference code:
` Ort::Env env;
std::string weightFile = "./model/r100model_fp16_nokit.onnx";
Ort::SessionOptions session_options;
OrtCUDAProviderOptions options;
options.device_id = 0;
options.arena_extend_strategy = 0;
options.cuda_mem_limit = (size_t)1 * 1024 * 1024 * 1024;
options.cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE;
options.do_copy_in_default_stream = 1;
session_options.AppendExecutionProvider_CUDA(options);
Ort::Session session_{env, weightFile.c_str(), session_options};
static constexpr const int width_ = 112;
static constexpr const int height_ = 112;
Ort::Value input_tensor_{nullptr};
std::array<int64_t, 4> input_shape_{1, 3, width_, height_};
Ort::Value output_tensor_{nullptr};
std::array<int64_t, 2> output_shape_{1, 512};
std::array<uint16_t, width_ * height_ * 3> input_image_{};
std::array<uint16_t, 512> results_{};
auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
input_tensor_ = Ort::Value::CreateTensor(memory_info, input_image_.data(), input_image_.size()*sizeof(uint16_t), input_shape_.data(), input_shape_.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16);
output_tensor_ = Ort::Value::CreateTensor(memory_info, results_.data(), results_.size()*sizeof(uint16_t), output_shape_.data(), output_shape_.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16);
const char* input_names[] = {"input.1"};
const char* output_names[] = {"1333"};
std::string imgPath = "./img/0.jpg";
cv::Mat img = cv::imread(imgPath);
double totaltime = 0;
int totalNum = 0;
int N = 3000;
for(int k = 0;k < N; k++)
{
double timetmp1 = (double)cv::getTickCount();
cv::Mat img_f32;
img.convertTo(img_f32, CV_32FC3);
for (int i = 0; i < img.rows; i++) {
for (int j = 0; j < img.cols; j++) {
input_image_[i * img.cols + j + 0] = float32_to_float16(img_f32.at<cv::Vec3f>(i, j)[2]);//R
input_image_[i * img.cols + j + 1 * img.cols * img.rows] = float32_to_float16(img_f32.at<cv::Vec3f>(i, j)[1]);//G
input_image_[i * img.cols + j + 2 * img.cols * img.rows] = float32_to_float16(img_f32.at<cv::Vec3f>(i, j)[0]);//B
}
}
session_.Run(Ort::RunOptions{nullptr}, input_names, &input_tensor_, 1, output_names, &output_tensor_, 1);
uint16_t* f = output_tensor_.GetTensorMutableData<uint16_t>();
for(int i = 0; i < dims[1]; i++)
{
//std::cout<< float16_to_float32(f[i])<< std::endl;
}
double timetmp2 = (double)cv::getTickCount();
double timediff = timetmp2 - timetmp1;
std::cout<<"time:"<<1000 * timediff / cv::getTickFrequency() << "ms"<<std::endl;
if(k >= 100)
{
totaltime+=timediff;
totalNum++;
}
}
std::cout<<"totalNum:"<< totalNum <<", avgtime:"<< 1000 * (totaltime/totalNum)/ cv::getTickFrequency() << "ms"<<std::endl;`
Contributor guide
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Assessment
This issue has not been assessed yet.