microsoft / microsoft/onnxruntime

Large numerical discrepancy in CPUExecutionProvider ReduceSum for large float32 tensor

Open
#28,450 0 comments 1 reaction 2 assignees Claimed by @justinchuby View on GitHub
Dominant language
C++
Stars
21.9k
Forks
4.2k
Avg merge
4d 11h
Merged PRs (30d)
184

Description

### Describe the issue

I observed a large numerical discrepancy in ONNX Runtime `CPUExecutionProvider` for a simple model containing a `float32` `ReduceSum`.

The model is exported from PyTorch to ONNX and then executed with ONNX Runtime. The discrepancy is isolated to the `float32` `ReduceSum` output. A parallel `float64` reduction path, cast back to `float32`, matches PyTorch exactly.

The same discrepancy is observed even with:

- `ORT_DISABLE_ALL`
- `ORT_SEQUENTIAL`
- `intra_op_num_threads = 1`
- `inter_op_num_threads = 1`

I understand that `float32` reductions are not necessarily bitwise identical across implementations because of different accumulation orders. However, in this case the ONNX Runtime result is much farther from both PyTorch and NumPy than expected for this simple all-constant input.

### To reproduce

### Minimal reproduction script
```python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import tempfile
import platform
from collections import Counter

import numpy as np
import torch
import torch.nn as nn
import onnx
import onnxruntime as ort

class MyModel(nn.Module):
def forward(self, x):
sum_f32 = x.sum()
sum_f64_cast = x.double().sum().float()
residual = sum_f64_cast - sum_f32
return sum_f32, sum_f64_cast, residual

def make_input():
return torch.ones(5, 68, 64, 64, dtype=torch.float32) * 0.1

def print_env():
print("Environment:")
print(f" Python: {platform.python_version()}")
print(f" Platform: {platform.platform()}")
print(f" Torch: {torch.__version__}")
print(f" ONNX: {onnx.__version__}")
print(f" ONNX Runtime: {ort.__version__}")
print(f" ORT providers: {ort.get_available_providers()}")
print()

def export_onnx(model, x, onnx_path):
torch.onnx.export(
model,
(x,),
onnx_path,
opset_version=18,
input_names=["input"],
output_names=[
"sum_float32",
"sum_float64_cast_float32",
"residual_float64_minus_float32",
],
do_constant_folding=True,
)

model_proto = onnx.load(onnx_path)
onnx.checker.check_model(model_proto)

op_counts = Counter(node.op_type for node in model_proto.graph.node)
print("Exported ONNX graph:")
print(f" path: {onnx_path}")
print(f" op counts: {dict(sorted(op_counts.items()))}")
print()

def make_ort_session(onnx_path, *, opt_level, execution_mode, intra_threads, inter_threads):
so = ort.SessionOptions()
so.graph_optimization_level = getattr(ort.GraphOptimizationLevel, opt_level)
so.execution_mode = getattr(ort.ExecutionMode, execution_mode)
so.intra_op_num_threads = intra_threads
so.inter_op_num_threads = inter_threads

return ort.InferenceSession(
onnx_path,
sess_options=so,
providers=["CPUExecutionProvider"],
)

def scalar(v):
if isinstance(v, torch.Tensor):
return float(v.detach().cpu().numpy().reshape(-1)[0])
return float(np.asarray(v).reshape(-1)[0])

def compare_outputs(eager_outputs, ort_outputs, case_name):
names = [
"sum_float32",
"sum_float64_cast_float32",
"residual_float64_minus_float32",
]

print(f"ORT case: {case_name}")
for name, eager, ort_out in zip(names, eager_outputs, ort_outputs):
eager_val = scalar(eager)
ort_val = scalar(ort_out)
abs_diff = abs(eager_val - ort_val)
rel_diff = abs_diff / max(abs(eager_val), abs(ort_val), 1e-12)

print(f" {name}:")
print(f" PyTorch eager: {eager_val}")
print(f" ONNX Runtime: {ort_val}")
print(f" abs diff: {abs_diff}")
print(f" rel diff: {rel_diff}")

print()

def main():
print_env()

torch.manual_seed(0)
np.random.seed(0)

model = MyModel().eval()
x = make_input()

with torch.no_grad():
eager_outputs = model(x)

x_np = x.detach().cpu().numpy()

print("Reference values:")
print(f" Num elements: {x.numel()}")
print(f" NumPy sum, dtype=float32: {float(np.sum(x_np, dtype=np.float32))}")
print(
" NumPy sum, dtype=float64 cast to float32: "
f"{float(np.array(np.sum(x_np, dtype=np.float64), dtype=np.float32))}"
)
print(f" PyTorch sum_float32: {scalar(eager_outputs[0])}")
print(f" PyTorch sum_float64_cast_float32: {scalar(eager_outputs[1])}")
print(f" PyTorch residual: {scalar(eager_outputs[2])}")
print()

with tempfile.TemporaryDirectory(prefix="ort_reducesum_repro_") as tmp:
onnx_path = os.path.join(tmp, "model.onnx")
export_onnx(model, x, onnx_path)

feed = {"input": x_np}

cases = [
{
"name": "ORT_DISABLE_ALL + ORT_SEQUENTIAL + single-thread",
"opt_level": "ORT_DISABLE_ALL",
"execution_mode": "ORT_SEQUENTIAL",
"intra_threads": 1,
"inter_threads": 1,
},
{
"name": "ORT_ENABLE_EXTENDED + ORT_PARALLEL",
"opt_level": "ORT_ENABLE_EXTENDED",
"execution_mode": "ORT_PARALLEL",
"intra_threads": 4,
"inter_threads": 2,
},
]

for case in cases:
sess = make_ort_session(
onnx_path,
opt_level=case["opt_level"],
execution_mode=case["execution_mode"],
intra_threads=case["intra_threads"],
inter_threads=case["inter_threads"],
)
ort_outputs = sess.run(None, feed)
compare_outputs(eager_outputs, ort_outputs, case["name"])

if __name__ == "__main__":
main()
```

### Output
```
Environment:
Python: 3.10.16
Platform: Linux-6.8.0-85-generic-x86_64-with-glibc2.35
Torch: 2.11.0+cu130
ONNX: 1.19.1
ONNX Runtime: 1.23.2
ORT providers: ['AzureExecutionProvider', 'CPUExecutionProvider']

Reference values:
Num elements: 1392640
NumPy sum, dtype=float32: 139264.234375
NumPy sum, dtype=float64 cast to float32: 139264.0
PyTorch sum_float32: 139264.015625
PyTorch sum_float64_cast_float32: 139264.0
PyTorch residual: -0.015625

Exported ONNX graph:
op counts: {'Cast': 2, 'ReduceSum': 2, 'Sub': 1}

ORT case: ORT_DISABLE_ALL + ORT_SEQUENTIAL + single-thread
sum_float32:
PyTorch eager: 139264.015625
ONNX Runtime: 139020.953125
abs diff: 243.0625
rel diff: 0.0017453360001804127
sum_float64_cast_float32:
PyTorch eager: 139264.0
ONNX Runtime: 139264.0
abs diff: 0.0
rel diff: 0.0
residual_float64_minus_float32:
PyTorch eager: -0.015625
ONNX Runtime: 243.046875
abs diff: 243.0625
rel diff: 1.0000642880102861

ORT case: ORT_ENABLE_EXTENDED + ORT_PARALLEL
sum_float32:
PyTorch eager: 139264.015625
ONNX Runtime: 139020.953125
abs diff: 243.0625
rel diff: 0.0017453360001804127
sum_float64_cast_float32:
PyTorch eager: 139264.0
ONNX Runtime: 139264.0
abs diff: 0.0
rel diff: 0.0
residual_float64_minus_float32:
PyTorch eager: -0.015625
ONNX Runtime: 243.046875
abs diff: 243.0625
rel diff: 1.0000642880102861
```

### Urgency

_No response_

### Platform

Linux

### OS Version

Ubuntu 22.04.4 LTS (x86_64)

### ONNX Runtime Installation

Released Package

### ONNX Runtime Version or Commit ID

1.23.2

### ONNX Runtime API

Python

### Architecture

X64

### Execution Provider

Default CPU

### Execution Provider Library Version

_No response_

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.