OpenVino inference isn't working with existing PyTorch pipeline """ TypeError: <lambda>() missing 1 required positional argument: 'x' """
- Dominant language
- Python
- Stars
- 2.3k
- Forks
- 365
- Avg merge
- 4h 6m
- Merged PRs (30d)
- 1
Description
### Checklist
- [X] I have searched for [similar issues](https://github.com/isl-org/Open3D-ML/issues).
- [X] I have tested with the [latest development wheel](http://www.open3d.org/docs/latest/getting_started.html#development-version-pip).
- [X] I have checked the [release documentation](http://www.open3d.org/docs/release/) and the [latest documentation](http://www.open3d.org/docs/latest/) (for `master` branch).
### Describe the issue
I have been trying to run provided OpenVino inference with the PyTorch RANLANET model trained over the S3DIS dataset. However, it generates the mentioned error.
I have tried different versions of PyTorch, Python, Open3D, OpenVino, etc., but the error outcome is the same.
Please download and adjust the path to these PCD folder for reprodcuing an error [https://drive.google.com/drive/folders/1bqqH0Lkt9fWIpzgE6Czy7ZEDBSkec7-c?usp=sharing](url)
### Steps to reproduce the bug
```python
import os #LD_LIBRARY_PATH=/home/maffan/miniconda3/envs/randlanet2/lib
import open3d as o3d
import open3d.ml as _ml3d
import open3d.ml.torch as ml3d
import numpy as np
import glob
from open3d._ml3d.torch.models import openvino_model
def custom_draw_geometry(pcd):
vis = o3d.visualization.Visualizer()
vis.create_window()
vis.get_render_option().point_size = 2.0
vis.get_render_option().background_color = np.asarray([1.0, 1.0, 1.0])
vis.add_geometry(pcd)
vis.run()
vis.destroy_window()
def load_custom_dataset(dataset_path):
print("Loading custom dataset")
#print("1*********+",dataset_path)
#dataset_path = "/home/maffan/Downloads/open3d_/open3d_experiments/pcds"
pcd_paths = glob.glob(dataset_path+"/*.pcd")
#print("2*********+",pcd_paths)
pcds = []
for pcd_path in pcd_paths:
pcds.append(o3d.io.read_point_cloud(pcd_path))
return pcds
def prepare_point_cloud_for_inference(pcd):
# Remove NaNs and infinity values
pcd.remove_non_finite_points()
# Extract the xyz points
xyz = np.asarray(pcd.points)
# Set the points to the correct format for inference
data = {"point":xyz, 'feat': None, 'label':np.zeros((len(xyz),), dtype=np.int32)}
return data, pcd
def load_point_cloud_for_inference(file_path, dataset_path):
pcd_path = dataset_path + "/" + file_path
# Load the file
pcd = o3d.io.read_point_cloud(pcd_path)
# Remove NaNs and infinity values
pcd.remove_non_finite_points()
# Extract the xyz points
xyz = np.asarray(pcd.points)
# Set the points to the correct format for inference
data = {"point":xyz, 'feat': None, 'label':np.zeros((len(xyz),), dtype=np.int32)}
return data, pcd
# Class colors, RGB values as ints for easy reading
COLOR_MAP = {
0: (0, 0, 0),
1: (245, 150, 100),
2: (245, 230, 100),
3: (150, 60, 30),
4: (180, 30, 80),
5: (255, 0., 0),
6: (30, 30, 255),
7: (200, 40, 255),
8: (90, 30, 150),
9: (255, 0, 255),
10: (255, 150, 255),
11: (75, 0, 75),
12: (75, 0., 175),
13: (0, 200, 255),
14: (50, 120, 255),
15: (0, 175, 0),
16: (0, 60, 135),
17: (80, 240, 150),
18: (150, 240, 255),
19: (0, 0, 255),
}
# ------ for custom data -------
kitti_labels = {
0: 'unlabeled',
1: 'car',
2: 'bicycle',
3: 'motorcycle',
4: 'truck',
5: 'other-vehicle',
6: 'person',
7: 'bicyclist',
8: 'motorcyclist',
9: 'road',
10: 'parking',
11: 'sidewalk',
12: 'other-ground',
13: 'building',
14: 'fence',
15: 'vegetation',
16: 'trunk',
17: 'terrain',
18: 'pole',
19: 'traffic-sign'
}
# Convert class colors to doubles from 0 to 1, as expected by the visualizer
for label in COLOR_MAP:
COLOR_MAP[label] = tuple(val/255 for val in COLOR_MAP[label])
# Load an ML configuration file
cfg_file = "/home/maffan/miniconda3/envs/randlanet2/lib/python3.7/site-packages/open3d/_ml3d/configs/randlanet_semantickitti.yml"
cfg = _ml3d.utils.Config.load_from_file(cfg_file)
# Load the RandLANet model
model = ml3d.models.RandLANet(**cfg.model, device='cpu')
model = openvino_model.OpenVINOModel(model)
# Add path to the SemanticKitti dataset and your own custom dataset
cfg.dataset['dataset_path'] = '/home/maffan/open3d_/open3d_experiments/pcds'
cfg.dataset['custom_dataset_path'] = '/home/maffan/open3d_/open3d_experiments/pcds'
#print(cfg.dataset)
# Load the datasets
dataset = ml3d.datasets.S3DIS(cfg.dataset.pop('dataset_path', None), **cfg.dataset)
#print("dataset = ",cfg.dataset)
custom_dataset = load_custom_dataset(cfg.dataset.pop('custom_dataset_path', None))
#print(custom_dataset)
# Create the ML pipeline
pipeline = ml3d.pipelines.SemanticSegmentation(model, dataset=dataset, device="gpu", **cfg.pipeline)
# Download the weights.
ckpt_folder = "./logs/"
os.makedirs(ckpt_folder, exist_ok=True)
ckpt_path = ckpt_folder + "randlanet_semantickitti_202201071330utc.pth"
randlanet_url = "https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_semantickitti_202201071330utc.pth"
if not os.path.exists(ckpt_path):
cmd = "wget {} -O {}".format(randlanet_url, ckpt_path)
os.system(cmd)
# Load the parameters of the model.
pipeline.load_ckpt(ckpt_path=ckpt_path)
# # Get one test point cloud from the SemanticKitti dataset
# pc_idx = 256 # change the index to get a different point cloud
# test_split = dataset.get_split("test")
# data = test_split.get_data(pc_idx)
# run inference on a single example.
# returns dict with 'predict_labels' and 'predict_scores'.
# result = pipeline.run_inference(data)
# Create a pcd to be visualized
# pcd = o3d.geometry.PointCloud()
# xyz = data["point"] # Get the points
# pcd.points = o3d.utility.Vector3dVector(xyz)
# colors = [COLOR_MAP[clr] for clr in list(result['predict_labels'])] # Get the color associated to each predicted label
# pcd.colors = o3d.utility.Vector3dVector(colors) # Add color data to the point cloud
# Create visualization
# custom_draw_geometry(pcd)
# Get one test point cloud from the custom dataset
pc_idx = 2 # change the index to get a different point cloud
custom_pc = custom_dataset[pc_idx]
print(np.array(custom_pc.points).shape)
data, pcd = prepare_point_cloud_for_inference(custom_pc)
# Run inference
result = pipeline.run_inference(data)[0]
# Colorize the point cloud with predicted labels
colors = [COLOR_MAP[clr] for clr in list(result['predict_labels'])]
pcd.colors = o3d.utility.Vector3dVector(colors)
# Create visualization
custom_draw_geometry(pcd)
# evaluate performance on the test set; this will write logs to './logs'.
#pipeline.run_test()
```
### Error message
Traceback (most recent call last):
File "/home/maffan/open3d_/open3d_experiments/semantic_torch_pcd_openvino.py", line 172, in
result = pipeline.run_inference(data)[0]
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/open3d/_ml3d/torch/pipelines/semantic_segmentation.py", line 161, in run_inference
results = model(inputs['data'])
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/open3d/_ml3d/torch/models/openvino_model.py", line 121, in __call__
return self.forward(inputs)
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/open3d/_ml3d/torch/models/openvino_model.py", line 96, in forward
self._read_torch_model(inputs)
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/open3d/_ml3d/torch/models/openvino_model.py", line 86, in _read_torch_model
input_names=input_names,
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/onnx/utils.py", line 519, in export
export_modules_as_functions=export_modules_as_functions,
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/onnx/utils.py", line 1539, in _export
dynamic_axes=dynamic_axes,
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/onnx/utils.py", line 1111, in _model_to_graph
graph, params, torch_out, module = _create_jit_graph(model, args)
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/onnx/utils.py", line 987, in _create_jit_graph
graph, torch_out = _trace_and_get_graph_from_model(model, args)
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/onnx/utils.py", line 896, in _trace_and_get_graph_from_model
_return_inputs_states=True,
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/jit/_trace.py", line 1184, in _get_trace_graph
outs = ONNXTracedModule(f, strict, _force_outplace, return_inputs, _return_inputs_states)(*args, **kwargs)
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(*input, **kwargs)
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/jit/_trace.py", line 132, in forward
self._force_outplace,
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/jit/_trace.py", line 118, in wrapper
outs.append(self.inner(*trace_inputs))
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(*input, **kwargs)
File "/home/maffan/miniconda3/envs/randlanet_pytorch_cpu/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1182, in _slow_forward
result = self.forward(*input, **kwargs)
TypeError: () missing 1 required positional argument: 'x'
### Expected behavior
_No response_
### Open3D, Python and System information
```markdown
- Operating system: Ubuntu 20.04
- Python version: 3.7.16
- Open3D version: 0.17.0
- OpenVino version: 2021.4.2
- PyTorch version: Version: 1.13.1+cpu
- System type: x84
- Is this a remote workstation?: no
- How did you install Open3D?: pip
- Compiler version (if built from source): gcc (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
```
### Additional information
_No response_
Contributor guide
No contributing guide indexed for this repository
Assessment
This issue has not been assessed yet.