apple / apple/coremltools

`tf.keras.layers.Resizing` can not use flexible shapes

Open
#1,865 8 comments 0 reactions 0 assignees View on GitHub
bug tf2.x / tf.keras triaged
Dominant language
Python
Stars
5.4k
Forks
850
Avg merge
4d 5h
Merged PRs (30d)
10

Description

## 🐞Describing the bug

I'm unable to use `coremltools` to convert a Tensorflow 2 model containing a `tf.keras.layers.Resizing` layer: I get a `Cannot add const` error. This layer is used to resize an input image to be the correct shape for the rest of the model.

## Stack Trace
```
ValueError Traceback (most recent call last)
/var/folders/7c/0cjr9j7j50l2qg_kpcslzyd40000gn/T/ipykernel_14861/4025639170.py in ()
----> 1 ct_model = ct.convert(
2 model,
3 convert_to='neuralnetwork',
4 source='tensorflow',
5 inputs=[ct.ImageType()],

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/_converters_entry.py in convert(model, source, inputs, outputs, classifier_config, minimum_deployment_target, convert_to, compute_precision, skip_model_load, compute_units, package_dir, debug, pass_pipeline)
490 specification_version = _set_default_specification_version(exact_target)
491
--> 492 mlmodel = mil_convert(
493 model,
494 convert_from=exact_source,

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/converter.py in mil_convert(model, convert_from, convert_to, compute_units, **kwargs)
186 See `coremltools.converters.convert`
187 """
--> 188 return _mil_convert(model, convert_from, convert_to, ConverterRegistry, MLModel, compute_units, **kwargs)
189
190

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/converter.py in _mil_convert(model, convert_from, convert_to, registry, modelClass, compute_units, **kwargs)
210 kwargs["weights_dir"] = weights_dir.name
211
--> 212 proto, mil_program = mil_convert_to_proto(
213 model,
214 convert_from,

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/converter.py in mil_convert_to_proto(model, convert_from, convert_to, converter_registry, main_pipeline, **kwargs)
283
284 frontend_converter = frontend_converter_type()
--> 285 prog = frontend_converter(model, **kwargs)
286 PipelineManager.apply_pipeline(prog, frontend_pipeline)
287

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/converter.py in __call__(self, *args, **kwargs)
96
97 tf2_loader = TF2Loader(*args, **kwargs)
---> 98 return tf2_loader.load()
99
100

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/frontend/tensorflow/load.py in load(self)
80 )
81
---> 82 program = self._program_from_tf_ssa()
83 logger.debug("program:\n{}".format(program))
84 return program

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/frontend/tensorflow2/load.py in _program_from_tf_ssa(self)
208 opset_version=self.kwargs["specification_version"],
209 )
--> 210 return converter.convert()
211
212 def _populate_sub_graph_input_shapes(self, graph, graph_fns):

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/frontend/tensorflow/converter.py in convert(self)
463 for g_name in self.graph_stack[1:]:
464 self.context.add_graph(g_name, self.tfssa.functions[g_name].graph)
--> 465 self.convert_main_graph(prog, graph)
466 return prog

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/frontend/tensorflow/converter.py in convert_main_graph(self, prog, graph)
387 input_var = mb.cast(x=input_var, dtype="fp32", name=name)
388 self.context.add(name, input_var)
--> 389 outputs = convert_graph(self.context, graph, self.output_names)
390 ssa_func.set_outputs(outputs)
391 prog.add_function("main", ssa_func)

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/frontend/tensorflow/convert_utils.py in convert_graph(context, graph, outputs)
189 )
190 raise NotImplementedError(msg)
--> 191 add_op(context, node)
192
193 if len(node.outputs) > 0:

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/frontend/tensorflow/ops.py in ResizeBilinear(context, node)
2744 scale_factor_width = Wout / Win if Wout % Win == 0 else (Wout + 1e-4) / Win
2745
-> 2746 x = mb.upsample_bilinear(
2747 x=x,
2748 scale_factor_height=scale_factor_height,

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/mil/ops/registry.py in add_op(cls, **kwargs)
180 op_cls_to_add = op_reg[op_type]
181
--> 182 return cls._add_op(op_cls_to_add, **kwargs)
183
184 setattr(Builder, op_type, add_op)

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/mil/builder.py in _add_op(cls, op_cls, **kwargs)
160 # Shallow copy list inputs to ensure op inputs are immutable
161 kwargs = {k: v if not isinstance(v, (list, tuple)) else v[:] for k, v in kwargs.items() if v is not None}
--> 162 kwargs.update(cls._create_vars(
163 input_spec=op_cls.input_spec,
164 op_name=kwargs["name"], before_op=before_op,

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/mil/builder.py in _create_vars(cls, input_spec, op_name, before_op, candidate_kv)
143
144 if isinstance(in_type, (TensorInputType, ListOrTensorInputType)):
--> 145 var = cls._add_const(val, new_var_name, before_op)
146 update_dict[k] = var
147

/opt/homebrew/Caskroom/miniforge/base/envs/ca-iq/lib/python3.9/site-packages/coremltools/converters/mil/mil/builder.py in _add_const(cls, val, name, before_op)
74 def _add_const(cls, val, name, before_op):
75 if not is_python_value(val):
---> 76 raise ValueError("Cannot add const {}".format(val))
77 if any_symbolic(val):
78 msg = (

ValueError: Cannot add const 128.0001/is63
```

## To Reproduce

```python
import tensorflow as tf
import coremltools as ct

model = tf.keras.Sequential(
[
tf.keras.layers.Resizing(128, 128),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(2)
]
)
model.build([1, None, None, 3])
model.compile()

ct_model = ct.convert(
model,
convert_to='neuralnetwork',
source='tensorflow',
inputs=[ct.ImageType()],
classifier_config=ct.ClassifierConfig(["a", "b"])
)
```

## System environment (please complete the following information):
- coremltools version: `6.3.0 `
- OS (e.g. MacOS version or Linux type): `MacOS 13.2.1`
- Any other relevant version information (e.g. PyTorch or TensorFlow version): TensorFlow `2.12.0`

## Additional context

If I remove the resizing layer then the conversion will work, i.e. the following converts without error:
```python
model = tf.keras.Sequential(
[
# tf.keras.layers.Resizing(128, 128),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(2)
]
)
model.build([1, 128, 128, 3])
model.compile()
```

Contributor guide

Open the contributing guide

Research direction

Reproduce the failure with the TensorFlow 2 model in the issue using a flexible [1, None, None, 3] shape. Start with the TensorFlow frontend's ResizeBilinear entry point in converters/mil/frontend/tensorflow/ops.py and the constant handling shown in mil/builder.py; done means conversion succeeds for the Resizing layer without the Cannot add const error.

Written by the indexing model from the issue text.

Assessment

Tech stack
python, tensorflow
Domain
machine-learning, tooling
Issue type
Bug
Difficulty
3/5
Estimated time
1-2 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
42/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.