apple / apple/coremltools

Keras CapsuleNetwork exported model fails to run.

Open
#1,743 5 comments 0 reactions 0 assignees View on GitHub
missing layer type tf2.x / tf.keras triaged
Dominant language
Python
Stars
5.4k
Forks
850
Avg merge
4d 5h
Merged PRs (30d)
10

Description

## 🐞Describing the bug
I am trying to convert a custom Keras layer that implements Capsule Network. The model exports but fails to load and run in Xcode.

## To Reproduce

```python
from keras.models import Model
from keras import Input
import numpy as np
import tensorflow as tf

import coremltools as ct

class Squash(tf.keras.layers.Layer):
def __init__(self, eps=10e-21, **kwargs):
super().__init__(**kwargs)
self.eps = eps

def call(self, s):
n = tf.norm(s,axis=-1,keepdims=True)
return (1 - 1/(tf.math.exp(n)+self.eps))*(s/(n+self.eps))

def get_config(self):
base_config = super().get_config()
return {**base_config}

def compute_output_shape(self, input_shape):
return input_shape

class FCCaps(tf.keras.layers.Layer):
"""
Fully-connected caps layer. It exploites the routing mechanism, explained in 'Efficient-CapsNet: Capsule Network with Self-Attention Routing',
to create a parent layer of capsules.

...

Attributes
----------
N: int
number of primary capsules
D: int
primary capsules dimension (number of properties)
kernel_initilizer: str
matrix W initialization strategy

Methods
-------
call(inputs)
compute the primary capsule layer
"""
def __init__(self, N, D, kernel_initializer='he_normal', **kwargs):
super(FCCaps, self).__init__(**kwargs)
self.N = N
self.D = D
self.kernel_initializer = tf.keras.initializers.get(kernel_initializer)

def build(self, input_shape):
input_N = input_shape[-2]
input_D = input_shape[-1]

self.W = self.add_weight(shape=[self.N, input_N, input_D, self.D],initializer=self.kernel_initializer,name='W')
self.b = self.add_weight(shape=[self.N, input_N,1], initializer=tf.zeros_initializer(), name='b')
self.built = True

def call(self, inputs, training=None):

u = tf.einsum('...ji,kjiz->...kjz',inputs,self.W) # u shape=(None,N,H*W*input_N,D)

c = tf.einsum('...ij,...kj->...i', u, u)[...,None] # b shape=(None,N,H*W*input_N,1) -> (None,j,i,1)
c = c/tf.sqrt(tf.cast(self.D, tf.float32))
c = tf.nn.softmax(c, axis=1) # c shape=(None,N,H*W*input_N,1) -> (None,j,i,1)
c = c + self.b
s = tf.reduce_sum(tf.multiply(u, c),axis=-2) # s shape=(None,N,D)
v = Squash()(s) # v shape=(None,N,D)

return v

def compute_output_shape(self, input_shape):
return (None, self.C, self.L)

def get_config(self):
config = {
'N': self.N,
'D': self.D
}
base_config = super(FCCaps, self).get_config()
return dict(list(base_config.items()) + list(config.items()))

class PrimaryCaps(tf.keras.layers.Layer):
"""
Create a primary capsule layer with the methodology described in 'Efficient-CapsNet: Capsule Network with Self-Attention Routing'.
Properties of each capsule s_n are exatracted using a 2D depthwise convolution.

...

Attributes
----------
F: int
depthwise conv number of features
K: int
depthwise conv kernel dimension
N: int
number of primary capsules
D: int
primary capsules dimension (number of properties)
s: int
depthwise conv strides
Methods
-------
call(inputs)
compute the primary capsule layer
"""
def __init__(self, F, K, N, D, s=1, **kwargs):
super(PrimaryCaps, self).__init__(**kwargs)
self.F = F
self.K = K
self.N = N
self.D = D
self.s = s

def build(self, input_shape):
self.DW_Conv2D = tf.keras.layers.Conv2D(self.F, self.K, self.s,
activation='linear', groups=self.F, padding='valid')

self.built = True

def call(self, inputs):
x = self.DW_Conv2D(inputs)
x = tf.keras.layers.Reshape((self.N, self.D))(x)
x = Squash()(x)
return x

def get_config(self):
config = {
'F': self.F,
'K': self.K,
'N': self.N,
'D': self.D,
's': self.s
}
base_config = super(PrimaryCaps, self).get_config()
return dict(list(base_config.items()) + list(config.items()))

inputs = Input((28,28,1))
x = tf.keras.layers.Conv2D(32,5,activation="relu", padding='valid', kernel_initializer='he_normal')(inputs)
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Conv2D(64,3, activation='relu', padding='valid', kernel_initializer='he_normal')(x)
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Conv2D(64,3, activation='relu', padding='valid', kernel_initializer='he_normal')(x)
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Conv2D(128,3,2, activation='relu', padding='valid', kernel_initializer='he_normal')(x)
x = tf.keras.layers.BatchNormalization()(x)
x = PrimaryCaps(128, 9, 16, 8)(x)
x = FCCaps(10,16)(x)
m = Model(inputs=inputs, outputs=[x], name='CapsNet_Example')
q = m.predict(np.zeros((1,28,28,1), dtype=np.float32))

print(m.summary())
print(q)

coreml_model = ct.convert(
m,
compute_precision=ct.precision.FLOAT32,
minimum_deployment_target=ct.target.iOS16,
source='tensorflow'
)
```

Output

```
Model: "CapsNet_Example"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) [(None, 28, 28, 1)] 0

conv2d (Conv2D) (None, 24, 24, 32) 832

batch_normalization (BatchN (None, 24, 24, 32) 128
ormalization)

conv2d_1 (Conv2D) (None, 22, 22, 64) 18496

batch_normalization_1 (Batc (None, 22, 22, 64) 256
hNormalization)

conv2d_2 (Conv2D) (None, 20, 20, 64) 36928

batch_normalization_2 (Batc (None, 20, 20, 64) 256
hNormalization)

conv2d_3 (Conv2D) (None, 9, 9, 128) 73856

batch_normalization_3 (Batc (None, 9, 9, 128) 512
hNormalization)

primary_caps (PrimaryCaps) (None, 16, 8) 10496

fc_caps (FCCaps) (None, 10, 16) 20640

=================================================================
Total params: 162,400
Trainable params: 161,824
Non-trainable params: 576
_________________________________________________________________
None
[[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]]]

Running TensorFlow Graph Passes: 0%| | 0/6 [00:00 MIL Ops: 61%|██████████████████████████████████████████████▎ | 56/92 [00:00<00:00, 4170.18 ops/s]
Traceback (most recent call last):
coremltools/converters/mil/frontend/tensorflow/convert_utils.py", line 190, in convert_graph
raise NotImplementedError(msg)
NotImplementedError: Conversion for TF op 'PartitionedCall' not implemented.

name: "CapsNet_Example/primary_caps/conv2d/PartitionedCall"
op: "PartitionedCall"
input: "CapsNet_Example/batch_normalization_3/FusedBatchNormV3"
input: "CapsNet_Example/primary_caps/conv2d/ReadVariableOp"
attr {
key: "Tin"
value {
list {
type: DT_FLOAT
type: DT_FLOAT
}
}
}
attr {
key: "Tout"
value {
list {
type: DT_FLOAT
}
}
}
attr {
key: "_XlaMustCompile"
value {
b: true
}
}
attr {
key: "_collective_manager_ids"
value {
list {
}
}
}
attr {
key: "_read_only_resource_inputs"
value {
list {
}
}
}
attr {
key: "config"
value {
s: ""
}
}
attr {
key: "config_proto"
value {
s: "\n\007\n\003CPU\020\001\n\007\n\003GPU\020\0002\002J\0008\001\202\001\000"
}
}
attr {
key: "executor_type"
value {
s: ""
}
}
attr {
key: "f"
value {
func {
name: "__inference__jit_compiled_convolution_op_278"
}
}
}

```

## System environment (please complete the following information):
- coremltools version 6.1
- OS Test on MacOS 13.1 – Exported on Linux Ubuntu 22.04:
- Tensorflow 2.10

Contributor guide

Open the contributing guide

Research direction

Start by running the provided TensorFlow 2.10 and coremltools 6.1 reproducer, then inspect coremltools/converters/mil/frontend/tensorflow/convert_utils.py around the reported PartitionedCall conversion failure. Done means this CapsuleNetwork model converts without the NotImplementedError and the exported model loads and runs in Xcode.

Written by the indexing model from the issue text.

Assessment

Tech stack
python, tensorflow
Domain
machine-learning, tooling
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
42/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.