apple / apple/coremltools

Unable to run model update when neural networks contain LSTM layer

Open
#2,074 4 comments 0 reactions 0 assignees View on GitHub
bug on-device update
Dominant language
Python
Stars
5.4k
Forks
850
Avg merge
4d 5h
Merged PRs (30d)
10

Description

## ❓Question
I converted a simple nerual network containing LSTM from Pytorch to mlmodel, and set only the last fully-connected layer to be updatable. Using this updatable model, **I can run prediction**, but model training gives error `libc++abi: terminating due to uncaught exception of type Espresso::invalid_argument_error: Espresso exception: "Invalid argument": generic_expand_dims_kernel: Output rank cannot be more than 5 in expand_dims:transpose_1_expanded`. Does it mean that, once LSTM is in the network, I cannot fine-tune the last layer even if it is just a fully-connected layer? Is there any temporary solution? Thanks.

I use `torch==1.13.1` and `coremltools==7.0`

Here is the network structure:
Screenshot 2023-11-28 at 9 13 00 PM

Here is the Pytorch code:
```python
import torch
import torch.nn as nn
import coremltools as ct
import coremltools
import numpy as np
from coremltools.models.neural_network import NeuralNetworkBuilder, SgdParams, AdamParams
from coremltools.models import datatypes

class LSTMClassificationModel(nn.Module):
def __init__(self, input_size, hidden_size, output_size, num_layers=1):
super(LSTMClassificationModel, self).__init__()
self.hidden_size = hidden_size
self.num_layers = num_layers

# LSTM layer
self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)

# Fully connected layer to get the final output
self.fc = nn.Linear(hidden_size, output_size)

def forward(self, x):
# Reshape the input to (batch_size, seq_length, input_size)
x = x.unsqueeze(-1) # Reshapes from (batch_size, 3) to (batch_size, 3, 1)

# Initialize hidden state and cell state
h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(x.device)
c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(x.device)

# Forward propagate through the LSTM layer
out, _ = self.lstm(x, (h0, c0))

# Take the output of the last time step
out = out[:, -1, :]

# Forward propagate through the fully connected layer
out = self.fc(out)

return out

# Example usage
input_size = 1 # Number of features in each time step (after reshaping)
hidden_size = 3 # Number of features in the hidden state
output_size = 5 # Number of output classes

model = LSTMClassificationModel(input_size, hidden_size, output_size)

# Create a sample input tensor
sample_input = torch.rand(1, 3) # Adjust the shape according to your model's input

# Trace the model with a sample input
traced_model = torch.jit.trace(model, sample_input)

# Convert the traced model to Core ML format
input_features = [ct.TensorType(shape=(1, 3))]
output_features = ["output"]
mlmodel = ct.convert(
traced_model,
inputs=input_features,
convert_to="neuralnetwork"
)
mlmodel.save("classification.mlmodel")

spec = coremltools.utils.load_spec('classification.mlmodel')

builder = coremltools.models.neural_network.NeuralNetworkBuilder(spec=spec)

# Load the model specification
spec = coremltools.utils.load_spec('classification.mlmodel')
builder = NeuralNetworkBuilder(spec=spec)

# Make layers updatable
builder.make_updatable(['linear_0'])

builder.add_softmax(name='output_prob', input_name='linear_0', output_name='output_prob')
builder.set_categorical_cross_entropy_loss(name='lossLayer', input='output_prob')

# define the optimizer (Adam in this example)
adam_params = AdamParams(lr=0.01, beta1=0.9, beta2=0.999, eps=1e-8, batch=32)
builder.set_adam_optimizer(adam_params)

# Set the number of epochs
builder.set_epochs(100)

# Optionally, set descriptions for your training inputs
spec.description.trainingInput[0].shortDescription = 'Input data'
spec.description.trainingInput[1].shortDescription = 'Target output data'

spec.description.output[0].name = 'output_prob'
spec.description.output[0].shortDescription = 'Probability distribution over output classes'

# Save the updated model
updated_model = coremltools.models.MLModel(spec)
updated_model.save('updatable_classification11.mlmodel')

adam_params = AdamParams(lr=0.01, beta1=0.9, beta2=0.999, eps=1e-8, batch=32)
builder.set_adam_optimizer(adam_params)

# Set the number of epochs
builder.set_epochs(100)

# Optionally, set descriptions for your training inputs
spec.description.trainingInput[0].shortDescription = 'Input data'
spec.description.trainingInput[1].shortDescription = 'Target output data'

spec.description.output[0].name = 'output_prob'
spec.description.output[0].shortDescription = 'Probability distribution over output classes'

# Save the updated model
updated_model = coremltools.models.MLModel(spec)
updated_model.save('updatable_classification11.mlmodel')
```

Here is my swift prediction code which works:
```swift
import CoreML

import GameKit

func generateSampleData(numSamples: Int, seed: UInt64) -> ([MLMultiArray], [MLMultiArray]) {
var inputArray = [MLMultiArray]()
var outputArray = [MLMultiArray]()

let randomSource = GKLinearCongruentialRandomSource(seed: seed)
let randomDistribution = GKRandomDistribution(randomSource: randomSource, lowestValue: 0, highestValue: 10)

for _ in 0.. Int {
let length = multiArray.count
let ptr = UnsafeMutablePointer(OpaquePointer(multiArray.dataPointer))
var maxValue: Float = ptr[0]
var maxIndex: Int = 0

for i in 1.. maxValue {
maxValue = ptr[i]
maxIndex = i
}
}

return maxIndex
}

func computeMetrics(model: MLModel, data: ([MLMultiArray], [MLMultiArray])) -> (loss: Double, accuracy: Double) {
let (inputData, outputData) = data
var totalLoss: Double = 0
var correctPredictions: Int = 0

for (index, input) in inputData.enumerated() {
let output = outputData[index]

if let prediction = try? model.prediction(from: MLDictionaryFeatureProvider(dictionary: ["x_1": MLFeatureValue(multiArray: input)])),
let predictedOutputProb = prediction.featureValue(for: "output_prob")?.multiArrayValue {

let trueClass = output[0].intValue
let predictedClass = argmax(multiArray: predictedOutputProb)
correctPredictions += (trueClass == predictedClass) ? 1 : 0

// Calculate cross-entropy loss
let predictedProb = predictedOutputProb[trueClass]
totalLoss += -log(max(Double(predictedProb.doubleValue), 1e-10))
}
}
let accuracy = Double(correctPredictions) / Double(inputData.count)
return (totalLoss / Double(inputData.count), accuracy)
}

func testModel() {
// Generate test data
let (inputData, outputData) = generateSampleData(numSamples: 500, seed: 8)

// Load the model from the main bundle
guard let modelURL = Bundle.main.url(forResource: "updatable_classification11", withExtension: "mlmodelc"),
let model = try? MLModel(contentsOf: modelURL) else {
print("Failed to load the model from the bundle")
return
}

// Make predictions
for i in 0..

Contributor guide

Open the contributing guide

Research direction

Start by reproducing the Swift trainModel() path with the posted PyTorch 1.13.1 and coremltools 7.0 conversion, focusing on the generated model's LSTM and updatable linear_0 layer. Compare the training failure's expand_dims rank error with the working prediction path; done means establishing whether this configuration is supported or documenting a concrete workaround.

Written by the indexing model from the issue text.

Assessment

Tech stack
python, pytorch
Domain
machine-learning
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
35/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.