Unable to run model update when neural networks contain LSTM layer
- Dominant language
- Python
- Stars
- 5.4k
- Forks
- 850
- Avg merge
- 4d 5h
- Merged PRs (30d)
- 10
Description
## ❓Question
I converted a simple nerual network containing LSTM from Pytorch to mlmodel, and set only the last fully-connected layer to be updatable. Using this updatable model, **I can run prediction**, but model training gives error `libc++abi: terminating due to uncaught exception of type Espresso::invalid_argument_error: Espresso exception: "Invalid argument": generic_expand_dims_kernel: Output rank cannot be more than 5 in expand_dims:transpose_1_expanded`. Does it mean that, once LSTM is in the network, I cannot fine-tune the last layer even if it is just a fully-connected layer? Is there any temporary solution? Thanks.
I use `torch==1.13.1` and `coremltools==7.0`
Here is the network structure:
Here is the Pytorch code:
```python
import torch
import torch.nn as nn
import coremltools as ct
import coremltools
import numpy as np
from coremltools.models.neural_network import NeuralNetworkBuilder, SgdParams, AdamParams
from coremltools.models import datatypes
class LSTMClassificationModel(nn.Module):
def __init__(self, input_size, hidden_size, output_size, num_layers=1):
super(LSTMClassificationModel, self).__init__()
self.hidden_size = hidden_size
self.num_layers = num_layers
# LSTM layer
self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
# Fully connected layer to get the final output
self.fc = nn.Linear(hidden_size, output_size)
def forward(self, x):
# Reshape the input to (batch_size, seq_length, input_size)
x = x.unsqueeze(-1) # Reshapes from (batch_size, 3) to (batch_size, 3, 1)
# Initialize hidden state and cell state
h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(x.device)
c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(x.device)
# Forward propagate through the LSTM layer
out, _ = self.lstm(x, (h0, c0))
# Take the output of the last time step
out = out[:, -1, :]
# Forward propagate through the fully connected layer
out = self.fc(out)
return out
# Example usage
input_size = 1 # Number of features in each time step (after reshaping)
hidden_size = 3 # Number of features in the hidden state
output_size = 5 # Number of output classes
model = LSTMClassificationModel(input_size, hidden_size, output_size)
# Create a sample input tensor
sample_input = torch.rand(1, 3) # Adjust the shape according to your model's input
# Trace the model with a sample input
traced_model = torch.jit.trace(model, sample_input)
# Convert the traced model to Core ML format
input_features = [ct.TensorType(shape=(1, 3))]
output_features = ["output"]
mlmodel = ct.convert(
traced_model,
inputs=input_features,
convert_to="neuralnetwork"
)
mlmodel.save("classification.mlmodel")
spec = coremltools.utils.load_spec('classification.mlmodel')
builder = coremltools.models.neural_network.NeuralNetworkBuilder(spec=spec)
# Load the model specification
spec = coremltools.utils.load_spec('classification.mlmodel')
builder = NeuralNetworkBuilder(spec=spec)
# Make layers updatable
builder.make_updatable(['linear_0'])
builder.add_softmax(name='output_prob', input_name='linear_0', output_name='output_prob')
builder.set_categorical_cross_entropy_loss(name='lossLayer', input='output_prob')
# define the optimizer (Adam in this example)
adam_params = AdamParams(lr=0.01, beta1=0.9, beta2=0.999, eps=1e-8, batch=32)
builder.set_adam_optimizer(adam_params)
# Set the number of epochs
builder.set_epochs(100)
# Optionally, set descriptions for your training inputs
spec.description.trainingInput[0].shortDescription = 'Input data'
spec.description.trainingInput[1].shortDescription = 'Target output data'
spec.description.output[0].name = 'output_prob'
spec.description.output[0].shortDescription = 'Probability distribution over output classes'
# Save the updated model
updated_model = coremltools.models.MLModel(spec)
updated_model.save('updatable_classification11.mlmodel')
adam_params = AdamParams(lr=0.01, beta1=0.9, beta2=0.999, eps=1e-8, batch=32)
builder.set_adam_optimizer(adam_params)
# Set the number of epochs
builder.set_epochs(100)
# Optionally, set descriptions for your training inputs
spec.description.trainingInput[0].shortDescription = 'Input data'
spec.description.trainingInput[1].shortDescription = 'Target output data'
spec.description.output[0].name = 'output_prob'
spec.description.output[0].shortDescription = 'Probability distribution over output classes'
# Save the updated model
updated_model = coremltools.models.MLModel(spec)
updated_model.save('updatable_classification11.mlmodel')
```
Here is my swift prediction code which works:
```swift
import CoreML
import GameKit
func generateSampleData(numSamples: Int, seed: UInt64) -> ([MLMultiArray], [MLMultiArray]) {
var inputArray = [MLMultiArray]()
var outputArray = [MLMultiArray]()
let randomSource = GKLinearCongruentialRandomSource(seed: seed)
let randomDistribution = GKRandomDistribution(randomSource: randomSource, lowestValue: 0, highestValue: 10)
for _ in 0.. Int {
let length = multiArray.count
let ptr = UnsafeMutablePointer(OpaquePointer(multiArray.dataPointer))
var maxValue: Float = ptr[0]
var maxIndex: Int = 0
for i in 1.. maxValue {
maxValue = ptr[i]
maxIndex = i
}
}
return maxIndex
}
func computeMetrics(model: MLModel, data: ([MLMultiArray], [MLMultiArray])) -> (loss: Double, accuracy: Double) {
let (inputData, outputData) = data
var totalLoss: Double = 0
var correctPredictions: Int = 0
for (index, input) in inputData.enumerated() {
let output = outputData[index]
if let prediction = try? model.prediction(from: MLDictionaryFeatureProvider(dictionary: ["x_1": MLFeatureValue(multiArray: input)])),
let predictedOutputProb = prediction.featureValue(for: "output_prob")?.multiArrayValue {
let trueClass = output[0].intValue
let predictedClass = argmax(multiArray: predictedOutputProb)
correctPredictions += (trueClass == predictedClass) ? 1 : 0
// Calculate cross-entropy loss
let predictedProb = predictedOutputProb[trueClass]
totalLoss += -log(max(Double(predictedProb.doubleValue), 1e-10))
}
}
let accuracy = Double(correctPredictions) / Double(inputData.count)
return (totalLoss / Double(inputData.count), accuracy)
}
func testModel() {
// Generate test data
let (inputData, outputData) = generateSampleData(numSamples: 500, seed: 8)
// Load the model from the main bundle
guard let modelURL = Bundle.main.url(forResource: "updatable_classification11", withExtension: "mlmodelc"),
let model = try? MLModel(contentsOf: modelURL) else {
print("Failed to load the model from the bundle")
return
}
// Make predictions
for i in 0..
Contributor guide
Research direction
Start by reproducing the Swift trainModel() path with the posted PyTorch 1.13.1 and coremltools 7.0 conversion, focusing on the generated model's LSTM and updatable linear_0 layer. Compare the training failure's expand_dims rank error with the working prediction path; done means establishing whether this configuration is supported or documenting a concrete workaround.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- python, pytorch
- Domain
- machine-learning
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 35/100