dotnet / dotnet/machinelearning
Unseen labels during retraining map to value "0", results in System.InvalidOperationException: 'No valid training instances found, all instances have missing features.'
- Dominant language
- C#
- Stars
- 9.4k
- Forks
- 2k
- Avg merge
- 2d 20h
- Merged PRs (30d)
- 11
Description
### System information
- Windows Server 2016
- .NET Version 3.1.300
### Issue
I'm trying to do MultiClass LbfgsMaximumEntropy Re-training
When trying to Fit new data, I get System.InvalidOperationException: 'No valid training instances found, all instances have missing features.' on row
ITransformer _keyToValueModel1 = _mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy("Label", "Features")
.Fit(transformedData, originalModelParameters.Model);
[data1.zip](https://github.com/dotnet/machinelearning/files/4744624/data1.zip)
I would appreciate either help or MultiClass LbfgsMaximumEntropy Re-training code sample.
### Source code / logs
```c#
[data1.zip](https://github.com/dotnet/machinelearning/files/4744619/data1.zip)
[data2.zip](https://github.com/dotnet/machinelearning/files/4744620/data2.zip)
public class GitHubIssueClassification
{
static List testDatas = new List()
{
new GitHubIssueTransformed() {Area="11", Title="WHIRLPOOL AWE 50610", Description="" },
new GitHubIssueTransformed() {Area="14", Title="FAGOR 4CC-130 E X", Description="" },
new GitHubIssueTransformed() {Area="19", Title="AEG T8DFE68SC", Description="" },
new GitHubIssueTransformed() {Area="999", Title="TEST 999", Description="" }
};
private static string _appPath => Path.GetDirectoryName(Environment.GetCommandLineArgs()[0]);
private static string _mainDataPath1 => Path.Combine(_appPath, "..", "..", "..", "Data", "data1.csv");
private static string _mainDataPath2 => Path.Combine(_appPath, "..", "..", "..", "Data", "data2.csv");
private static string _mainDataPath3 => Path.Combine(_appPath, "..", "..", "..", "Data", "data3.csv");
private static string _modelPath => Path.Combine(_appPath, "..", "..", "..", "Models", "trainedModel.zip");
private static string _keyToValueModelPath => Path.Combine(_appPath, "..", "..", "..", "Models", "keyToValueModel.zip");
static DataOperationsCatalog.TrainTestData splittedData;
private static MLContext _mlContext;
private static PredictionEngine _predEngine;
private static ITransformer _trainedModel { get; set; }
private static ITransformer _keyToValueModel { get; set; }
static IDataView _trainingDataView;
public static void Run()
{
_mlContext = new MLContext(seed: 0);
var allData = _mlContext.Data.LoadFromTextFile(_mainDataPath1, hasHeader: true);
splittedData = _mlContext.Data.TrainTestSplit(allData, testFraction: 0.09);
_trainingDataView = splittedData.TrainSet;
Console.WriteLine($"=============== Finished Loading Dataset ===============");
var pipeline = ProcessData();
var transformedData = BuildAndTrainModel(_trainingDataView, pipeline);
Evaluate(_trainingDataView.Schema, transformedData, splittedData.TestSet);
PredictIssue_FirstLoadModelFromDisk();
SecondLap(_mlContext);
}
public static IEstimator ProcessData()
{
Console.WriteLine($"=============== Processing Data ===============");
var pipeline = _mlContext.Transforms.Conversion.MapValueToKey(inputColumnName: "Area", outputColumnName: "Label")
.Append(_mlContext.Transforms.Text.FeaturizeText(inputColumnName: "Title", outputColumnName: "TitleFeaturized"))
.Append(_mlContext.Transforms.Text.FeaturizeText(inputColumnName: "Description", outputColumnName: "DescriptionFeaturized"))
.Append(_mlContext.Transforms.Concatenate("Features", "TitleFeaturized", "DescriptionFeaturized"))
.AppendCacheCheckpoint(_mlContext);
Console.WriteLine($"=============== Finished Processing Data ===============");
return pipeline;
}
public static IDataView BuildAndTrainModel(IDataView trainingDataView, IEstimator pipeline)
{
//var trainingPipeline = pipeline.Append(_mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy/*.SdcaMaximumEntropy*/("Label", "Features"))
// .Append(_mlContext.Transforms.Conversion.MapKeyToValue("PredictedLabel"));
var trainingPipeline = pipeline.Append(_mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy/*.SdcaMaximumEntropy*/("Label", "Features"));
//var keyToValuePipeline = _mlContext.Transforms.Conversion.MapKeyToValue("PredictedLabel");
var keyToValuePipeline = trainingPipeline.Append(_mlContext.Transforms.Conversion.MapKeyToValue("Area", "PredictedLabel"));
Console.WriteLine($"=============== Training the model ===============");
_trainedModel = trainingPipeline.Fit(trainingDataView);
var transformedData = _trainedModel.Transform(trainingDataView);
_keyToValueModel = keyToValuePipeline.Fit(transformedData);
_mlContext.Model.Save(_trainedModel, trainingDataView.Schema, _modelPath);
_mlContext.Model.Save(_keyToValueModel, transformedData.Schema, _keyToValueModelPath);
Console.WriteLine($"=============== Finished Training the model Ending time: {DateTime.Now.ToString()} ===============");
// (OPTIONAL) Try/test a single prediction with the "just-trained model" (Before saving the model)
Console.WriteLine($"=============== Single Prediction just-trained-model ===============");
_predEngine = _mlContext.Model.CreatePredictionEngine(_keyToValueModel);
foreach (var testIssue in testDatas)
{
var prediction = _predEngine.Predict(testIssue);
if (prediction.Area.ToString() != testIssue.Area.ToString())
Console.ForegroundColor = ConsoleColor.Red;
else
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine($"=============== Single Prediction just-trained-model - Result: {prediction.Area}/{testIssue.Area} {testIssue.Title} ===============");
}
Console.ResetColor();
return transformedData;
}
public static void Evaluate(DataViewSchema trainingDataViewSchema, IDataView transformedData, IDataView testDataView2 = null)
{
// STEP 5: Evaluate the model in order to get the model's accuracy metrics
Console.WriteLine($"=============== Evaluating to get model's accuracy metrics - Starting time: {DateTime.Now.ToString()} ===============");
IDataView testDataView = testDataView2;
var testMetrics = _mlContext.MulticlassClassification.Evaluate(_trainedModel.Transform(testDataView));
Console.WriteLine($"=============== Evaluating to get model's accuracy metrics - Ending time: {DateTime.Now.ToString()} ===============");
Console.WriteLine($"*************************************************************************************************************");
Console.WriteLine($"* Metrics for Multi-class Classification model - Test Data ");
Console.WriteLine($"*------------------------------------------------------------------------------------------------------------");
Console.WriteLine($"* MicroAccuracy: {testMetrics.MicroAccuracy:0.###}");
Console.WriteLine($"* MacroAccuracy: {testMetrics.MacroAccuracy:0.###}");
Console.WriteLine($"* LogLoss: {testMetrics.LogLoss:#.###}");
Console.WriteLine($"* LogLossReduction: {testMetrics.LogLossReduction:#.###}");
Console.WriteLine($"*************************************************************************************************************");
SaveModelAsFile(_mlContext, trainingDataViewSchema, transformedData, _trainedModel, _keyToValueModel);
}
public static void PredictIssue_FirstLoadModelFromDisk()
{
//ITransformer loadedModel = _mlContext.Model.Load(_modelPath, out var modelInputSchema);
ITransformer loadedModel = _mlContext.Model.Load(_keyToValueModelPath, out var modelInputSchema);
foreach (var testIssue in testDatas)
{
_predEngine = _mlContext.Model.CreatePredictionEngine(loadedModel);
var prediction = _predEngine.Predict(testIssue);
if (prediction.Area.ToString() != testIssue.Area.ToString())
Console.ForegroundColor = ConsoleColor.Red;
else
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine($"=============== Single Prediction - Result: {prediction.Area}/{testIssue.Area} {testIssue.Title} ===============");
Console.ResetColor();
}
}
private static void SaveModelAsFile(MLContext mlContext, DataViewSchema trainingDataViewSchema,
IDataView transformedData, ITransformer _trainedModel, ITransformer _keyToValueModel)
{
//mlContext.Model.Save(_trainedModel, trainingDataViewSchema, _modelPath);
//mlContext.Model.Save(_keyToValueModel, transformedData.Schema, _keyToValueModelPath);
Console.WriteLine("The model is saved to {0}", _modelPath);
}
static void SecondLap(MLContext _mlContext)
{
var allData = _mlContext.Data.LoadFromTextFile(_mainDataPath2, hasHeader: true);
splittedData = _mlContext.Data.TrainTestSplit(allData, testFraction: 0.09);
_trainingDataView = splittedData.TrainSet;
ITransformer dataPrepPipeline = _mlContext.Model.Load(_modelPath, out var dataPrepPipelineSchema);
var originalModelParameters = (dataPrepPipeline as TransformerChain).LastTransformer as MulticlassPredictionTransformer;
int rowsCount = splittedData.TrainSet.Preview().RowView.Count();
//var transformedData = dataPrepPipeline.Transform(splittedData.TrainSet);
//var transformedData = _keyToValueModel.Transform(splittedData.TrainSet);
var transformedData = _trainedModel.Transform(splittedData.TrainSet);
ITransformer _keyToValueModel1 = _mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy("Label", "Features")
.Fit(transformedData, originalModelParameters.Model);
}
}
public class GitHubIssue
{
[LoadColumn(0)]
public string ID { get; set; }
[LoadColumn(1)]
public string Area { get; set; }
[LoadColumn(2)]
public string Title { get; set; }
[LoadColumn(3)]
public string Description { get; set; }
}
public class GitHubIssueTransformed: GitHubIssue
{
//[ColumnName("PredictedLabel")]
//public string XX;
}
public class IssuePrediction
{
//[ColumnName("PredictedLabel")]
public string Area;
}
```
Contributor guide
Assessment
This issue has not been assessed yet.