dotnet / dotnet/machinelearning

Unseen labels during retraining map to value "0", results in System.InvalidOperationException: 'No valid training instances found, all instances have missing features.'

Open
#5,214 14 comments 0 reactions 0 assignees View on GitHub
area-Trainers enhancement Priority:2
Dominant language
C#
Stars
9.4k
Forks
2k
Avg merge
2d 20h
Merged PRs (30d)
11

Description

### System information

- Windows Server 2016
- .NET Version 3.1.300

### Issue
I'm trying to do MultiClass LbfgsMaximumEntropy Re-training
When trying to Fit new data, I get System.InvalidOperationException: 'No valid training instances found, all instances have missing features.' on row
ITransformer _keyToValueModel1 = _mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy("Label", "Features")
.Fit(transformedData, originalModelParameters.Model);
[data1.zip](https://github.com/dotnet/machinelearning/files/4744624/data1.zip)

I would appreciate either help or MultiClass LbfgsMaximumEntropy Re-training code sample.

### Source code / logs

```c#
[data1.zip](https://github.com/dotnet/machinelearning/files/4744619/data1.zip)
[data2.zip](https://github.com/dotnet/machinelearning/files/4744620/data2.zip)

public class GitHubIssueClassification
{
static List testDatas = new List()
{
new GitHubIssueTransformed() {Area="11", Title="WHIRLPOOL AWE 50610", Description="" },
new GitHubIssueTransformed() {Area="14", Title="FAGOR 4CC-130 E X", Description="" },
new GitHubIssueTransformed() {Area="19", Title="AEG T8DFE68SC", Description="" },
new GitHubIssueTransformed() {Area="999", Title="TEST 999", Description="" }
};

private static string _appPath => Path.GetDirectoryName(Environment.GetCommandLineArgs()[0]);
private static string _mainDataPath1 => Path.Combine(_appPath, "..", "..", "..", "Data", "data1.csv");
private static string _mainDataPath2 => Path.Combine(_appPath, "..", "..", "..", "Data", "data2.csv");
private static string _mainDataPath3 => Path.Combine(_appPath, "..", "..", "..", "Data", "data3.csv");
private static string _modelPath => Path.Combine(_appPath, "..", "..", "..", "Models", "trainedModel.zip");
private static string _keyToValueModelPath => Path.Combine(_appPath, "..", "..", "..", "Models", "keyToValueModel.zip");

static DataOperationsCatalog.TrainTestData splittedData;

private static MLContext _mlContext;
private static PredictionEngine _predEngine;
private static ITransformer _trainedModel { get; set; }
private static ITransformer _keyToValueModel { get; set; }
static IDataView _trainingDataView;
public static void Run()
{
_mlContext = new MLContext(seed: 0);

var allData = _mlContext.Data.LoadFromTextFile(_mainDataPath1, hasHeader: true);
splittedData = _mlContext.Data.TrainTestSplit(allData, testFraction: 0.09);
_trainingDataView = splittedData.TrainSet;
Console.WriteLine($"=============== Finished Loading Dataset ===============");

var pipeline = ProcessData();
var transformedData = BuildAndTrainModel(_trainingDataView, pipeline);
Evaluate(_trainingDataView.Schema, transformedData, splittedData.TestSet);
PredictIssue_FirstLoadModelFromDisk();

SecondLap(_mlContext);
}

public static IEstimator ProcessData()
{
Console.WriteLine($"=============== Processing Data ===============");
var pipeline = _mlContext.Transforms.Conversion.MapValueToKey(inputColumnName: "Area", outputColumnName: "Label")
.Append(_mlContext.Transforms.Text.FeaturizeText(inputColumnName: "Title", outputColumnName: "TitleFeaturized"))
.Append(_mlContext.Transforms.Text.FeaturizeText(inputColumnName: "Description", outputColumnName: "DescriptionFeaturized"))
.Append(_mlContext.Transforms.Concatenate("Features", "TitleFeaturized", "DescriptionFeaturized"))
.AppendCacheCheckpoint(_mlContext);

Console.WriteLine($"=============== Finished Processing Data ===============");

return pipeline;
}

public static IDataView BuildAndTrainModel(IDataView trainingDataView, IEstimator pipeline)
{
//var trainingPipeline = pipeline.Append(_mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy/*.SdcaMaximumEntropy*/("Label", "Features"))
// .Append(_mlContext.Transforms.Conversion.MapKeyToValue("PredictedLabel"));
var trainingPipeline = pipeline.Append(_mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy/*.SdcaMaximumEntropy*/("Label", "Features"));
//var keyToValuePipeline = _mlContext.Transforms.Conversion.MapKeyToValue("PredictedLabel");
var keyToValuePipeline = trainingPipeline.Append(_mlContext.Transforms.Conversion.MapKeyToValue("Area", "PredictedLabel"));

Console.WriteLine($"=============== Training the model ===============");

_trainedModel = trainingPipeline.Fit(trainingDataView);
var transformedData = _trainedModel.Transform(trainingDataView);
_keyToValueModel = keyToValuePipeline.Fit(transformedData);

_mlContext.Model.Save(_trainedModel, trainingDataView.Schema, _modelPath);
_mlContext.Model.Save(_keyToValueModel, transformedData.Schema, _keyToValueModelPath);

Console.WriteLine($"=============== Finished Training the model Ending time: {DateTime.Now.ToString()} ===============");

// (OPTIONAL) Try/test a single prediction with the "just-trained model" (Before saving the model)
Console.WriteLine($"=============== Single Prediction just-trained-model ===============");

_predEngine = _mlContext.Model.CreatePredictionEngine(_keyToValueModel);
foreach (var testIssue in testDatas)
{
var prediction = _predEngine.Predict(testIssue);
if (prediction.Area.ToString() != testIssue.Area.ToString())
Console.ForegroundColor = ConsoleColor.Red;
else
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine($"=============== Single Prediction just-trained-model - Result: {prediction.Area}/{testIssue.Area} {testIssue.Title} ===============");
}
Console.ResetColor();

return transformedData;
}

public static void Evaluate(DataViewSchema trainingDataViewSchema, IDataView transformedData, IDataView testDataView2 = null)
{
// STEP 5: Evaluate the model in order to get the model's accuracy metrics
Console.WriteLine($"=============== Evaluating to get model's accuracy metrics - Starting time: {DateTime.Now.ToString()} ===============");

IDataView testDataView = testDataView2;

var testMetrics = _mlContext.MulticlassClassification.Evaluate(_trainedModel.Transform(testDataView));

Console.WriteLine($"=============== Evaluating to get model's accuracy metrics - Ending time: {DateTime.Now.ToString()} ===============");
Console.WriteLine($"*************************************************************************************************************");
Console.WriteLine($"* Metrics for Multi-class Classification model - Test Data ");
Console.WriteLine($"*------------------------------------------------------------------------------------------------------------");
Console.WriteLine($"* MicroAccuracy: {testMetrics.MicroAccuracy:0.###}");
Console.WriteLine($"* MacroAccuracy: {testMetrics.MacroAccuracy:0.###}");
Console.WriteLine($"* LogLoss: {testMetrics.LogLoss:#.###}");
Console.WriteLine($"* LogLossReduction: {testMetrics.LogLossReduction:#.###}");
Console.WriteLine($"*************************************************************************************************************");

SaveModelAsFile(_mlContext, trainingDataViewSchema, transformedData, _trainedModel, _keyToValueModel);
}

public static void PredictIssue_FirstLoadModelFromDisk()
{
//ITransformer loadedModel = _mlContext.Model.Load(_modelPath, out var modelInputSchema);
ITransformer loadedModel = _mlContext.Model.Load(_keyToValueModelPath, out var modelInputSchema);

foreach (var testIssue in testDatas)
{
_predEngine = _mlContext.Model.CreatePredictionEngine(loadedModel);
var prediction = _predEngine.Predict(testIssue);
if (prediction.Area.ToString() != testIssue.Area.ToString())
Console.ForegroundColor = ConsoleColor.Red;
else
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine($"=============== Single Prediction - Result: {prediction.Area}/{testIssue.Area} {testIssue.Title} ===============");
Console.ResetColor();
}
}

private static void SaveModelAsFile(MLContext mlContext, DataViewSchema trainingDataViewSchema,
IDataView transformedData, ITransformer _trainedModel, ITransformer _keyToValueModel)
{
//mlContext.Model.Save(_trainedModel, trainingDataViewSchema, _modelPath);
//mlContext.Model.Save(_keyToValueModel, transformedData.Schema, _keyToValueModelPath);
Console.WriteLine("The model is saved to {0}", _modelPath);
}

static void SecondLap(MLContext _mlContext)
{
var allData = _mlContext.Data.LoadFromTextFile(_mainDataPath2, hasHeader: true);
splittedData = _mlContext.Data.TrainTestSplit(allData, testFraction: 0.09);
_trainingDataView = splittedData.TrainSet;

ITransformer dataPrepPipeline = _mlContext.Model.Load(_modelPath, out var dataPrepPipelineSchema);
var originalModelParameters = (dataPrepPipeline as TransformerChain).LastTransformer as MulticlassPredictionTransformer;

int rowsCount = splittedData.TrainSet.Preview().RowView.Count();
//var transformedData = dataPrepPipeline.Transform(splittedData.TrainSet);
//var transformedData = _keyToValueModel.Transform(splittedData.TrainSet);
var transformedData = _trainedModel.Transform(splittedData.TrainSet);

ITransformer _keyToValueModel1 = _mlContext.MulticlassClassification.Trainers.LbfgsMaximumEntropy("Label", "Features")
.Fit(transformedData, originalModelParameters.Model);
}
}
public class GitHubIssue
{
[LoadColumn(0)]
public string ID { get; set; }
[LoadColumn(1)]
public string Area { get; set; }
[LoadColumn(2)]
public string Title { get; set; }
[LoadColumn(3)]
public string Description { get; set; }
}

public class GitHubIssueTransformed: GitHubIssue
{
//[ColumnName("PredictedLabel")]
//public string XX;
}

public class IssuePrediction
{
//[ColumnName("PredictedLabel")]
public string Area;
}
```

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.