Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update test data #2356

Merged
merged 8 commits into from
Jan 10, 2020
Merged

Update test data #2356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
db84058
Update test data
snnn Nov 8, 2019
d5453c0
update
snnn Jan 8, 2020
2cd5bbd
a
snnn Jan 9, 2020
e2ef558
update
snnn Jan 9, 2020
6737b2f
update
snnn Jan 10, 2020
63d03df
update
snnn Jan 10, 2020
feb6785
update
snnn Jan 10, 2020
a1ec95c
Merge remote-tracking branch 'origin/master' into snnn/20191107.zip
snnn Jan 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 68 additions & 65 deletions csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,8 @@ private static Dictionary<string, string> GetSkippedModels()
{ "mlperf_ssd_mobilenet_300", "Could not find file output_0.pb" },
{ "tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied" },
{ "tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied" },
{ "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" }
{ "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" },
{ "mask_rcnn_keras", "Model should be edited to remove the extra outputs" },
};

// The following models fails on nocontribops win CI
Expand Down Expand Up @@ -437,84 +438,86 @@ private void TestPreTrainedModels(string opset, string modelName)
{
testDataDirNamePattern = "seq_lens*"; // discrepency in data directory
}
var testDataDir = modelDir.EnumerateDirectories(testDataDirNamePattern).First();
var inputContainer = new List<NamedOnnxValue>();
var outputContainer = new List<NamedOnnxValue>();
foreach (var f in testDataDir.EnumerateFiles("input_*.pb"))
{
inputContainer.Add(LoadTensorFromFilePb(f.FullName, inMeta));
}
foreach (var f in testDataDir.EnumerateFiles("output_*.pb"))
foreach (var testDataDir in modelDir.EnumerateDirectories(testDataDirNamePattern))
{
outputContainer.Add(LoadTensorFromFilePb(f.FullName, session.OutputMetadata));
}
var inputContainer = new List<NamedOnnxValue>();
var outputContainer = new List<NamedOnnxValue>();
foreach (var f in testDataDir.EnumerateFiles("input_*.pb"))
{
inputContainer.Add(LoadTensorFromFilePb(f.FullName, inMeta));
}
foreach (var f in testDataDir.EnumerateFiles("output_*.pb"))
{
outputContainer.Add(LoadTensorFromFilePb(f.FullName, session.OutputMetadata));
}

using (var resultCollection = session.Run(inputContainer))
{
foreach (var result in resultCollection)
using (var resultCollection = session.Run(inputContainer))
{
Assert.True(session.OutputMetadata.ContainsKey(result.Name));
var outputMeta = session.OutputMetadata[result.Name];
NamedOnnxValue outputValue = null;
foreach (var o in outputContainer)
{
if (o.Name == result.Name)
{
outputValue = o;
break;
}
}
if (outputValue == null)
{
outputValue = outputContainer.First(); // in case the output data file does not contain the name
}
if (outputMeta.IsTensor)
foreach (var result in resultCollection)
{
if (outputMeta.ElementType == typeof(float))
{
Assert.Equal(result.AsTensor<float>(), outputValue.AsTensor<float>(), new floatComparer());
}
else if (outputMeta.ElementType == typeof(int))
{
Assert.Equal(result.AsTensor<int>(), outputValue.AsTensor<int>(), new ExactComparer<int>());
}
else if (outputMeta.ElementType == typeof(uint))
{
Assert.Equal(result.AsTensor<uint>(), outputValue.AsTensor<uint>(), new ExactComparer<uint>());
}
else if (outputMeta.ElementType == typeof(short))
Assert.True(session.OutputMetadata.ContainsKey(result.Name));
var outputMeta = session.OutputMetadata[result.Name];
NamedOnnxValue outputValue = null;
foreach (var o in outputContainer)
{
Assert.Equal(result.AsTensor<short>(), outputValue.AsTensor<short>(), new ExactComparer<short>());
if (o.Name == result.Name)
{
outputValue = o;
break;
}
}
else if (outputMeta.ElementType == typeof(ushort))
if (outputValue == null)
{
Assert.Equal(result.AsTensor<ushort>(), outputValue.AsTensor<ushort>(), new ExactComparer<ushort>());
outputValue = outputContainer.First(); // in case the output data file does not contain the name
}
else if (outputMeta.ElementType == typeof(long))
if (outputMeta.IsTensor)
{
Assert.Equal(result.AsTensor<long>(), outputValue.AsTensor<long>(), new ExactComparer<long>());
}
else if (outputMeta.ElementType == typeof(ulong))
{
Assert.Equal(result.AsTensor<ulong>(), outputValue.AsTensor<ulong>(), new ExactComparer<ulong>());
}
else if (outputMeta.ElementType == typeof(byte))
{
Assert.Equal(result.AsTensor<byte>(), outputValue.AsTensor<byte>(), new ExactComparer<byte>());
}
else if (outputMeta.ElementType == typeof(bool))
{
Assert.Equal(result.AsTensor<bool>(), outputValue.AsTensor<bool>(), new ExactComparer<bool>());
if (outputMeta.ElementType == typeof(float))
{
Assert.Equal(result.AsTensor<float>(), outputValue.AsTensor<float>(), new floatComparer());
}
else if (outputMeta.ElementType == typeof(int))
{
Assert.Equal(result.AsTensor<int>(), outputValue.AsTensor<int>(), new ExactComparer<int>());
}
else if (outputMeta.ElementType == typeof(uint))
{
Assert.Equal(result.AsTensor<uint>(), outputValue.AsTensor<uint>(), new ExactComparer<uint>());
}
else if (outputMeta.ElementType == typeof(short))
{
Assert.Equal(result.AsTensor<short>(), outputValue.AsTensor<short>(), new ExactComparer<short>());
}
else if (outputMeta.ElementType == typeof(ushort))
{
Assert.Equal(result.AsTensor<ushort>(), outputValue.AsTensor<ushort>(), new ExactComparer<ushort>());
}
else if (outputMeta.ElementType == typeof(long))
{
Assert.Equal(result.AsTensor<long>(), outputValue.AsTensor<long>(), new ExactComparer<long>());
}
else if (outputMeta.ElementType == typeof(ulong))
{
Assert.Equal(result.AsTensor<ulong>(), outputValue.AsTensor<ulong>(), new ExactComparer<ulong>());
}
else if (outputMeta.ElementType == typeof(byte))
{
Assert.Equal(result.AsTensor<byte>(), outputValue.AsTensor<byte>(), new ExactComparer<byte>());
}
else if (outputMeta.ElementType == typeof(bool))
{
Assert.Equal(result.AsTensor<bool>(), outputValue.AsTensor<bool>(), new ExactComparer<bool>());
}
else
{
Assert.True(false, "The TestPretrainedModels does not yet support output of type " + nameof(outputMeta.ElementType));
}
}
else
{
Assert.True(false, "The TestPretrainedModels does not yet support output of type " + nameof(outputMeta.ElementType));
Assert.True(false, "TestPretrainedModel cannot handle non-tensor outputs yet");
}
}
else
{
Assert.True(false, "TestPretrainedModel cannot handle non-tensor outputs yet");
}
}
}
}
Expand Down
Loading