https://github.com/Microsoft/CNTK
Tip revision: 16a41cef30894ca92667bd93079cd6fa11b3e92d authored by Sayan Pathak on 02 November 2017, 16:10:10 UTC
Added super resolution tutorial contributed by Borna with added code to minimize test downloads, fix tests, added documentation and small editorial changes to LSGAN tutorial
Added super resolution tutorial contributed by Borna with added code to minimize test downloads, fix tests, added documentation and small editorial changes to LSGAN tutorial
Tip revision: 16a41ce
Program.cs
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
//
// Program.cs -- main C# file that contains client code to call the CLI Wrapper class.
//
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Threading;
using System.Threading.Tasks;
using CNTKImageProcessing;
namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
{
/// <summary>
/// Program for demonstrating how to run model evaluations using the CLIWrapper
/// </summary>
/// <description>
/// This program is a managed client using the CLIWrapper to run the model evaluator in CNTK.
/// There are four cases shown in this program related to model loading, network creation and evaluation.
///
/// To run this program from the CNTK binary drop, you must add the NuGet package for model evaluation first.
/// Refer to <see cref="https://docs.microsoft.com/en-us/cognitive-toolkit/NuGet-Package"/> for information regarding the NuGet package for model evaluation.
///
/// EvaluateModelSingleLayer and EvaluateModelMultipleLayers
/// --------------------------------------------------------
/// These two cases require the 01_OneHidden model which is part of the <CNTK>/Examples/Image/GettingStarted example.
/// Refer to <see cref="https://github.com/Microsoft/CNTK/blob/master/Examples/Image/GettingStarted/README.md"/> for how to train
/// the model used in these examples.
///
/// EvaluateNetworkSingleLayer and EvaluateNetworkSingleLayerNoInput
/// ----------------------------------------------------------------
/// These two cases do not required a trained model (just the network description). These cases show how to extract values from a single forward-pass
/// without any input to the model.
///
/// EvaluateMultipleModels
/// ----------------------
/// This case requires the 02_Convolution model and the Test-28x28_cntk_text.txt test file which are part of the <CNTK>/Examples/Image/GettingStarted example.
/// Refer to <see cref="https://github.com/Microsoft/CNTK/blob/master/Examples/Image/GettingStarted/README.md"/> for how to train
/// the model used in this example.
///
/// EvaluateImageClassificationModel
/// -----------------------
/// This case requires the ResNet_18 trained model which can be downloaded from <see cref="https://www.cntk.ai/resnet/ResNet_18.model"/>.
/// This case shows how to evaluate a model that was trained with the ImageReader.
/// The input for evaluation needs to be transformed in a similar manner as the ImageReader did during training.
///
/// </description>
class Program
{
private static string initialDirectory;
/// The location of the Resnet model file that is required for the image API tests.
private static string resnetModelFilePath;
/// The location of the test image that is using in the image API tests.
private static string imageFileName;
// The width and height of the images that go into ResNet.
private static int resNetImageSize = 224;
/// <summary>
/// Program entry point
/// </summary>
/// <param name="args">Program arguments (ignored)</param>
private static void Main(string[] args)
{
initialDirectory = Environment.CurrentDirectory;
Console.WriteLine("====== EvaluateModelSingleLayer ========");
EvaluateModelSingleLayer();
Console.WriteLine("\n====== EvaluateModelMultipleLayers ========");
EvaluateModelMultipleLayers();
Console.WriteLine("\n====== EvaluateExtendedNetworkSingleLayerNoInput ========");
EvaluateExtendedNetworkSingleLayerNoInput();
Console.WriteLine("\n====== EvaluateMultipleModels ========");
EvaluateMultipleModels();
// The image tests require the Resnet model.
// The model can be downloaded from <see cref="https://www.cntk.ai/resnet/ResNet_18.model"/>
// The model is assumed to be located at: <CNTK>\Examples\Image\Classification\ResNet
// along with a sample image file named "zebra.jpg".
var resnetDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\Classification\ResNet");
resnetModelFilePath = Path.Combine(resnetDirectory, "ResNet_18.model");
ThrowIfFileNotExist(resnetModelFilePath,
string.Format("Error: The model '{0}' does not exist. Please download the model from https://www.cntk.ai/resnet/ResNet_18.model and save it under ..\\..\\Examples\\Image\\Classification\\ResNet.", resnetModelFilePath));
imageFileName = Path.Combine(resnetDirectory, "zebra.jpg");
ThrowIfFileNotExist(imageFileName, string.Format("Error: The test image file '{0}' does not exist.", imageFileName));
Console.WriteLine("\n====== EvaluateImageInputUsingFeatureVector ========");
var outputs1 = EvaluateImageInputUsingFeatureVector();
Console.WriteLine("\n====== EvaluateImageInputUsingImageApi ========");
var outputs2 = EvaluateImageInputUsingImageApi();
Console.WriteLine("\n====== CompareImageApiResults ========");
CompareImageApiResults(outputs1, outputs2);
// This pattern is used by End2EndTests to check whether the program runs to complete.
Console.WriteLine("\n====== Evaluation Complete ========");
}
private static void CompareImageApiResults(List<float> outputs1,List<float> outputs2)
{
if (outputs1.Count != outputs2.Count)
{
throw new Exception("Both APIs must return the same number of output values.");
}
foreach (var i in Enumerable.Range(0, outputs1.Count))
{
if (Math.Abs(outputs1[i] - outputs2[i]) > 1e-5f)
{
throw new Exception(String.Format("Output value mismatch at position {0}", i));
}
}
Console.WriteLine("Both image API calls returned the same output vector.");
}
/// <summary>
/// Checks whether the file exists. If not, write the error message on the console and throw FileNotFoundException.
/// </summary>
/// <param name="filePath">The file to check.</param>
/// <param name="errorMsg">The message to write on console if the file does not exist.</param>
private static void ThrowIfFileNotExist(string filePath, string errorMsg)
{
if (!File.Exists(filePath))
{
if (!string.IsNullOrEmpty(errorMsg))
{
Console.WriteLine(errorMsg);
}
throw new FileNotFoundException(string.Format("File '{0}' not found.", filePath));
}
}
/// <summary>
/// Handle CNTK exceptions.
/// </summary>
/// <param name="ex">The exception to be handled.</param>
private static void OnCNTKException(CNTKException ex)
{
// The pattern "Inner Exception" is used by End2EndTests to catch test failure.
Console.WriteLine("Error: {0}\nNative CallStack: {1}\n Inner Exception: {2}", ex.Message, ex.NativeCallStack, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
throw ex;
}
/// <summary>
/// Handle general exceptions.
/// </summary>
/// <param name="ex">The exception to be handled.</param>
private static void OnGeneralException(Exception ex)
{
// The pattern "Inner Exception" is used by End2EndTests to catch test failure.
Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
throw ex;
}
/// <summary>
/// Evaluates a trained model and obtains a single layer output
/// </summary>
/// <remarks>
/// This example requires the 01_OneHidden trained model
/// </remarks>
private static void EvaluateModelSingleLayer()
{
try
{
string outputLayerName;
// The examples assume the executable is running from the data folder
// We switch the current directory to the data folder (assuming the executable is in the <CNTK>/x64/Debug|Release folder
Environment.CurrentDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\GettingStarted");
List<float> outputs;
using (var model = new IEvaluateModelManagedF())
{
// Load model
string modelFilePath = Path.Combine(Environment.CurrentDirectory, @".\Output\Models\01_OneHidden");
ThrowIfFileNotExist(modelFilePath,
string.Format("Error: The model '{0}' does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/GettingStarted to create the model.", modelFilePath));
model.CreateNetwork(string.Format("modelPath=\"{0}\"", modelFilePath), deviceId: -1);
// Generate random input values in the appropriate structure and size
var inDims = model.GetNodeDimensions(NodeGroup.Input);
var inputs = GetDictionary(inDims.First().Key, inDims.First().Value, 255);
// We request the output layer names(s) and dimension, we'll use the first one.
var outDims = model.GetNodeDimensions(NodeGroup.Output);
outputLayerName = outDims.First().Key;
// We can call the evaluate method and get back the results (single layer)...
outputs = model.Evaluate(inputs, outputLayerName);
}
OutputResults(outputLayerName, outputs);
}
catch (CNTKException ex)
{
OnCNTKException(ex);
}
catch (Exception ex)
{
OnGeneralException(ex);
}
}
/// <summary>
/// Evaluates a trained model and obtains multiple layers output (including hidden layer)
/// </summary>
/// <remarks>
/// This example requires the 01_OneHidden trained model
/// </remarks>
private static void EvaluateModelMultipleLayers()
{
try
{
// The examples assume the executable is running from the data folder
// We switch the current directory to the data folder (assuming the executable is in the <CNTK>/x64/Debug|Release folder
Environment.CurrentDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\GettingStarted");
Dictionary<string, List<float>> outputs;
using (var model = new IEvaluateModelManagedF())
{
// Desired output layers
const string hiddenLayerName = "out.h1";
const string outputLayerName = "out.z";
// Load model
string modelFilePath = Path.Combine(Environment.CurrentDirectory, @".\Output\Models\01_OneHidden");
ThrowIfFileNotExist(modelFilePath,
string.Format("Error: The model '{0}' does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/GettingStarted to create the model.", modelFilePath));
var desiredOutputLayers = new List<string>() { hiddenLayerName, outputLayerName };
model.CreateNetwork(string.Format("modelPath=\"{0}\"", modelFilePath), deviceId: -1, outputNodeNames: desiredOutputLayers);
// Generate random input values in the appropriate structure and size
var inDims = model.GetNodeDimensions(NodeGroup.Input);
var inputs = GetDictionary(inDims.First().Key, inDims.First().Value, 255);
// We request the output layer names(s) and dimension, we'll get both the hidden layer and the output layer
var outDims = model.GetNodeDimensions(NodeGroup.Output);
// We can preallocate the output structure and pass it in (multiple output layers)
outputs = new Dictionary<string, List<float>>()
{
{ hiddenLayerName, GetFloatArray(outDims[hiddenLayerName], 1) },
{ outputLayerName, GetFloatArray(outDims[outputLayerName], 1) }
};
model.Evaluate(inputs, outputs);
}
OutputResults(outputs);
}
catch (CNTKException ex)
{
OnCNTKException(ex);
}
catch (Exception ex)
{
OnGeneralException(ex);
}
}
/// <summary>
/// Evaluates an extended network (without a model and without input) and obtains a single layer output
/// </summary>
private static void EvaluateExtendedNetworkSingleLayerNoInput()
{
const string modelDefinition = @"precision = ""float""
traceLevel = 1
run=NDLNetworkBuilder
NDLNetworkBuilder=[
v1 = Constant(1)
v2 = Constant(2, tag=""output"")
ol = Plus(v1, v2, tag=""output"")
FeatureNodes = (v1)
]";
try
{
using (var model = new ModelEvaluationExtendedF())
{
// Create the network
model.CreateNetwork(modelDefinition);
VariableSchema outputSchema = model.GetOutputSchema();
var outputNodeNames = outputSchema.Select(s => s.Name).ToList<string>();
model.StartForwardEvaluation(outputNodeNames);
var outputBuffer = outputSchema.CreateBuffers<float>();
var inputBuffer = new ValueBuffer<float>[0];
// We can call the evaluate method and get back the results...
model.ForwardPass(inputBuffer, outputBuffer);
// We expect two outputs: the v2 constant, and the ol Plus result
var expected = new float[][] { new float[] { 2 }, new float[] { 3 } };
Console.WriteLine("Expected values: {0}", string.Join(" - ", expected.Select(b => string.Join(", ", b)).ToList<string>()));
Console.WriteLine("Actual Values : {0}", string.Join(" - ", outputBuffer.Select(b => string.Join(", ", b.Buffer)).ToList<string>()));
}
}
catch (CNTKException ex)
{
OnCNTKException(ex);
}
catch (Exception ex)
{
OnGeneralException(ex);
}
}
/// <summary>
/// Evaluates multiple instances of a model in the same process.
/// </summary>
/// <remarks>
/// Although all models execute concurrently (multiple tasks), each model is evaluated with a single task at a time.
/// </remarks>
private static void EvaluateMultipleModels()
{
// Specifies the number of models in memory as well as the number of parallel tasks feeding these models (1 to 1)
int numConcurrentModels = 4;
// Specifies the number of times to iterate through the test file (epochs)
int numRounds = 1;
// Counts the number of evaluations across all models
int count = 0;
// Counts the number of failed evaluations (output != expected) across all models
int errorCount = 0;
// The examples assume the executable is running from the data folder
// We switch the current directory to the data folder (assuming the executable is in the <CNTK>/x64/Debug|Release folder
Environment.CurrentDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\GettingStarted");
// Load model
string modelFilePath = Path.Combine(Environment.CurrentDirectory, @".\Output\Models\02_OneConv");
ThrowIfFileNotExist(modelFilePath,
string.Format("Error: The model '{0}' does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/GettingStarted to create the model.", modelFilePath));
// Initializes the model instances
ModelEvaluator.Initialize(numConcurrentModels, modelFilePath);
string testfile = Path.Combine(Environment.CurrentDirectory, @"..\DataSets\MNIST\Test-28x28_cntk_text.txt");
ThrowIfFileNotExist(testfile,
string.Format("Error: The test file '{0}' does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/GettingStarted to download the data.", testfile));
Stopwatch sw = new Stopwatch();
sw.Start();
try
{
for (int i = 0; i < numRounds; i++)
{
// Feed each line to a single model in parallel
Parallel.ForEach(File.ReadLines(testfile), new ParallelOptions() { MaxDegreeOfParallelism = numConcurrentModels }, (line) =>
{
Interlocked.Increment(ref count);
// The file format correspond to the CNTK Text Format Reader format (https://docs.microsoft.com/en-us/cognitive-toolkit/Brainscript-CNTKTextFormat-Reader)
var sets = line.Split('|');
var labels = sets[1].Trim().Split(' ').Skip(1);
var features = sets[2].Trim().Split(' ').Skip(1);
// Retrieve the 1-hot vector with the label index
var expected = labels.Select(float.Parse).Select((v, index) => new { Value = v, Index = index })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
// Retrieve the features
var inputs = features.Select(float.Parse).ToList();
// We can call the evaluate method and get back the results (single layer)...
var outputs = ModelEvaluator.Evaluate(inputs);
// Retrieve the outcome index (so we can compare it with the expected index)
var max = outputs.Select((v, index) => new { Value = v, Index = index })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
// Count the errors
if (expected != max)
{
Interlocked.Increment(ref errorCount);
}
});
}
}
catch (CNTKException ex)
{
OnCNTKException(ex);
}
catch (Exception ex)
{
OnGeneralException(ex);
}
sw.Stop();
ModelEvaluator.DisposeAll();
Console.WriteLine("The file {0} was processed using {1} concurrent model(s) with an error rate of: {2:P2} ({3} error(s) out of {4} record(s)), and a throughput of {5:N2} records/sec", @"Test-28x28_cntk_text.txt",
numConcurrentModels, (float)errorCount / count, errorCount, count, (count + errorCount) * 1000.0 / sw.ElapsedMilliseconds);
}
/// <summary>
/// This method shows how to evaluate a trained image classification model, with
/// explicitly created feature vectors.
/// </summary>
public static List<float> EvaluateImageInputUsingFeatureVector()
{
List<float> outputs = null;
try
{
// This example requires the RestNet_18 model.
// The model can be downloaded from <see cref="https://www.cntk.ai/resnet/ResNet_18.model"/>
// The model is assumed to be located at: <CNTK>\Examples\Image\Classification\ResNet
// along with a sample image file named "zebra.jpg".
Environment.CurrentDirectory = initialDirectory;
using (var model = new IEvaluateModelManagedF())
{
model.CreateNetwork(string.Format("modelPath=\"{0}\"", resnetModelFilePath), deviceId: -1);
// Prepare input value in the appropriate structure and size
var inDims = model.GetNodeDimensions(NodeGroup.Input);
if (inDims.First().Value != resNetImageSize * resNetImageSize * 3)
{
throw new CNTKRuntimeException(string.Format("The input dimension for {0} is {1} which is not the expected size of {2}.", inDims.First(), inDims.First().Value, 224 * 224 * 3), string.Empty);
}
// Transform the image
Bitmap bmp = new Bitmap(Bitmap.FromFile(imageFileName));
var resized = bmp.Resize(resNetImageSize, resNetImageSize, true);
var resizedCHW = resized.ParallelExtractCHW();
var inputs = new Dictionary<string, List<float>>() { {inDims.First().Key, resizedCHW } };
// We can call the evaluate method and get back the results (single layer output)...
var outDims = model.GetNodeDimensions(NodeGroup.Output);
outputs = model.Evaluate(inputs, outDims.First().Key);
}
// Retrieve the outcome index (so we can compare it with the expected index)
var max = outputs.Select((value, index) => new { Value = value, Index = index })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
Console.WriteLine("EvaluateImageInputUsingFeatureVector: Outcome = {0}", max);
}
catch (CNTKException ex)
{
OnCNTKException(ex);
}
catch (Exception ex)
{
OnGeneralException(ex);
}
return outputs;
}
/// <summary>
/// This method shows how to evaluate a trained image classification model, where the
/// creation of the CNTK feature vector is happening in native code inside the EvalWrapper.
/// </summary>
public static List<float> EvaluateImageInputUsingImageApi()
{
List<float> outputs = null;
try
{
// This example requires the RestNet_18 model.
// The model can be downloaded from <see cref="https://www.cntk.ai/resnet/ResNet_18.model"/>
// The model is assumed to be located at: <CNTK>\Examples\Image\Classification\ResNet
// along with a sample image file named "zebra.jpg".
Environment.CurrentDirectory = initialDirectory;
using (var model = new IEvaluateModelManagedF())
{
model.CreateNetwork(string.Format("modelPath=\"{0}\"", resnetModelFilePath), deviceId: -1);
// Prepare input value in the appropriate structure and size
var inDims = model.GetNodeDimensions(NodeGroup.Input);
if (inDims.First().Value != resNetImageSize * resNetImageSize * 3)
{
throw new CNTKRuntimeException(string.Format("The input dimension for {0} is {1} which is not the expected size of {2}.", inDims.First(), inDims.First().Value, 224 * 224 * 3), string.Empty);
}
// Transform the image
Bitmap bmp = new Bitmap(Bitmap.FromFile(imageFileName));
var resized = bmp.Resize(resNetImageSize, resNetImageSize, true);
// Now evaluate using the alternative API, where we directly pass the
// native bitmap data to the unmanaged code.
var outDims = model.GetNodeDimensions(NodeGroup.Output);
var outputNodeName = outDims.First().Key;
outputs = model.EvaluateRgbImage(resized, outputNodeName);
}
// Retrieve the outcome index (so we can compare it with the expected index)
var max = outputs.Select((value, index) => new { Value = value, Index = index })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
Console.WriteLine("EvaluateImageInputUsingImageApi: Outcome = {0}", max);
}
catch (CNTKException ex)
{
OnCNTKException(ex);
}
catch (Exception ex)
{
OnGeneralException(ex);
}
return outputs;
}
/// <summary>
/// Dumps the output to the console
/// </summary>
/// <param name="outputs">The structure containing the output layers</param>
private static void OutputResults(Dictionary<string, List<float>> outputs)
{
Console.WriteLine("--- Output results ---");
foreach (var item in outputs)
{
OutputResults(item.Key, item.Value);
}
}
/// <summary>
/// Dumps the output of a layer to the console
/// </summary>
/// <param name="layer">The display name for the layer</param>
/// <param name="values">The layer values</param>
private static void OutputResults(string layer, List<float> values)
{
if (values == null)
{
Console.WriteLine("No Output for layer: {0}", layer);
return;
}
Console.WriteLine("Output layer: {0}", layer);
foreach (var entry in values)
{
Console.WriteLine(entry);
}
}
/// <summary>
/// Creates a Dictionary for input entries or output allocation
/// </summary>
/// <param name="key">The key for the mapping</param>
/// <param name="size">The number of element entries associated to the key</param>
/// <param name="maxValue">The maximum value for random generation values</param>
/// <returns>A dictionary with a single entry for the key/values</returns>
static Dictionary<string, List<float>> GetDictionary(string key, int size, int maxValue)
{
var dict = new Dictionary<string, List<float>>();
if (key != string.Empty && size >= 0 && maxValue > 0)
{
dict.Add(key, GetFloatArray(size, maxValue));
}
return dict;
}
/// <summary>
/// Creats a list of random numbers
/// </summary>
/// <param name="size">The size of the list</param>
/// <param name="maxValue">The maximum value for the generated values</param>
/// <returns>A list of random numbers</returns>
static List<float> GetFloatArray(int size, int maxValue)
{
List<float> list = new List<float>();
if (size > 0 && maxValue >= 0)
{
Random rnd = new Random();
list.AddRange(Enumerable.Range(1, size).Select(i => (float)rnd.Next(maxValue)).ToList());
}
return list;
}
}
}