MLFDataDeserializer.cpp
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
//
#include "stdafx.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <limits>
#include "MLFDataDeserializer.h"
#include "ConfigHelper.h"
#include "../HTKMLFReader/htkfeatio.h"
#include "../HTKMLFReader/msra_mgram.h"
#include "latticearchive.h"
#undef max // max is defined in minwindef.h
namespace Microsoft { namespace MSR { namespace CNTK {
using namespace std;
static float s_oneFloat = 1.0;
static double s_oneDouble = 1.0;
// Currently we only have a single mlf chunk that contains a vector of all labels.
// TODO: In the future MLF should be converted to a more compact format that is amenable to chunking.
class MLFDataDeserializer::MLFChunk : public Chunk
{
MLFDataDeserializer* m_parent;
public:
MLFChunk(MLFDataDeserializer* parent) : m_parent(parent)
{}
virtual void GetSequence(size_t sequenceId, vector<SequenceDataPtr>& result) override
{
m_parent->GetSequenceById(sequenceId, result);
}
};
// Inner class for an utterance.
struct MLFUtterance : SequenceDescription
{
size_t m_sequenceStart;
};
MLFDataDeserializer::MLFDataDeserializer(CorpusDescriptorPtr corpus, const ConfigParameters& cfg, bool primary)
{
// TODO: This should be read in one place, potentially given by SGD.
m_frameMode = (ConfigValue)cfg("frameMode", "true");
// MLF cannot control chunking.
if (primary)
{
LogicError("Mlf deserializer does not support primary mode - it cannot control chunking.");
}
argvector<ConfigValue> inputs = cfg("input");
if (inputs.size() != 1)
{
LogicError("MLFDataDeserializer supports a single input stream only.");
}
ConfigParameters input = inputs.front();
auto inputName = input.GetMemberIds().front();
ConfigParameters streamConfig = input(inputName);
ConfigHelper config(streamConfig);
size_t dimension = config.GetLabelDimension();
wstring labelMappingFile = streamConfig(L"labelMappingFile", L"");
InitializeChunkDescriptions(corpus, config, labelMappingFile, dimension);
InitializeStream(inputName, dimension);
}
MLFDataDeserializer::MLFDataDeserializer(CorpusDescriptorPtr corpus, const ConfigParameters& labelConfig, const wstring& name)
{
// The frame mode is currently specified once per configuration,
// not in the configuration of a particular deserializer, but on a higher level in the configuration.
// Because of that we are using find method below.
m_frameMode = labelConfig.Find("frameMode", "true");
ConfigHelper config(labelConfig);
config.CheckLabelType();
size_t dimension = config.GetLabelDimension();
if (dimension > numeric_limits<IndexType>::max())
{
RuntimeError("Label dimension (%" PRIu64 ") exceeds the maximum allowed "
"value (%" PRIu64 ")\n", dimension, (size_t)numeric_limits<IndexType>::max());
}
wstring labelMappingFile = labelConfig(L"labelMappingFile", L"");
InitializeChunkDescriptions(corpus, config, labelMappingFile, dimension);
InitializeStream(name, dimension);
}
// Currently we create a single chunk only.
void MLFDataDeserializer::InitializeChunkDescriptions(CorpusDescriptorPtr corpus, const ConfigHelper& config, const wstring& stateListPath, size_t dimension)
{
// TODO: Similarly to the old reader, currently we assume all Mlfs will have same root name (key)
// restrict MLF reader to these files--will make stuff much faster without having to use shortened input files
// TODO: currently we do not use symbol and word tables.
const msra::lm::CSymbolSet* wordTable = nullptr;
unordered_map<const char*, int>* symbolTable = nullptr;
vector<wstring> mlfPaths = config.GetMlfPaths();
// TODO: Currently we still use the old IO module. This will be refactored later.
const double htkTimeToFrame = 100000.0; // default is 10ms
msra::asr::htkmlfreader<msra::asr::htkmlfentry, msra::lattices::lattice::htkmlfwordsequence> labels(mlfPaths, set<wstring>(), stateListPath, wordTable, symbolTable, htkTimeToFrame);
// Make sure 'msra::asr::htkmlfreader' type has a move constructor
static_assert(
is_move_constructible<
msra::asr::htkmlfreader<msra::asr::htkmlfentry,
msra::lattices::lattice::htkmlfwordsequence >> ::value,
"Type 'msra::asr::htkmlfreader' should be move constructible!");
m_elementType = config.GetElementType();
MLFUtterance description;
size_t numClasses = 0;
size_t totalFrames = 0;
const auto& stringRegistry = corpus->GetStringRegistry();
// TODO resize m_keyToSequence with number of IDs from string registry
for (const auto& l : labels)
{
// Currently the string registry contains only utterances described in scp.
// So here we skip all others.
size_t id = 0;
if (!stringRegistry.TryGet(msra::strfun::utf8(l.first), id))
continue;
description.m_key.m_sequence = id;
const auto& utterance = l.second;
description.m_sequenceStart = m_classIds.size();
uint32_t numberOfFrames = 0;
foreach_index(i, utterance)
{
const auto& timespan = utterance[i];
if ((i == 0 && timespan.firstframe != 0) ||
(i > 0 && utterance[i - 1].firstframe + utterance[i - 1].numframes != timespan.firstframe))
{
RuntimeError("Labels are not in the consecutive order MLF in label set: %ls", l.first.c_str());
}
if (timespan.classid >= dimension)
{
RuntimeError("Class id %d exceeds the model output dimension %d.", (int)timespan.classid, (int)dimension);
}
if (timespan.classid != static_cast<msra::dbn::CLASSIDTYPE>(timespan.classid))
{
RuntimeError("CLASSIDTYPE has too few bits");
}
if (SEQUENCELEN_MAX < timespan.firstframe + timespan.numframes)
{
RuntimeError("Maximum number of sample per sequence exceeded.");
}
numClasses = max(numClasses, (size_t)(1u + timespan.classid));
for (size_t t = timespan.firstframe; t < timespan.firstframe + timespan.numframes; t++)
{
m_classIds.push_back(timespan.classid);
numberOfFrames++;
}
}
description.m_numberOfSamples = numberOfFrames;
m_utteranceIndex.push_back(totalFrames);
totalFrames += numberOfFrames;
if (m_keyToSequence.size() <= description.m_key.m_sequence)
{
m_keyToSequence.resize(description.m_key.m_sequence + 1, SIZE_MAX);
}
assert(m_keyToSequence[description.m_key.m_sequence] == SIZE_MAX);
m_keyToSequence[description.m_key.m_sequence] = m_utteranceIndex.size() - 1;
m_numberOfSequences++;
}
m_utteranceIndex.push_back(totalFrames);
m_totalNumberOfFrames = totalFrames;
fprintf(stderr, "MLFDataDeserializer::MLFDataDeserializer: %" PRIu64 " utterances with %" PRIu64 " frames in %" PRIu64 " classes\n",
m_numberOfSequences,
m_totalNumberOfFrames,
numClasses);
// Initializing array of labels.
m_categories.reserve(dimension);
m_categoryIndices.reserve(dimension);
for (size_t i = 0; i < dimension; ++i)
{
SparseSequenceDataPtr category = make_shared<SparseSequenceData>();
m_categoryIndices.push_back(static_cast<IndexType>(i));
category->m_indices = &(m_categoryIndices[i]);
category->m_nnzCounts.resize(1);
category->m_nnzCounts[0] = 1;
category->m_totalNnzCount = 1;
category->m_numberOfSamples = 1;
if (m_elementType == ElementType::tfloat)
{
category->m_data = &s_oneFloat;
}
else
{
assert(m_elementType == ElementType::tdouble);
category->m_data = &s_oneDouble;
}
m_categories.push_back(category);
}
}
void MLFDataDeserializer::InitializeStream(const wstring& name, size_t dimension)
{
// Initializing stream description - a single stream of MLF data.
StreamDescriptionPtr stream = make_shared<StreamDescription>();
stream->m_id = 0;
stream->m_name = name;
stream->m_sampleLayout = make_shared<TensorShape>(dimension);
stream->m_storageType = StorageType::sparse_csc;
stream->m_elementType = m_elementType;
m_streams.push_back(stream);
}
void InitializeFeatureInformation();
void InitializeAugmentationWindow(ConfigHelper& config);
// Currently MLF has a single chunk.
// TODO: This will be changed when the deserializer properly supports chunking.
ChunkDescriptions MLFDataDeserializer::GetChunkDescriptions()
{
auto cd = make_shared<ChunkDescription>();
cd->m_id = 0;
cd->m_numberOfSequences = m_frameMode ? m_totalNumberOfFrames : m_numberOfSequences;
cd->m_numberOfSamples = m_totalNumberOfFrames;
return ChunkDescriptions{cd};
}
// Gets sequences for a particular chunk.
void MLFDataDeserializer::GetSequencesForChunk(ChunkIdType, vector<SequenceDescription>& result)
{
UNUSED(result);
LogicError("Mlf deserializer does not support primary mode - it cannot control chunking.");
}
ChunkPtr MLFDataDeserializer::GetChunk(ChunkIdType chunkId)
{
UNUSED(chunkId);
assert(chunkId == 0);
return make_shared<MLFChunk>(this);
};
// Sparse labels for an utterance.
template <class ElemType>
struct MLFSequenceData : SparseSequenceData
{
vector<ElemType> m_values;
unique_ptr<IndexType[]> m_indicesPtr;
MLFSequenceData(size_t numberOfSamples) :
m_values(numberOfSamples, 1),
m_indicesPtr(new IndexType[numberOfSamples])
{
if (numberOfSamples > numeric_limits<IndexType>::max())
{
RuntimeError("Number of samples in an MLFSequence (%" PRIu64 ") "
"exceeds the maximum allowed value (%" PRIu64 ")\n",
numberOfSamples, (size_t)numeric_limits<IndexType>::max());
}
m_nnzCounts.resize(numberOfSamples, static_cast<IndexType>(1));
m_numberOfSamples = (uint32_t) numberOfSamples;
m_totalNnzCount = static_cast<IndexType>(numberOfSamples);
m_indices = m_indicesPtr.get();
m_data = m_values.data();
}
};
void MLFDataDeserializer::GetSequenceById(size_t sequenceId, vector<SequenceDataPtr>& result)
{
if (m_frameMode)
{
size_t label = m_classIds[sequenceId];
assert(label < m_categories.size());
result.push_back(m_categories[label]);
}
else
{
// Packing labels for the utterance into sparse sequence.
size_t startFrameIndex = m_utteranceIndex[sequenceId];
size_t numberOfSamples = m_utteranceIndex[sequenceId + 1] - startFrameIndex;
SparseSequenceDataPtr s;
if (m_elementType == ElementType::tfloat)
{
s = make_shared<MLFSequenceData<float>>(numberOfSamples);
}
else
{
assert(m_elementType == ElementType::tdouble);
s = make_shared<MLFSequenceData<double>>(numberOfSamples);
}
for (size_t i = 0; i < numberOfSamples; i++)
{
size_t frameIndex = startFrameIndex + i;
size_t label = m_classIds[frameIndex];
s->m_indices[i] = static_cast<IndexType>(label);
}
result.push_back(s);
}
}
bool MLFDataDeserializer::GetSequenceDescriptionByKey(const KeyType& key, SequenceDescription& result)
{
auto sequenceId = key.m_sequence < m_keyToSequence.size() ? m_keyToSequence[key.m_sequence] : SIZE_MAX;
if (sequenceId == SIZE_MAX)
{
return false;
}
result.m_chunkId = 0;
result.m_key = key;
if (m_frameMode)
{
size_t index = m_utteranceIndex[sequenceId] + key.m_sample;
result.m_id = index;
result.m_numberOfSamples = 1;
}
else
{
assert(result.m_key.m_sample == 0);
result.m_id = sequenceId;
result.m_numberOfSamples = (uint32_t) (m_utteranceIndex[sequenceId + 1] - m_utteranceIndex[sequenceId]);
}
return true;
}
}}}