//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
//
// BinaryFile.cpp : Define the Binary File class
//
#include "stdafx.h"
#include "DataReader.h"
#include "BinaryReader.h"
#include <limits.h>
#include <stdint.h>
namespace Microsoft { namespace MSR { namespace CNTK {
// HIGH and LOW DWORD functions
DWORD HIDWORD(size_t size)
{
return size >> 32;
}
DWORD LODWORD(size_t size)
{
return size & 0xFFFFFFFF;
}
// BinaryFile Constructor
// fileName - file to read or create (if it doesn't exist)
// options - file options, (fileOptionsReadWrite and fileOptionsRead are accepted)
// size - size of the file to map, will expand/contract existing files to given size. zero means keep current size
BinaryFile::BinaryFile(std::wstring fileName, FileOptions options, size_t size)
{
SYSTEM_INFO sysInfo;
GetSystemInfo(&sysInfo);
m_viewAlignment = sysInfo.dwAllocationGranularity;
/* If file created, continue to map file. */
m_writeFile = options == fileOptionsReadWrite;
m_name = fileName;
m_maxViewSize = 0x10000000; // 256MB initial max size
m_hndFile = CreateFile(fileName.c_str(), m_writeFile ? (GENERIC_WRITE | GENERIC_READ) : GENERIC_READ,
FILE_SHARE_READ, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if (m_hndFile == INVALID_HANDLE_VALUE)
{
char message[256];
sprintf_s(message, "Unable to Open/Create file %ls, error %x", fileName.c_str(), GetLastError());
RuntimeError(message);
}
// code to detect type of file (network/local)
// std::wstring path;
// auto found = fileName.find_last_of(L"\\/");
// if (found == npos)
// path = fileName + L"\\";
// else
// path = fileName.substr(0, found);
// auto driveType = GetDriveType(path.c_str());
// get the actual size of the file
if (size == 0)
{
GetFileSizeEx(m_hndFile, (PLARGE_INTEGER) &size);
}
m_filePositionMax = size;
m_hndMapped =
CreateFileMapping(m_hndFile, NULL,
m_writeFile ? PAGE_READWRITE : PAGE_READONLY,
HIDWORD(size), LODWORD(size),
NULL);
if (m_hndMapped == NULL)
{
char message[256];
sprintf_s(message, "Unable to map file %ls, error 0x%x", fileName.c_str(), GetLastError());
RuntimeError(message);
}
m_mappedSize = size;
// if writing the file, the inital size of the file is zero
if (m_writeFile)
{
m_filePositionMax = 0;
}
}
// Destructor - destroy the BinaryFile
BinaryFile::~BinaryFile()
{
for (auto iter = m_views.begin(); iter != m_views.end();)
{
// the view
iter = ReleaseView(iter, true);
}
CloseHandle(m_hndMapped);
// if we are writing the file, truncate to actual size
if (m_writeFile)
{
SetFilePointerEx(m_hndFile, *(LARGE_INTEGER*) &m_filePositionMax, NULL, FILE_BEGIN);
SetEndOfFile(m_hndFile);
}
CloseHandle(m_hndFile);
}
void BinaryFile::SetFilePositionMax(size_t filePositionMax)
{
m_filePositionMax = filePositionMax;
if (m_filePositionMax > m_mappedSize)
{
char message[128];
sprintf_s(message, "Setting max position larger than mapped file size: %ld > %ld", m_filePositionMax, m_mappedSize);
RuntimeError(message);
}
}
// Mapped - Check to see if the current address is mapped
// filePosition - file position to search for
// size - [in]size of the mapped region
// [out] size of region that is actually mapped starting at filePosition
// returns - true if the address is already mapped completely
vector<ViewPosition>::iterator BinaryFile::Mapped(size_t filePosition, size_t& size)
{
auto iter = m_views.begin();
for (; iter != m_views.end(); ++iter)
{
if (filePosition >= iter->filePosition &&
filePosition + size < iter->filePosition + iter->size)
{
// this is safe to put in a size_t (unsigned type)
// because we know the answer will be positive (from the if above)
size = iter->filePosition + iter->size - filePosition;
break;
}
}
return iter;
}
// FindDataView - Find view that contains the pointer value
// data - data pointer to search for
// returns - iterator to the found view, or end() if not found
vector<ViewPosition>::iterator BinaryFile::FindDataView(void* data)
{
// search for the view we want
auto iter = m_views.begin();
for (; iter != m_views.end(); ++iter)
{
byte* viewBegin = (byte*) iter->view;
if (viewBegin <= data && viewBegin + iter->size > data)
break;
}
return iter;
}
// FindView - Find view by mapped pointer (pointer to beginning)
// view - pointer to view to search for
// returns - iterator to the found view, or end() if not found
vector<ViewPosition>::iterator BinaryFile::FindView(void* view)
{
auto iter = m_views.begin();
for (; iter != m_views.end(); ++iter)
{
if (iter->view == view)
break;
}
return iter;
}
// FindView - Find view by file Position
// filePosition - file position to search for
// returns - iterator to the found view, or end() if not found
vector<ViewPosition>::iterator BinaryFile::FindView(size_t filePosition)
{
auto iter = m_views.begin();
for (; iter != m_views.end(); ++iter)
{
if (iter->filePosition == filePosition)
break;
}
return iter;
}
// ReleaseView - release a view, and return the next iterator
// iter - iterator into the view array
// returns - next element in the view vector that is valid
vector<ViewPosition>::iterator BinaryFile::ReleaseView(vector<ViewPosition>::iterator iter, bool force)
{
if (!force && iter->refCount > 1)
{
iter->refCount--;
iter++;
}
else
{
if (m_writeFile)
FlushViewOfFile(iter->view, iter->size);
bool ret = UnmapViewOfFile(iter->view) != FALSE;
ret;
iter = m_views.erase(iter);
}
return iter;
}
// GetView - Get a view of the file
// filePosition - file position where the view starts
// size - size of the view (in bytes)
// returns - pointer to the view
void* BinaryFile::GetView(size_t filePosition, size_t size)
{
void* pBuf = MapViewOfFile(m_hndMapped, // handle to map object
m_writeFile ? FILE_MAP_WRITE : FILE_MAP_READ, // get correct permissions
HIDWORD(filePosition),
LODWORD(filePosition),
size);
if (pBuf == NULL)
{
char message[128];
sprintf_s(message, "Unable to map file %ls @ %lld, error %x", m_name.c_str(), filePosition, GetLastError());
RuntimeError(message);
}
m_views.push_back(ViewPosition(pBuf, filePosition, size));
// update file position max if neccesary
size_t filePositionEnd = filePosition + size;
if (m_filePositionMax < filePositionEnd)
m_filePositionMax = filePositionEnd;
return pBuf;
}
// ReleaseView - Release a view of the file
// view - view to release (must have been returned from GetView() previously
void BinaryFile::ReleaseView(void* view)
{
bool found = false;
for (auto iter = m_views.begin(); iter != m_views.end(); ++iter)
{
if (iter->view == view)
{
ReleaseView(iter);
found = true;
break;
}
}
// view not released, could not be found
assert(found);
}
// ReallocateView - Reallocate View
// view - pointer to the view
// size - size of the view
// returns - pointer to the reallocated view
// NOTE: on reallocate, the view address may change
void* BinaryFile::ReallocateView(void* view, size_t size)
{
auto viewPos = FindView(view);
if (viewPos != m_views.end())
{
size_t filePosition = viewPos->filePosition;
ReleaseView(viewPos);
view = GetView(filePosition, size);
}
else
{
// if we can't find the view, can't reallocate it, return NULL
assert(false);
view = NULL;
}
return view;
}
// EnsureViewSize - Ensure that a view is at least a particular size
// view - pointer to the view
// size - minimum size of the view
// returns - pointer to the view that is at least size bytes large
// NOTE: on reallocate, the view address may change
void* BinaryFile::EnsureViewSize(void* view, size_t size)
{
auto viewPos = FindView(view);
if (viewPos != m_views.end())
{
if (viewPos->size < size)
{
size_t filePosition = viewPos->filePosition;
ReleaseView(viewPos);
view = GetView(filePosition, size);
}
}
else
{
// if we can't find the view, can't reallocate it, return NULL
assert(false);
view = NULL;
}
return view;
}
// EnsureMapped - Ensure that a view is at least a particular size
// dataPointer - pointer to the data
// size - allocated size of the data
// returns - pointer to the data location that is at least size bytes large
// NOTE: on reallocate, the view address may change
void* BinaryFile::EnsureMapped(void* data, size_t size)
{
auto viewPos = FindDataView(data);
if (viewPos != m_views.end())
{
int64_t offset = (byte*) data - (byte*) viewPos->view;
int64_t dataEnd = offset + size;
// if our end of data is beyond the size of the view, need to reallocate
if (dataEnd > (int64_t) viewPos->size)
{
// TODO: this view change only accomidates this request
size_t filePosition = viewPos->filePosition;
ReleaseView(viewPos);
byte* view = (byte*) GetView(filePosition, dataEnd);
data = view + offset;
}
}
else
{
// if we can't find the view, can't reallocate it, return NULL
assert(false);
data = NULL;
}
return data;
}
// RoundUp - round the file position to the next mappable location if we intend on mapping the location separately
// filePosition - position in the file we want to round up
size_t BinaryFile::RoundUp(size_t filePosition)
{
size_t roundTo = m_viewAlignment;
return ((filePosition + roundTo - 1) / roundTo) * roundTo;
}
// SectionFile Constructor
// fileName - file to read or create (if it doesn't exist)
// options - file options, (fileOptionsReadWrite and fileOptionsRead are accepted)
// size - size of the file to map, will expand/contract existing files to given size. zero means keep current size
SectionFile::SectionFile(std::wstring fileName, FileOptions options, size_t size)
: BinaryFile(fileName, options, size)
{
m_fileSection = new Section(this, 0, NULL, mappingFile, sectionHeaderMin);
if (m_writeFile)
{
m_fileSection->InitHeader(sectionTypeFile, string("Binary Data File"), sectionDataNone, 0);
}
// check for a file header
if (!m_fileSection->ValidateHeader(m_writeFile))
{
char message[128];
sprintf_s(message, "Invalid File format for binary file %ls", fileName.c_str());
RuntimeError(message);
}
}
// SectionFile Destructor
SectionFile::~SectionFile()
{
// remove each section
SectionHeader* header = m_fileSection->GetHeader();
// finishing up the file, put the magic number on it now. (but only if
if (m_writeFile)
{
header->wMagic = magicFile;
}
// now delete the file section
delete m_fileSection;
m_fileSection = NULL;
// after this BinaryFile destructor will take care of cleaning up sections mappings, memory mapped file, etc.
}
void Section::InitHeader(SectionType type, std::string description, SectionData dataType, WORD elementSize)
{
memset(m_sectionHeader, 0, sectionHeaderMin);
m_sectionHeader->wMagic = type == sectionTypeFile ? magicIncomplete : magicSection; // magic number ACE9/4ACE
m_sectionHeader->version = 0x0100; // version number ##.## in hex
m_sectionHeader->sizeHeader = sectionHeaderMin; // size of this header (rounded up to nearest 64 byte boundary)
m_sectionHeader->dataSections = 0; // number of data sub-sections (for nesting)
m_sectionHeader->sectionType = (WORD) type; // what is the type of the data in this section
m_sectionHeader->sectionData = (WORD) dataType; // type of section (SectionData enum)
m_sectionHeader->bytesPerElement = elementSize; // number of bytes per element, (0 means variable)
m_sectionHeader->customStructureID = customStructureNone; // ID for custom structure
m_sectionHeader->elementsPerRecord = 0; // number of elements per Record
m_sectionHeader->flags = flagNone; // bit flags, dependent on sectionType
m_sectionHeader->elementsCount = 0; // number of total elements stored
memset(m_sectionHeader->nameDescription, 0, descriptionSize); // clear out the string buffer to all zeros first
strcpy_s(m_sectionHeader->nameDescription, description.c_str()); // name and description of section contents in this format (name: description) (string, with extra bytes zeroed out, at least one null terminator required)
m_sectionHeader->size = sectionHeaderMin; // size of this section (including header)
m_sectionHeader->sizeAll = sectionHeaderMin; // size of this section (including header and all sub-sections)
m_sectionHeader->sectionFilePosition[0] = 0; // sub-section file offsets (if needed), assumed to be in File Position order
return;
}
// ValidateHeader - make sure everything in the header is valid
bool Section::ValidateHeader(bool writing) const
{
bool valid = true;
valid &= (m_sectionHeader->wMagic == magicFile || m_sectionHeader->wMagic == magicSection || (writing && m_sectionHeader->wMagic == magicIncomplete)); // magic number ACE9/4ACE
valid &= (m_sectionHeader->version >= 0x0100); // version number ##.## in hex
valid &= (m_sectionHeader->sizeHeader >= sectionHeaderMin && m_sectionHeader->sizeHeader < sectionHeaderMin * 2); // size of this header (rounded up to nearest 64 byte boundary)
valid &= (m_sectionHeader->dataSections < (m_sectionHeader->sizeHeader - offset(SectionHeader, sectionFilePosition)) / sizeof(size_t)); // number of data sub-sections (for nesting)
valid &= (m_sectionHeader->sectionType < sectionTypeMax * 2); // what is the type of the data in this section
valid &= (m_sectionHeader->sectionData < sectionDataMax * 2); // type of section (SectionData enum)
// m_sectionHeader->bytesPerElement = elementSize; // number of bytes per element, (0 means variable)
valid &= (m_sectionHeader->customStructureID < customStructureMax * 2); // ID for custom structure
// m_sectionHeader->elementsPerRecord = 0; // number of elements per Record
valid &= (m_sectionHeader->flags < flagMax * 2); // bit flags, dependent on sectionType
// m_sectionHeader->elementsCount = 0; // number of total elements stored
// strcpy_s(m_sectionHeader->nameDescription, description.c_str()); // name and description of section contents in this format (name: description) (string, with extra bytes zeroed out, at least one null terminator required)
// the size of the section must at least accomidate the header and all it's elements
valid &= (m_sectionHeader->size >= m_sectionHeader->sizeHeader + m_sectionHeader->elementsCount * m_sectionHeader->bytesPerElement); // size of this section (including header and all sub-sections)
valid &= (m_sectionHeader->sizeAll >= m_sectionHeader->sizeHeader + m_sectionHeader->elementsCount * m_sectionHeader->bytesPerElement); // size of this section (including header and all sub-sections)
// There should be an integral number of records in the file (elementsCount/elementsPerRecord))
valid &= (m_sectionHeader->elementsPerRecord == 0 || m_sectionHeader->elementsCount % m_sectionHeader->elementsPerRecord == 0);
// file positions must be in order
size_t filePositionLast = 0;
for (int i = 0; i < m_sectionHeader->dataSections; ++i)
{
valid &= (m_sectionHeader->sectionFilePosition[i] > filePositionLast); // sub-section file offsets (if needed), assumed to be in File Position order
}
assert(valid);
return valid;
}
// GetName - Get the name of this section, value can be overridden by user
const std::wstring& Section::GetName()
{
if (!m_name.empty())
return m_name;
// if name is not set yet, get it from the description header
std::wstring nameDescription(msra::strfun::utf16(m_sectionHeader->nameDescription));
auto firstColon = nameDescription.find_first_of(L':');
if (firstColon != npos && nameDescription.size() >= firstColon)
{
m_name = nameDescription.substr(0, firstColon);
}
// return name (or empty string if description had no colon)
return m_name;
}
// GetSection - Get a sub section of this section
// index - index to this section
// mapping - type of mapping desired for this section
// sizeElements - size of the mapping window (records) if using mappingElementWindow
Section* Section::ReadSection(size_t index, MappingType mapping, size_t sizeElements)
{
// check for valid index
if (m_sections.size() <= index)
{
RuntimeError("ReadSection:Invalid index");
}
// already loaded, so return
if (m_sections[index] != NULL)
return m_sections[index];
// load it if it hasn't been loaded (mapped)
size_t filePosition = m_sectionHeader->sectionFilePosition[index];
Section* section = NULL;
size_t loadSize = 0;
SectionHeader* header = GetSectionHeader(filePosition, mapping, loadSize);
// makes sure the requested records isn't more than we have
sizeElements = min(header->elementsCount / header->elementsPerRecord, sizeElements);
// convert from records to bytes
sizeElements *= header->elementsPerRecord * header->bytesPerElement;
switch (header->sectionType)
{
default:
section = new Section(m_file, this, header, filePosition, mapping, loadSize);
break;
case sectionTypeLabelMapping:
section = new SectionString(m_file, this, header, filePosition, mapping, loadSize);
// variable sized section, size is just the size of the section in bytes
// for now we don't support separate window for variable sized data
sizeElements = section->GetSize() - section->GetHeaderSize();
break;
case sectionTypeLabel:
section = new SectionLabel(m_file, this, header, filePosition, mapping, loadSize);
break;
case sectionTypeStats:
section = new SectionStats(m_file, this, header, filePosition, mapping, loadSize);
break;
}
// make sure the header is valid
if (!section->ValidateHeader())
{
char message[256];
sprintf_s(message, "Invalid header in file %ls, in header %s\n", m_file->GetName(), section->GetName());
RuntimeError(message);
}
// setup the element mapping and pointers as needed
section->EnsureElements(0, sizeElements);
// add section to parent section array
m_sections[index] = section;
return section;
}
// Destructor - destroy the section type
Section::~Section()
{
// go through the sections array and delete all the sections
for (auto iter = m_sections.begin(); iter != m_sections.end(); ++iter)
{
Section* section = *iter;
// a section that is not mapped is skipped
if (section == NULL)
continue;
delete section;
}
m_sections.clear();
}
// EnsureElements - Make sure that the elements range is available to read/write
// element - beginning element to access
// bytesRequested - bytes requested
// returns: pointer to the element requested
char* Section::EnsureElements(size_t element, size_t bytesRequested)
{
// everything is already setup, so just return the pointer
assert(!(GetFlags() & flagVariableSized)); // does not support variable size entries, those should be handled by Section subclasses
size_t elementsRequested = bytesRequested / GetElementSize();
if (element + elementsRequested > GetElementCount())
{
char message[256];
sprintf_s(message, "Element out of range, error accesing element %lld, size=%lld\n", element, bytesRequested);
RuntimeError(message);
}
// make sure we have the buffer in the range to handle the request
if (element < m_elemMin || element + elementsRequested >= m_elemMax || m_elementBuffer == NULL)
{
GetElementBuffer(element, bytesRequested);
}
return (char*) m_elementBuffer + (element - m_elemMin) * GetElementSize();
}
// GetElementBuffer - get the element buffer for the passed element and size
// element - element we want the elementBuffer to start from
// windowSize - minimum size of the window in bytes for Element Window (will not resize smaller)
// returns: pointer to the element buffer
char* Section::GetElementBuffer(size_t element, size_t windowSize)
{
// check element range
if (!m_file->Writing() && element >= GetElementCount())
{
char message[256];
sprintf_s(message, "Element out of range, error accesing element %lld, max element=%lld\n", element, GetElementCount());
RuntimeError(message);
}
// section is mapped as a whole, so no separate mapping for element buffer
if (m_mappingType != mappingElementWindow)
{
// mapped together with header, just offset
m_elementBuffer = (char*) m_sectionHeader + m_sectionHeader->sizeHeader;
m_elemMin = 0;
if (m_file->Writing())
{
m_elemMax = windowSize;
// variable sized sections won't have an element size, so just use bytes
if (GetElementSize())
m_elemMax /= GetElementSize();
}
else
{
m_elemMax = GetElementCount();
}
m_mappedElementSize = 0; // only used for element window mapping
// save off the header size so we can reestablish pointers if necessary
// size_t headerSize = m_sectionHeader->sizeHeader;
// assert((char*)m_elementBuffer - (char*)m_sectionHeader);
// void* elementBuffer =
EnsureMapped(m_elementBuffer, windowSize);
// check to see if the mapping changed, if so update pointers
// if (elementBuffer != m_elementBuffer)
// {
// m_elementBuffer = elementBuffer;
// m_sectionHeader -= headerSize;
// }
return (char*) m_elementBuffer;
}
// if the window size is larger, increase mapped size
if (m_mappedElementSize < windowSize)
m_mappedElementSize = windowSize;
// if we have no element size specified us a default size
else if (m_mappedElementSize == 0)
m_mappedElementSize = 16 * 1024 * 1024; // default size 16Meg for now
// get number of elements mapped
size_t elementMappedCount = m_mappedElementSize / GetElementSize();
elementMappedCount = min(elementMappedCount, GetElementCount());
m_mappedElementSize = elementMappedCount * GetElementSize();
// get the file position of the element
if (element + elementMappedCount > GetElementCount())
{
m_elemMin = GetElementCount() - elementMappedCount;
m_elemMax = GetElementCount();
}
else
{
m_elemMin = element;
m_elemMax = m_elemMin + elementMappedCount;
}
size_t sectionPosition = GetHeaderSize() + m_elemMin * GetElementSize();
// now get the aligned view location
size_t alignBoundary = m_file->GetViewAlignment();
size_t viewPosition = ((sectionPosition + m_filePosition) / alignBoundary) * alignBoundary;
size_t offset = (sectionPosition + m_filePosition) % alignBoundary;
// release the old view if we have a current view
if (m_elementView != NULL)
{
m_file->ReleaseView(m_elementView);
}
m_elementView = m_file->GetView(viewPosition, m_mappedElementSize + offset);
m_elementBuffer = (char*) m_elementView + offset;
return (char*) m_elementBuffer;
}
// RemapHeader - Update view pointers if view changed for all sections in that view
// view - new view pointer to the passed filePosition
// filePosition - filePosition view refers to
void Section::RemapHeader(void* view, size_t filePosition)
{
for (auto iter = m_sections.begin(); iter != m_sections.end(); ++iter)
{
Section* section = *iter;
// a section that is not mapped is skipped
if (section == NULL)
continue;
if (section->m_mappingType == mappingParent)
{
int64_t offset = section->m_filePosition - filePosition;
if (offset < 0)
RuntimeError("RemapHeader:Invalid mapping, children must follow parent in mapping space");
// update our header location
SectionHeader* header = section->m_sectionHeader = (SectionHeader*) ((char*) view + offset);
section->m_elementBuffer = (char*) header + header->sizeHeader;
// now update any children
section->RemapHeader(view, filePosition);
}
}
}
// EnsureMappingSize - make sure the mapping for this section is at least this size
// size - size to ensure we have
void Section::EnsureMappingSize(size_t size)
{
// only call on mapping owners, that are mapped at least by section
assert(m_mappingType != mappingElementWindow);
assert(this == SectionMappingOwner());
void* view = m_file->EnsureViewSize(m_sectionHeader, size);
// check for remap
if (view != m_sectionHeader)
{
m_sectionHeader = (SectionHeader*) view;
m_elementBuffer = (char*) m_sectionHeader + m_sectionHeader->sizeHeader;
RemapHeader(view, m_filePosition);
}
}
// EnsureMapped - make sure the pointer passed is mapped appropriately
// dataStart - starting pointer of the location we want mapped
// size - size to ensure we have starting at this pointer
// returns: new mapping if applicable
void* Section::EnsureMapped(void* dataStart, size_t size)
{
// only call on mapping owners
void* view = m_file->EnsureMapped(dataStart, size);
// check for remap, if this section isn't mapping itself, we may need to
// check to see if the mapping owner is different
if (view != dataStart)
{
// Element Window is mapped separately so won't no need to remap
if (m_mappingType != mappingElementWindow)
{
int64_t offset = (byte*) view - (byte*) dataStart;
m_sectionHeader = (SectionHeader*) ((char*) m_sectionHeader + offset);
m_elementBuffer = (char*) m_sectionHeader + m_sectionHeader->sizeHeader;
RemapHeader(m_sectionHeader, m_filePosition);
}
}
return view;
}
// GetSectionHeader - Retrieve a section header (and possibly entire section)
// filePosition - position in the file where the section starts
// mappingType - [in,out] mapping type to use for this header, may be changed depending on the header
// size - [in] max size to read, if 0, read the entire section (size in the header)
// [out] size actually mapped for this section
// returns pointer to the section header
SectionHeader* Section::GetSectionHeader(size_t filePosition, MappingType& mappingType, size_t& size)
{
SectionHeader* sectionHeader = NULL;
switch (mappingType)
{
case mappingParent:
{
auto sectionOwner = SectionMappingOwner();
if (filePosition < sectionOwner->GetFilePosition())
RuntimeError("invalid fileposition, cannot be earlier in the file than mapping parent");
size_t offset = filePosition - sectionOwner->GetFilePosition();
size_t totalSize = offset + max(size, (size_t) sectionHeaderMin);
// make sure we can at least get to the header
if (sectionOwner->GetMappedSize() < totalSize)
{
sectionOwner->EnsureMappingSize(totalSize);
}
sectionHeader = (SectionHeader*) ((char*) sectionOwner->GetHeader() + offset);
// if we are reading we can actually see the REAL header size now, realloc if needed to get that as well
if (!m_file->Writing())
{
size = sectionHeader->size;
totalSize = offset + size;
if (sectionOwner->GetMappedSize() < totalSize)
{
sectionOwner->EnsureMappingSize(totalSize);
}
}
break;
}
case mappingFile:
if (filePosition != 0)
RuntimeError("invalid fileposition, file mapping sections must start at filePostion zero");
// intentional fall-through - same case, just at beginning of file
case mappingSectionAll:
sectionHeader = m_file->GetSection(filePosition, size);
if (!m_file->Writing())
{
size_t newSize = sectionHeader->sizeAll;
if (newSize > size)
sectionHeader = (SectionHeader*) m_file->ReallocateView(sectionHeader, newSize);
size = newSize;
}
break;
case mappingSection:
{
sectionHeader = m_file->GetSection(filePosition, size);
break;
}
case mappingElementWindow:
size_t sizeHeader = sectionHeaderMin;
sectionHeader = m_file->GetSection(filePosition, sizeHeader);
size = sizeHeader;
// we don't allocate the element view at this time, we will later
// This allows us to not map the areas of the file we don't care about...
// if we are reading we can actually see the REAL header size now, realloc if needed to get that as well
if (!m_file->Writing())
{
// however we always map auxiliary Sections, now that we know we have one switch over to mappingSection and map it.
if (sectionHeader->flags & (flagAuxilarySection | flagVariableSized))
{
size = sectionHeader->size;
m_file->ReleaseSection(sectionHeader);
sectionHeader = m_file->GetSection(filePosition, size);
mappingType = mappingSection;
}
}
break;
}
// Now return the header
return sectionHeader;
}
// GetSection - Retrieve a section header (and possibly entire section)
// filePosition - position in the file where the section starts
// size - [in] max size to read, if 0, read the entire section (size in the header)
// [out] size actually mapped for this section
// returns pointer to the section header
SectionHeader* SectionFile::GetSection(size_t filePosition, size_t& size)
{
bool determineSize = size == 0;
if (determineSize)
size = sectionHeaderMin;
SectionHeader* binHeader = (SectionHeader*) GetView(filePosition, size);
// if we aren't writing a file figure out what size we need
if (!m_writeFile)
{
// mapped size not large enough for this section, and they want us to allocate for the entire thing
if (binHeader->size > size && determineSize)
{
size = binHeader->size;
binHeader = (SectionHeader*) ReallocateView(binHeader, size);
}
// size not big enough for the header, need at least that much
else if (binHeader->sizeHeader > size)
{
size = binHeader->sizeHeader;
binHeader = (SectionHeader*) ReallocateView(binHeader, size);
}
// max size is bigger than entire section, reduce to actual size mapped
else if (binHeader->sizeAll < size)
{
size = binHeader->size;
binHeader = (SectionHeader*) ReallocateView(binHeader, size);
}
}
else
{
memset(binHeader, 0, sectionHeaderMin);
binHeader->sectionType = sectionTypeData;
binHeader->wMagic = magicSection;
binHeader->version = 0x0100;
binHeader->sizeHeader = sectionHeaderMin;
binHeader->size = size; // set the currently mapped size for this header
binHeader->sizeAll = size; // set size with all sub-elements to this size too
}
return binHeader;
}
// ReleaseSection - Release a Section of the file
// section - Section to release (must have been returned from GetSection() previously)
void SectionFile::ReleaseSection(SectionHeader* section)
{
ReleaseView(section);
}
// SectionMappingOwner - find the section that owns the mapping view for this section
// returns - section that owns the mapping
Section* Section::SectionMappingOwner()
{
Section* section = this;
// if the current section is not a parent mapping, then this is the mapping owner
if (section->m_mappingType != mappingParent)
{
assert(section->m_mappingType == mappingSectionAll || section->m_mappingType == mappingFile);
return this;
}
// search backwards looking for a mapping owner
while (section->m_parent != NULL)
{
section = section->m_parent;
if (section->m_mappingType == mappingSectionAll || section->m_mappingType == mappingFile)
{
return section;
}
}
// oops, nothing found, mapping error
assert("mapping error, no mapping parent found");
return this;
}
// Section Constructor
// initializes all the section variables
// for writing, it does NOT initialize the header, this must be done by the caller
Section::Section(SectionFile* sectionFile, Section* parentSection, size_t filePosition, MappingType mappingType, size_t size)
{
Init(sectionFile, parentSection, NULL, filePosition, mappingType, size);
}
// Section Constructor
// initializes all the section variables
// for writing, it does NOT initialize the header, this must be done by the caller
Section::Section(SectionFile* sectionFile, Section* parentSection, SectionHeader* sectionHeader, size_t filePosition, MappingType mappingType, size_t size)
{
Init(sectionFile, parentSection, sectionHeader, filePosition, mappingType, size);
}
// Init Section
// file - section file to which we belong
// parentSection - parent section
// sectionHeader - sectionHeader pointer to pre-mapped section, or NULL if constructor should do mapping
// filePosition - position in file where header starts
// mappingType - type of mapping we are using for this section
// size - size for this section, for reading this should be zero so actual size of section will be used
// NOTE: for writing, it does NOT initialize the header, call InitHeader() for header initalization
void Section::Init(SectionFile* file, Section* parentSection, SectionHeader* sectionHeader, size_t filePosition, MappingType mappingType, size_t size)
{
m_file = file;
m_filePosition = file->RoundUp(filePosition);
m_parent = parentSection;
m_mappingType = mappingType;
if (size < sectionHeaderMin && file->Writing())
RuntimeError("Insufficient size requested for section in write mode");
// clear out element window mapping variables
m_elementView = NULL; // pointer to beginning of the valid view
m_elementBuffer = NULL; // no buffer pointer yet.
m_mappedElementSize = 0; // size of mapped element view, only used for mappingElementWindow mappingType
m_elemMin = 0; // currently mapped minimum value (first valid)
m_elemMax = 0; // currently mapped maximum value (one past last valid)
if (sectionHeader == NULL)
{
sectionHeader = GetSectionHeader(filePosition, mappingType, size);
m_mappingType = mappingType; // in case it was modified in GetSectionHeader
}
m_sectionHeader = sectionHeader;
// resize the number of sections so it is correct, initialize with zero to show we haven't actually mapped that area yet
if (m_sectionHeader->dataSections > 0)
m_sections.resize(m_sectionHeader->dataSections, 0);
m_mappedSize = size;
}
// AddSection - Add a new subsection to this section.
// This handles any allocation that may be necessary as well as updating headers with valid file positions
// Currently limited to adding sections at the end of a parent section
// sectionToAdd - subsection we are adding on this section
Section* Section::AddSection(Section* sectionToAdd)
{
// Get file position
Section* section = this;
size_t filePosition = sectionToAdd->GetFilePosition(); // get file position
size_t sizeRounded = m_file->RoundUp(section->GetSize());
// set parent
sectionToAdd->SetParent(this);
// add to parents header array of sections if they are in the same file
if (sectionToAdd->GetSectionFile() == section->GetSectionFile())
{
SectionHeader* header = section->GetHeader();
header->sectionFilePosition[header->dataSections++] = filePosition;
if (header->dataSections && header->sectionFilePosition[header->dataSections - 1] < filePosition)
RuntimeError("invalid fileposition for section, subsection cannot start earlier in the file than parent section");
// now update size for all pervious ancestor sections
do
{ // add in the size of the new section to parent setion, always assumed to go on the end
sizeRounded = m_file->RoundUp(header->sizeAll);
sizeRounded += sectionToAdd->GetSize(); // put it on the end of this section
header->sizeAll = sizeRounded;
// if this section is the file level, we don't need to update past here
if (section->GetMappingType() == mappingFile)
break;
section = section->GetParent();
header = section->GetHeader();
} while (section != NULL && sectionToAdd->GetSectionFile() == section->GetSectionFile());
}
// add to sections array (in memory only)
m_sections.push_back(sectionToAdd);
return sectionToAdd;
}
// SaveData - save data in the file/files
// recordStart - Starting record number, will be > datasetSize for multiple passes
// matricies - a map of section name (section:subsection) to data pointer. Data sepcifications from config file will be used to determine where and how to save data
// numRecords - number of records we are saving, can be zero if not applicable
// datasetSize - Size of the dataset
// byteVariableSized - for variable sized data, size of current block to be written, zero when not used, or ignored if not variable sized data
// returns: true if more data is desired, false if no more data is required (signal that file may be closed)
bool Section::SaveData(size_t recordStart, const std::map<std::wstring, void*, nocase_compare>& matrices, size_t numRecords, size_t datasetSize, size_t byteVariableSized)
{
Section* section = this;
// if the section not in the data stream, by default we don't save anything
// (override for implementations that do stats, and other processing)
auto iterator = matrices.find(GetName());
if (iterator == matrices.end())
{
return false;
}
void* dataSource = iterator->second;
// only want to save data on first iteration
if (recordStart >= datasetSize)
{
return false;
}
size_t index = recordStart * section->GetElementsPerRecord();
size_t size;
if (byteVariableSized > 0 && (GetFlags() & flagVariableSized))
size = byteVariableSized;
else if (section->GetElementSize() > 0)
size = section->GetElementSize() * section->GetElementsPerRecord() * numRecords;
else
{
// hum, is this allowable?
RuntimeError("Invalid size in Binary Writer, variable sized data with no length");
}
char* data = section->EnsureElements(index, size);
// void* data = section->GetElement(index);
memcpy_s(data, size, dataSource, size);
return recordStart + numRecords < GetRecordCount();
}
SectionString::SectionString(SectionFile* file, Section* parentSection, SectionHeader* sectionHeader, size_t filePosition, MappingType mappingType, size_t size)
: Section(file, parentSection, sectionHeader, filePosition, mappingType, size)
{
}
SectionString::SectionString(SectionFile* file, Section* parentSection, size_t filePosition, MappingType mappingType, size_t size)
: Section(file, parentSection, filePosition, mappingType, size)
{
}
// GetLabelMapping - Get the label mapping from the current file
// translates from a string representation in the file
// returns - a map from LabelId to Label string
const std::map<SectionString::LabelIdType, SectionString::LabelType>& SectionString::GetLabelMapping()
{
if (m_mapIdToLabel.size() == 0)
{
char* str = (char*) GetElement(0);
SectionString::LabelIdType elements = (SectionString::LabelIdType) GetElementCount();
for (SectionString::LabelIdType element = 0; element < elements; ++element)
{
std::string stringLabel(str);
m_mapIdToLabel[element] = stringLabel;
m_mapLabelToId[stringLabel] = element;
// advance to the next string (stops on the NULL)
while (*str++)
;
}
}
return m_mapIdToLabel;
}
// SetLabelMapping - set the mapping data
void SectionString::SetLabelMapping(const std::map<SectionString::LabelIdType, SectionString::LabelType>& labelMapping)
{
Section* section = this;
if (labelMapping.size() != GetElementCount())
RuntimeError("SetLabelMapping called with mapping table that doesn't match file defined element count");
if (GetSectionType() != sectionTypeLabelMapping)
RuntimeError("label type invalid");
// free the old mapping tables
m_mapLabelToId.clear();
m_mapIdToLabel.clear();
// now write out the passed mapping table
char* curStr = section->GetElement<char>(0);
size_t size = section->GetSize() - section->GetHeaderSize();
size_t originalSize = size;
for (unsigned i = 0; i < section->GetElementCount(); ++i)
{
auto iter = labelMapping.find(i);
if (iter == labelMapping.end())
{
char message[256];
sprintf_s(message, "Mapping table doesn't contain an entry for label Id#%d\n", i);
RuntimeError(message);
}
// add to reverse mapping table
m_mapLabelToId[iter->second] = i;
const string& str = iter->second;
errno_t err = strcpy_s(curStr, size, str.c_str());
if (err)
{
char message[256];
sprintf_s(message, "Not enough room in mapping buffer, %lld bytes insufficient for string %d - %s\n", originalSize, i, str.c_str());
RuntimeError(message);
}
size_t len = str.length() + 1; // don't forget the null
size -= len;
curStr += len;
}
// now zero out the rest of the free space
memset(curStr, 0, size);
// store the resulting map
m_mapIdToLabel = labelMapping;
}
// GetElement - Get the element at the given index
void* SectionString::GetElement(size_t index) const
{
char* str = (char*) m_elementBuffer;
if (index >= GetElementCount())
{
char message[256];
sprintf_s(message, "GetElement: invalid index, %lld requested when there are only %lld elements\n", index, GetElementCount());
RuntimeError(message);
}
// now skip all the strings before the one that we want
for (SectionString::LabelIdType element = 0; element < index; ++element)
{
// advance to the next string (stops on the NULL)
while (*str++)
;
}
return str;
}
// EnsureElements - Make sure that the elements range is available to read/write
// element - beginning element to access
// bytesRequested - bytes requested
// returns: pointer to the element requested
char* SectionString::EnsureElements(size_t element, size_t bytesRequested)
{
// everything is already setup, so just return the pointer
assert((GetFlags() & flagVariableSized)); // does support variable size entries
assert(GetMappingType() != mappingElementWindow); // not supported for string tables currently
if (element >= GetElementCount())
{
char message[256];
sprintf_s(message, "Element out of range, error accesing element %lld, size=%lld\n", element, bytesRequested);
RuntimeError(message);
}
// make sure we have the buffer in the range to handle the request
if (m_elementBuffer == NULL)
{
GetElementBuffer(element, bytesRequested);
}
return (char*) GetElement(element);
}
SectionLabel::SectionLabel(SectionFile* file, Section* parentSection, SectionHeader* sectionHeader, size_t filePosition, MappingType mappingType, size_t size)
: Section(file, parentSection, sectionHeader, filePosition, mappingType, size)
{
}
SectionLabel::SectionLabel(SectionFile* file, Section* parentSection, size_t filePosition, MappingType mappingType, size_t size)
: Section(file, parentSection, filePosition, mappingType, size)
{
}
// Store - store the stats in the file
void SectionStats::Store()
{
for (int i = 0; i < GetElementCount(); i++)
{
auto stat = GetElement<NumericStatistics>(i);
if (!_stricmp(stat->statistic, "sum"))
{
stat->value = m_sum;
}
else if (!_stricmp(stat->statistic, "count"))
{
stat->value = (double) m_count;
}
else if (!_stricmp(stat->statistic, "mean"))
{
stat->value = m_mean;
}
else if (!_stricmp(stat->statistic, "max"))
{
stat->value = m_max;
}
else if (!_stricmp(stat->statistic, "min"))
{
stat->value = m_min;
}
else if (!_stricmp(stat->statistic, "range"))
{
stat->value = abs(m_max - m_min);
}
else if (!_stricmp(stat->statistic, "rootmeansquare"))
{
stat->value = m_rms;
}
else if (!_stricmp(stat->statistic, "variance"))
{
stat->value = m_variance;
}
else if (!_stricmp(stat->statistic, "stddev"))
{
stat->value = m_stddev;
}
else
{
fprintf(stderr, "Unknown statistic requested in file %s\n", stat->statistic);
}
}
}
SectionStats::SectionStats(SectionFile* file, Section* parentSection, size_t filePosition, MappingType mappingType, size_t size)
: Section(file, parentSection, filePosition, mappingType, size)
{
}
SectionStats::SectionStats(SectionFile* file, Section* parentSection, SectionHeader* sectionHeader, size_t filePosition, MappingType mappingType, size_t size)
: Section(file, parentSection, sectionHeader, filePosition, mappingType, size)
{
}
// SetCompute - Initialize the statistics gathering locations
// compute - config array of the names of the stats we want recorded
void SectionStats::InitCompute(const ConfigArray& compute)
{
SetElementCount(compute.size());
for (int i = 0; i < compute.size(); i++)
{
std::string name = compute[i];
auto stat = GetElement<NumericStatistics>(i);
strcpy_s(stat->statistic, name.c_str());
stat->value = 0.0;
}
// initialize all the internal variables
m_count = 0; // number of elements
m_max = -DBL_MAX; // maximum value we have seen
m_min = DBL_MAX; // minimum value we have seen
m_sum = 0.0; // sum of all numbers we have seen
m_sum2 = 0.0; // sum of the squares of all numbers we have seen
// compute after single pass
m_mean = 0.0; // mean of all values
m_rms = 0.0; // root mean square
// second pass measures
m_varSum = 0.0; // accumulated sum of difference between the mean and and the value squared
// compute after second pass
m_variance = 0.0;
m_stddev = 0.0;
// progressive variables
m_pmean = 0.0;
m_pvariance = 0.0;
}
// SetCompute - Set a compute variable to a value
// name - name of the compute variable
// value - value to set
void SectionStats::SetCompute(const std::string& name, double value)
{
for (int i = 0; i < GetElementCount(); i++)
{
auto stat = GetElement<NumericStatistics>(i);
if (!_stricmp(stat->statistic, name.c_str()))
{
stat->value = value;
break;
}
}
}
// GetCompute - Get a compute variable based on a name
// name - name of the compute variable
// returns - value of the variable
double SectionStats::GetCompute(const std::string& name)
{
for (int i = 0; i < GetElementCount(); i++)
{
auto stat = GetElement<NumericStatistics>(i);
if (!_stricmp(stat->statistic, name.c_str()))
{
return stat->value;
}
}
return 0.0;
}
// SaveData - save data in the file/files
// recordStart - Starting record number, will be > datasetSize for multiple passes
// matricies - a map of section name (section:subsection) to data pointer. Data sepcifications from config file will be used to determine where and how to save data
// numRecords - number of records we are saving, can be zero if not applicable
// datasetSize - Size of the dataset
// byteVariableSized - for variable sized data, size of current block to be written, zero when not used, or ignored if not variable sized data
// returns: true if more data is desired, false if no more data is required (signal that file may be closed)
bool SectionStats::SaveData(size_t recordStart, const std::map<std::wstring, void*, nocase_compare>& matrices, size_t numRecords, size_t datasetSize, size_t /*byteVariableSized*/)
{
// if the section is not the data stream we are a child of ignore it
auto iterator = matrices.find(m_parent->GetName());
if (iterator == matrices.end())
{
return false;
}
void* dataSource = iterator->second;
size_t elementSize = m_parent->GetElementSize();
bool retVal = false;
if (elementSize == sizeof(float))
retVal = AccumulateData((float*) dataSource, numRecords, recordStart, datasetSize);
else if (elementSize == sizeof(double))
retVal = AccumulateData((double*) dataSource, numRecords, recordStart, datasetSize);
return retVal;
}
#define ONE_PASS
// AccumulateData - accumulate the data for the dataset
// Templated to handle both double and float values
// dataSource - pointer to the data
// numRecords - number of records in the dataSource
// recordStart - record we are starting on
// datasetSize - size of the dataset
template <typename ElemType>
bool SectionStats::AccumulateData(ElemType* dataSource, size_t numRecords, size_t recordStart, size_t datasetSize)
{
// on first pass we want to calculate sums and squared sums
if (recordStart < datasetSize)
{
ElemType* elemP = (ElemType*) dataSource;
ElemType* elemPMax = elemP + m_parent->GetElementsPerRecord() * numRecords;
// sum up all the values
while (elemP < elemPMax)
{
ElemType elem = *elemP;
// keep current and next count for easier math below
size_t curCount = m_count;
m_count++;
// calculate progressive mean and variance
// see: http://mathworld.wolfram.com/SampleVarianceComputation.html
double prevMean = m_pmean;
m_pmean = m_pmean + (elem - m_pmean) / (m_count);
if (curCount)
{
double meanDifference = (m_pmean - prevMean);
m_pvariance = (1.0 - 1.0 / curCount) * m_pvariance + m_count * meanDifference * meanDifference;
}
// accumulate sums
m_sum += elem;
m_sum2 += elem * elem;
if (elem < m_min)
m_min = elem;
if (elem > m_max)
m_max = elem;
elemP++;
}
// check for the last chunk in the first pass
if (recordStart + numRecords == datasetSize)
{
m_mean = m_sum / m_count;
m_rms = m_sum2 / m_count;
m_rms = sqrt(m_rms);
m_variance = m_pvariance;
m_stddev = sqrt(m_variance);
double error = m_mean - m_pmean;
if (error > 0.00001)
printf("substantial difference in progressive mean\n");
#if defined(ONE_PASS)
// since we are done now we can store the final values
Store();
return false;
#endif
}
return true;
}
#if !defined(ONE_PASS)
else if (recordStart < 2 * datasetSize)
{
ElemType* elemP = (ElemType*) dataSource;
ElemType* elemPMax = elemP + m_parent->GetElementsPerRecord() * numRecords;
// sum up all the values
while (elemP < elemPMax)
{
ElemType elem = *elemP++;
double diff = elem - m_mean;
m_varSum += diff * diff;
}
// check for the last chunk in the first pass
if (recordStart + numRecords == 2 * datasetSize)
{
m_variance = m_varSum / m_count;
m_stddev = sqrt(m_variance);
double error = m_pvariance - m_variance;
if (error > 0.00001)
printf("substantial difference in progressive variance\n");
// since we are done now we can store the final values
Store();
return false;
}
return true;
}
#endif
// done with data
return false;
}
} } }