// // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE.md file in the project root for full license information. // #pragma once #include "Basics.h" // for RuntimeError() #include "Matrix.h" #include "File.h" #include "Helpers.h" #include "CommonMatrix.h" #include "CPURNGHandle.h" #include #include #include #include #include "QuantizedOperations.h" //#include "GPUMatrix.h" //#include "CPUSparseMatrix.h" //#include "GPUSparseMatrix.h" // NOTE NOTE NOTE: // use CPUSingleMatrix and CPUDoubleMatrix instead of using the template directly /////////////////////////////////////////////// // This class is exported from the Math.dll namespace Microsoft { namespace MSR { namespace CNTK { double logadd(double x, double y); // To comply with BLAS libraries matrices are stored in ColMajor. However, by default C/C++/C# use RowMajor // conversion is need when passing data between CPUMatrix and C++ matrices template class MATH_API CPUMatrix : public BaseMatrix { typedef BaseMatrix Base; using Base::m_numRows; using Base::m_numCols; using Base::m_sliceViewOffset; using Base::HasExternalBuffer; using Base::SetBuffer; using Base::SetComputeDeviceId; using Base::SetSizeAllocated; using Base::GetSizeAllocated; using Base::ZeroInit; using Base::ZeroValues; using Base::m_sob; using Base::ShallowCopyFrom; using Base::VerifyResizable; public: using Base::VerifyWritable; using Base::GetComputeDeviceId; using Base::Buffer; using Base::GetNumRows; using Base::GetNumCols; using Base::GetNumElements; using Base::OwnBuffer; using Base::GetFormat; using Base::SetFormat; using Base::IsEmpty; using Base::VerifySize; public: CPUMatrix(); CPUMatrix(const CPUMatrix& shallowCopyFrom, bool shallow); // copy constructor, shallow CPUMatrix(const size_t numRows, const size_t numCols); CPUMatrix(const size_t numRows, const size_t numCols, ElemType* pArray, const size_t matrixFlags = matrixFlagNormal); CPUMatrix(const CPUMatrix& deepCopyFrom); // copy constructor, deep copy CPUMatrix& operator=(const CPUMatrix& deepCopyFrom); // assignment operator, deep copy CPUMatrix(CPUMatrix&& moveFrom); // move constructor, shallow copy CPUMatrix& operator=(CPUMatrix&& moveFrom); // move assignment operator, shallow copy public: size_t BufferSize() const { return m_numRows * m_numCols * sizeof(ElemType); } // Returns pointer into underlying data buffer corresponding to slice-view. This makes it different from method Buffer() ElemType* Data() const { return Buffer() + m_sliceViewOffset; } CPUMatrix ColumnSlice(size_t startColumn, size_t numCols) const; CPUMatrix& AssignColumnSlice(const CPUMatrix& fromMatrix, size_t startColumn, size_t numCols); CPUMatrix& SetColumnSlice(const CPUMatrix& fromMatrix, size_t startColumn, size_t numCols); void CopyColumnsStrided(const CPUMatrix& fromMatrix, size_t numCols, size_t srcNumColsStride, size_t destNumColsStride); CPUMatrix Diagonal() const; ElemType Adagrad(CPUMatrix& gradients, const bool needAveMultiplier); void FSAdagrad(CPUMatrix& gradients, CPUMatrix& functionValues, ElemType learnRatePerSample, ElemType momentum, ElemType adaWeight, ElemType adaMul); ElemType RmsProp(CPUMatrix& gradients, ElemType RMS_GAMMA, ElemType RMS_WGT_INC, ElemType RMS_WGT_MAX, ElemType RMS_WGT_DEC, ElemType RMS_WGT_MIN, const bool needAveMultiplier); void Reshape(const size_t numRows, const size_t numCols); // RequireSize is now the new preferred method of ensuring the correct size inside of the Matrix class. Since Resize will fail if the storage object has // multiple views, RequireSize will first check to see if Resize is required. If it is not, then it short-circuits and is a noop. Otherwise, RequireSize // will call Resize, which may fail if the matrix has multiple views. void RequireSize(const size_t numRows, const size_t numCols, bool growOnly = true); // by default we only reallocate if need to grow // Resize first checks to ensure that the caller has the authority to call Resize (i.e., it checks to ensure the underlying data is owned by only this matrix), and then // actually resizes the underlying matrix, doing any allocation as required. void Resize(const size_t numRows, const size_t numCols, bool growOnly = true); // by default we only reallocate if need to grow ElemType* CopyToArray() const; // allocated by the callee but need to be deleted by the caller size_t CopyToArray(ElemType*& arrayCopyTo, size_t& currentArraySize) const; // allocated by the callee but need to be deleted by the caller void CopySection(size_t numRows, size_t numCols, ElemType* dst, size_t colStride) const; inline ElemType& operator()(const size_t row, const size_t col) { return Data()[LocateElement(row, col)]; } inline const ElemType& operator()(const size_t row, const size_t col) const { return Data()[LocateElement(row, col)]; } inline ElemType Get00Element() const { return operator()(0, 0); } void SetValue(const ElemType v); void SetValue(const CPUMatrix& deepCopyFrom); //void SetValue(const GPUMatrix& deepCopyFrom); //void SetValue(const CPUSparseMatrix& deepCopyFrom); //void SetValue(const GPUSparseMatrix& deepCopyFrom); void SetValue(const size_t numRows, const size_t numCols, ElemType* pArray, size_t matrixFlags = matrixFlagNormal); void MaskColumnsValue(const CPUMatrix& columnsMask, ElemType val); void SetColumn(const ElemType* colPointer, size_t colInd); void SetColumn(const CPUMatrix& valMat, size_t colInd); void SetColumn(const ElemType val, size_t j); void SetDiagonalValue(const ElemType v); void SetDiagonalValue(const CPUMatrix& vector); void SetUniformRandomValue(const ElemType low, const ElemType high, unsigned long seed = USE_TIME_BASED_SEED); void SetGaussianRandomValue(const ElemType mean, const ElemType sigma, unsigned long seed = USE_TIME_BASED_SEED); void SetUniformRandomMask(const ElemType maskRate, const ElemType scaleValue, RNGHandle& rngHandle); void AddGaussianRandomValue(const ElemType mean, const ElemType sigma, unsigned long seed = USE_TIME_BASED_SEED); CPUMatrix Transpose(); CPUMatrix& AssignTransposeOf(const CPUMatrix& a); CPUMatrix& DoGatherColumnsOf (ElemType beta, const CPUMatrix& idx, const CPUMatrix& a, ElemType alpha); CPUMatrix& DoScatterColumnsOf(ElemType beta, const CPUMatrix& idx, const CPUMatrix& a, ElemType alpha); CPUMatrix& operator+=(const ElemType alpha); CPUMatrix operator+(const ElemType alpha) const; CPUMatrix& AssignSumOf(const ElemType alpha, const CPUMatrix& a); CPUMatrix& operator+=(const CPUMatrix& a); CPUMatrix operator+(const CPUMatrix& a) const; CPUMatrix& AssignSumOf(const CPUMatrix& a, const CPUMatrix& b); CPUMatrix& operator-=(const ElemType alpha); CPUMatrix operator-(const ElemType alpha) const; CPUMatrix& AssignDifferenceOf(const ElemType alpha, const CPUMatrix& a); CPUMatrix& AssignDifferenceOf(const CPUMatrix& a, const ElemType alpha); CPUMatrix& operator-=(const CPUMatrix& a); CPUMatrix operator-(const CPUMatrix& a) const; CPUMatrix& AssignDifferenceOf(const CPUMatrix& a, const CPUMatrix& b); CPUMatrix& operator*=(const ElemType alpha); CPUMatrix operator*(const ElemType alpha) const; CPUMatrix& AssignProductOf(const ElemType alpha, const CPUMatrix& a); CPUMatrix operator*(const CPUMatrix& a) const; CPUMatrix& AssignProductOf(const CPUMatrix& a, const bool transposeA, const CPUMatrix& b, const bool transposeB); CPUMatrix& operator/=(ElemType alpha); CPUMatrix operator/(ElemType alpha) const; CPUMatrix& operator^=(ElemType alpha); // element-wise power CPUMatrix operator^(ElemType alpha) const; // element-wise power CPUMatrix& AssignElementPowerOf(const CPUMatrix& a, const ElemType power); CPUMatrix& ElementMultiplyWith(const CPUMatrix& a); CPUMatrix& AssignElementProductOf(const CPUMatrix& a, const CPUMatrix& b); CPUMatrix& AddElementProductOf(const CPUMatrix& a, const CPUMatrix& b); CPUMatrix& AssignElementDivisionOf(const CPUMatrix& a, const CPUMatrix& b); CPUMatrix& ElementDivideBy(const CPUMatrix& a); CPUMatrix& ColumnElementMultiplyWith(const CPUMatrix& a); CPUMatrix& RowElementMultiplyWith(const CPUMatrix& a); CPUMatrix& ColumnElementDivideBy(const CPUMatrix& a); CPUMatrix& RowElementDivideBy(const CPUMatrix& a); CPUMatrix& ElementInverse(); CPUMatrix& AssignElementInverseOf(const CPUMatrix& a); CPUMatrix& InplaceSigmoid(); CPUMatrix& AssignSigmoidOf(const CPUMatrix& a); CPUMatrix& InplaceLinearRectifierDerivative(); CPUMatrix& AssignLinearRectifierDerivativeOf(const CPUMatrix& a); CPUMatrix& InplaceSigmoidDerivative(); CPUMatrix& AssignSigmoidDerivativeOf(const CPUMatrix& a); CPUMatrix& InplaceTanh(); CPUMatrix& AssignTanhOf(const CPUMatrix& a); CPUMatrix& InplaceLogSoftmax(const bool isColWise); CPUMatrix& AssignLogSoftmaxOf(const CPUMatrix& a, const bool isColWise); CPUMatrix& InplaceHardmax(const bool isColWise); CPUMatrix& AssignHardmaxOf(const CPUMatrix& a, const bool isColWise); // sequence training CPUMatrix& DropFrame(const CPUMatrix& label, const CPUMatrix& gamma, const ElemType& threshhold); CPUMatrix& AssignSequenceError(const ElemType hsmoothingWeight, const CPUMatrix& label, const CPUMatrix& dnnoutput, const CPUMatrix& gamma, ElemType alpha); CPUMatrix& InplaceSqrt(); CPUMatrix& AssignSqrtOf(const CPUMatrix& a); CPUMatrix& InplaceExp(); CPUMatrix& AssignExpOf(const CPUMatrix& a); CPUMatrix& InplaceLog(); CPUMatrix& AssignLogOf(const CPUMatrix& a); CPUMatrix& InplaceLog10(); CPUMatrix& AssignLog10Of(const CPUMatrix& a); CPUMatrix& InplaceCosine(); CPUMatrix& AssignCosineOf(const CPUMatrix& a); CPUMatrix& InplaceNegativeSine(); CPUMatrix& AssignNegativeSineOf(const CPUMatrix& a); CPUMatrix& InplaceAbs(); CPUMatrix& AssignAbsOf(const CPUMatrix& a); CPUMatrix& InplaceTruncateBottom(const ElemType threshold); CPUMatrix& AssignTruncateBottomOf(const CPUMatrix& a, const ElemType threshold); CPUMatrix& InplaceTruncateTop(const ElemType threshold); CPUMatrix& AssignTruncateTopOf(const CPUMatrix& a, const ElemType threshold); CPUMatrix& InplaceTruncate(const ElemType threshold); CPUMatrix& InplaceSoftThreshold(const ElemType threshold); CPUMatrix& SetToZeroIfAbsLessThan(const ElemType threshold); ElemType SumOfAbsElements() const; // sum of all abs(elements) ElemType SumOfElements() const; // sum of all elements CPUMatrix& AssignSumOfElements(const CPUMatrix& a); bool IsEqualTo(const CPUMatrix& a, const ElemType threshold = 1e-8) const; static void VectorSum(const CPUMatrix& a, CPUMatrix& c, const bool isColWise); void VectorNorm1(CPUMatrix& c, const bool isColWise) const; CPUMatrix& AssignVectorNorm1Of(CPUMatrix& a, const bool isColWise); void VectorNorm2(CPUMatrix& c, const bool isColWise) const; CPUMatrix& AssignVectorNorm2Of(CPUMatrix& a, const bool isColWise); void AssignNoiseContrastiveEstimation(const CPUMatrix& a, const CPUMatrix& b, const CPUMatrix& bias, CPUMatrix& tmp, CPUMatrix& c); void AssignSoftmaxSum(const CPUMatrix& a, CPUMatrix& softmax); void AssignNCEUnnormalizedEval(const CPUMatrix& a, const CPUMatrix& b, const CPUMatrix& bias, CPUMatrix& c); CPUMatrix& AssignNCEDerivative(const CPUMatrix& tmp, const CPUMatrix& a, const CPUMatrix& b, size_t inputIndex, CPUMatrix& c); void VectorNormInf(CPUMatrix& c, const bool isColWise) const; CPUMatrix& AssignVectorNormInfOf(CPUMatrix& a, const bool isColWise); CPUMatrix& AssignInnerProductOf(const CPUMatrix& a, const CPUMatrix& b, const bool isColWise); CPUMatrix& AssignKhatriRaoProductOf(const CPUMatrix& a, const CPUMatrix& b); CPUMatrix& AddColumnReshapeProductOf(const CPUMatrix& a, const CPUMatrix& b, const bool transposeAColumn); CPUMatrix& AddWithScaleOf(ElemType alpha, const CPUMatrix& a); ElemType FrobeniusNorm() const; CPUMatrix& AssignFrobeniusNormOf(const CPUMatrix& a); ElemType MatrixNormInf() const; ElemType MatrixNorm1() const; ElemType MatrixNorm0() const; // number of non-zero elemets CPUMatrix& AssignSignOf(const CPUMatrix& a); CPUMatrix& AddSignOf(const CPUMatrix& a); CPUMatrix& AssignRowSliceValuesOf(const CPUMatrix& a, const size_t startIndex, const size_t numRows); CPUMatrix& AddToRowSliceValuesOf(const CPUMatrix& a, const size_t startIndex, const size_t numRows); CPUMatrix& AddWithRowSliceValuesOf(const CPUMatrix& a, const size_t startIndex, const size_t numRows); // CPUMatrix& AssignRowStackValuesOf(const std::vector*>& inputMatrices, const size_t sliceStartCol, const size_t sliceNumCols); CPUMatrix& AssignToRowSliceValuesOf(const CPUMatrix& a, const size_t startIndex, const size_t numRows); CPUMatrix& AssignRepeatOf(const CPUMatrix& a, const size_t numRowRepeats, const size_t numColRepeats); CPUMatrix& AddToRowRepeatValuesOf(const CPUMatrix& a, const size_t numRowRepeats); CPUMatrix& AssignPositiveAndShiftedNegSample(const CPUMatrix& a, const size_t posNumber, const size_t negNumber, const size_t shiftNumber); CPUMatrix& AddFoldedPositiveAndShiftedNegSample(const CPUMatrix& a, const size_t posNumber, const size_t negNumber, const size_t shiftNumber); void VectorMax(CPUMatrix& maxIndexes, CPUMatrix& maxValues, const bool isColWise, int topK = 1) const; void VectorMin(CPUMatrix& minIndexes, CPUMatrix& minValues, const bool isColWise) const; CPUMatrix& AssignNumOfDiff(const CPUMatrix& a, const CPUMatrix& b, bool searchInCol = false); void Print(const char* matrixName, ptrdiff_t rowStart, ptrdiff_t rowEnd, ptrdiff_t colStart, ptrdiff_t colEnd) const; void Print(const char* matrixName = nullptr) const; // print whole matrix. can be expensive void ReadFromFile(FILE* f, const char* matrixName); // matrixName is used to verify that correct matrix is read. void WriteToFile(FILE* f, const char* matrixName); // matrixName is used to verify that correct matrix is read. CPUMatrix& AssignPackedConvolutionInput(const CPUMatrix& inputSubBatch, const size_t inputWidth, const size_t inputHeight, const size_t inputChannels, const size_t outputWidth, const size_t outputHeight, const size_t outputChannels, const size_t kernelWidth, const size_t kernelHeight, const size_t horizontalSubsample, const size_t verticalSubsample, const bool zeroPadding = false); CPUMatrix& UnpackConvolutionInput(CPUMatrix& inputSubBatch, const size_t inputWidth, const size_t inputHeight, const size_t inputChannels, const size_t outputWidth, const size_t outputHeight, const size_t outputChannels, const size_t kernelWidth, const size_t kernelHeight, const size_t horizontalSubsample, const size_t verticalSubsample, const bool zeroPadding = false) const; CPUMatrix& AssignMaxPoolingResult(const CPUMatrix& inputBatch, const size_t channels, const size_t inputWidth, const size_t inputHeight, const size_t inputSizePerSample, const size_t outputWidth, const size_t outputHeight, const size_t outputSizePerSample, const size_t windowWidth, const size_t windowHeight, const size_t horizontalSubsample, const size_t verticalSubsample); CPUMatrix& AddMaxPoolingGradient(const CPUMatrix& outputGradientBatch, const CPUMatrix& inputBatch, const CPUMatrix& outputBatch, const size_t channels, const size_t inputWidth, const size_t inputHeight, const size_t inputSizePerSample, const size_t outputWidth, const size_t outputHeight, const size_t outputSizePerSample, const size_t windowWidth, const size_t windowHeight, const size_t horizontalSubsample, const size_t verticalSubsample); CPUMatrix& AssignAveragePoolingResult(const CPUMatrix& inputBatch, const size_t channels, const size_t inputWidth, const size_t inputHeight, const size_t inputSizePerSample, const size_t outputWidth, const size_t outputHeight, const size_t outputSizePerSample, const size_t windowWidth, const size_t windowHeight, const size_t horizontalSubsample, const size_t verticalSubsample); CPUMatrix& AddAveragePoolingGradient(const CPUMatrix& outputGradientBatch, const size_t channels, const size_t inputWidth, const size_t inputHeight, const size_t inputSizePerSample, const size_t outputWidth, const size_t outputHeight, const size_t outputSizePerSample, const size_t windowWidth, const size_t windowHeight, const size_t horizontalSubsample, const size_t verticalSubsample); void ConvolutionForward(const CPUMatrix& kernel, const CPUMatrix& mpRowCol, const CPUMatrix& mpRowIwht, const CPUMatrix& mpRowRun, const CPUMatrix& runs, CPUMatrix& output) const; void ConvolutionBackwardData(const CPUMatrix& kernel, const CPUMatrix& mpRowCol, const CPUMatrix& mpRowIwht, const CPUMatrix& mpRowRun, const CPUMatrix& runs, CPUMatrix& grad) const; void ConvolutionBackwardKernel(const CPUMatrix& in, const CPUMatrix& mpRowCol, const CPUMatrix& mpRowIwht, const CPUMatrix& mpRowRun, const CPUMatrix& runs, CPUMatrix& kernelGrad) const; void UnrollConvolutionInput(size_t unrollCols, size_t mapOutSize, const CPUMatrix& mpRowCol, const CPUMatrix& mpRowRun, const CPUMatrix& runs, CPUMatrix& output) const; void UnrollConvolutionOutput(size_t unrollCols, size_t mapInCount, size_t mapOutCount, const CPUMatrix& mpRowCol, const CPUMatrix& mpRowRun, const CPUMatrix& runs, CPUMatrix& output) const; void UnrollConvolutionInputForKernelBackprop(size_t mapOutSize, const CPUMatrix& mpRowCol, const CPUMatrix& mpRowRun, const CPUMatrix& runs, CPUMatrix& output) const; void MaxPoolingForward(const CPUMatrix& mpRowCol, const CPUMatrix& mpRowIndices, const CPUMatrix& indices, CPUMatrix& output) const; void MaxPoolingBackward(const CPUMatrix& out, const CPUMatrix& in, const CPUMatrix& mpRowCol, const CPUMatrix& mpRowIndices, const CPUMatrix& indices, CPUMatrix& grad) const; void ROIPoolingForward(const size_t numRois, const size_t numImg, const size_t channels, const size_t width, const size_t height, const size_t pooledWidth, const size_t pooledHeight, const CPUMatrix& roiData, CPUMatrix& output, CPUMatrix& argmax) const; void ROIPoolingBackward(const size_t numRois, const size_t numImg, const size_t channels, const size_t width, const size_t height, const size_t pooledWidth, const size_t pooledHeight, const CPUMatrix& roiData, CPUMatrix& grad, CPUMatrix& argmax) const; void MaxUnpooling(const CPUMatrix& mpRowCol, const CPUMatrix& mpRowIndices, const CPUMatrix& indices, const CPUMatrix& poolInput, CPUMatrix& input) const; void AveragePoolingForward(const CPUMatrix& mpRowCol, const CPUMatrix& mpRowIndices, const CPUMatrix& indices, CPUMatrix& output) const; void AveragePoolingBackward(const CPUMatrix& mpRowCol, const CPUMatrix& mpRowIndices, const CPUMatrix& indices, CPUMatrix& grad) const; void BatchNormalizationForward(const CPUMatrix& scale, const CPUMatrix& bias, bool inferenceOnly, double expAvgFactor, double blendFactor, CPUMatrix& runMean, CPUMatrix& runVariance, CPUMatrix& out, double epsilon, CPUMatrix& saveMean, CPUMatrix& saveInvStdDev) const; void BatchNormalizationBackward(const CPUMatrix& in, CPUMatrix& grad, const CPUMatrix& scale, double blendFactor, const CPUMatrix& saveMean, const CPUMatrix& saveInvStdDev, CPUMatrix& scaleGrad, CPUMatrix& biasGrad) const; public: // This functions do not depend on , i.e. you can call them on any static int SetNumThreads(int numThreads); static int GetMaxNumThreads(); static void SetCompatibleMode(); // static BLAS functions static void SVD(const CPUMatrix& A, CPUMatrix& SIGMA, CPUMatrix& U, CPUMatrix& VT, CPUMatrix& W); static void MultiplyAndWeightedAdd(ElemType alpha, const CPUMatrix& a, const bool transposeA, const CPUMatrix& b, const bool transposeB, ElemType beta, CPUMatrix& c, shared_ptr> pQuantizedMultiplier=nullptr); static void MultiplyAndAdd(const CPUMatrix& a, const bool transposeA, const CPUMatrix& b, const bool transposeB, CPUMatrix& c); static void Multiply(const CPUMatrix& a, const bool transposeA, const CPUMatrix& b, const bool transposeB, CPUMatrix& c); static void Multiply(const CPUMatrix& a, const CPUMatrix& b, CPUMatrix& c); static void Multiply1x1AndWeightedAdd(ElemType alpha, const CPUMatrix& a, const CPUMatrix& b, ElemType beta, CPUMatrix& c); static void ScaleAndAdd(ElemType alpha, const CPUMatrix& a, CPUMatrix& c); static void AddScaledDifference(const ElemType alpha, const CPUMatrix& a, const CPUMatrix& b, CPUMatrix& c); static void AssignScaledDifference(const ElemType alpha, const CPUMatrix& a, const CPUMatrix& b, CPUMatrix& c); static void AddScaledDifference(const CPUMatrix& alpha, const CPUMatrix& a, const CPUMatrix& b, CPUMatrix& c); // alpha must be 1X1 static void AssignScaledDifference(const CPUMatrix& alpha, const CPUMatrix& a, const CPUMatrix& b, CPUMatrix& c); // alpha must be 1X1 static void AddElementToElement(ElemType beta, const CPUMatrix& a, const size_t ai, const size_t aj, CPUMatrix& c, const size_t ci, const size_t cj); static void MinusOneAt(CPUMatrix& c, const size_t position); static void Scale(ElemType alpha, CPUMatrix& a); static void Scale(CPUMatrix alpha, CPUMatrix& a); // In this case Matrix alpha must be 1x1 static void Scale(ElemType alpha, const CPUMatrix& a, CPUMatrix& c); static void InnerProduct(const CPUMatrix& a, const CPUMatrix& b, CPUMatrix& c, const bool isColWise); static ElemType InnerProductOfMatrices(const CPUMatrix& a, const CPUMatrix& b); static void ElementWisePower(ElemType alpha, const CPUMatrix& a, CPUMatrix& c); static bool AreEqual(const CPUMatrix& a, const CPUMatrix& b, const ElemType threshold = 1e-8); static void TensorShuffleScaleAndAdd(ElemType keepWeight, const CPUMatrix& a, size_t D, size_t S, size_t M, size_t K, size_t T, ElemType scaleFactor, const CPUMatrix& b, CPUMatrix& c); void TensorOp(ElemType beta, const CPUMatrix& a, ElemType alpha, ElementWiseOperator op, ElementWiseOperator reductionOp, const std::array& offsets, const SmallVector& regularOpDims, const std::array, 2>& regularStrides, const SmallVector& reducingOpDims, const std::array, 2>& reducingStrides); void TensorOp(ElemType beta, const CPUMatrix& a, const CPUMatrix& b, ElemType alpha, ElementWiseOperator op, ElementWiseOperator reductionOp, const std::array& offsets, const SmallVector& regularOpDims, const std::array, 3>& regularStrides, const SmallVector& reducingOpDims, const std::array, 3>& reducingStrides); void TensorOp(ElemType beta, const CPUMatrix& a, const CPUMatrix& b, const CPUMatrix& c, ElemType alpha, ElementWiseOperator op, ElementWiseOperator reductionOp, const std::array& offsets, const SmallVector& regularOpDims, const std::array, 4>& regularStrides, const SmallVector& reducingOpDims, const std::array, 4>& reducingStrides); static CPUMatrix Ones(const size_t rows, const size_t cols); static CPUMatrix Zeros(const size_t rows, const size_t cols); static CPUMatrix