local cunntest = torch.TestSuite()
local ffi = require 'ffi'
local precision_forward = 1e-4
local precision_backward = 1e-2
local nloop = 1
local times = {}

--e.g.: th -lcunn -e "nn.testcuda{'Sigmoid_forward'}"

local typenames = {
  'torch.CudaTensor',
  'torch.CudaDoubleTensor',
}

local t2cpu = {
  ['torch.CudaTensor'] = 'torch.FloatTensor',
  ['torch.CudaDoubleTensor'] = 'torch.DoubleTensor',

}

local function checkHalf()
   if cutorch.hasHalf then
       table.insert(typenames, 'torch.CudaHalfTensor')
       t2cpu['torch.CudaHalfTensor'] = 'torch.FloatTensor'
   end
end

-- workarounds for non-existant functions
function torch.CudaHalfTensor:mean()
   return self:cuda():mean()
end

function torch.CudaDoubleTensor:mean()
   return self:cuda():mean()
end

local function half_max_error(maxabs)
  -- arbitrarily double the precision limit
  return 2 * ((maxabs and (2^(math.floor(math.log(maxabs) / math.log(2)))) * (2^(-10))) or 0)
end

-- half has additional error on top of double/float
local function precision_forward_type(precision_f, tensor_type, maxabs)
   if (tensor_type == 'torch.CudaHalfTensor') then
      return 1e-2 + precision_f + half_max_error(maxabs)
   else
      return precision_f
   end
end

local function precision_backward_type(precision_b, tensor_type, maxabs)
   if (tensor_type == 'torch.CudaHalfTensor') then
      return 1e-1 + precision_b + half_max_error(maxabs)
   else
      return precision_b
   end
end

local function precision_backward_conv_weightbias(precision_b, tensor_type, maxabs)
   if (tensor_type == 'torch.CudaHalfTensor') then
      -- cudnn uses 8 here
      return 2 + precision_b + half_max_error(maxabs)
   else
      return precision_b
   end
end

local function pointwise_forward(proto_module, name, max_error)
   local size = math.random(1,100)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size):type(typename)
      local ctype = t2cpu[typename]
      local input = input:type(ctype)
      if name == 'Sqrt' then input:abs() end
      local sconv = proto_module:type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = proto_module:clone():type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(max_error, typename),
        string.format('error on state (forward) with %s', typename))
    end
end

local function pointwise_backward(proto_module, name, max_error)
   local size = math.random(1,100)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size):type(typename)
      local gradOutput = torch.randn(size):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      if name == 'Sqrt' then input:abs() end
      local sconv = proto_module:type(ctype)
      sconv:forward(input)
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = proto_module:clone():type(typename)
      gconv:forward(input)
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(max_error, typename),
        string.format('error on state (backward) with %s', typename))
    end
end

local function pointwise_backward_inplace(proto_module, name)
   local size = math.random(1,100)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      if name == 'Sqrt' then input:abs() end
      local gradOutput = torch.randn(size)
      gradOutput = gradOutput:type(ctype)
      local sconv = proto_module:type(ctype)
      local groundgrad = sconv:backward(input, gradOutput)
      mytester:assertTensorEq(groundgrad:double(),
                              gradOutput:double(),
                              0.000001,
                              string.format("inplace not respected for %s", ctype))

      input = torch.randn(size)
      input = input:type(typename)
      if name == 'Sqrt' then input:abs() end
      gradOutput = torch.randn(size)
      gradOutput = gradOutput:type(typename)
      local sconv = proto_module:clone():type(typename)
      local groundgrad = sconv:backward(input, gradOutput)
      mytester:assertTensorEq(groundgrad:double(),
                              gradOutput:double(),
                              0.000001,
                              string.format("cuda inplace not respected for %s", typename))
    end
end

local function pointwise_transposed(proto_module, name, max_error)
   max_error = max_error or 1e-7

   for k, typename in ipairs(typenames) do
      local ctype = t2cpu[typename]
      local input = torch.Tensor(11, 19):uniform(-1, 1):type(typename)
      input = input:type(ctype)
      local proto_module = proto_module:type(ctype)
      if name == 'Sqrt' then
        input:uniform(0.1, 1)
      end
      local inputCUDA = input:clone():type(typename)

      local cuda_module = proto_module:clone():type(typename)

      -- transpose the inputs and DON'T make contiguous
      input = input:transpose(1, 2)
      inputCUDA = inputCUDA:transpose(1, 2)

      local output = proto_module:forward(input)
      local outputCUDA = cuda_module:forward(inputCUDA)

      local error = outputCUDA:double() - output:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(max_error, typename),
        string.format('error on state (forward) for %s', typename))

      local gradOutput = torch.Tensor(11, 19):uniform(-1, 1):type(ctype)
      local gradOutputCUDA = gradOutput:clone():type(typename)

      gradOutput = gradOutput:transpose(1, 2)
      gradOutputCUDA = gradOutputCUDA:transpose(1, 2)

      local gradInput = proto_module:backward(input, gradOutput)
      local gradInputCUDA  = cuda_module:backward(inputCUDA, gradOutputCUDA)

      local error = gradInputCUDA:double() - gradInput:double()
      mytester:assertlt(error:abs():max(), precision_backward_type(max_error, typename),
        string.format('error on state (backward) for %s', typename))
    end
end

function cunntest.Tanh_forward()
   pointwise_forward(nn.Tanh(), 'Tanh', precision_forward)
end

function cunntest.Tanh_backward()
   pointwise_backward(nn.Tanh(), 'Tanh', precision_backward)
end

function cunntest.Tanh_transposed()
   pointwise_transposed(nn.Tanh(), 'Tanh', 1.8e-7)
end

function cunntest.HardTanh_forward()
   pointwise_forward(nn.HardTanh(), 'HardTanh', precision_forward)
end

function cunntest.HardTanh_backward()
   pointwise_backward(nn.HardTanh(), 'HardTanh', precision_backward)
end

function cunntest.HardTanh_backward_inplace()
   pointwise_backward_inplace(nn.HardTanh(nil, nil, true), 'HardTanh')
end

function cunntest.HardTanh_transposed()
   pointwise_transposed(nn.HardTanh(), 'HardTanh', 1.5e-7)
end

function cunntest.Abs_forward()
   pointwise_forward(nn.Abs(), 'Abs', precision_forward)
end

function cunntest.Abs_backward()
   pointwise_backward(nn.Abs(), 'Abs', precision_backward)
end

function cunntest.Abs_transposed()
   pointwise_transposed(nn.Abs(), 'Abs')
end

function cunntest.Sigmoid_forward()
   pointwise_forward(nn.Sigmoid(), 'Sigmoid', precision_forward)
end

function cunntest.Sigmoid_backward()
   pointwise_backward(nn.Sigmoid(), 'Sigmoid', precision_backward)
end

function cunntest.Sigmoid_transposed()
   pointwise_transposed(nn.Sigmoid(), 'Sigmoid')
end

function cunntest.LogSigmoid_forward()
   pointwise_forward(nn.LogSigmoid(), 'LogSigmoid', precision_forward)
end

function cunntest.LogSigmoid_backward()
   pointwise_backward(nn.LogSigmoid(), 'LogSigmoid', precision_backward)
end

function cunntest.LogSigmoid_transposed()
   pointwise_transposed(nn.LogSigmoid(), 'LogSigmoid', 1e-6)
end

function cunntest.Threshold_forward()
  pointwise_forward(nn.Threshold(), 'Threshold', precision_forward)
  pointwise_forward(nn.Threshold(nil, nil, true), 'Threshold_inplace', precision_forward)
end

function cunntest.Threshold_backward()
  pointwise_backward(nn.Threshold(), 'Threshold', precision_backward)
  pointwise_backward(nn.Threshold(nil, nil, true), 'Threshold_inplace', precision_backward)
end

function cunntest.ReLU6_forward()
  for inplace = 0, 1 do
    local net = nn.Sequential()
    -- pointwise_forward uses randn, so add a big constant to make sure some
    -- of the values saturate.
    net:add(nn.MulConstant(6))
    net:add(nn.ReLU6(inplace == 1))
    pointwise_forward(net, 'ReLU6 inplace ' .. inplace, precision_forward)
  end
end

function cunntest.ReLU6_backward()
  for inplace = 0, 1 do
    local net = nn.Sequential()
    net:add(nn.MulConstant(6))
    net:add(nn.ReLU6(inplace == 1))
    pointwise_backward(net, 'ReLU6 inplace ' .. inplace, precision_backward)
  end
end

function cunntest.LeakyReLU_forward()
   pointwise_forward(nn.LeakyReLU(), 'LeakyReLU', precision_forward)
end

function cunntest.LeakyReLU_backward()
   pointwise_backward(nn.LeakyReLU(), 'LeakyReLU', precision_backward)
end

function cunntest.LeakyReLU_transposed()
   pointwise_transposed(nn.LeakyReLU(), 'LeakyReLU', 1.5e-7)
end

function cunntest.Sqrt_forward()
   pointwise_forward(nn.Sqrt(), 'Sqrt', precision_forward)
end

function cunntest.Sqrt_backward()
   pointwise_backward(nn.Sqrt(), 'Sqrt', precision_backward)
end

function cunntest.Sqrt_zero()
   local size = math.random(1, 100)

   for k, typename in ipairs(typenames) do
      -- Test zero inputs; we will avoid a div-by-zero by setting to zero
      local module_gpu = nn.Sqrt():type(typename)
      local input_gpu = torch.CudaTensor(size, size):zero():type(typename)
      module_gpu:forward(input_gpu)

      local gradOutput_gpu = torch.CudaTensor(size, size):fill(1):type(typename)
      local gradInput_gpu = module_gpu:backward(input_gpu, gradOutput_gpu)

      mytester:assertTensorEq(gradInput_gpu:double(),
                              torch.DoubleTensor(size, size):zero(),
                              0.000001, "error in sqrt backward singularity")

      -- Verify CPU and GPU zero behavior equivalency
      local ctype = t2cpu[typename]
      local module_cpu = nn.Sqrt():type(ctype)
      local input_cpu = input_gpu:type(ctype)
      module_cpu:forward(input_cpu)

      local gradOutput_cpu = gradOutput_gpu:type(ctype)
      local gradInput_cpu = module_cpu:backward(input_cpu, gradOutput_cpu)

      mytester:assertTensorEq(gradInput_gpu:double(),
                            gradInput_cpu:double(),
                            0.000001, "Sqrt_zero CPU and GPU not equivalent")
    end
end

function cunntest.Sqrt_transposed()
   pointwise_transposed(nn.Sqrt(), 'Sqrt')
end

function cunntest.Square_forward()
   pointwise_forward(nn.Square(), 'Square', precision_forward)
end

function cunntest.Square_backward()
   pointwise_backward(nn.Square(), 'Square', precision_backward)
end

function cunntest.Square_transposed()
   pointwise_transposed(nn.Square(), 'Square')
end

function cunntest.SoftShrink_forward()
  local r = ffi.C.THC_half2float(ffi.C.THC_float2half(math.random()))
  pointwise_forward(nn.SoftShrink(r), 'SoftShrink', precision_forward)
end

function cunntest.SoftShrink_backward()
  local r = ffi.C.THC_half2float(ffi.C.THC_float2half(math.random()))
  pointwise_backward(nn.SoftShrink(r), 'SoftShrink', precision_backward)
end

function cunntest.SoftShrink_transposed()
  local r = ffi.C.THC_half2float(ffi.C.THC_float2half(math.random()))
  pointwise_transposed(nn.SoftShrink(r), 'SoftShrink', precision_backward)
end

function cunntest.ELU_forward()
   pointwise_forward(nn.ELU(), 'ELU', precision_forward)
end

function cunntest.ELU_backward()
   pointwise_backward(nn.ELU(), 'ELU', precision_backward)
end

function cunntest.ELU_transposed()
   pointwise_transposed(nn.ELU(), 'ELU', 1e-6)
end

function cunntest.SoftMax_forward()
   pointwise_forward(nn.SoftMax(), 'SoftMax', precision_forward)
end

function cunntest.SoftMax_backward()
   pointwise_backward(nn.SoftMax(), 'SoftMax', precision_backward)
end

function cunntest.LogSoftMax_forward()
   pointwise_forward(nn.LogSoftMax(), 'LogSoftMax', precision_forward*10)
end

function cunntest.LogSoftMax_backward()
   pointwise_backward(nn.LogSoftMax(), 'LogSoftMax', precision_backward)
end

function cunntest.SpatialSoftMax()
   local bs = math.random(32,256)
   local dim = torch.random(1, 50)
   local h = torch.random(1, 50)
   local w = torch.random(1, 50)

   local input = torch.randn(bs, dim, h, w)
   local sconv = nn.SpatialSoftMax()
   local groundtruth = sconv:forward(input)
   local gradOutput = groundtruth:clone():fill(0.5)
   local gradInput = sconv:backward(input, gradOutput)

   input = input:cuda()
   gradOutput = gradOutput:cuda()
   local gconv = nn.SpatialSoftMax():cuda()
   local rescuda = gconv:forward(input)
   local gradcuda = gconv:backward(input, gradOutput)

   local error = rescuda:float() - groundtruth
   mytester:assertlt(error:abs():max(), precision_forward*10, 'error on state (forward) ')

   local error = gradcuda:float() - gradInput
   mytester:assertlt(error:abs():max(), precision_backward*10, 'error on state (backward) ')
end

function cunntest.LogSoftMax_forward_batch()
   local size = math.random(1,256)
   local bs = math.random(32,256)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs, size):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.LogSoftMax():type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.LogSoftMax():type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward*10, typename),
          string.format('error on state (forward) with %s', typename))
    end
end

function cunntest.LogSoftMax_backward_batch()
   local size = math.random(1,256)
   local bs = math.random(32,256)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs, size):type(typename)
      local gradOutput = torch.randn(bs, size):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.LogSoftMax():type(ctype)
      sconv:forward(input)
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = sconv:clone():type(typename)
      gconv:forward(input)
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.SpatialLogSoftMax_forward()
   local size = math.random(1,256)
   local ini = math.random(8,32)
   local inj = math.random(8,32)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size, inj, ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialLogSoftMax():type(ctype)
      local groundtruth = sconv:forward(input):type(ctype)

      input = input:type(typename)
      local gconv = nn.SpatialLogSoftMax():type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(),
          precision_forward_type(precision_forward*25, typename),
          string.format('error on state (forward) with %s', typename))
    end
end

function cunntest.SpatialLogSoftMax_backward()
   local size = math.random(1,256)
   local ini = math.random(8,32)
   local inj = math.random(8,32)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size, inj, ini):type(typename)
      local gradOutput = torch.randn(size, inj, ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialLogSoftMax():type(ctype)
      sconv:forward(input)
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = sconv:clone():type(typename)
      gconv:forward(input)
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.SpatialLogSoftMax_forward_batch()
   local size = math.random(1,256)
   local bs = math.random(8,32)
   local ini = math.random(8,32)
   local inj = math.random(8,32)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs, size, inj, ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialLogSoftMax():type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialLogSoftMax():type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(),
          precision_forward_type(precision_forward*25, typename),
          string.format('error on state (forward) with %s', typename))
    end
end

function cunntest.SpatialLogSoftMax_backward_batch()
   local size = math.random(1,256)
   local bs = math.random(8,32)
   local ini = math.random(8,32)
   local inj = math.random(8,32)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs, size, inj, ini):type(typename)
      local gradOutput = torch.randn(bs, size, inj, ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialLogSoftMax():type(ctype)
      sconv:forward(input)
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = sconv:clone():type(typename)
      gconv:forward(input)
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
    end
end


function cunntest.Euclidean_forward_batch()
   local bs = math.random(8,32)
   local nin = math.random(1,100)
   local nout = math.random(1,100)

   local tm = {}
   local title = string.format('Euclidean forward %d %d -> %d %d', bs, nin, bs, nout)
   times[title] = tm

   local input = torch.randn(bs, nin)
   local sconv = nn.Euclidean(nin, nout)
   local groundtruth = sconv:forward(input)
   local a = torch.Timer()
   for i = 1,nloop do
      groundtruth = sconv:forward(input)
   end
   tm.cpu = a:time().real

   input = input:cuda()
   local gconv = sconv:clone():cuda()
   local rescuda = gconv:forward(input)
   a:reset()
   for i = 1,nloop do
      rescuda = gconv:forward(input)
   end
   cutorch.synchronize()
   tm.gpu = a:time().real

   local error = rescuda:float() - groundtruth
   mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) batch ')
end

function cunntest.Euclidean_backward_batch()
   local bs = math.random(8,32)
   local nin = math.random(1,100)
   local nout = math.random(1,100)

   local tm = {}
   local title = string.format('Euclidean backward %d %d <- %d %d', bs, nin, bs, nout)
   times[title] = tm

   local input = torch.randn(bs, nin)
   local gradOutput = torch.randn(bs, nout)
   local sconv = nn.Euclidean(nin, nout)
   sconv:forward(input)
   sconv:zeroGradParameters()
   local groundgrad = sconv:backward(input, gradOutput)
   local a = torch.Timer()
   for i = 1,nloop do
      sconv:zeroGradParameters()
      groundgrad = sconv:backward(input, gradOutput)
   end
   local groundweight = sconv.gradWeight
   tm.cpu = a:time().real

   input = input:cuda()
   gradOutput = gradOutput:cuda()
   local gconv = sconv:clone():cuda()
   gconv:forward(input)
   gconv:zeroGradParameters()
   local rescuda = gconv:backward(input, gradOutput)
   a:reset()
   for i = 1,nloop do
      gconv:zeroGradParameters()
      rescuda = gconv:backward(input, gradOutput)
   end
   cutorch.synchronize()
   tm.gpu = a:time().real

   local weightcuda = gconv.gradWeight

   local error = rescuda:float() - groundgrad
   local werror = weightcuda:float() - groundweight

   mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
   mytester:assertlt(werror:abs():max(), precision_backward, 'error on weight (backward) ')
end

function cunntest.WeightedEuclidean_forward_batch()
   local bs = math.random(8,32)
   local nin = math.random(1,100)
   local nout = math.random(1,100)

   local tm = {}
   local title = string.format('WeightedEuclidean forward %d %d -> %d %d', bs, nin, bs, nout)
   times[title] = tm

   local input = torch.randn(bs, nin)
   local sconv = nn.WeightedEuclidean(nin, nout)
   local groundtruth = sconv:forward(input)
   local a = torch.Timer()
   for i = 1,nloop do
      groundtruth = sconv:forward(input)
   end
   tm.cpu = a:time().real

   input = input:cuda()
   local gconv = sconv:clone():cuda()
   local rescuda = gconv:forward(input)
   a:reset()
   for i = 1,nloop do
      rescuda = gconv:forward(input)
   end
   cutorch.synchronize()
   tm.gpu = a:time().real

   local error = rescuda:float() - groundtruth
   mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) batch ')
end

function cunntest.WeightedEuclidean_backward_batch()
   local bs = math.random(8,32)
   local nin = math.random(1,100)
   local nout = math.random(1,100)

   local tm = {}
   local title = string.format('WeightedEuclidean backward %d %d <- %d %d', bs, nin, bs, nout)
   times[title] = tm

   local input = torch.randn(bs, nin)
   local gradOutput = torch.randn(bs, nout)
   local sconv = nn.WeightedEuclidean(nin, nout)
   sconv:forward(input)
   sconv:zeroGradParameters()
   local groundgrad = sconv:backward(input, gradOutput)
   local a = torch.Timer()
   for i = 1,nloop do
      sconv:zeroGradParameters()
      groundgrad = sconv:backward(input, gradOutput)
   end
   local groundweight = sconv.gradWeight
   local grounddiagCov = sconv.gradDiagCov
   tm.cpu = a:time().real

   input = input:cuda()
   gradOutput = gradOutput:cuda()
   local gconv = sconv:clone():cuda()
   gconv:forward(input)
   gconv:zeroGradParameters()
   local rescuda = gconv:backward(input, gradOutput)
   a:reset()
   for i = 1,nloop do
      gconv:zeroGradParameters()
      rescuda = gconv:backward(input, gradOutput)
   end
   cutorch.synchronize()
   tm.gpu = a:time().real

   local weightcuda = gconv.gradWeight
   local diagCovcuda = gconv.gradDiagCov

   local error = rescuda:float() - groundgrad
   local werror = weightcuda:float() - groundweight
   local derror = diagCovcuda:float() - grounddiagCov

   mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
   mytester:assertlt(werror:abs():max(), precision_backward, 'error on weight (backward) ')
   mytester:assertlt(derror:abs():max(), precision_backward, 'error on diagCov (backward) ')
end

function cunntest.SparseLinear_forward()
    local inb = math.random(5,10)
    local ini = math.random(50,100)
    local inj = math.random(5,10)

    for k, typename in ipairs(typenames) do
        if typename ~= "torch.CudaHalfTensor" then
            local ctype = t2cpu[typename]
            local module = nn.SparseLinear(ini,inj):type(ctype)
            local sslin = module
            local gslin = module:clone():type(typename)

            -- Create a random sparse vector
            local input = {}
            for i=1,inb do
                local nnz = math.random(5, 10)
                local inds = torch.randperm(ini)[{{1,nnz}}]
                input[i] = torch.Tensor(nnz, 2):type(ctype)
                input[i]:select(2,1):copy(inds)
                input[i]:select(2,2):copy(torch.rand(nnz):type(typename):type(ctype))
            end

            local groundtruth = sslin:forward(input)
            sslin:zeroGradParameters()

            for i,v in ipairs(input) do input[i] = input[i]:type(typename) end
            local rescuda = gslin:forward(input)
            gslin:zeroGradParameters()

            local error = rescuda:double() - groundtruth:double()
            mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
                string.format('error on state (forward) with %s', typename))
        end
    end
end

function cunntest.SparseLinear_backward()
    local inb = math.random(5,10)
    local ini = math.random(50,100)
    local inj = math.random(5,10)

    for k, typename in ipairs(typenames) do
        if typename ~= "torch.CudaHalfTensor" then
            local ctype = t2cpu[typename]
            local gslin = nn.SparseLinear(ini,inj):type(typename)
            local sslin = nn.Linear(ini,inj):type(ctype)
            gslin.weight = sslin.weight:clone():type(typename)
            gslin.bias = sslin.bias:clone():type(typename)

            -- Create a random sparse vector
            local input = {}
            local nonsparse = torch.zeros(inb, ini):type(ctype)
            for i=1,inb do
                local nnz = math.random(3, 5)
                local inds = torch.randperm(ini)[{{1,nnz}}]
                input[i] = torch.Tensor(nnz, 2):type(ctype)
                input[i]:select(2,1):copy(inds)
                input[i]:select(2,2):copy(torch.rand(nnz):type(typename):type(ctype))
                nonsparse[i]:scatter(1, input[i]:select(2,1):long(), input[i]:select(2,2))
            end

            local gradOutput = torch.randn(inb, inj):type(typename):type(ctype)
            sslin:forward(nonsparse)
            local groundgrad = sslin:backward(nonsparse, gradOutput)
            sslin:zeroGradParameters()
            local groundweight = sslin.gradWeight
            local groundbias = sslin.gradBias

            for i,v in ipairs(input) do input[i] = input[i]:type(typename) end
            gradOutput = gradOutput:type(typename)
            gslin:forward(input)
            local rescuda = gslin:backward(input, gradOutput)
            gslin:zeroGradParameters()
            local weightcuda = gslin.gradWeight
            local biascuda = gslin.gradBias

            local werror = weightcuda:double() - groundweight:double()
            local berror = biascuda:double() - groundbias:double()

            mytester:assertlt(werror:abs():max(), precision_backward_type(precision_backward, typename),
                string.format('error on weight (backward) with %s', typename))
            mytester:assertlt(berror:abs():max(), precision_backward_type(precision_backward, typename),
                string.format('error on bias (backward) with %s', typename))

            gslin:updateParameters(.1)
            sslin:updateParameters(.1)
            werror = gslin.weight:double() - sslin.weight:double()
            berror = gslin.bias:double() - sslin.bias:double()

            mytester:assertlt(werror:abs():max(), precision_backward_type(precision_backward, typename),
                string.format('error on weight (update) with %s', typename))
            mytester:assertlt(berror:abs():max(), precision_backward_type(precision_backward, typename),
                string.format('error on bias (update) with %s', typename))

            gslin:zeroGradParameters()
        end
    end
end

local function BatchNormalization_forward(moduleName, inputSize)
   local planes = inputSize[2]

   for k, typename in ipairs(typenames) do
      local input = torch.randn(table.unpack(inputSize)):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sbnorm = nn[moduleName](planes):type(ctype)
      local groundtruth = sbnorm:forward(input)

      input = input:type(typename)
      local gbnorm = nn[moduleName](planes):type(typename)
      gbnorm.weight = sbnorm.weight:type(typename)
      gbnorm.bias = sbnorm.bias:type(typename)
      local rescuda = gbnorm:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename, rescuda:abs():max()),
         string.format('error on state (forward) with %s', typename))
      mytester:assertlt((gbnorm.running_mean:double() - sbnorm.running_mean:double()):abs():max(),
         precision_forward_type(precision_forward, typename, gbnorm.running_mean:abs():max()),
         string.format('error on running_mean (forward) with %s', typenanme))
      mytester:assertlt((gbnorm.running_var:double() - sbnorm.running_var:double()):abs():max(),
         precision_forward_type(precision_forward, typename, gbnorm.running_var:abs():max()),
         string.format('error on running_var (forward) with %s', typename))
   end
end

local function BatchNormalization_forward_inference(moduleName, inputSize)
   local planes = inputSize[2]

   for k, typename in ipairs(typenames) do
      local input = torch.randn(table.unpack(inputSize)):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sbnorm = nn[moduleName](planes):type(ctype)
      sbnorm.running_mean:normal(1, 2)
      sbnorm.running_var:uniform(1e-3, 2)
      sbnorm.running_var = sbnorm.running_var:type(typename):type(ctype)
      sbnorm.running_mean = sbnorm.running_mean:type(typename):type(ctype)

      sbnorm:evaluate()
      local groundtruth = sbnorm:forward(input)

      input = input:type(typename)
      local gbnorm = nn[moduleName](planes):type(typename)
      gbnorm:evaluate()
      gbnorm.weight = sbnorm.weight:type(typename)
      gbnorm.bias = sbnorm.bias:type(typename)
      gbnorm.running_mean = sbnorm.running_mean:type(typename)
      gbnorm.running_var = sbnorm.running_var:type(typename)
      local rescuda = gbnorm:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename, rescuda:abs():max()),
         string.format('error on state (forward evaluate) with %s', typename))
   end
end

local function BatchNormalization_backward(moduleName, mode, inputSize, backwardFn)
   assert(mode == 'training' or mode == 'evaluation', 'invalid mode')

   local planes = inputSize[2]

   for k, typename in ipairs(typenames) do
      local input = torch.randn(table.unpack(inputSize)):type(typename)
      local gradOutput = torch.randn(table.unpack(inputSize)):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sbnorm = nn[moduleName](planes):type(ctype)
      if mode == 'training' then
        sbnorm:training()
      else
        sbnorm:evaluate()
      end
      sbnorm:forward(input)
      sbnorm:zeroGradParameters()
      local groundgrad = backwardFn(sbnorm, input, gradOutput)
      local groundweight = sbnorm.gradWeight
      local groundbias = sbnorm.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gbnorm = nn[moduleName](planes):type(typename)
      if mode == 'training' then
        gbnorm:training()
      else
        gbnorm:evaluate()
      end
      gbnorm.weight = sbnorm.weight:type(typename)
      gbnorm.bias = sbnorm.bias:type(typename)
      gbnorm:forward(input)
      gbnorm:zeroGradParameters()
      local rescuda = backwardFn(gbnorm, input, gradOutput)
      local weightcuda = gbnorm.gradWeight
      local biascuda = gbnorm.gradBias

      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      local backerror = precision_backward_type(precision_backward, typename, rescuda:abs():max())
      if typename == 'torch.CudaHalfTensor' and (mode == 'training') then
        -- this correction is empirical; mean can be off by roughly 4e-4, multiplied by roughly stdval^2.
        backerror = backerror + (sbnorm.save_std:max())^2 * 4e-4
      end
      mytester:assertlt(error:abs():max(),
        backerror,
        string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
        precision_backward_type(precision_backward, typename, weightcuda:abs():max()),
        string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
        precision_backward_type(precision_backward, typename, biascuda:abs():max()),
        string.format('error on bias (backward) with %s', typename))
    end
end

local function testBatchNormalization(name, dim, k)
   local function inputSize()
      local inputSize = { torch.random(2,32), torch.random(1, k) }
      for i=1,dim do
         table.insert(inputSize, torch.random(1,k))
      end
      return inputSize
   end
   local function backward1(m, input, gradOutput)
      return m:backward(input, gradOutput)
   end
   local function backward2(m, input, gradOutput)
      local gradInput = m:updateGradInput(input, gradOutput)
      m:accGradParameters(input, gradOutput)
      return gradInput
   end

   BatchNormalization_forward(name, inputSize())
   BatchNormalization_forward_inference(name, inputSize())
   BatchNormalization_backward(name, 'training', inputSize(), backward1)
   BatchNormalization_backward(name, 'training', inputSize(), backward2)
   BatchNormalization_backward(name, 'evaluation', inputSize(), backward1)
   BatchNormalization_backward(name, 'evaluation', inputSize(), backward2)
end

function cunntest.BatchNormalization()
   testBatchNormalization('BatchNormalization', 0, 128)
end

function cunntest.SpatialBatchNormalization()
   testBatchNormalization('SpatialBatchNormalization', 2, 64)
   -- check with large image size (32*32 = 1024)
   BatchNormalization_forward('SpatialBatchNormalization', {2, 2, 32, 32})
end

function cunntest.VolumetricBatchNormalization()
   testBatchNormalization('VolumetricBatchNormalization', 3, 16)
end

function cunntest.SpatialConvolutionMM_forward_single()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local outi = math.random(1,64)
   local outj = math.random(1,64)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local ini = (outi-1)*si+ki-padW*2
   local inj = (outj-1)*sj+kj-padH*2

   local function jacTests(noBias)
      noBias = noBias or false
      for k, typename in ipairs(typenames) do
         local input = torch.randn(from,inj,ini):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         local sconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj,padW,padH):type(ctype)
         if noBias then
            sconv:noBias()
         end
         local groundtruth = sconv:forward(input)

         input = input:type(typename)
         local gconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj,padW,padH):type(typename)
         if noBias then
            gconv:noBias()
         end
         gconv.weight = sconv.weight:type(typename)
         if gconv.bias then
            gconv.bias = sconv.bias:type(typename)
         end
         local rescuda = gconv:forward(input)

         local error = rescuda:double() - groundtruth:double()
         mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
            string.format('error on state (forward) with %s', typename))
      end
   end

   jacTests(false)
   jacTests(true)
end

function cunntest.SpatialConvolutionMM_forward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local outi = math.random(1,64)
   local outj = math.random(1,64)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local ini = (outi-1)*si+ki-padW*2
   local inj = (outj-1)*sj+kj-padH*2

   local function jacTests(noBias)
      noBias = noBias or false
      for k, typename in ipairs(typenames) do
         local input = torch.randn(bs,from,inj,ini):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         local sconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj,padW,padH):type(ctype)
         if noBias then
            sconv:noBias()
         end
         local groundtruth = sconv:forward(input)

         input = input:type(typename)
         local gconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj,padW,padH):type(typename)
         if noBias then
            gconv:noBias()
         end
         gconv.weight = sconv.weight:type(typename)
         if gconv.bias then
            gconv.bias = sconv.bias:type(typename)
         end
         local rescuda = gconv:forward(input)

         local error = rescuda:double() - groundtruth:double()
         mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
            string.format('error on state (forward) with %s', typename))
      end
   end


end

function cunntest.SpatialConvolutionMM_backward_single()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local outi = math.random(1,64)
   local outj = math.random(1,64)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local ini = (outi-1)*si+ki-padW*2
   local inj = (outj-1)*sj+kj-padH*2

   local function jacTests(noBias)
      noBias = noBias or false

      for k, typename in ipairs(typenames) do
         local input = torch.randn(from,inj,ini):type(typename)
         local gradOutput = torch.randn(to,outj,outi):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         gradOutput = gradOutput:type(ctype)
         local sconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj,padW,padH):type(ctype)
         if noBias then
            sconv:noBias()
         end
         sconv:forward(input)
         sconv:zeroGradParameters()
         local groundgrad = sconv:backward(input, gradOutput)
         local groundweight = sconv.gradWeight
         local groundbias = sconv.gradBias

         input = input:type(typename)
         gradOutput = gradOutput:type(typename)
         local gconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj,padW,padH):type(typename)
         if noBias then
            gconv:noBias()
         end
         gconv.weight = sconv.weight:type(typename)
         if gconv.bias then
            gconv.bias = sconv.bias:type(typename)
         end
         gconv:forward(input)
         gconv:zeroGradParameters()
         local rescuda = gconv:backward(input, gradOutput)
         local weightcuda = gconv.gradWeight

         local error = rescuda:double() - groundgrad:double()
         local werror = weightcuda:double() - groundweight:double()

         mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
            string.format('error on state (backward) with %s', typename))
         mytester:assertlt(werror:abs():max(),
            precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
            string.format('error on weight (backward) with %s', typename))

         if gconv.bias then
            local berror = gconv.gradBias:double() - groundbias:double()
            mytester:assertlt(berror:abs():max(),
                precision_backward_conv_weightbias(precision_backward, typename, gconv.gradBias:abs():max()),
                string.format('error on bias (backward) with %s', typename))
         end
      end
   end

   jacTests(false)
   jacTests(true)
end

function cunntest.SpatialConvolutionMM_backward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local outi = math.random(1,64)
   local outj = math.random(1,64)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local ini = (outi-1)*si+ki-padW*2
   local inj = (outj-1)*sj+kj-padH*2

   local function jacTests(noBias)
      noBias = noBias or false

      for k, typename in ipairs(typenames) do
         local input = torch.randn(bs,from,inj,ini)
         local gradOutput = torch.randn(bs,to,outj,outi)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         gradOutput = gradOutput:type(ctype)
         local sconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj,padW,padH):type(ctype)
         if noBias then
            sconv:noBias()
         end
         sconv:forward(input)
         sconv:zeroGradParameters()
         local groundgrad = sconv:backward(input, gradOutput)
         local groundweight = sconv.gradWeight
         local groundbias = sconv.gradBias

         input = input:type(typename)
         gradOutput = gradOutput:type(typename)
         local gconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj,padW,padH):type(typename)
         if noBias then
            gconv:noBias()
         end
         gconv.weight = sconv.weight:type(typename)
         if gconv.bias then
            gconv.bias = sconv.bias:type(typename)
         end
         gconv:forward(input)
         gconv:zeroGradParameters()
         local rescuda = gconv:backward(input, gradOutput)
         local weightcuda = gconv.gradWeight

         local error = rescuda:double() - groundgrad:double()
         local werror = weightcuda:double() - groundweight:double()

         mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
            string.format('error on state (backward) with %s', typename))
         mytester:assertlt(werror:abs():max(),
            precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
            string.format('error on weight (backward) with %s', typename))
         if gconv.bias then
            local berror = gconv.gradBias:double() - groundbias:double()
            mytester:assertlt(berror:abs():max(),
                precision_backward_conv_weightbias(precision_backward, typename, gconv.gradBias:abs():max()),
                string.format('error on bias (backward) with %s', typename))
         end
      end
   end

   jacTests(false)
   jacTests(true)
end

function cunntest.SpatialConvolutionLocal_forward_single()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local outi = math.random(1,48)
   local outj = math.random(1,48)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local ini = (outi-1)*si+ki-padW*2
   local inj = (outj-1)*sj+kj-padH*2

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialConvolutionLocal(from,to,ini,inj,ki,kj,si,sj,padW,padH):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialConvolutionLocal(from,to,ini,inj,ki,kj,si,sj,padW,padH):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialConvolutionLocal_forward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local outi = math.random(1,48)
   local outj = math.random(1,48)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local ini = (outi-1)*si+ki-padW*2
   local inj = (outj-1)*sj+kj-padH*2

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialConvolutionLocal(from,to,ini,inj,ki,kj,si,sj,padW,padH):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialConvolutionLocal(from,to,ini,inj,ki,kj,si,sj,padW,padH):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialConvolutionLocal_backward_single()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local outi = math.random(1,48)
   local outj = math.random(1,48)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local ini = (outi-1)*si+ki-padW*2
   local inj = (outj-1)*sj+kj-padH*2

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)
      local gradOutput = torch.randn(to,outj,outi):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialConvolutionLocal(from,to,ini,inj,ki,kj,si,sj,padW,padH):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialConvolutionLocal(from,to,ini,inj,ki,kj,si,sj,padW,padH):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias

      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
          precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
          string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
          precision_backward_conv_weightbias(precision_backward, typename, biascuda:abs():max()),
          string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.SpatialConvolutionLocal_backward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local outi = math.random(1,48)
   local outj = math.random(1,48)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local ini = (outi-1)*si+ki-padW*2
   local inj = (outj-1)*sj+kj-padH*2

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local gradOutput = torch.randn(bs,to,outj,outi):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialConvolutionLocal(from,to,ini,inj,ki,kj,si,sj,padW,padH):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialConvolutionLocal(from,to,ini,inj,ki,kj,si,sj,padW,padH):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias

      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
          precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
          string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
          precision_backward_conv_weightbias(precision_backward, typename, biascuda:abs():max()),
          string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.SpatialFullConvolution_forward_single()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local outi = math.random(ki, 64)
   local outj = math.random(kj, 64)
   local adjW = (outi + padW*2 - ki) % si
   local adjH = (outj + padH*2 - kj) % sj
   local ini = math.floor((outi + 2 * padW - ki) / si + 1)
   local inj = math.floor((outj + 2 * padH - kj) / sj + 1)

   local function jacTests(noBias)
      noBias = noBias or false
      for k, typename in ipairs(typenames) do
         local input = torch.randn(from,inj,ini):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         local sconv = nn.SpatialFullConvolution(from,to,ki,kj,si,sj,padW,padH,adjW,adjH):type(ctype)
         if noBias then
            sconv:noBias()
         end
         local groundtruth = sconv:forward(input)

         input = input:type(typename)
         local gconv = nn.SpatialFullConvolution(from,to,ki,kj,si,sj,padW,padH,adjW,adjH):type(typename)
         if noBias then
            gconv:noBias()
         end
         gconv.weight = sconv.weight:type(typename)
         if gconv.bias then
            gconv.bias = sconv.bias:type(typename)
         end
         local rescuda = gconv:forward(input)

         local error = rescuda:double() - groundtruth:double()
         mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
            string.format('error on state (forward) with %s', typename))
      end
   end

   jacTests(false)
   jacTests(true)
end

function cunntest.SpatialFullConvolution_forward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local outi = math.random(ki, 64)
   local outj = math.random(kj, 64)
   local adjW = (outi + padW*2 - ki) % si
   local adjH = (outj + padH*2 - kj) % sj
   local ini = math.floor((outi + 2 * padW - ki) / si + 1)
   local inj = math.floor((outj + 2 * padH - kj) / sj + 1)

   local function jacTests(noBias)
      noBias = noBias or false
      for k, typename in ipairs(typenames) do
         local input = torch.randn(bs,from,inj,ini):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         local sconv = nn.SpatialFullConvolution(from,to,ki,kj,si,sj,padW,padH,adjW,adjH):type(ctype)
         if noBias then
            sconv:noBias()
         end
         local groundtruth = sconv:forward(input)

         input = input:type(typename)
         local gconv = nn.SpatialFullConvolution(from,to,ki,kj,si,sj,padW,padH,adjW,adjH):type(typename)
         if noBias then
            gconv:noBias()
         end
         gconv.weight = sconv.weight:type(typename)
         if gconv.bias then
            gconv.bias = sconv.bias:type(typename)
         end
         local rescuda = gconv:forward(input)

         local error = rescuda:double() - groundtruth:double()
         mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
              string.format('error on state (forward) with %s', typename))
      end
   end

   jacTests(false)
   jacTests(true)
end

function cunntest.SpatialFullConvolution_backward_single()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local outi = math.random(ki, 64)
   local outj = math.random(kj, 64)
   local adjW = (outi + padW*2 - ki) % si
   local adjH = (outj + padH*2 - kj) % sj
   local ini = math.floor((outi + 2 * padW - ki) / si + 1)
   local inj = math.floor((outj + 2 * padH - kj) / sj + 1)

   local function jacTests(noBias)
      noBias = noBias or false
      for k, typename in ipairs(typenames) do
         local input = torch.randn(from,inj,ini):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         local sconv = nn.SpatialFullConvolution(from,to,ki,kj,si,sj,padW,padH,adjW,adjH):type(ctype)
         if noBias then
            sconv:noBias()
         end
         local output = sconv:forward(input)
         local gradOutput = output:clone():normal()
         sconv:zeroGradParameters()
         local groundgrad = sconv:backward(input, gradOutput)
         local groundweight = sconv.gradWeight
         local groundbias = sconv.gradBias

         input = input:type(typename)
         gradOutput = gradOutput:type(typename)
         local gconv = nn.SpatialFullConvolution(from,to,ki,kj,si,sj,padW,padH,adjW,adjH):type(typename)
         if noBias then
            gconv:noBias()
         end
         gconv.weight = sconv.weight:type(typename)
         if gconv.bias then
            gconv.bias = sconv.bias:type(typename)
         end
         gconv:forward(input)
         gconv:zeroGradParameters()
         local rescuda = gconv:backward(input, gradOutput)
         local weightcuda = gconv.gradWeight

         local error = rescuda:double() - groundgrad:double()
         local werror = weightcuda:double() - groundweight:double()

         mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
            string.format('error on state (backward) with %s', typename))
         mytester:assertlt(werror:abs():max(),
            precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
            string.format('error on weight (backward) with %s', typename))

         if gconv.bias then
            local berror = gconv.gradBias:double() - groundbias:double()
            mytester:assertlt(berror:abs():max(),
               precision_backward_conv_weightbias(precision_backward, typename, gconv.gradBias:abs():max()),
               string.format('error on bias (backward) with %s', typename))
         end
      end
   end

  jacTests(false)
  jacTests(true)
end

function cunntest.SpatialFullConvolution_backward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local outi = math.random(ki, 64)
   local outj = math.random(kj, 64)
   local adjW = (outi + padW*2 - ki) % si
   local adjH = (outj + padH*2 - kj) % sj
   local ini = math.floor((outi + 2 * padW - ki) / si + 1)
   local inj = math.floor((outj + 2 * padH - kj) / sj + 1)

   local function jacTests(noBias)
      noBias = noBias or false

      for k, typename in ipairs(typenames) do
         local input = torch.randn(bs,from,inj,ini):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         local sconv = nn.SpatialFullConvolution(from,to,ki,kj,si,sj,padW,padH,adjW,adjH):type(ctype)
         if noBias then
            sconv:noBias()
         end
         local output = sconv:forward(input)
         local gradOutput = output:clone():normal()
         sconv:zeroGradParameters()
         local groundgrad = sconv:backward(input, gradOutput)
         local groundweight = sconv.gradWeight
         local groundbias = sconv.gradBias

         input = input:type(typename)
         gradOutput = gradOutput:type(typename)
         local gconv = nn.SpatialFullConvolution(from,to,ki,kj,si,sj,padW,padH,adjW,adjH):type(typename)
         if noBias then
            gconv:noBias()
         end
         gconv.weight = sconv.weight:type(typename)
         if gconv.bias then
            gconv.bias = sconv.bias:type(typename)
         end
         gconv:forward(input)
         gconv:zeroGradParameters()
         local rescuda = gconv:backward(input, gradOutput)
         local weightcuda = gconv.gradWeight

         local error = rescuda:double() - groundgrad:double()
         local werror = weightcuda:double() - groundweight:double()

         mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
            string.format('error on state (backward) with %s', typename))
         mytester:assertlt(werror:abs():max(),
            precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
            string.format('error on weight (backward) with %s', typename))
         if gconv.bias then
            local berror = gconv.gradBias:double() - groundbias:double()
            mytester:assertlt(berror:abs():max(),
               precision_backward_conv_weightbias(precision_backward, typename, gconv.gradBias:abs():max()),
               string.format('error on bias (backward) with %s', typename))
         end
      end
   end

   jacTests(false)
   jacTests(true)
end

function cunntest.SpatialDilatedConvolution_forward_single()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local outi = math.random(ki, 64)
   local outj = math.random(kj, 64)
   local dilationW = math.random(1,10)
   local dilationH = math.random(1,10)
   local ini = (outi - 1) * si - 2 * padW + dilationW * (ki-1) + 1
   local inj = (outj - 1) * sj - 2 * padH + dilationH * (kj-1) + 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialDilatedConvolution(from,to,ki,kj,si,sj,padW,padH,dilationW,dilationH):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialDilatedConvolution(from,to,ki,kj,si,sj,padW,padH,dilationW,dilationH):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
         string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialDilatedConvolution_forward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local outi = math.random(ki, 64)
   local outj = math.random(kj, 64)
   local dilationW = math.random(1,10)
   local dilationH = math.random(1,10)
   local ini = (outi - 1) * si - 2 * padW + dilationW * (ki-1) + 1
   local inj = (outj - 1) * sj - 2 * padH + dilationH * (kj-1) + 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialDilatedConvolution(from,to,ki,kj,si,sj,padW,padH,dilationW,dilationH):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialDilatedConvolution(from,to,ki,kj,si,sj,padW,padH,dilationW,dilationH):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
         string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialDilatedConvolution_backward_single()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local outi = math.random(ki, 64)
   local outj = math.random(kj, 64)
   local dilationW = math.random(1,10)
   local dilationH = math.random(1,10)
   local ini = (outi - 1) * si - 2 * padW + dilationW * (ki-1) + 1
   local inj = (outj - 1) * sj - 2 * padH + dilationH * (kj-1) + 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialDilatedConvolution(from,to,ki,kj,si,sj,padW,padH,dilationW,dilationH):type(ctype)
      local output = sconv:forward(input)
      local gradOutput = output:clone():normal()
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialDilatedConvolution(from,to,ki,kj,si,sj,padW,padH,dilationW,dilationH):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias

      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
         string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
         precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
         string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
         precision_backward_conv_weightbias(precision_backward, typename, biascuda:abs():max()),
         string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.SpatialDilatedConvolution_backward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local outi = math.random(ki, 64)
   local outj = math.random(kj, 64)
   local dilationW = math.random(1,10)
   local dilationH = math.random(1,10)
   local ini = (outi - 1) * si - 2 * padW + dilationW * (ki-1) + 1
   local inj = (outj - 1) * sj - 2 * padH + dilationH * (kj-1) + 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialDilatedConvolution(from,to,ki,kj,si,sj,padW,padH,dilationW,dilationH):type(ctype)
      local output = sconv:forward(input)
      local gradOutput = output:clone():normal()
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialDilatedConvolution(from,to,ki,kj,si,sj,padW,padH,dilationW,dilationH):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias

      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
         string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
         precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
         string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
         precision_backward_conv_weightbias(precision_backward, typename, biascuda:abs():max()),
         string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.SpatialSubSampling_forward()
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialSubSampling(from,ki,kj,si,sj):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialSubSampling(from,ki,kj,si,sj):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.Sampling_forward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialSubSampling(from,ki,kj,si,sj):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialSubSampling(from,ki,kj,si,sj):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialSubSampling_backward()
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outi = math.random(32,64)
   local outj = math.random(32,64)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj

   for k, typename in ipairs(typenames) do
      -- FIXME: SpatialSubSampling accumulates directly to real, causes
      -- precision issues with half
      precision_backward_old = precision_backward
      if typename == 'torch.CudaHalfTensor' then
          precision_backward = 0.4
      end
      local input = torch.randn(from,inj,ini):type(typename)
      local gradOutput = torch.randn(to,outj,outi):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialSubSampling(from,ki,kj,si,sj):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialSubSampling(from,ki,kj,si,sj):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias

      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on bias (backward) with %s', typename))

      precision_backward = precision_backward_old
   end
end

function cunntest.SpatialSubSampling_backward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outi = math.random(32,64)
   local outj = math.random(32,64)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local gradOutput = torch.randn(bs,to,outj,outi):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialSubSampling(from,ki,kj,si,sj):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialSubSampling(from,ki,kj,si,sj):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias

      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      -- FIXME: SpatialSubSampling accumulates directly to real, causes
      -- precision issues with half, so we double the error tolerance
      mytester:assertlt(error:abs():max(),
          2*precision_backward_type(precision_backward, typename, rescuda:abs():max()),
          string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
          2*precision_backward_type(precision_backward, typename, weightcuda:abs():max()),
          string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
          2*precision_backward_type(precision_backward, typename, biascuda:abs():max()),
          string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.SpatialMaxPooling_forward()
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(1,4)
   local sj = math.random(1,4)
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ini = (outi-1)*si+ki - padi*2
   local inj = (outj-1)*sj+kj - padj*2
   local ceil_mode = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then sconv:ceil() end
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then gconv:ceil() end
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
      local error_ind = gconv.indices:long() - sconv.indices
      mytester:asserteq(error_ind:max(), 0,
          string.format('error on indices (forward) with %s', typename))
    end
end

function cunntest.SpatialMaxPooling_forward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ini = (outi-1)*si+ki - padi*2
   local inj = (outj-1)*sj+kj - padj*2
   local ceil_mode = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then sconv:ceil() end
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then gconv:ceil() end
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
    end
end

function cunntest.SpatialMaxUnpooling_forward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = ki
   local sj = kj
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ceil_mode = math.random(0,1) == 1
   local fun = ceil_mode and torch.ceil or torch.floor
   local ini = fun((outi + padi*2 - ki)/si) +1
   local inj = fun((outj + padj*2 - kj)/sj) +1

   for k, typename in ipairs(typenames) do
      local ctype = t2cpu[typename]
      local pooler = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then pooler:ceil() end
      local sunpool = nn.SpatialMaxUnpooling(pooler):type(ctype)

      local original = torch.randn(bs,from,outj,outi):type(typename)
      original = original:type(ctype)
      local input = pooler:forward(original)
      local groundtruth = sunpool:forward(input)

      original = original:type(typename)
      pooler = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then pooler:ceil() end
      local gunpool = nn.SpatialMaxUnpooling(pooler):type(typename)

      input = pooler:forward(original)
      local rescuda = gunpool:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
    end
end

function cunntest.SpatialMaxPooling_backward()
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(1,4)
   local sj = math.random(1,4)
   local outi = math.random(32,64)
   local outj = math.random(32,64)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ini = (outi-1)*si+ki - padi*2
   local inj = (outj-1)*sj+kj - padj*2
   local ceil_mode = true--math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)
      local gradOutput = torch.randn(to,outj,outi):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)

      local sconv = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then sconv:ceil() end
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then gconv:ceil() end
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.SpatialMaxPooling_backward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outi = math.random(32,64)
   local outj = math.random(32,64)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ini = (outi-1)*si+ki - padi*2
   local inj = (outj-1)*sj+kj - padj*2
   local ceil_mode = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local gradOutput = torch.randn(bs,to,outj,outi):type(typename)
      local ctype = t2cpu[typename]
      local input = input:type(ctype)
      local gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then sconv:ceil() end
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then gconv:ceil() end
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.SpatialMaxUnpooling_backward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = ki
   local sj = kj
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ceil_mode = math.random(0,1) == 1
   local fun = ceil_mode and torch.ceil or torch.floor
   local ini = fun((outi + padi*2 - ki)/si) +1
   local inj = fun((outj + padj*2 - kj)/sj) +1

   for k, typename in ipairs(typenames) do
      local ctype = t2cpu[typename]
      local pooler = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then pooler:ceil() end
      local sunpool = nn.SpatialMaxUnpooling(pooler):type(ctype)

      local original = torch.randn(bs,from,outj,outi):type(typename)
      original = original:type(ctype)
      local input = pooler:forward(original)
      local gradOutput = torch.randn(original:size()):type(typename)
      gradOutput = gradOutput:type(ctype)
      sunpool:forward(input)
      sunpool:zeroGradParameters()
      local groundgrad = sunpool:backward(input, gradOutput)

      pooler = nn.SpatialMaxPooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then pooler:ceil() end
      local gunpool = nn.SpatialMaxUnpooling(pooler):type(typename)

      original = original:type(typename)
      input = pooler:forward(original)
      gunpool:forward(input)

      gradOutput = gradOutput:type(typename)
      gunpool:zeroGradParameters()
      local rescuda = gunpool:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.SpatialDilatedMaxPooling_forward()
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(1,4)
   local sj = math.random(1,4)
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local dilationi = math.random(1,10)
   local dilationj = math.random(1,10)
   local ini = (outi-1)*si+(dilationi*(ki-1)+1)-2*padi
   local inj = (outj-1)*sj+(dilationj*(kj-1)+1)-2*padj
   local ceil_mode = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialDilatedMaxPooling(ki,kj,si,sj,padi,padj,dilationi,dilationj):type(ctype)
      if ceil_mode then sconv:ceil() end
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialDilatedMaxPooling(ki,kj,si,sj,padi,padj,dilationi,dilationj):type(typename)
      if ceil_mode then gconv:ceil() end
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
      local error_ind = gconv.indices:long() - sconv.indices
      mytester:asserteq(error_ind:max(), 0,
          string.format('error on indices (forward) with %s', typename))
    end
end

function cunntest.SpatialDilatedMaxPooling_forward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local dilationi = math.random(1,10)
   local dilationj = math.random(1,10)
   local ini = (outi-1)*si+(dilationi*(ki-1)+1)-2*padi
   local inj = (outj-1)*sj+(dilationj*(kj-1)+1)-2*padj
   local ceil_mode = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialDilatedMaxPooling(ki,kj,si,sj,padi,padj,dilationi,dilationj):type(ctype)
      if ceil_mode then sconv:ceil() end
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialDilatedMaxPooling(ki,kj,si,sj,padi,padj,dilationi,dilationj):type(typename)
      if ceil_mode then gconv:ceil() end
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
    end
end

function cunntest.SpatialDilatedMaxPooling_backward()
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(1,4)
   local sj = math.random(1,4)
   local outi = math.random(32,64)
   local outj = math.random(32,64)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local dilationi = math.random(1,10)
   local dilationj = math.random(1,10)
   local ini = (outi-1)*si+(dilationi*(ki-1)+1)-2*padi
   local inj = (outj-1)*sj+(dilationj*(kj-1)+1)-2*padj
   local ceil_mode = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)
      local gradOutput = torch.randn(to,outj,outi):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialDilatedMaxPooling(ki,kj,si,sj,padi,padj,dilationi,dilationj):type(ctype)
      if ceil_mode then sconv:ceil() end
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialDilatedMaxPooling(ki,kj,si,sj,padi,padj,dilationi,dilationj):type(typename)
      if ceil_mode then gconv:ceil() end
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.SpatialDilatedMaxPooling_backward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outi = math.random(32,64)
   local outj = math.random(32,64)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local dilationi = math.random(1,10)
   local dilationj = math.random(1,10)
   local ini = (outi-1)*si+(dilationi*(ki-1)+1)-2*padi
   local inj = (outj-1)*sj+(dilationj*(kj-1)+1)-2*padj
   local ceil_mode = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local gradOutput = torch.randn(bs,to,outj,outi):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialDilatedMaxPooling(ki,kj,si,sj,padi,padj,dilationi,dilationj):type(ctype)
      if ceil_mode then sconv:ceil() end
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialDilatedMaxPooling(ki,kj,si,sj,padi,padj,dilationi,dilationj):type(typename)
      if ceil_mode then gconv:ceil() end
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.SpatialFractionalMaxPooling_forward()
    local batch = math.random(1, 3)
    local plane = math.random(1, 3)
    local outW = math.random(1, 7)
    local outH = math.random(1, 7)
    local poolSizeW = math.random(2, 4)
    local poolSizeH = math.random(2, 4)

    local minInW = outW + poolSizeW
    local minInH = outH + poolSizeH

    local inW = math.random(minInW, minInW + 6)
    local inH = math.random(minInH, minInH + 6)

    local useRatio = (math.random(1, 2) == 1)
    local ratioW = outW / inW
    local ratioH = outH / inH

    for k, typename in ipairs(typenames) do
        local input = nil
        if batch == 1 then
            input = torch.Tensor(plane, inH, inW):uniform():type(typename)
        else
            input = torch.Tensor(batch, plane, inH, inW):uniform():type(typename)
        end

        local ctype = t2cpu[typename]
        input = input:type(ctype)
        local module = nil
        if useRatio then
            module =
                nn.SpatialFractionalMaxPooling(poolSizeW, poolSizeH, ratioW, ratioH):type(ctype)
        else
            module =
                nn.SpatialFractionalMaxPooling(poolSizeW, poolSizeH, outW, outH):type(ctype)
        end

        module:fixPoolingRegions()

        local groundtruth = module:forward(input)

        input = input:type(typename)

        local gmodule = nil
        if useRatio then
            gmodule =
                nn.SpatialFractionalMaxPooling(poolSizeW, poolSizeH, ratioW, ratioH)
        else
            gmodule =
                nn.SpatialFractionalMaxPooling(poolSizeW, poolSizeH, outW, outH)
        end

        gmodule = gmodule:fixPoolingRegions():type(typename)

        -- For comparison purposes, make sure we are using the same random pooling regions
        -- as the CPU
        gmodule.randomSamples = module.randomSamples:type(typename)

        local rescuda = gmodule:forward(input)

        local error = rescuda:double() - groundtruth:double()
        mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
            string.format('error on state (forward) with %s', typename))
        local error_ind = gmodule.indices:long() - module.indices
        mytester:asserteq(error_ind:abs():max(), 0,
            string.format('error on indices (forward) with %s', typename))
    end
end

function cunntest.SpatialFractionalMaxPooling_backward()
    local batch = math.random(1, 3)
    local plane = math.random(1, 3)
    local outW = math.random(1, 7)
    local outH = math.random(1, 7)
    local poolSizeW = math.random(2, 4)
    local poolSizeH = math.random(2, 4)

    local minInW = outW + poolSizeW
    local minInH = outH + poolSizeH

    local inW = math.random(minInW, minInW + 6)
    local inH = math.random(minInH, minInH + 6)

    for k, typename in ipairs(typenames) do
        local input = nil
        local gradOutput = nil
        if batch == 1 then
            input = torch.Tensor(plane, inH, inW):uniform():type(typename)
            gradOutput = torch.Tensor(plane, outH, outW):uniform():type(typename)
        else
            input = torch.Tensor(batch, plane, inH, inW):uniform():type(typename)
            gradOutput = torch.Tensor(batch, plane, outH, outW):uniform():type(typename)
        end

        local ctype = t2cpu[typename]
        input = input:type(ctype)
        gradOutput = gradOutput:type(ctype)
        local module =
            nn.SpatialFractionalMaxPooling(poolSizeW, poolSizeH, outW, outH)
            :fixPoolingRegions():type(ctype)

        -- convert type of randomSamples and ensure we don't resample
        module:initSampleBuffer_(input)
        module:fixPoolingRegions()
        module.randomSamples = module.randomSamples:type(typename):type(ctype)
        module:forward(input)
        module:zeroGradParameters()
        local groundgrad = module:backward(input, gradOutput)

        input = input:type(typename)
        gradOutput = gradOutput:type(typename)

        local gmodule =
            nn.SpatialFractionalMaxPooling(poolSizeW, poolSizeH, outW, outH)
            :fixPoolingRegions():type(typename)
        -- For comparison purposes, make sure we are using the same random pooling regions
        -- as the CPU
        gmodule.randomSamples = module.randomSamples:type(typename)

        gmodule:forward(input)
        gmodule:zeroGradParameters()
        local rescuda = gmodule:backward(input, gradOutput)

        local error = rescuda:double() - groundgrad:double()
        mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
            string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.SpatialAveragePooling_includepad()
   for k, typename in ipairs(typenames) do
      local net = nn.SpatialAveragePooling(2, 2, 1, 1, 1, 1):type(typename)
      local net_no_include_pad = net:clone()
      net_no_include_pad:setCountExcludePad()
      local net_include_pad = net:clone()
      net_include_pad:setCountIncludePad()

      local input = torch.FloatTensor(1, 1, 1, 1):type(typename)
      input[1][1][1][1] = 3
      local out_noinclude = net_no_include_pad:forward(input)
      local out_include = net_include_pad:forward(input)

      local noinc_out = out_noinclude[1][1][1][1]
      local inc_out = out_include[1][1][1][1]
      mytester:assertne(noinc_out, inc_out)
      mytester:asserteq(3, noinc_out)
      mytester:asserteq(3/4, inc_out)
   end
end

function cunntest.SpatialAveragePooling_forward()
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(1,ki)
   local sj = math.random(1,kj)
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ini = (outi-1)*si+ki - padi*2
   local inj = (outj-1)*sj+kj - padj*2
   local ceil_mode = math.random(0,1) == 1
   local count_exclude_pad = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialAveragePooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then sconv:ceil() end
      if count_exclude_pad then sconv:setCountExcludePad() end
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialAveragePooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then gconv:ceil() end
      if count_exclude_pad then gconv:setCountExcludePad() end
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialAveragePooling_forward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(1,ki)
   local sj = math.random(1,kj)
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ini = (outi-1)*si+ki - padi*2
   local inj = (outj-1)*sj+kj - padj*2
   local ceil_mode = math.random(0,1) == 1
   local count_exclude_pad = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local ctype = t2cpu[typename]

      input = input:type(ctype)
      local sconv = nn.SpatialAveragePooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then sconv:ceil() end
      if count_exclude_pad then sconv:setCountExcludePad() end
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialAveragePooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then gconv:ceil() end
      if count_exclude_pad then gconv:setCountExcludePad() end
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialAveragePooling_backward()
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(1,ki)
   local sj = math.random(1,kj)
   local outi = math.random(32,64)
   local outj = math.random(32,64)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ini = (outi-1)*si+ki - padi*2
   local inj = (outj-1)*sj+kj - padj*2
   local ceil_mode = math.random(0,1) == 1
   local count_exclude_pad = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)
      local gradOutput = torch.randn(to,outj,outi):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialAveragePooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then sconv:ceil() end
      if count_exclude_pad then sconv:setCountExcludePad() end
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialAveragePooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then gconv:ceil() end
      if count_exclude_pad then gconv:setCountExcludePad() end
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialAveragePooling_backward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = math.random(1,ki)
   local sj = math.random(1,kj)
   local outi = math.random(32,64)
   local outj = math.random(32,64)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local ini = (outi-1)*si+ki - padi*2
   local inj = (outj-1)*sj+kj - padj*2
   local ceil_mode = math.random(0,1) == 1
   local count_exclude_pad = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local gradOutput = torch.randn(bs,to,outj,outi):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialAveragePooling(ki,kj,si,sj,padi,padj):type(ctype)
      if ceil_mode then sconv:ceil() end
      if count_exclude_pad then sconv:setCountExcludePad() end
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialAveragePooling(ki,kj,si,sj,padi,padj):type(typename)
      if ceil_mode then gconv:ceil() end
      if count_exclude_pad then gconv:setCountExcludePad() end
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialAdaptiveMaxPooling_forward()
   local from = math.random(1,64)
   local to = from
   local outi = math.random(2,64)
   local outj = math.random(2,64)
   local ini = math.random(10,256)
   local inj = math.random(10,256)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(ctype)
      local groundtruth = sconv:forward(input):type(ctype)

      input = input:type(typename)
      local gconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
      local error_ind = gconv.indices:long() - sconv.indices
      mytester:asserteq(error_ind:max(), 0,
          string.format('error on indices (forward) with %s', typename))
   end
end

function cunntest.SpatialAdaptiveMaxPooling_forward_noncontig()
   local from = math.random(1,64)
   local to = from
   local outi = math.random(2,64)
   local outj = math.random(2,64)
   local ini = math.random(10,256)
   local inj = math.random(10,256)

   for k, typename in ipairs(typenames) do
      local input0 = torch.randn(from,ini,inj):type(typename)
      local ctype = t2cpu[typename]
      local input = input0:type(ctype):transpose(2,3)
      local sconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input0:type(typename):transpose(2,3)
      local gconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
      local error_ind = gconv.indices:long() - sconv.indices
      mytester:asserteq(error_ind:max(), 0,
          string.format('error on indices (forward) with %s', typename))
   end
end

function cunntest.SpatialAdaptiveMaxPooling_forward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,48)
   local to = from
   local outi = math.random(2,48)
   local outj = math.random(2,48)
   local ini = math.random(10,256)
   local inj = math.random(10,256)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialAdaptiveMaxPooling_backward()
   local from = math.random(1,64)
   local to = from
   local outi = math.random(2,64)
   local outj = math.random(2,64)
   local ini = math.random(10,256)
   local inj = math.random(10,256)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,inj,ini):type(typename)
      local gradOutput = torch.randn(to,outj,outi):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialAdaptiveMaxPooling_backward_noncontig()
   local from = math.random(1,64)
   local to = from
   local outi = math.random(2,64)
   local outj = math.random(2,64)
   local ini = math.random(10,256)
   local inj = math.random(10,256)

   for k, typename in ipairs(typenames) do
      local input0 = torch.randn(from,ini,inj):type(typename)
      local gradOutput = torch.randn(to,outj,outi):type(typename)
      local ctype = t2cpu[typename]
      local input = input0:type(ctype):transpose(2,3)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input0:type(typename):transpose(2,3)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialAdaptiveMaxPooling_backward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local outi = math.random(2,64)
   local outj = math.random(2,64)
   local ini = math.random(10,256)
   local inj = math.random(10,256)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,inj,ini):type(typename)
      local gradOutput = torch.randn(bs,to,outj,outi):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.SpatialAdaptiveMaxPooling(outi,outj):type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialLPPooling_forward()
   local from = math.random(1,64)
   local to = from
   local pnorm = 2
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = ki
   local sj = kj
   local outi = math.random(32,256)
   local outj = math.random(32,256)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj

   local tm = {}
   local title = string.format('SpatialLPPooling.forward (P=2 only) %dx%dx%d o %dx%d -> %dx%dx%d',
                               from, inj, ini, kj, ki, to, outj, outi)
   times[title] = tm

   local input = torch.randn(from,inj,ini)
   local sconv = nn.SpatialLPPooling(from,pnorm,ki,kj,si,sj)
   local groundtruth = sconv:forward(input)
   local a = torch.Timer()
   for i = 1,nloop do
      groundtruth = sconv:forward(input)
   end
   tm.cpu = a:time().real

   input = input:cuda()
   local gconv = nn.SpatialLPPooling(from,pnorm,ki,kj,si,sj):cuda()
   local rescuda = gconv:forward(input)
   a:reset()
   for i = 1,nloop do
      rescuda = gconv:forward(input)
   end
   cutorch.synchronize()
   tm.gpu = a:time().real

   local error = rescuda:float() - groundtruth
   mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
end

function cunntest.SpatialLPPooling_backward()
   local from = math.random(1,64)
   local to = from
   local pnorm = 2
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local si = ki
   local sj = kj
   local outi = math.random(32,64)
   local outj = math.random(32,64)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj

   local tm = {}
   local title = string.format('SpatialLPPooling.backward (P=2 only) %dx%dx%d o %dx%d -> %dx%dx%d',
                               from, inj, ini, kj, ki, to, outj, outi)
   times[title] = tm

   local input = torch.randn(from,inj,ini)
   local gradOutput = torch.randn(to,outj,outi)
   local sconv = nn.SpatialLPPooling(from,pnorm,ki,kj,si,sj)
   sconv:forward(input)
   sconv:zeroGradParameters()
   local groundgrad = sconv:backward(input, gradOutput)
   local a = torch.Timer()
   for i = 1,nloop do
      sconv:zeroGradParameters()
      groundgrad = sconv:backward(input, gradOutput)
   end
   tm.cpu = a:time().real

   input = input:cuda()
   gradOutput = gradOutput:cuda()
   local gconv = sconv:clone():cuda()
   gconv:forward(input)
   gconv:zeroGradParameters()
   local rescuda = gconv:backward(input, gradOutput)
   a:reset()
   for i = 1,nloop do
      gconv:zeroGradParameters()
      rescuda = gconv:backward(input, gradOutput)
   end
   cutorch.synchronize()
   tm.gpu = a:time().real

   local error = rescuda:float() - groundgrad

   mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
end


-- Criterion tests

local function BCECriterion_forward_truth(buffer, input, target, weights, sizeAverage)

  local eps = 1e-12
  local output

  buffer:resizeAs(input)

  if weights ~= nil and target:dim() ~= 1 then
    weights = weights:view(1, target:size(2)):expandAs(target)
  end

  -- log(input) * target
  buffer:add(input, eps):log()
  if weights ~= nil then buffer:cmul(weights) end

  output = torch.dot(target, buffer)

  -- log(1 - input) * (1 - target)
  buffer:mul(input, -1):add(1):add(eps):log()
  if weights ~= nil then buffer:cmul(weights) end

  output = output + torch.sum(buffer)
  output = output - torch.dot(target, buffer)

  if sizeAverage then
    output = output / input:nElement()
  end

  output = - output

  return output

end

function cunntest.BCECriterion_forward()
  local size = math.random(1,100)

  for k, typename in ipairs(typenames) do
     local input = torch.Tensor(size):uniform():type(typename)
     local target = torch.Tensor(size):uniform():gt(0.5):type(torch.type(input))

     local ctype = t2cpu[typename]
     input = input:type(ctype)
     target = target:type(ctype)
     local crit = nn.BCECriterion():type(ctype)
     local rescpu = crit:forward(input, target)

     input = input:type(typename)
     target = target:type(typename)
     local g_crit = nn.BCECriterion():type(typename)
     local rescuda = g_crit:forward(input, target)
     local errorVal = rescuda - rescpu
     mytester:assertlt(errorVal, precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))

     -- test vs lua implementation
     input = input:type(ctype)
     target = target:type(ctype)
     buffer = input.new()
     local restruth = BCECriterion_forward_truth(buffer, input, target, nil, true)
     errorVal = rescpu - restruth
     mytester:assertlt(errorVal, precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
     errorVal = rescuda - restruth
     mytester:assertlt(errorVal, precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
  end
end

function cunntest.BCECriterionWeights_forward()
  local size = math.random(1,100)
  for k, typename in ipairs(typenames) do
     local input = torch.Tensor(size):uniform():type(typename)
     local target = torch.Tensor(size):uniform():gt(0.5):type(torch.type(input))
     local weights = torch.Tensor(size):uniform():type(typename)

     local ctype = t2cpu[typename]
     input = input:type(ctype)
     target = target:type(ctype)
     weights = weights:type(ctype)
     local crit = nn.BCECriterion(weights):type(ctype)
     local rescpu = crit:forward(input, target)

     input = input:type(typename)
     target = target:type(typename)
     weights = weights:type(typename)
     local g_crit = nn.BCECriterion(weights):type(typename)
     local rescuda = g_crit:forward(input, target)

     local errorVal = rescuda - rescpu
     mytester:assertlt(errorVal, precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))

     -- test vs lua implementation
     -- FIXME: half does not support dot without CUDA 8.0, so can't compare to lua implementation.
     if typename ~= 'torch.CudaHalfTensor' then
        buffer = input.new()
        restruth = BCECriterion_forward_truth(buffer, input, target, weights, true)
        errorVal = rescpu - restruth
        mytester:assertlt(errorVal, precision_forward_type(precision_forward, typename),
           string.format('error on state (forward) with %s', typename))
        errorVal = rescuda - restruth
        mytester:assertlt(errorVal, precision_forward_type(precision_forward, typename),
           string.format('error on state (forward) with %s', typename))
     end
  end
end

function cunntest.MarginCriterion_forward()
  local size = math.random(1,100)

  for k, typename in ipairs(typenames) do
    local input = ((torch.rand(size)-0.5) * 2):type(typename) -- data spread from -1 to 1
    local target = ((torch.round(torch.rand(size))*2)-1):type(typename) -- generate random labels -1, 1

    local ctype = t2cpu[typename]
    input = input:type(ctype)
    target = input:type(ctype)
    local crit = nn.MarginCriterion():type(ctype)
    local groundtruth= crit:forward(input, target)

    input = input:type(typename)
    target = target:type(typename)
    local g_crit = nn.MarginCriterion():type(typename)
    local rescuda = g_crit:forward(input, target)
    local errorVal = rescuda - groundtruth
    mytester:assertlt(errorVal, precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
  end
end

function cunntest.MultiLabelMarginCriterion_forward()
  local size = math.random(1,100)

  for k, typename in ipairs(typenames) do
     local input = ((torch.rand(size)-0.5) * 2):type(typename) -- data spread from -1 to 1
     local target = torch.round(torch.rand(size)*(size-1)):add(1) -- generate random labels > 0
     local zero = math.random(0,size) -- turn some labels into 0 targets
     if zero > 0 then
        target:sub(size-zero+1,size):zero()
     end

     local ctype = t2cpu[typename]
     input = input:type(ctype)
     local crit = nn.MultiLabelMarginCriterion():type(ctype)
     local groundtruth= crit:forward(input, target)
     input = input:type(typename)
     target = target:type(typename)
     local g_crit = nn.MultiLabelMarginCriterion():type(typename)
     local rescuda = g_crit:forward(input, target)
     local errorVal = rescuda - groundtruth
     mytester:assertlt(errorVal, precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
  end
end

function cunntest.MultiLabelMarginCriterion_backward()
   local size = math.random(1,100)

   for k, typename in ipairs(typenames) do
      local input = ((torch.rand(size)-0.5) * 2):type(typename) -- data spread from -1 to 1
      local target = torch.round(torch.rand(size)*(size-1)):add(1) -- generate random labels > 0
      local zero = math.random(0,size) -- turn some labels into 0 targets
      if zero > 0 then
         target:sub(size-zero+1,size):zero()
      end

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local crit = nn.MultiLabelMarginCriterion():type(ctype)
      local pred = crit:forward(input, target)
      local groundgrad = crit:backward(input, target)

      input = input:type(typename)
      target = target:type(typename)
      local g_crit = nn.MultiLabelMarginCriterion():type(typename)
      g_crit:forward(input, target)
      local rescuda = g_crit:backward(input, target)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
         string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialCrossMapLRN_forward_batch()
   local bs = math.random(4,10)
   local inputSize = math.random(6,9)
   local size = math.random(1,3)*2+1
   local nbfeatures = math.random(3,8)
   local alpha = math.random(1,100)/100
   local beta  = math.random(0,100)/100
   local k = math.random(1,3)

   for k, typename in ipairs(typenames) do
      local input = torch.rand(bs, nbfeatures, inputSize, inputSize):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialCrossMapLRN(size, alpha, beta, k):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SpatialCrossMapLRN(size, alpha, beta, k):type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialCrossMapLRN_backward_batch()
   local bs = math.random(4,10)
   local inputSize = math.random(6,9)
   local size = math.random(1,3)*2+1
   local nbfeatures = math.random(3,8)
   local alpha = math.random(1,100)/100
   local beta  = math.random(0,100)/100
   local k = math.random(1,3)

   for k, typename in ipairs(typenames) do
      local input = torch.rand(bs, nbfeatures, inputSize, inputSize):type(typename)
      local gradOutput = torch.rand(input:size()):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialCrossMapLRN(size, alpha, beta, k):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local gconv = nn.SpatialCrossMapLRN(size, alpha, beta, k):type(ctype)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward),
          string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.MarginCriterion_backward()
   local size = math.random(1,100)

   for k, typename in ipairs(typenames) do
      local input = ((torch.rand(size)-0.5) * 2):type(typename) -- data spread from -1 to 1
      local target = ((torch.round(torch.rand(size))*2)-1):type(typename) -- generate random labels -1, 1

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      target = target:type(ctype)
      local crit = nn.MarginCriterion():type(ctype)
      crit:forward(input, target)
      local groundgrad = crit:backward(input, target)

      input = input:type(typename)
      target = target:type(typename)
      local g_crit = nn.MarginCriterion():type(typename)
      g_crit:forward(input, target)
      local rescuda = g_crit:backward(input, target)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward),
         string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.BCECriterion_backward()
   local size = math.random(1,100)

   for k, typename in ipairs(typenames) do
      local input = torch.Tensor(size):uniform():type(typename)
      local target = torch.Tensor(size):uniform():gt(0.5):type(torch.type(input))

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      target = target:type(ctype)
      local crit = nn.BCECriterion():type(ctype)
      crit:forward(input, target)
      local groundgrad = crit:backward(input, target)

      input = input:type(typename)
      target = target:type(typename)
      local g_crit = nn.BCECriterion():type(typename)
      g_crit:forward(input, target)
      local rescuda = g_crit:backward(input, target)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
         string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.BCECriterionWeights_backward()
  local size = math.random(1,100)

  for k, typename in ipairs(typenames) do
     local input = torch.Tensor(size):uniform():type(typename)
     local target = torch.Tensor(size):uniform():gt(0.5):type(torch.type(input))
     local weights = torch.Tensor(size):uniform():type(typename)

     local ctype = t2cpu[typename]
     input = input:type(ctype)
     target = target:type(ctype)
     weights = weights:type(ctype)
     local crit = nn.BCECriterion(weights):type(ctype)
     crit:forward(input, target)
     local groundgrad = crit:backward(input, target)

     input = input:type(typename)
     target = target:type(typename)
     weights = weights:type(typename)
     local g_crit = nn.BCECriterion(weights):type(typename)
     g_crit:forward(input, target)
     local rescuda = g_crit:backward(input, target)

     local error = rescuda:double() - groundgrad:double()

     mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
        string.format('error on state (backward) with %s', typename))
  end
end

function cunntest.mse()
   for sizeAverage = 0, 1 do
      for k, typename in ipairs(typenames) do
         local size = math.random(3000,5000)
         local input = torch.randn(size,1,1):type(typename)
         local target = torch.randn(size):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         target = target:type(ctype)
         local mod = nn.MSECriterion(sizeAverage == 1):type(ctype)

         local fout = mod:forward(input,target)
         local fgin = mod:backward(input,target):clone()

         local cinput = input:type(typename)
         local ctarget = target:type(typename)
         local cmod = nn.MSECriterion(sizeAverage == 1):type(typename)
         local cout = cmod:forward(cinput,ctarget)
         local cgin = cmod:backward(cinput,ctarget)

         if (typename == 'torch.CudaHalfTensor') then
            fout = ffi.C.THC_half2float(ffi.C.THC_float2half(fout))
         end
         mytester:assertlt(math.abs(fout-cout), precision_forward_type(0.02, typename),
            string.format('error  on output with %s', typename))
         local gerr = cgin:double() - fgin:double()
         mytester:assertlt(gerr:abs():max(), precision_forward_type(precision_forward, typename),
            string.format('error  on gradInput with %s', typename))
      end
   end
end

function cunntest.SmoothL1()
   for sizeAverage = 0, 1 do
      local size = math.random(3000,5000)

      for k, typename in ipairs(typenames) do
         local input = torch.randn(size,1,1):type(typename)
         local target = torch.randn(size):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         target = target:type(ctype)
         local mod = nn.SmoothL1Criterion(sizeAverage == 1):type(ctype)

         local fout = mod:forward(input,target)
         local fgin = mod:backward(input,target):clone()

         local cinput = input:type(typename)
         local ctarget = target:type(typename)
         local cmod = nn.SmoothL1Criterion(sizeAverage == 1):type(typename)
         local cout = cmod:forward(cinput,ctarget)
         local cgin = cmod:backward(cinput,ctarget)

         if (typename == 'torch.CudaHalfTensor') then
            fout = ffi.C.THC_half2float(ffi.C.THC_float2half(fout))
         end
         mytester:assertlt(math.abs(fout-cout), 0.01, string.format('error  on output with %s', typename))
         local gerr = cgin:double() - fgin:double()
         mytester:assertlt(gerr:abs():max(), precision_forward_type(precision_forward, typename),
            string.format('error  on gradInput with %s', typename))
      end
   end
end

function cunntest.SoftMarginCriterion()
   for sizeAverage = 0, 1 do
      for k, typename in ipairs(typenames) do
         local size = math.random(3000,5000)
         local input = torch.randn(size,1,1):type(typename)
         local target = torch.randn(size):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         target = target:type(ctype)
         local mod = nn.SoftMarginCriterion(sizeAverage == 1):type(ctype)

         local fout = mod:forward(input,target)
         local fgin = mod:backward(input,target):clone()

         local cinput = input:type(typename)
         local ctarget = target:type(typename)
         local cmod = nn.SoftMarginCriterion(sizeAverage == 1):type(typename)
         local cout = cmod:forward(cinput,ctarget)
         local cgin = cmod:backward(cinput,ctarget)

        mytester:assertlt(math.abs(fout-cout), 0.01, 'error  on output')
        local gerr = cgin:double() - fgin:double()
        mytester:assertlt(gerr:abs():max(), precision_forward_type(precision_forward, typename),
           string.format('error  on gradInput with %s', typename))
      end
   end
end


function cunntest.distkldiv()
   for sizeAverage = 0, 1 do
      local size = math.random(3000,5000)

      for k, typename in ipairs(typenames) do
         local input = torch.randn(size):type(typename) -- TODO, make it back to (size, 1, 1), see https://github.com/torch/cunn/issues/245#issuecomment-209260954
         local target = torch.randn(size):type(typename)

         local ctype = t2cpu[typename]
         input = input:type(ctype)
         target = target:type(ctype)
         local mod = nn.DistKLDivCriterion(sizeAverage == 1):type(ctype)

         local fout = mod:forward(input,target)
         local fgin = mod:backward(input,target):clone()

         local cinput = input:type(typename)
         local ctarget = target:type(typename)
         local cmod = nn.DistKLDivCriterion(sizeAverage == 1):type(typename)
         local cout = cmod:forward(cinput,ctarget)
         local cgin = cmod:backward(cinput,ctarget)

         mytester:assertlt(math.abs(fout-cout), precision_forward_type(precision_forward, typename),
            string.format('error  on output with %s', typename))
         local gerr = cgin:double() - fgin:double()
         mytester:assertlt(gerr:abs():max(), precision_backward_type(precision_backward, typename),
            string.format('error  on gradInput with %s', typename))
      end
   end
end

function cunntest.TemporalConvolution_forward()
   local from = math.random(1,64) -- inputFrameSize
   local to = math.random(1,64) -- outputFrameSize
   local ki = math.random(3,15) -- kernelWidth (kW)
   local si = math.random(1,2) -- stepSize (dW)
   local outi = math.random(1,256) -- nOutputFrame
   local ini = (outi-1)*si+ki -- nInputFrame

   for k, typename in ipairs(typenames) do
      local input = torch.randn(ini,from):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.TemporalConvolution(from,to,ki,si):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.TemporalConvolution(from,to,ki,si):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.TemporalConvolution_forward_batch()
   local bs = math.random(4,16)
   local from = math.random(1,64)
   local to = math.random(1,64)
   local ki = math.random(3,15)
   local si = math.random(1,2)
   local outi = math.random(1,256)
   local ini = (outi-1)*si+ki

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,ini,from):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.TemporalConvolution(from,to,ki,si):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.TemporalConvolution(from,to,ki,si):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.TemporalConvolution_backward()
  local from = math.random(1,64)
   local to = math.random(1,64)
   local ki = math.random(3,15)
   local si = math.random(1,2)
   local outi = math.random(1,256)
   local ini = (outi-1)*si+ki

   for k, typename in ipairs(typenames) do
      local input = torch.randn(ini,from):type(typename)
      local gradOutput = torch.randn(outi,to):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.TemporalConvolution(from,to,ki,si):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.TemporalConvolution(from,to,ki,si):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias

      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
        string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
        string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, biascuda:abs():max()),
        string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.TemporalConvolution_backward_batch()
   local bs = math.random(4,16)
   local from = math.random(1,64)
   local to = math.random(1,64)
   local ki = math.random(3,15)
   local si = math.random(1,2)
   local outi = math.random(1,256)
   local ini = (outi-1)*si+ki

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,ini,from):type(typename)
      local gradOutput = torch.randn(bs,outi,to):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.TemporalConvolution(from,to,ki,si):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.TemporalConvolution(from,to,ki,si):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias

      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
        string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
        string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, biascuda:abs():max()),
        string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.Dropout()
   local p = 0.2 --prob of droping out a neuron
   local input = torch.CudaTensor(1000):fill((1-p))
   local module = nn.Dropout(p)
   module:cuda()
   -- version 2
   local output = module:forward(input)
   mytester:assert(math.abs(output:mean() - (1-p)) < 0.05, 'dropout output')
   local gradInput = module:backward(input, input)
   mytester:assert(math.abs(gradInput:mean() - (1-p)) < 0.05, 'dropout gradInput')
   -- version 1 (old nnx version)
   local input = input:fill(1)
   local module = nn.Dropout(p,true)
   module:cuda()
   local output = module:forward(input)
   mytester:assert(math.abs(output:mean() - (1-p)) < 0.05, 'dropout output')
   local gradInput = module:backward(input, input)
   mytester:assert(math.abs(gradInput:mean() - (1-p)) < 0.05, 'dropout gradInput')
end

function cunntest.Dropout_forward()
   local size = math.random(1,200)

   local tm = {}
   local title = string.format('Dropout forward %d -> %d', size, size)
   times[title] = tm

   local input = torch.randn(size)
   local sconv = nn.Dropout()
   local groundtruth = sconv:forward(input)
   local a = torch.Timer()
   for i = 1,nloop do
      groundtruth = sconv:forward(input)
   end
   tm.cpu = a:time().real

   input = input:cuda()
   local gconv = nn.Dropout():cuda()
   local rescuda = gconv:forward(input)
   a:reset()
   for i = 1,nloop do
      rescuda = gconv:forward(input)
   end
   cutorch.synchronize()
   tm.gpu = a:time().real

end

function cunntest.SoftPlus_forward()
   local size = math.random(1,100)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SoftPlus():type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.SoftPlus():type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward,typename),
          string.format('error on state (forward) with %s', typename))
    end
end

function cunntest.SoftPlus_backward()
   local size = math.random(1,100)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size):type(typename)
      local gradOutput = torch.randn(size):type(typename)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SoftPlus():type(ctype)
      sconv:forward(input)
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = sconv:clone():type(typename)
      gconv:forward(input)
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()
      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.SpatialUpSamplingNearest_forward()
   local f = torch.random(3, 15)
   local h = torch.random(3, 15)
   local w = torch.random(3, 15)
   local scale = torch.random(2,5)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(f, h, w):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialUpSamplingNearest(scale):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = sconv:clone():type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialUpSamplingNearest_forward_batch()
   local nbatch = torch.random(3, 15)
   local f = torch.random(3, 15)
   local h = torch.random(3, 15)
   local w = torch.random(3, 15)
   local scale = torch.random(2,5)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(nbatch, f, h, w):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialUpSamplingNearest(scale):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = sconv:clone():type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialUpSamplingNearest_backward()
   local f = torch.random(3, 15)
   local h = torch.random(3, 15)
   local w = torch.random(3, 15)
   local scale = torch.random(2,5)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(f, h, w):type(typename)
      local gradOutput = torch.randn(f, h*scale, w*scale):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialUpSamplingNearest(scale):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = sconv:clone():type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialUpSamplingNearest_backward_batch()
   local nbatch = torch.random(3, 15)
   local f = torch.random(3, 15)
   local h = torch.random(3, 15)
   local w = torch.random(3, 15)
   local scale = torch.random(2,5)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(nbatch, f, h, w):type(typename)
      local gradOutput = torch.randn(nbatch, f, h*scale, w*scale):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialUpSamplingNearest(scale):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = sconv:clone():type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialUpSamplingBilinear_forward()
   local f = torch.random(3, 15)
   local h = torch.random(3, 15)
   local w = torch.random(3, 15)
   local scale = torch.random(2,5)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(f, h, w):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialUpSamplingBilinear(scale):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = sconv:clone():type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
                        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialUpSamplingBilinear_forward_batch()
   local nbatch = torch.random(3, 15)
   local f = torch.random(3, 15)
   local h = torch.random(3, 15)
   local w = torch.random(3, 15)
   local scale = torch.random(2,5)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(nbatch, f, h, w):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.SpatialUpSamplingBilinear(scale):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = sconv:clone():type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
                        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialUpSamplingBilinear_backward()
   local f = torch.random(3, 15)
   local h = torch.random(3, 15)
   local w = torch.random(3, 15)
   local scale = torch.random(2,5)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(f, h, w):type(typename)
      local gradOutput = torch.randn(f, h*scale, w*scale):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialUpSamplingBilinear(scale):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = sconv:clone():type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
                        string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialUpSamplingBilinear_backward_batch()
   local nbatch = torch.random(3, 15)
   local f = torch.random(3, 15)
   local h = torch.random(3, 15)
   local w = torch.random(3, 15)
   local scale = torch.random(2,5)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(nbatch, f, h, w):type(typename)
      local gradOutput = torch.randn(nbatch, f, h*scale, w*scale):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.SpatialUpSamplingBilinear(scale):type(ctype)
      local output = sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = sconv:clone():type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local err = rescuda:double() - groundgrad:double()

      mytester:assertlt(err:abs():max(), precision_backward_type(precision_backward, typename),
                        string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.l1cost()
   local size = math.random(300,500)

   for k, typename in ipairs(typenames) do
     local input = torch.randn(size):type(typename)

     local ctype = t2cpu[typename]
     input = input:type(ctype)
     local mod = nn.L1Cost():type(ctype)

     local fout = mod:forward(input)
     local fgin = mod:backward(input):clone()

     local cinput = input:type(typename)
     local cmod = nn.L1Cost():type(typename)
     local cout = cmod:forward(cinput)
     local cgin = cmod:backward(cinput)

     if (typename == 'torch.CudaHalfTensor') then
        fout = ffi.C.THC_half2float(ffi.C.THC_float2half(fout))
     end
     mytester:assertlt(math.abs(fout-cout), precision_forward_type(precision_forward, typename),
        string.format('error  on output with %s', typename))
     local gerr = cgin:double() - fgin:double()
     mytester:assertlt(gerr:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error  on gradInput with %s', typename))
   end
end


function cunntest.ClassNLLCriterionSingleTarget()
   local size = math.random(3000,5000)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size):type(typename)
      local target = 1

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local mod = nn.ClassNLLCriterion():type(ctype)

      local fout = mod:forward(input, target)
      local fgin = mod:backward(input, target):clone()

      local cinput = input:type(typename)
      local ctarget = torch.CudaTensor(1):fill(target)
      local cmod = nn.ClassNLLCriterion():type(typename)
      local cout = cmod:forward(cinput,ctarget)
      local cgin = cmod:backward(cinput,ctarget)

      mytester:assertlt(
         math.abs(fout-cout), precision_forward_type(precision_forward, typename),
            string.format('error  on output with %s', typename))
      local gerr = cgin:double() - fgin:double()
      mytester:assertlt(gerr:abs():max(), precision_forward_type(precision_forward, typename),
         string.format('error  on gradInput with %s', typename))
   end
end

function cunntest.ClassNLLCriterionSingleTargetWeights()
   local size = math.random(3000,5000)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size):type(typename)
      local target = 1
      local weights = torch.rand(size):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      weights = weights:type(ctype)
      local mod = nn.ClassNLLCriterion(weights):type(ctype)

      local fout = mod:forward(input, target)
      local fgin = mod:backward(input, target):clone()

      local cinput = input:type(typename)
      local cweights = weights:type(typename)
      local ctarget = torch.CudaTensor(1):fill(target)
      local cmod = nn.ClassNLLCriterion(cweights):type(typename)
      local cout = cmod:forward(cinput,ctarget)
      local cgin = cmod:backward(cinput,ctarget)

      mytester:assertlt(
         math.abs(fout-cout), precision_forward_type(precision_forward, typename),
            string.format('error  on output with %s', typename))
      local gerr = cgin:double() - fgin:double()
      mytester:assertlt(gerr:abs():max(), precision_forward_type(precision_forward, typename),
         string.format('error  on gradInput with %s', typename))
   end
end

function cunntest.ClassNLLCriterionMultipleTarget()
   local size = math.random(3000,5000)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size, size):type(typename)
      local target = torch.randperm(size)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local mod = nn.ClassNLLCriterion():type(ctype)

      local fout = mod:forward(input, target)
      local fgin = mod:backward(input, target):clone()

      local cinput = input:type(typename)
      local ctarget = target:cuda()

      local cmod = nn.ClassNLLCriterion():type(typename)
      local cout = cmod:forward(cinput,ctarget)
      local cgin = cmod:backward(cinput,ctarget)

      mytester:assertlt(
        math.abs(fout-cout), precision_forward_type(precision_forward, typename),
          string.format('error on output with %s', typename))

      local gerr = cgin:double() - fgin:double()
      mytester:assertlt(gerr:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error  on gradInput with %s', typename))
   end
end

function cunntest.SpatialClassNLLCriterion()
   local batchSize = math.random(5, 10)
   local h = math.random(300, 500)
   local w = math.random(300, 800)
   local classes = math.random(10,30)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(batchSize, classes, h, w):type(typename)
      local target = torch.Tensor(batchSize, h, w)
      target:apply(function() return math.random(1, classes) end)
      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local mod = nn.SpatialClassNLLCriterion():type(ctype)
      local fout = mod:forward(input, target)
      local fgin = mod:backward(input, target):clone()

      local cinput = input:type(typename)
      local ctarget = target:type(typename)

      local cmod = nn.SpatialClassNLLCriterion():type(typename)
      local cout = cmod:forward(cinput,ctarget)
      local cgin = cmod:backward(cinput,ctarget)
      cutorch.synchronize()

      mytester:assertlt(
        math.abs(fout-cout), precision_forward_type(precision_forward, typename),
          string.format('error on output with %s', typename))

      local gerr = cgin:double() - fgin:double()
      mytester:assertlt(gerr:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error  on gradInput with %s', typename))
    end
end

function cunntest.ClassNLLCriterionMultipleTargetWeights()
   local size = math.random(3000,5000)

   for k, typename in ipairs(typenames) do
      local input = torch.randn(size, size):type(typename)
      local target = torch.randperm(size)
      local weights = torch.rand(size):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      weights = weights:type(ctype)
      local mod = nn.ClassNLLCriterion(weights):type(ctype)

      local fout = mod:forward(input, target)
      local fgin = mod:backward(input, target):clone()

      local cinput = input:type(typename)
      local ctarget = target:cuda()
      local cweights = weights:type(typename)

      local cmod = nn.ClassNLLCriterion(cweights):type(typename)
      local cout = cmod:forward(cinput,ctarget)
      local cgin = cmod:backward(cinput,ctarget)

      mytester:assertlt(
        math.abs(fout-cout), precision_forward_type(precision_forward, typename),
          string.format('error on output with %s', typename))

      local gerr = cgin:double() - fgin:double()
      mytester:assertlt(gerr:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error  on gradInput with %s', typename))
   end
end

function cunntest.TemporalMaxPooling()
   local settings = {{2, 2}, {3, 3}, {4, 2}, {2, 4}, {3, 5}}

   for i, setting in ipairs(settings) do
      for k, typename in ipairs(typenames) do
        local input = torch.rand(16, 18, 3):type(typename)

        local ctype = t2cpu[typename]
        input = input:type(ctype)
        local mod = nn.TemporalMaxPooling(setting[1], setting[2]):type(ctype)

        local fout = mod:forward(input)
        local fgout = torch.rand(fout:size()):type(typename):type(ctype)
        local fgin = mod:backward(input, fgout):clone()

        local cinput = input:type(typename)
        local cgout = fgout:type(typename)
        local cmod = nn.TemporalMaxPooling(setting[1], setting[2]):type(typename)
        local cout = cmod:forward(cinput)
        local cgin = cmod:backward(cinput, cgout)

        local outerror = cout:double() - fout:double()
        mytester:assertlt(outerror:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on output with %s', typename))

        local ginerror = cgin:double() - fgin:double()
        mytester:assertlt(ginerror:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on gradInput with %s', typename))
      end
   end
end

function cunntest.VolumetricConvolution_forward_single()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(3,15)
   local kj = math.random(3,15)
   local kk = math.random(3,15)
   local si = math.random(1,ki)
   local sj = math.random(1,kj)
   local sk = math.random(1,kk)
   local outi = math.random(1,20)
   local outj = math.random(1,20)
   local outk = math.random(1,20)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj
   local ink = (outk-1)*sk+kk

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,ini,inj,ink):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.VolumetricConvolution(from,to,ki,kk,kj,si,sk,sj):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.VolumetricConvolution(from,to,ki,kk,kj,si,sk,sj):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
      mytester:assert(groundtruth:isSize(rescuda:size()),
        string.format('size mismatch on state (forward) with %s', typename))
   end
end

function cunntest.VolumetricConvolution_forward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,8)
   local to = math.random(1,4) * 4
   local ki = math.random(3,8)
   local kj = math.random(3,8)
   local kk = math.random(3,8)
   local si = math.random(1,ki)
   local sj = math.random(1,kj)
   local sk = math.random(1,kk)
   local outi = math.random(1,16)
   local outj = math.random(1,16)
   local outk = math.random(1,16)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj
   local ink = (outk-1)*sk+kk

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,ini,inj, ink):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.VolumetricConvolution(from,to,ki,kk,kj,si,sj,sk):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.VolumetricConvolution(from,to,ki,kk,kj,si,sj,sk):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
      mytester:assert(groundtruth:isSize(rescuda:size()),
        string.format('size mismatch on state (forward) with %s', typename))
   end
end

function cunntest.VolumetricConvolution_backward_single()
   local from = math.random(1,4)
   local to = math.random(1,3) * 8
   local ki = math.random(3,8)
   local kj = math.random(3,8)
   local kk = math.random(3,8)
   local si = math.random(1,ki)
   local sj = math.random(1,kj)
   local sk = math.random(1,kk)
   local outi = math.random(1,16)
   local outj = math.random(1,16)
   local outk = math.random(1,16)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj
   local ink = (outk-1)*sk+kk

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from, ini, inj, ink):type(typename)
      local gradOutput = torch.randn(to, outi, outj, outk):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.VolumetricConvolution(from,to,ki,kk,kj,si,sk,sj):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.VolumetricConvolution(from,to,ki,kk,kj,si,sk,sj):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias
      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()
      mytester:assert(groundgrad:isSize(rescuda:size()),
        string.format('size mismatch on state (forward) with %s', typename))
      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
        string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
        string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, biascuda:abs():max()),
        string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.VolumetricConvolution_backward_batch()
   local bs = math.random(1,4) * 4
   local from = math.random(1,4)
   local to = math.random(1,3) * 8
   local ki = math.random(3,8)
   local kj = math.random(3,8)
   local kk = math.random(3,8)
   local si = math.random(1,ki)
   local sj = math.random(1,kj)
   local sk = math.random(1,kk)
   local outi = math.random(1,16)
   local outj = math.random(1,16)
   local outk = math.random(1,16)
   local ini = (outi-1)*si+ki
   local inj = (outj-1)*sj+kj
   local ink = (outk-1)*sk+kk

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs, from, ini, inj, ink):type(typename)
      local gradOutput = torch.randn(bs, to, outi, outj, outk):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.VolumetricConvolution(from,to,ki,kk,kj,si,sk,sj):type(ctype)
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.VolumetricConvolution(from,to,ki,kk,kj,si,sk,sj):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias
      local error = rescuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()
      mytester:assert(groundgrad:isSize(rescuda:size()),
        string.format('size mismatch on state (forward) with %s', typename))
      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
        string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
        string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, biascuda:abs():max()),
        string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.VolumetricMaxPooling_forward()
   local kT = math.random(3, 7)
   local kH = math.random(3, 7)
   local kW = math.random(3, 7)
   local dT = math.random(1, 13)
   local dH = math.random(1, 13)
   local dW = math.random(1, 13)
   local iT = math.random(kT*2, 60)
   local iH = math.random(kH*2, 60)
   local iW = math.random(kW*2, 60)
   local padT = math.random(0,kT/2-1)
   local padH = math.random(0,kH/2-1)
   local padW = math.random(0,kW/2-1)
   local iF = math.random(1, 16) -- features
   local oT = math.floor((iT - kT + 2*padT) / dT + 1)
   local oH = math.floor((iH - kH + 2*padH) / dH + 1)
   local oW = math.floor((iW - kW + 2*padW) / dW + 1)

   for k, typename in ipairs(typenames) do
      local input = torch.Tensor(iF, iT, iH, iW):float():uniform(-1, 1):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local layer = nn.VolumetricMaxPooling(kT, kW, kH, dT, dW, dH, padT, padW, padH):type(ctype)
      local output = layer:forward(input)

      local inputCUDA = input:type(typename)
      local layerCUDA = layer:clone():type(typename)
      local outputCUDA = layerCUDA:forward(inputCUDA)

      local error = outputCUDA:double() - output:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.VolumetricMaxPooling_backward()
   local kT = math.random(3, 7)
   local kH = math.random(3, 7)
   local kW = math.random(3, 7)
   local dT = math.random(1, 13)
   local dH = math.random(1, 13)
   local dW = math.random(1, 13)
   local iT = math.random(kT*2, 60)
   local iH = math.random(kH*2, 60)
   local iW = math.random(kW*2, 60)
   local padT = math.random(0,kT/2-1)
   local padH = math.random(0,kH/2-1)
   local padW = math.random(0,kW/2-1)
   local iF = math.random(1, 16) -- features
   local oT = math.floor((iT - kT + 2*padT) / dT + 1)
   local oH = math.floor((iH - kH + 2*padH) / dH + 1)
   local oW = math.floor((iW - kW + 2*padW) / dW + 1)

   for k, typename in ipairs(typenames) do
      local input = torch.Tensor(iF, iT, iH, iW):float():uniform(-1, 1):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local layer = nn.VolumetricMaxPooling(kT, kW, kH, dT, dW, dH, padT, padW, padH):type(ctype)
      local output = layer:forward(input)
      local gradOutput = output:clone():uniform(-1, 1)

      local gradInput = layer:backward(input, gradOutput)

      local inputCUDA = input:type(typename)
      local layerCUDA = layer:clone():type(typename)
      local outputCUDA = layerCUDA:forward(inputCUDA)
      local gradOutputCUDA = gradOutput:type(typename)
      local gradInputCUDA = layerCUDA:backward(inputCUDA, gradOutputCUDA)

      local error = gradInputCUDA:double() - gradInput:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.VolumetricDilatedMaxPooling_forward_batch()
   local bs = math.random(8,16)
   local from = math.random(8,16)
   local to = from
   local kt = math.random(2,4)
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local st = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outt = math.random(1,10)
   local outi = math.random(1,33)
   local outj = math.random(1,33)
   local padt = math.random(0,kt/2-1)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local dilationt = math.random(1,10)
   local dilationi = math.random(1,10)
   local dilationj = math.random(1,10)
   local int = (outt-1)*st+(dilationt*(kt-1)+1)-2*padt
   local ini = (outi-1)*si+(dilationi*(ki-1)+1)-2*padi
   local inj = (outj-1)*sj+(dilationj*(kj-1)+1)-2*padj
   local ceil_mode = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,int,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.VolumetricDilatedMaxPooling(kt,ki,kj,st,si,sj,padt,padi,padj,dilationt,dilationi,dilationj):type(ctype)
      if ceil_mode then sconv:ceil() end
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = nn.VolumetricDilatedMaxPooling(kt,ki,kj,st,si,sj,padt,padi,padj,dilationt,dilationi,dilationj):type(typename)
      if ceil_mode then gconv:ceil() end
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.VolumetricDilatedMaxPooling_backward_batch()
   local bs = math.random(8,16)
   local from = math.random(8,16)
   local to = from
   local kt = math.random(2,4)
   local ki = math.random(2,4)
   local kj = math.random(2,4)
   local st = math.random(2,4)
   local si = math.random(2,4)
   local sj = math.random(2,4)
   local outt = math.random(32,60)
   local outi = math.random(32,60)
   local outj = math.random(32,60)
   local padt = math.random(0,kt/2-1)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local dilationt = math.random(1,10)
   local dilationi = math.random(1,10)
   local dilationj = math.random(1,10)
   local int = (outt-1)*st+(dilationt*(kt-1)+1)-2*padt
   local ini = (outi-1)*si+(dilationi*(ki-1)+1)-2*padi
   local inj = (outj-1)*sj+(dilationj*(kj-1)+1)-2*padj
   local ceil_mode = math.random(0,1) == 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(bs,from,int,inj,ini):type(typename)
      local gradOutput = torch.randn(bs,to,outt,outj,outi):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local sconv = nn.VolumetricDilatedMaxPooling(kt,ki,kj,st,si,sj,padt,padi,padj,dilationt,dilationi,dilationj):type(ctype)
      if ceil_mode then sconv:ceil() end
      sconv:forward(input)
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.VolumetricDilatedMaxPooling(kt,ki,kj,st,si,sj,padt,padi,padj,dilationt,dilationi,dilationj):type(typename)
      if ceil_mode then gconv:ceil() end
      gconv:forward(input)
      gconv:zeroGradParameters()
      local rescuda = gconv:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
        string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.VolumetricMaxUnpooling_forward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local kt = math.random(3,7)
   local ki = math.random(3,7)
   local kj = math.random(3,7)
   local st, si, sj = kt, ki, kj
   local outt = math.random(32,128)
   local outi = math.random(32,128)
   local outj = math.random(32,128)
   local padt = math.random(0,kt/2-1)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local it = math.max(((outt + padt*2 - kt)/st) +1, kt)
   local ii = math.max(((outi + padi*2 - ki)/si) +1, ki)
   local ij = math.max(((outj + padj*2 - kj)/sj) +1, kj)

   for k, typename in ipairs(typenames) do
      local ctype = t2cpu[typename]

      local pooler = nn.VolumetricMaxPooling(kt,ki,kj,st,si,sj,padt,padi,padj):type(ctype)
      local sunpool = nn.VolumetricMaxUnpooling(pooler):type(ctype)

      local original = torch.randn(bs,from,it,ij,ii):type(typename):type(ctype)
      local input = pooler:forward(original)
      local groundtruth = sunpool:forward(input)

      original = original:type(typename)
      pooler = nn.VolumetricMaxPooling(kt,ki,kj,st,si,sj,padt,padi,padj):type(typename)
      local gunpool = nn.VolumetricMaxUnpooling(pooler):type(typename)

      input = pooler:forward(original)
      local rescuda = gunpool:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.VolumetricMaxUnpooling_backward_batch()
   local bs = math.random(4,10)
   local from = math.random(1,64)
   local to = from
   local kt = math.random(3,7)
   local ki = math.random(3,7)
   local kj = math.random(3,7)
   local st, si, sj = kt, ki, kj
   local outt = math.random(32,128)
   local outi = math.random(32,128)
   local outj = math.random(32,128)
   local padt = math.random(0,kt/2-1)
   local padi = math.random(0,ki/2-1)
   local padj = math.random(0,kj/2-1)
   local it = math.max(((outt + padt*2 - kt)/st) +1, kt)
   local ii = math.max(((outi + padi*2 - ki)/si) +1, ki)
   local ij = math.max(((outj + padj*2 - kj)/sj) +1, kj)

   for k, typename in ipairs(typenames) do
      local ctype = t2cpu[typename]

      local pooler = nn.VolumetricMaxPooling(kt,ki,kj,st,si,sj,padt,padi,padj):type(ctype)
      local sunpool = nn.VolumetricMaxUnpooling(pooler):type(ctype)

      local original = torch.randn(bs,from,it,ij,ii):type(typename):type(ctype)
      local input = pooler:forward(original)
      local gradOutput = torch.randn(original:size()):type(typename):type(ctype)
      sunpool:forward(input)
      sunpool:zeroGradParameters()
      local groundgrad = sunpool:backward(input, gradOutput)

      pooler = nn.VolumetricMaxPooling(kt,ki,kj,st,si,sj,padt,padi,padj):type(typename)
      local gunpool = nn.VolumetricMaxUnpooling(pooler):type(typename)

      original = original:type(typename)
      input = pooler:forward(original)
      gunpool:forward(input)

      gradOutput = gradOutput:type(typename)
      gunpool:zeroGradParameters()
      local rescuda = gunpool:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()

      mytester:assertlt(error:abs():max(), precision_backward_type(precision_backward, typename),
        string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.VolumetricAveragePooling_forward()
   local kT = math.random(3, 7)
   local kH = math.random(3, 7)
   local kW = math.random(3, 7)
   local dT = math.random(1, 13)
   local dH = math.random(1, 13)
   local dW = math.random(1, 13)
   local oT = math.random(1, 20)
   local oH = math.random(1, 20)
   local oW = math.random(1, 20)
   local iF = math.random(1, 16) -- features
   local iT = (oT - 1) * dT + kT
   local iH = (oH - 1) * dH + kH
   local iW = (oW - 1) * dW + kW

   for k, typename in ipairs(typenames) do
      local input = torch.Tensor(iF, iT, iH, iW):float():uniform(-1, 1):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local layer = nn.VolumetricAveragePooling(kT, kW, kH, dT, dW, dH):type(ctype)
      local output = layer:forward(input)

      local inputCUDA = input:type(typename)
      local layerCUDA = layer:clone():type(typename)
      local outputCUDA = layerCUDA:forward(inputCUDA)

      local error = outputCUDA:double() - output:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.VolumetricAveragePooling_backward()
   local kT = math.random(3, 7)
   local kH = math.random(3, 7)
   local kW = math.random(3, 7)
   local dT = math.random(1, 13)
   local dH = math.random(1, 13)
   local dW = math.random(1, 13)
   local oT = math.random(1, 20)
   local oH = math.random(1, 20)
   local oW = math.random(1, 20)
   local iF = math.random(1, 16) -- features
   local iT = (oT - 1) * dT + kT
   local iH = (oH - 1) * dH + kH
   local iW = (oW - 1) * dW + kW

   for k, typename in ipairs(typenames) do
      local input = torch.Tensor(iF, iT, iH, iW):float():uniform(-1, 1):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local layer = nn.VolumetricAveragePooling(kT, kW, kH, dT, dW, dH):type(ctype)
      local output = layer:forward(input)
      local gradOutput = output:clone():uniform(-1, 1)

      local gradInput = layer:backward(input, gradOutput)

      local inputCUDA = input:type(typename)  local layerCUDA = layer:clone():type(typename)
      local outputCUDA = layerCUDA:forward(inputCUDA)   local gradOutputCUDA = gradOutput:type(typename)
      local gradInputCUDA = layerCUDA:backward(inputCUDA, gradOutputCUDA)

      local error = gradInputCUDA:double() - gradInput:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.CMul_forward_batch()
   local bs = math.random(8,32)
   local nini = math.random(1,100)
   local ninj = math.random(1,100)
   local nink = math.random(1,100)

   local tm = {}
   local title = string.format('CMul forward %d %d %d %d', bs, nini, ninj, nink)
   times[title] = tm

   local input = torch.randn(bs, nini, ninj, nink)
   local sconv = nn.CMul(nini, ninj, nink)
   local groundtruth = sconv:forward(input)
   local a = torch.Timer()
   for i = 1,nloop do
      groundtruth = sconv:forward(input)
   end
   tm.cpu = a:time().real

   input = input:cuda()
   local gconv = sconv:clone():cuda()
   local rescuda = gconv:forward(input)
   a:reset()
   for i = 1,nloop do
      rescuda = gconv:forward(input)
   end
   cutorch.synchronize()
   tm.gpu = a:time().real

   local error = rescuda:float() - groundtruth
   mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) batch ')
end

function cunntest.CMul_backward_batch()
   local bs = math.random(8,32)
   local nini = math.random(1,100)
   local ninj = math.random(1,100)
   local nink = math.random(1,100)

   local tm = {}
   local title = string.format('CMul backward %d %d %d %d', bs, nini, ninj, nink)
   times[title] = tm

   local input = torch.randn(bs, nini, ninj, nink)
   local gradOutput = torch.randn(bs, nini, ninj, nink)
   local sconv = nn.CMul(nini, ninj, nink)
   sconv:forward(input)
   sconv:zeroGradParameters()
   local groundgrad = sconv:backward(input, gradOutput)
   local a = torch.Timer()
   for i = 1,nloop do
      sconv:zeroGradParameters()
      groundgrad = sconv:backward(input, gradOutput)
   end
   local groundweight = sconv.gradWeight
   tm.cpu = a:time().real

   input = input:cuda()
   gradOutput = gradOutput:cuda()
   local gconv = sconv:clone():cuda()
   gconv:forward(input)
   gconv:zeroGradParameters()
   local rescuda = gconv:backward(input, gradOutput)
   a:reset()
   for i = 1,nloop do
      gconv:zeroGradParameters()
      rescuda = gconv:backward(input, gradOutput)
   end
   cutorch.synchronize()
   tm.gpu = a:time().real

   local weightcuda = gconv.gradWeight

   local error = rescuda:float() - groundgrad
   local werror = weightcuda:float() - groundweight

   mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
   mytester:assertlt(werror:abs():max(), precision_backward, 'error on weight (backward) ')
end

function cunntest.PReLU_forward()
    local nOutputPlane = 8
    local w = math.random(1,100)
    local h = math.random(1,100)
    local input = torch.randn(nOutputPlane,h,w)

    for k, typename in ipairs(typenames) do
      local ctype = t2cpu[typename]
      local input = input:type(ctype)
      local sconv = nn.PReLU(nOutputPlane):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:type(typename)
      local gconv = sconv:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
          string.format('error on state with %s', typename))
    end
end

function cunntest.PReLU_backward()
    local nOutputPlane = 8
    local w = math.random(1,10)
    local h = math.random(1,10)

    for k, typename in ipairs(typenames) do
        local input = torch.randn(nOutputPlane, h, w):type(typename)
        local gradOutput = torch.randn(#input):type(typename)
        local ctype = t2cpu[typename]
        input = input:type(ctype)
        gradOutput = gradOutput:type(ctype)
        local sconv = nn.PReLU(nOutputPlane):type(ctype)
        local gconv = sconv:clone():type(typename)

        sconv:forward(input)
        sconv:zeroGradParameters()
        local groundgrad = sconv:backward(input, gradOutput)

        input = input:type(typename)
        gradOutput = gradOutput:type(typename)
        gconv:forward(input)
        gconv:zeroGradParameters()
        local rescuda = gconv:backward(input, gradOutput)

        local err = rescuda:double() - groundgrad:double()
        local weightGradError = gconv.gradWeight:double() - sconv.gradWeight:double()

        mytester:assertlt(err:abs():max(), precision_backward_type(precision_backward, typename),
            string.format('error on state %s', typename))
        mytester:assertlt(weightGradError:abs():max(), precision_backward_type(precision_backward, typename),
            string.format('error on weight %s', typename))
    end
end


function cunntest.RReLU_forward()
    local nOutputPlane = 8
    local w = math.random(1,100)
    local h = math.random(1,100)

    for k, typename in ipairs(typenames) do
       for _,train in ipairs({true,false}) do
          for _,inplace in ipairs({false,true}) do
              local input = torch.randn(nOutputPlane, h, w):type(typename) - 0.5
              local ctype = t2cpu[typename]
              input = input:type(ctype)
              local sconv = nn.RReLU(1/8, 1/3, inplace):type(ctype)
              if not train then
                  sconv:evaluate()
              end
              local groundtruth = sconv:forward(input:clone())

              input = input:type(typename)
              local gconv = sconv:type(typename)
              local rescuda = gconv:forward(input:clone())

              if not train then
                  local error = rescuda:double() - groundtruth:double()
                  mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
                      string.format('error on state %s', typename))
              end
          end
      end
    end
end

function cunntest.RReLU_backward()
    local nOutputPlane = 8
    local w = math.random(1,10)
    local h = math.random(1,10)

    for k, typename in ipairs(typenames) do
        for _,train in ipairs({true,false}) do
            for _,inplace in ipairs({false,true}) do
                local ctype = t2cpu[typename]
                local input = torch.randn(nOutputPlane, h, w):type(typename)
                local gradOutput = torch.randn(#input):type(typename) - 0.5
                input = input:type(ctype)
                gradOutput = gradOutput:type(ctype)
                local sconv = nn.RReLU(1/8, 1/3, inplace):type(ctype)
                if not train then
                  sconv:evaluate()
                end

                sconv:forward(input:clone())
                local groundgrad = sconv:backward(input, gradOutput:clone())

                local gconv = sconv:clone():type(typename)
                input = input:type(typename)
                gradOutput = gradOutput:type(typename)
                gconv:forward(input:clone())
                local rescuda = gconv:backward(input, gradOutput:clone())

                if not train then
                  local err = rescuda:double() - groundgrad:double()
                  mytester:assertlt(err:abs():max(), precision_backward_type(precision_backward, typename),
                    string.format('error on state', typename))
                end

                input = -torch.rand(1000):type(typename)
                gconv:forward(input) -- fill internal noise tensor
                local g = gconv:backward(input, torch.ones(1000):type(typename))
                local err = math.abs(g[input:le(0)]:mean()-(gconv.lower+gconv.upper)/2)
                mytester:assertlt(err, 0.05, 'mean deviation of gradient for negative inputs')
          end
       end
    end
end

function cunntest.VolumetricFullConvolution_pair_test()

    local kT = 2 * math.random(1,3) + 1  -- odd number
    local kH = 2 * math.random(1,3) + 1  -- odd number
    local kW = kH
    local dT = math.random(1,3)
    local dH = math.random(1,3)
    local dW = dH
    local pT = (kT-1)/2
    local pH = (kH-1)/2
    local pW = pH

    local inChan = math.random(1,32)
    local outChan = math.random(1,32)

    for k, typename in ipairs(typenames) do
      local ctype = t2cpu[typename]
      local module = nn.VolumetricFullConvolution(inChan, outChan, kT, kH, kW,
                                                  dT, dH, dW, pT, pH, pW):type(ctype);
      module.weight:fill(1);
      module.bias:fill(0.1);
      module.weight = module.weight:type(typename):type(ctype)
      module.bias = module.bias:type(typename):type(ctype)

      local bs = math.random(8,32)
      local inD = math.random(8,32)
      local inH = math.random(8,32)
      local inW = math.random(8,32)
      local outD = (inD - 1) * dT - 2 * pT + kT
      local outH = (inH - 1) * dH - 2 * pH + kH
      local outW = (inW - 1) * dW - 2 * pW + kW
      local input = torch.Tensor(bs, inChan, inD, inH, inW):fill(1):type(typename):type(ctype)
      local gradOut = torch.randn(bs, outChan, outD, outH, outW):type(typename):type(ctype)

      local outcpu = module:forward(input)
      local gradcpu = module:backward(input, gradOut)
      module:type(typename)
      local outgpu = module:forward(input:type(typename))
      local gradgpu = module:backward(input:type(typename), gradOut:type(typename))

      local error = outgpu:type(typename) - outcpu:type(typename)
      mytester:assertlt(error:abs():max(),
                        precision_forward_type(precision_forward, typename, outgpu:abs():max()),
                        string.format('error on state (forward) with %s', typename))

      local error = gradgpu:type(typename) - gradcpu:type(typename)
      mytester:assertlt(error:abs():max(),
                        precision_backward_type(precision_backward, typename),
                        string.format('error on state (backward) with %s', typename))
    end
end

function cunntest.VolumetricFullConvolution()
    for k, typename in ipairs(typenames) do
        local ctype = t2cpu[typename]
        local module = nn.VolumetricFullConvolution(3, 1, 3, 3, 3, 3, 3, 3):type(ctype);
        module.weight:fill(1);
        module.bias:fill(0.1);
        module:type(typename);

        local input = torch.Tensor(1, 3, 2, 2, 2):zero();
        for c = 1,3 do
            input[1][c][1][1][1] = 1
        end
        local output = module:forward(input:type(typename))
        for t = 1,6 do
            for h = 1,6 do
                for w = 1,6 do
                    if t <= 3 and h <= 3 and w <= 3 then
                        mytester:assertlt(output[1][1][t][h][w] - 3.1, precision_forward_type(precision_forward, typename),
                          string.format('error on forward with %s', typename))
                    else
                        mytester:assertlt(output[1][1][t][h][w] - 0.1, precision_forward_type(precision_forward, typename),
                          string.format('error on forward with %s', typename))
                    end
                end
            end
        end

        module:zeroGradParameters()
        local gradOut = torch.Tensor(1, 1, 6, 6, 6):fill(0.1);
        local gradIn = module:backward(input:type(typename), gradOut:type(typename))
        for t = 1,2 do
            for h = 1,2 do
                for w = 1,2 do
                    mytester:assertlt(gradIn[1][1][t][h][w] - 2.7, precision_backward_type(precision_backward, typename),
                                      string.format('error on backward input gradients with %s', typename))
                end
            end
        end

        mytester:assertlt(module.gradBias[1] - 21.6, precision_backward_type(precision_backward, typename),
                          string.format('error on backward gradBias with %s', typename))
        for c = 1,3 do
            for t = 1,3 do
                for h = 1,3 do
                    for w = 1,3 do
                        mytester:assertlt(module.gradWeight[c][1][t][h][w] - 0.1, precision_backward_type(precision_backward, typename),
                                          string.format('error on backward weight gradients with %s', typename))
                    end
                end
            end
        end
    end
end

function cunntest.VolumetricDilatedConvolution()
   local from = math.random(1,32)
   local to = math.random(1,8) * 8
   local ki = math.random(1,15)
   local kj = math.random(1,15)
   local kk = math.random(1,3)
   local si = math.random(1,3)
   local sj = math.random(1,3)
   local sk = math.random(1,3)
   local padW = math.random(0,1)
   local padH = math.random(0,1)
   local padT = math.random(0,1)
   local outi = math.random(ki, 64)
   local outj = math.random(kj, 64)
   local outk = math.random(kk, kk+5)
   local dilationW = math.random(1,10)
   local dilationH = math.random(1,10)
   local dilationT = math.random(1,10)
   local ini = (outi - 1) * si - 2 * padW + dilationW * (ki-1) + 1
   local inj = (outj - 1) * sj - 2 * padH + dilationH * (kj-1) + 1
   local ink = (outk - 1) * sk - 2 * padT + dilationT * (kk-1) + 1

   for k, typename in ipairs(typenames) do
      local input = torch.randn(from,ink,inj,ini):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local sconv = nn.VolumetricDilatedConvolution(from,to,kk,ki,kj,sk,si,sj,padT,padW,padH,dilationT,dilationW,dilationH):type(ctype)
      local output = sconv:forward(input)
      local gradOutput = output:clone():normal()
      sconv:zeroGradParameters()
      local groundgrad = sconv:backward(input, gradOutput)
      local groundweight = sconv.gradWeight
      local groundbias = sconv.gradBias

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gconv = nn.VolumetricDilatedConvolution(from,to,kk,ki,kj,sk,si,sj,padT,padW,padH,dilationT,dilationW,dilationH):type(typename)
      gconv.weight = sconv.weight:type(typename)
      gconv.bias = sconv.bias:type(typename)
      local rescuda = gconv:forward(input)
      gconv:zeroGradParameters()
      local gradcuda = gconv:backward(input, gradOutput)
      local weightcuda = gconv.gradWeight
      local biascuda = gconv.gradBias

      local error = rescuda:double() - output:double()
      local gerror = gradcuda:double() - groundgrad:double()
      local werror = weightcuda:double() - groundweight:double()
      local berror = biascuda:double() - groundbias:double()

      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state (forward) with %s', typename))
      mytester:assertlt(gerror:abs():max(), precision_backward_type(precision_backward, typename),
        string.format('error on state (backward) with %s', typename))
      mytester:assertlt(werror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, weightcuda:abs():max()),
        string.format('error on weight (backward) with %s', typename))
      mytester:assertlt(berror:abs():max(),
        precision_backward_conv_weightbias(precision_backward, typename, biascuda:abs():max()),
        string.format('error on bias (backward) with %s', typename))
   end
end

function cunntest.LookupTable_forward()
   local nVocab = 10000
   local nDim = 100
   local nInput = 1000

   for k, typename in ipairs(typenames) do
      local input = torch.LongTensor(nInput):random(nVocab)

      local ctype = t2cpu[typename]
      local sconv = nn.LookupTable(nVocab, nDim):type(ctype)
      local groundtruth = sconv:forward(input)

      input = input:cuda()
      local gconv = sconv:type(typename)
      local rescuda = gconv:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(), precision_forward_type(precision_forward, typename),
        string.format('error on state with %s', typename))
   end
end

function cunntest.LookupTable_backward()
   local grid = {
      nInput = {10, 101, 1000, 10007},
      nVocab = {100, 10000},
      nDim = {97, 255},
      scaleGradByFreq = {false, true},
      batch = {false, true},
      paddingValue = {0, 1},
   }

   for itr = 1, 10 do
      -- Randomly sample from grid of parameters
      local s = {}
      for k, v in pairs(grid) do
         s[k] = v[torch.random(#v)]
      end

      for k, typename in ipairs(typenames) do
          local ctype = t2cpu[typename]
          local input, gradOutput
          if s.batch then
              input = torch.LongTensor(s.nInput, 5):random(s.nVocab)
              gradOutput = torch.randn(s.nInput, 5, s.nDim):type(typename):type(ctype)
          else
              input = torch.LongTensor(s.nInput):random(s.nVocab)
              gradOutput = torch.randn(s.nInput, s.nDim):type(typename):type(ctype)
          end

          local sconv = nn.LookupTable(s.nVocab, s.nDim, s.paddingValue):type(ctype)
          local gconv = sconv:clone():type(typename)
          if s.scaleGradByFreq then
              sconv = sconv:scaleGradByFreq()
              gconv = gconv:scaleGradByFreq()
          end

          sconv:forward(input)
          sconv:backward(input, gradOutput)

          input = input:cuda()
          gradOutput = gradOutput:type(typename)
          gconv:forward(input)
          gconv:backward(input, gradOutput)

          local weightGradError = gconv.gradWeight:double() - sconv.gradWeight:double()
          mytester:assertlt(weightGradError:abs():max(),
              precision_backward_conv_weightbias(precision_backward, typename, gconv.gradWeight:abs():max()),
              'error on weight for size ' .. tostring(s.nInput) ..
              ' nVocab: ' .. tostring(s.nVocab) ..
              ' nDim ' .. tostring(s.nDim) ..
              ' scaleGradByFreq: ' .. tostring(s.scaleGradByFreq) ..
              ' batch: ' .. tostring(s.batch) ..
              ' paddingValue: ' .. tostring(s.paddingValue) ..
              ' type:' .. typename)
      end
   end

   local nVocab = 10000
   local nDim = 128
   local nInput = 1000

   for k, typename in ipairs(typenames) do
      local input = torch.LongTensor(nInput):random(nVocab)

      local ctype = t2cpu[typename]
      local gradOutput = torch.randn(nInput, nDim):type(ctype)
      local sconv = nn.LookupTable(nVocab, nDim):type(ctype)
      local gconv = sconv:clone():type(typename)

      sconv:forward(input)
      sconv:backward(input, gradOutput)

      input = input:cuda()
      gradOutput = gradOutput:type(typename)
      gconv:forward(input)
      gconv:backward(input, gradOutput)

      local weightGradError = gconv.gradWeight:double() - sconv.gradWeight:double()
      mytester:assertlt(weightGradError:abs():max(), precision_backward_type(precision_backward, typename),
          string.format('error on weight with %s', typename))
   end
end

function cunntest.getParameters()
  -- tensors are non-contiguous but compact; they can be gathered
  local L = nn.Linear(10,10):cuda()
  L.weight = torch.CudaTensor(10,10):t():fill(1)
  local tmp = torch.CudaTensor(10,10):fill(2)
  L.bias = tmp:select(1,2)
  local P = L:getParameters()
  mytester:asserteq(L.weight:mean(), 1)
  mytester:asserteq(L.bias:mean(), 2)
  mytester:asserteq(L.weight:storage(), L.bias:storage())
  mytester:asserteq(P:nElement(), 110)
  mytester:asserteq(P:storage():size(), 110)
  mytester:assertlt(L.bias[{ {10} }]:storageOffset() - 1, L.bias:storage():size())
end

function cunntest.SpatialReflectionPadding_forward()
   local batch = math.random(1,3)
   local plane = math.random(1,3)
   local sizeY = math.random(7,16)
   local sizeX = math.random(7,16)
   local padL = math.random(-3,3)
   local padR = math.random(-3,3)
   local padT = math.random(-3,3)
   local padB = math.random(-3,3)

   for k, typename in ipairs(typenames) do
      local input = torch.rand(batch, plane, sizeY, sizeX):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local module = nn.SpatialReflectionPadding(padL, padR, padT, padB):type(ctype)
      local groundtruth = module:forward(input)

      input = input:type(typename)
      local gmodule = nn.SpatialReflectionPadding(padL, padR, padT, padB):type(typename)
      local rescuda = gmodule:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(),
                        precision_forward_type(precision_forward, typename),
                        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialReflectionPadding_backward()
   local batch = math.random(1,3)
   local plane = math.random(1,3)
   local sizeY = math.random(7,16)
   local sizeX = math.random(7,16)
   local padL = math.random(-3,3)
   local padR = math.random(-3,3)
   local padT = math.random(-3,3)
   local padB = math.random(-3,3)

   for k, typename in ipairs(typenames) do
      local input = torch.rand(batch, plane, sizeY, sizeX):type(typename)
      local gradOutput = torch.rand(
          batch, plane, sizeY + padT + padB, sizeX + padL + padR
       ):type(typename)

       local ctype = t2cpu[typename]
       input = input:type(ctype)
       gradOutput = gradOutput:type(ctype)
       local module = nn.SpatialReflectionPadding(padL, padR, padT, padB):type(ctype)
       module:forward(input)
       module:zeroGradParameters()
       local groundgrad = module:backward(input, gradOutput)

       input = input:type(typename)
       gradOutput = gradOutput:type(typename)
       local gmodule = nn.SpatialReflectionPadding(padL, padR, padT, padB):type(typename)
       gmodule:forward(input)
       gmodule:zeroGradParameters()
       local rescuda = gmodule:backward(input, gradOutput)

       local error = rescuda:double() - groundgrad:double()
       mytester:assertlt(error:abs():max(),
                         precision_backward_type(precision_backward, type),
                         string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.SpatialReplicationPadding_forward()
   local batch = math.random(1,3)
   local plane = math.random(1,3)
   local sizeY = math.random(7,16)
   local sizeX = math.random(7,16)
   local padL = math.random(-3,3)
   local padR = math.random(-3,3)
   local padT = math.random(-3,3)
   local padB = math.random(-3,3)

   for k, typename in ipairs(typenames) do
      local input = torch.rand(batch, plane, sizeY, sizeX):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local module = nn.SpatialReplicationPadding(padL, padR, padT, padB):type(ctype)
      local groundtruth = module:forward(input)

      input = input:type(typename)
      local gmodule = nn.SpatialReplicationPadding(padL, padR, padT, padB):type(typename)
      local rescuda = gmodule:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(),
                        precision_forward_type(precision_forward, type),
                        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.SpatialReplicationPadding_backward()
   local batch = math.random(1,3)
   local plane = math.random(1,3)
   local sizeY = math.random(7,16)
   local sizeX = math.random(7,16)
   local padL = math.random(-3,3)
   local padR = math.random(-3,3)
   local padT = math.random(-3,3)
   local padB = math.random(-3,3)

   for k, typename in ipairs(typenames) do
      local input = torch.rand(batch, plane, sizeY, sizeX):type(typename)
      local gradOutput = torch.rand(
          batch, plane, sizeY + padT + padB, sizeX + padL + padR
       ):type(typename)

       local ctype = t2cpu[typename]
       input = input:type(ctype)
       gradOutput = gradOutput:type(ctype)
       local module = nn.SpatialReplicationPadding(padL, padR, padT, padB):type(ctype)
       module:forward(input)
       module:zeroGradParameters()
       local groundgrad = module:backward(input, gradOutput)

       input = input:type(typename)
       gradOutput = gradOutput:type(typename)
       local gmodule = nn.SpatialReplicationPadding(padL, padR, padT, padB):type(typename)
       gmodule:forward(input)
       gmodule:zeroGradParameters()
       local rescuda = gmodule:backward(input, gradOutput)

       local error = rescuda:double() - groundgrad:double()
       mytester:assertlt(error:abs():max(),
                         precision_backward_type(precision_backward, typename),
                         string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.VolumetricReplicationPadding_forward()
   local batch = math.random(1,3)
   local plane = math.random(1,3)
   local sizeZ = math.random(7,16)
   local sizeY = math.random(7,16)
   local sizeX = math.random(7,16)
   local pleft = math.random(-3,3)
   local pright = math.random(-3,3)
   local ptop = math.random(-3,3)
   local pbottom = math.random(-3,3)
   local pfront = math.random(-3,3)
   local pback = math.random(-3,3)

   for k, typename in ipairs(typenames) do
      local input = torch.rand(batch, plane, sizeZ, sizeY, sizeX):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      local module = nn.VolumetricReplicationPadding(pleft, pright, ptop, pbottom,
                                                     pfront, pback):type(ctype)
      local groundtruth = module:forward(input)

      input = input:type(typename)
      local gmodule = nn.VolumetricReplicationPadding(pleft, pright, ptop, pbottom,
                                                      pfront, pback):type(typename)
      local rescuda = gmodule:forward(input)

      local error = rescuda:double() - groundtruth:double()
      mytester:assertlt(error:abs():max(),
                        precision_forward_type(precision_forward, typename),
                        string.format('error on state (forward) with %s', typename))
   end
end

function cunntest.VolumetricReplicationPadding_backward()
   local batch = math.random(1,3)
   local plane = math.random(1,3)
   local sizeZ = math.random(7,16)
   local sizeY = math.random(7,16)
   local sizeX = math.random(7,16)
   local pleft = math.random(-3,3)
   local pright = math.random(-3,3)
   local ptop = math.random(-3,3)
   local pbottom = math.random(-3,3)
   local pfront = math.random(-3,3)
   local pback = math.random(-3,3)

   for k, typename in ipairs(typenames) do
      local input = torch.rand(batch, plane, sizeZ, sizeY, sizeX):type(typename)
      local gradOutput = torch.rand(
        batch, plane, sizeZ + pfront + pback, sizeY + ptop + pbottom,
        sizeX + pleft + pright
      ):type(typename)

      local ctype = t2cpu[typename]
      input = input:type(ctype)
      gradOutput = gradOutput:type(ctype)
      local module = nn.VolumetricReplicationPadding(pleft, pright, ptop, pbottom,
                                                     pfront, pback):type(ctype)
      module:forward(input)
      module:zeroGradParameters()
      local groundgrad = module:backward(input, gradOutput)

      input = input:type(typename)
      gradOutput = gradOutput:type(typename)
      local gmodule = nn.VolumetricReplicationPadding(pleft, pright, ptop, pbottom,
                                                      pfront, pback):type(typename)
      gmodule:forward(input)
      gmodule:zeroGradParameters()
      local rescuda = gmodule:backward(input, gradOutput)

      local error = rescuda:double() - groundgrad:double()
      mytester:assertlt(error:abs():max(),
                        precision_backward_type(precision_backward, typename),
                        string.format('error on state (backward) with %s', typename))
   end
end

function cunntest.GPU()
   local ndevice = cutorch.getDeviceCount()
   if ndevice < 2 then
      return
   end
   assert(nn.GPU, "Please update nn to latest version")

   local originaldevice = cutorch.getDevice()

   cutorch.setDevice(1)
   local linear = nn.Linear(3,4)
   local linear2 = linear:clone():float()
   linear.mybuffer = {torch.CudaTensor(3)}

   local gpu = nn.GPU(linear, 2, 1)
   gpu:cuda()

   mytester:assert(linear.mybuffer[1]:getDevice() == 2)
   mytester:assert(linear.weight:getDevice() == 2)
   mytester:assert(cutorch.getDevice() == originaldevice)

   local input = torch.CudaTensor(2,3):uniform(0,1)
   local output = gpu:forward(input)

   mytester:assert(linear.output:getDevice() == 2)
   mytester:assert(output:getDevice() == 1)
   mytester:assert(gpu._input:getDevice() == 2)

   local gradOutput = torch.CudaTensor(2,4):uniform(0,1)
   gpu:zeroGradParameters()
   mytester:assert(cutorch.getDevice() == 1)
   local gradInput = gpu:backward(input, gradOutput)

   mytester:assert(cutorch.getDevice() == 1)
   mytester:assert(gpu._gradOutput:getDevice() == 2)
   mytester:assert(linear.gradInput:getDevice() == 2)
   mytester:assert(gradInput:getDevice() == 1)

   mytester:assert(cutorch.getDevice() == 1)
   local input2, gradOutput2 = input:float(), gradOutput:float()
   local output2 = linear2:forward(input2)
   linear2:zeroGradParameters()
   local gradInput2 = linear2:backward(input2, gradOutput2)


   mytester:assertTensorEq(input2, input:float(), 0.000001)
   mytester:assertTensorEq(gradInput2, gradInput:float(), 0.000001)

   local params, gradParams = gpu:parameters()
   local params2, gradParams2 = linear2:parameters()

   for i=1,#params do
      mytester:assertTensorEq(params2[i], params[i]:float(), 0.000001)
      mytester:assertTensorEq(gradParams2[i], gradParams[i]:float(), 0.000001)
   end

   -- test serialize/deserialize

   local gpustr = torch.serialize(gpu)
   mytester:assert(cutorch.getDevice() == 1)
   local gpu2 = torch.deserialize(gpustr)
   mytester:assert(cutorch.getDevice() == 1)

   local output2 = gpu2:forward(input)

   mytester:assert(gpu2.modules[1].output:getDevice() == 2)
   mytester:assert(output2:getDevice() == 1)
   mytester:assert(gpu2._input:getDevice() == 2)

   gpu2:zeroGradParameters()
   mytester:assert(cutorch.getDevice() == 1)
   local gradInput2 = gpu2:backward(input, gradOutput)

   mytester:assert(cutorch.getDevice() == 1)
   mytester:assert(gpu2._gradOutput:getDevice() == 2)
   mytester:assert(gpu2.modules[1].gradInput:getDevice() == 2)
   mytester:assert(gradInput2:getDevice() == 1)

   mytester:assertTensorEq(input2, input2, 0.000001)
   mytester:assertTensorEq(gradInput2, gradInput2, 0.000001)

   local params, gradParams = gpu:parameters()
   local params2, gradParams2 = gpu2:parameters()

   for i=1,#params do
      mytester:assert(params2[i]:getDevice() == params[i]:getDevice())
      mytester:assert(gradParams2[i]:getDevice() == gradParams[i]:getDevice())
      mytester:assertTensorEq(params2[i]:float(), params[i]:float(), 0.000001)
      mytester:assertTensorEq(gradParams2[i]:float(), gradParams[i]:float(), 0.000001)
   end


   -- test table input/output
   local lin1, lin2 = nn.Linear(3,4), nn.Linear(3,4)
   local para = nn.ParallelTable():add(lin1):add(lin2)
   local para2 = para:clone():float()
   local gpu = nn.GPU(para, 2, 1)

   gpu:cuda()
   mytester:assert(lin1.weight:getDevice() == 2)
   mytester:assert(lin2.weight:getDevice() == 2)
   mytester:assert(cutorch.getDevice() == 1)

   local device3 = cutorch.getDeviceCount()
   local input = {
      torch.CudaTensor(2,3):uniform(0,1),
      cutorch.withDevice(device3, function() return torch.CudaTensor(2,3):uniform(0,1) end) -- tests input from multiple devices
   }
   local output = gpu:forward(input)

   mytester:assert(para.output[1]:getDevice() == 2)
   mytester:assert(para.output[2]:getDevice() == 2)
   mytester:assert(output[1]:getDevice() == 1)
   mytester:assert(output[2]:getDevice() == 1)
   mytester:assert(gpu._input[1]:getDevice() == 2)
   mytester:assert(gpu._input[2]:getDevice() == 2)

   local gradOutput = {
      torch.CudaTensor(2,4):uniform(0,1),
      cutorch.withDevice(device3, function() return torch.CudaTensor(2,4):uniform(0,1) end) -- tests gradOutput from multiple devices
   }

   gpu:zeroGradParameters()
   mytester:assert(cutorch.getDevice() == 1)
   local gradInput = gpu:backward(input, gradOutput)

   mytester:assert(cutorch.getDevice() == 1)
   mytester:assert(gpu._gradOutput[1]:getDevice() == 2)
   mytester:assert(gpu._gradOutput[2]:getDevice() == 2)
   mytester:assert(para.gradInput[1]:getDevice() == 2)
   mytester:assert(para.gradInput[2]:getDevice() == 2)
   mytester:assert(gradInput[1]:getDevice() == 1)
   mytester:assert(gradInput[2]:getDevice() == device3)

   local input2, gradOutput2 = {input[1]:float(), input[2]:float()}, {gradOutput[1]:float(), gradOutput[2]:float()}
   local output2 = para2:forward(input2)
   para2:zeroGradParameters()
   local gradInput2 = para2:backward(input2, gradOutput2)

   mytester:assertTensorEq(input2[1], input[1]:float(), 0.000001)
   mytester:assertTensorEq(input2[2], input[2]:float(), 0.000001)
   mytester:assertTensorEq(gradInput2[1], gradInput[1]:float(), 0.000001)
   mytester:assertTensorEq(gradInput2[2], gradInput[2]:float(), 0.000001)

   local params, gradParams = gpu:parameters()
   local params2, gradParams2 = para2:parameters()

   for i=1,#params do
      mytester:assertTensorEq(params2[i], params[i]:float(), 0.000001)
      mytester:assertTensorEq(gradParams2[i], gradParams[i]:float(), 0.000001)
   end

   -- test that it handles reduction in input/output size

   input[2], gradOutput[2] = nil, nil
   para.modules[2] = nil
   para.output[2] = nil
   para.gradInput[2] = nil

   local output = gpu:forward(input)

   mytester:assert(#gpu._input == 1)
   mytester:assert(#output == 1)

   local gradInput = gpu:backward(input, gradOutput)

   mytester:assert(#gpu._gradOutput == 1)
   mytester:assert(#gradInput == 1)

   -- test sequential multi-GPUs

   local mlp = nn.Sequential()
   for device=1,ndevice do
      local outdevice = device == ndevice and 1 or device
      mlp:add(nn.GPU(nn.Linear(3,3), device, outdevice))
      mytester:assert(cutorch.getDevice() == 1)
   end
   mlp:cuda()
   mytester:assert(cutorch.getDevice() == 1)

   local input = torch.CudaTensor(2,3):uniform(0,1)
   local gradOutput = torch.CudaTensor(2,3):uniform(0,1)

   local output = mlp:forward(input)
   mlp:zeroGradParameters()
   local gradInput = mlp:backward(input, gradOutput)

   -- test CPU only

   local params, gradParams = mlp:parameters()

   mlp:float()

   local input2, gradOutput2 = input:float(), gradOutput:float()

   local _cutorch = cutorch
   cutorch = nil

   local output2 = mlp:forward(input2)
   mlp:zeroGradParameters()
   local gradInput2 = mlp:backward(input2, gradOutput2)

   cutorch = _cutorch

   mytester:assertTensorEq(output:float(), output2, 0.000001)
   mytester:assertTensorEq(gradInput:float(), gradInput2, 0.000001)

   local params2, gradParams2 = mlp:parameters()

   for i=1,#params do
      mytester:assertTensorEq(params[i]:float(), params2[i], 0.000001)
      mytester:assertTensorEq(gradParams[i]:float(), gradParams2[i], 0.000001)
   end

   cutorch.setDevice(originaldevice)
end

local function setUp()
   cutorch.setDevice(1)
end

for k,v in pairs(cunntest.__tests) do
   cunntest.__tests[k] = function()
      setUp()
      v()
   end
end

local function initSeed(seed)
   seed = seed or math.floor((torch.tic() * 1e5) % 1e9)
   -- ensure that you can reproduce a failing test
   print('seed: ', seed)
   math.randomseed(seed)
   torch.manualSeed(seed)
   cutorch.manualSeedAll(seed)
end

function nn.testcuda(tests, print_timing, n_loop, seed)
   nloop = n_loop or nloop
   local oldtype = torch.getdefaulttensortype()
   torch.setdefaulttensortype('torch.FloatTensor')
   checkHalf()
   initSeed(seed)
   mytester = torch.Tester()
   mytester:add(cunntest)
   mytester:run(tests)
   torch.setdefaulttensortype(oldtype)
   if print_timing then
       print ''
       print ' ------------------------------------------------------------------------------------------------'
       print '|  Module                                                                          |  Speedup    |'
       print ' ------------------------------------------------------------------------------------------------'
       for module,tm in pairs(times) do
           local str = string.format('| %-80s | %4.2f        |', module, (tm.cpu / (tm.gpu or 1e6)))
           print(str)
       end
       print ' ------------------------------------------------------------------------------------------------'
   end
end

-- add alias, in same format as eg cutorch.test()
cunn = cunn or {}
cunn.test = nn.testcuda