#include "DomainInfo.h" #include #include #include #include #include #include #include #include "VtkUtil/VtkUtilArray.h" #include "VtkUtil/VtkUtilMPI.h" namespace { template std::vector calcCellSizes(const std::vector& data) { if (data.size() < 1) { throw std::runtime_error("Bad array size!"); } std::vector result(data.size() - 1); for (std::size_t i = 0; i < data.size() - 1; i++) { result[i] = data[i + 1] - data[i]; } return result; } template std::vector calcCellCenters(const std::vector& data) { if (data.size() < 1) { throw std::runtime_error("Bad array size!"); } std::vector result(data.size() - 1); for (std::size_t i = 0; i < data.size() - 1; i++) { result[i] = static_cast(0.5) * (data[i] + data[i + 1]); } return result; } template bool isUniformCoords(const std::vector& data, T eps = std::numeric_limits::epsilon()) { if (data.size() < 2) { return true; } T min_dist = std::numeric_limits::max(); T max_dist = std::numeric_limits::lowest(); for (std::size_t i = 1; i < data.size(); i++) { const T dist = data[i] - data[i - 1]; min_dist = std::min(min_dist, dist); max_dist = std::max(max_dist, dist); } return (max_dist - min_dist) < eps; } } // namespace VofFlow::DomainInfo::DomainInfo(vtkRectilinearGrid* grid, vtkMPIController* mpiController) : localExtent_{}, localZeroExtent_{}, globalExtent_{}, localBounds_{}, localZeroBounds_{}, globalBounds_{}, dims_{}, isUniform_(false), isParallel_(false), ghostArray_(nullptr) { if (grid == nullptr) { return; } // Save a permanent copy of the grid structure without any data to have access to coords. grid_ = vtkSmartPointer::New(); grid_->CopyStructure(grid); auto numberOfPieces = grid->GetInformation()->Get(vtkDataObject::DATA_NUMBER_OF_PIECES()); isParallel_ = mpiController != nullptr && mpiController->GetCommunicator() != nullptr && numberOfPieces > 1; // Basic input domain ranges grid->GetExtent(localExtent_.data()); grid->GetBounds(localBounds_.data()); // cell dimensions (grid->GetDimensions() will return node dimensions) dims_ = Grid::extentDimensions(localExtent_); // cell dimensions coordsX_ = getArrayValues(grid_->GetXCoordinates()); coordsY_ = getArrayValues(grid_->GetYCoordinates()); coordsZ_ = getArrayValues(grid_->GetZCoordinates()); cellSizesX_ = calcCellSizes(coordsX_); cellSizesY_ = calcCellSizes(coordsY_); cellSizesZ_ = calcCellSizes(coordsZ_); cellCentersX_ = calcCellCenters(coordsX_); cellCentersY_ = calcCellCenters(coordsY_); cellCentersZ_ = calcCellCenters(coordsZ_); // Validate assumptions for fast ComputeStructuredCoordinates if (coordsX_.size() < 2 || coordsY_.size() < 2 || coordsZ_.size() < 2) { throw std::runtime_error("All coordinates arrays must have at least size 2!"); } if (!std::is_sorted(coordsX_.begin(), coordsX_.end()) || !std::is_sorted(coordsY_.begin(), coordsY_.end()) || !std::is_sorted(coordsZ_.begin(), coordsZ_.end())) { throw std::runtime_error("All coordinates arrays must be sorted!"); } // Uniform Grid isUniform_ = isUniformCoords(coordsX_) && isUniformCoords(coordsY_) && isUniformCoords(coordsZ_); // No MPI / Single process case // // Global and zero domain are the same as local domain (no ghost cells and no other threads). if (!isParallel_) { globalExtent_ = localExtent_; localZeroExtent_ = localExtent_; globalBounds_ = localBounds_; localZeroBounds_ = localBounds_; return; } // MPI case extent // // GetExtent will include ghost cells, but we also need the local extent without ghost cells ("zero extent"). // The only official way to find ghost cells seems to lookup within the 'vtkGhostType' array. But here, we want // to avoid the array traversal and parsing, and calculate the zero extent directly. Unfortunately, only using the // local + global extent and the number of ghost cells, we cannot reconstruct the zero extent. There are many // edge cases, where calculations with only these values will fail, i.e. domain boundary reduces number of ghost // levels. // Therefore, we copy the VTK internal extent calculation which is used to set up the 'vtkGhostType' array. // Source: // https://gitlab.kitware.com/vtk/vtk/-/blob/fc5d7cb1748b7b123c7d97bce88cf9385b39eeb6/Common/ExecutionModel/vtkStreamingDemandDrivenPipeline.cxx#L976-986 // (VTK commit hash referenced by ParaView v5.9.0 tag) grid->GetInformation()->Get(vtkDataObject::ALL_PIECES_EXTENT(), globalExtent_.data()); auto pieceNumber = grid->GetInformation()->Get(vtkDataObject::DATA_PIECE_NUMBER()); vtkExtentTranslator* et = vtkExtentTranslator::New(); et->PieceToExtentThreadSafe(pieceNumber, numberOfPieces, 0, globalExtent_.data(), localZeroExtent_.data(), vtkExtentTranslator::BLOCK_MODE, 0); et->Delete(); // MPI case bounds vtkDataArray* coordsX = grid->GetXCoordinates(); vtkDataArray* coordsY = grid->GetYCoordinates(); vtkDataArray* coordsZ = grid->GetZCoordinates(); localZeroBounds_[0] = coordsX->GetComponent(localZeroExtent_[0] - localExtent_[0], 0); localZeroBounds_[1] = coordsX->GetComponent(localZeroExtent_[1] - localExtent_[0], 0); localZeroBounds_[2] = coordsY->GetComponent(localZeroExtent_[2] - localExtent_[2], 0); localZeroBounds_[3] = coordsY->GetComponent(localZeroExtent_[3] - localExtent_[2], 0); localZeroBounds_[4] = coordsZ->GetComponent(localZeroExtent_[4] - localExtent_[4], 0); localZeroBounds_[5] = coordsZ->GetComponent(localZeroExtent_[5] - localExtent_[4], 0); // Save inverse of upper bounds to use combined minimum operation std::array sendBounds{ localBounds_[0], -localBounds_[1], localBounds_[2], -localBounds_[3], localBounds_[4], -localBounds_[5], }; std::array recvBounds{}; mpiController->AllReduce(sendBounds.data(), recvBounds.data(), 6, vtkCommunicator::MIN_OP); globalBounds_[0] = recvBounds[0]; globalBounds_[1] = -recvBounds[1]; globalBounds_[2] = recvBounds[2]; globalBounds_[3] = -recvBounds[3]; globalBounds_[4] = recvBounds[4]; globalBounds_[5] = -recvBounds[5]; // Check if isUniform is true over all segments. // Assuming ghost cells, we can also be sure that there is no slicing change at a process border. unsigned char sendIsUniform = isUniform_ ? 1 : 0; unsigned char recvIsUniform = 0; mpiController->AllReduce(&sendIsUniform, &recvIsUniform, 1, vtkCommunicator::MIN_OP); isUniform_ = recvIsUniform == 1; // While we above avoided the array traversal, the ghost array is still useful to check individual cells without // calculating if they are within the zero extent. ghostArray_ = vtkSmartPointer::New(); ghostArray_->DeepCopy(grid->GetCellGhostArray()); // Find neighbors const int processId = mpiController->GetLocalProcessId(); const int numProcesses = mpiController->GetNumberOfProcesses(); // Send zero extents std::vector allZeroExtents(6 * numProcesses); std::vector allZeroBounds(6 * numProcesses); VofFlow::AllGatherVSameLength(mpiController, localZeroExtent_.data(), allZeroExtents.data(), 6); VofFlow::AllGatherVSameLength(mpiController, localZeroBounds_.data(), allZeroBounds.data(), 6); // Find neighbours neighborsByDirection_ = std::array, 6>(); for (int i = 0; i < numProcesses; ++i) { if (i == processId) { continue; } extent_t const& zeroExtent_i = *reinterpret_cast(&allZeroExtents[6 * i]); bounds_t const& zeroBounds_i = *reinterpret_cast(&allZeroBounds[6 * i]); // Neighbors are all processes which zero extent overlaps with the ghost cells of the current process. // We can assume, that zero extents do not overlap. Therefore, we can just intersect the full extent of the // current process with the zero extent of the other processes. if (Grid::intersect(localExtent_, zeroExtent_i)) { // Classify location of each neighbor. // Store a flag for each of the 6 sides, if the neighbor can be found in this direction. std::bitset<6> flags; flags.set(0, zeroExtent_i[0] < localZeroExtent_[0]); // low X flags.set(1, zeroExtent_i[1] > localZeroExtent_[1]); // up X flags.set(2, zeroExtent_i[2] < localZeroExtent_[2]); // low Y flags.set(3, zeroExtent_i[3] > localZeroExtent_[3]); // up Y flags.set(4, zeroExtent_i[4] < localZeroExtent_[4]); // low Z flags.set(5, zeroExtent_i[5] > localZeroExtent_[5]); // up Z neighbors_.push_back({i, flags, zeroExtent_i, zeroBounds_i}); // Convert to by-direction list format for (int j = 0; j < 6; j++) { if (flags[j]) { neighborsByDirection_[j].push_back(i); } } } } // Check if neighbors are symmetric between all processes. This should theoretically always be true, but do some // paranoid extra error checking here. // Build a matrix filled with 0 and 1 which processes are neighbors. std::vector row(numProcesses, 0); for (const auto& n : neighbors_) { row[n.processId] = 1; } std::vector neighborMatrix(numProcesses * numProcesses, 0); VofFlow::AllGatherVSameLength(mpiController, row, neighborMatrix); for (int i = 0; i < numProcesses; i++) { if (neighborMatrix[i * numProcesses + i] != 0) { throw std::runtime_error("Bad neighbor finding!"); } for (int j = i + 1; j < numProcesses; j++) { if (neighborMatrix[i * numProcesses + j] != neighborMatrix[j * numProcesses + i]) { throw std::runtime_error("Bad neighbor finding!"); } } } }