From b39e91f3bab0edfc20534e0d2000a79626aff03d Mon Sep 17 00:00:00 2001 From: Tom Meltzer Date: Fri, 19 Jun 2026 13:15:19 +0000 Subject: [PATCH 1/3] refactor(partitioner): unify periodic and non-periodic neighbour info Previously, neighbours across periodic boundaries were tracked in separate data structures (_neighbours_p, _sendPos_p, _recvPos_p, _cornerNeighbours_p, _cornerSendPos_p) with a dedicated getNeighbourInfoPeriodic() method and separate NetCDF output paths. This change merges periodic neighbour info into the same maps used for non-periodic neighbours, removing redundant code and data structures: - Removed getNeighbourInfoPeriodic() entirely - Removed periodic dimension/variable definitions from saveMetadata() - Removed periodic data writes from saveMetadata() - Consolidated periodic and regular neighbours into _neighbours, _sendPos, _recvPos, _cornerNeighbours, _cornerSendPos --- Partitioner.cpp | 152 +++--------------------------------------------- Partitioner.hpp | 34 ----------- 2 files changed, 8 insertions(+), 178 deletions(-) diff --git a/Partitioner.cpp b/Partitioner.cpp index 51c979e..ce25d39 100644 --- a/Partitioner.cpp +++ b/Partitioner.cpp @@ -281,33 +281,6 @@ void Partitioner::getNeighbourInfo(std::array, N_EDGE>& ids, } } -void Partitioner::getNeighbourInfoPeriodic(std::array, N_EDGE>& ids, - std::array, N_EDGE>& haloSizes, std::array, N_EDGE>& haloSend, - std::array, N_EDGE>& haloRecv, - std::array, N_CORNER>& cornerIds, - std::array, N_CORNER>& cornerSend) const -{ - for (auto edge : edges) { - if (((edge == LEFT || edge == RIGHT) && _px) || ((edge == TOP || edge == BOTTOM) && _py)) { - - for (auto it = _neighbours_p[edge].begin(); it != _neighbours_p[edge].end(); ++it) { - ids[edge].push_back(it->first); - haloSizes[edge].push_back(it->second); - haloSend[edge].push_back(_sendPos_p[edge].at(it->first)); - haloRecv[edge].push_back(_recvPos_p[edge].at(it->first)); - } - } - } - - for (auto corner : corners) { - for (auto it = _cornerNeighbours_p[corner].begin(); it != _cornerNeighbours_p[corner].end(); - ++it) { - cornerIds[corner].push_back(it->first); - cornerSend[corner].push_back(_cornerSendPos_p[corner].at(it->first)); - } - } -} - void Partitioner::saveMask(const std::string& filename) const { // Use C API for parallel I/O @@ -388,29 +361,6 @@ void Partitioner::saveMetadata(const std::string& filename) const &numCornerNeighbours[corner], &corner_offsets[corner], 1, MPI_INT, MPI_SUM, _comm)); } - // Prepare periodic neighbour data - std::array, N_EDGE> ids_p, halos_p, haloSend_p, haloRecv_p; - std::array, N_CORNER> corner_ids_p, cornerSend_p; - getNeighbourInfoPeriodic(ids_p, halos_p, haloSend_p, haloRecv_p, corner_ids_p, cornerSend_p); - std::vector num_neighbours_p(N_EDGE), dims_p(N_EDGE, 0), offsets_p(N_EDGE, 0); - for (auto edge : edges) { - num_neighbours_p[edge] = (int)ids_p[edge].size(); - CHECK_MPI( - MPI_Allreduce(&num_neighbours_p[edge], &dims_p[edge], 1, MPI_INT, MPI_SUM, _comm)); - CHECK_MPI( - MPI_Exscan(&num_neighbours_p[edge], &offsets_p[edge], 1, MPI_INT, MPI_SUM, _comm)); - } - - std::vector numCornerNeighbours_p(N_CORNER), corner_dims_p(N_CORNER, 0), - corner_offsets_p(N_CORNER, 0); - for (auto corner : corners) { - numCornerNeighbours_p[corner] = (int)corner_ids_p[corner].size(); - CHECK_MPI(MPI_Allreduce( - &numCornerNeighbours_p[corner], &corner_dims_p[corner], 1, MPI_INT, MPI_SUM, _comm)); - CHECK_MPI(MPI_Exscan( - &numCornerNeighbours_p[corner], &corner_offsets_p[corner], 1, MPI_INT, MPI_SUM, _comm)); - } - // Define dimensions in netCDF file int dimid; std::vector dimids(N_EDGE); @@ -425,19 +375,6 @@ void Partitioner::saveMetadata(const std::string& filename) const nc_id, corner_dir_chars[corner].c_str(), corner_dims[corner], &corner_dimids[corner])); } - // Define periodic dimensions in netCDF file - std::vector dimids_p(N_EDGE); - for (auto edge : edges) { - NC_CHECK(nc_def_dim( - nc_id, (dir_chars[edge] + "_periodic").c_str(), dims_p[edge], &dimids_p[edge])); - } - - std::vector corner_dimids_p(N_CORNER); - for (auto edge : edges) { - NC_CHECK(nc_def_dim(nc_id, (corner_dir_chars[edge] + "_periodic").c_str(), - corner_dims_p[edge], &corner_dimids_p[edge])); - } - // Define groups in netCDF file int bbox_gid, connectivity_gid; NC_CHECK(nc_def_grp(nc_id, "bounding_boxes", &bbox_gid)); @@ -487,44 +424,6 @@ void Partitioner::saveMetadata(const std::string& filename) const NC_INT, 1, &corner_dimids[corner], &cornerSend_vid[corner])); } - int num_vid_p[N_EDGE]; - int ids_vid_p[N_EDGE]; - int halos_vid_p[N_EDGE]; - int haloSend_vid_p[N_EDGE]; - int haloRecv_vid_p[N_EDGE]; - for (auto edge : edges) { - // Periodic members of connectivity group - NC_CHECK(nc_def_var(connectivity_gid, (dir_names[edge] + "_neighbours_periodic").c_str(), - NC_INT, 1, &dimid, &num_vid_p[edge])); - NC_CHECK(nc_def_var(connectivity_gid, (dir_names[edge] + "_neighbour_ids_periodic").c_str(), - NC_INT, 1, &dimids_p[edge], &ids_vid_p[edge])); - NC_CHECK( - nc_def_var(connectivity_gid, (dir_names[edge] + "_neighbour_halos_periodic").c_str(), - NC_INT, 1, &dimids_p[edge], &halos_vid_p[edge])); - NC_CHECK(nc_def_var(connectivity_gid, - (dir_names[edge] + "_neighbour_halo_send_periodic").c_str(), NC_INT, 1, &dimids_p[edge], - &haloSend_vid_p[edge])); - NC_CHECK(nc_def_var(connectivity_gid, - (dir_names[edge] + "_neighbour_halo_recv_periodic").c_str(), NC_INT, 1, &dimids_p[edge], - &haloRecv_vid_p[edge])); - } - - int num_corner_vid_p[N_CORNER]; - int ids_corner_vid_p[N_CORNER]; - int cornerSend_vid_p[N_CORNER]; - for (auto corner : corners) { - // Connectivity group - NC_CHECK(nc_def_var(connectivity_gid, - (corner_dir_names[corner] + "_neighbours_periodic").c_str(), NC_INT, 1, &dimid, - &num_corner_vid_p[corner])); - NC_CHECK(nc_def_var(connectivity_gid, - (corner_dir_names[corner] + "_neighbour_ids_periodic").c_str(), NC_INT, 1, - &corner_dimids_p[corner], &ids_corner_vid_p[corner])); - NC_CHECK(nc_def_var(connectivity_gid, - (corner_dir_names[corner] + "_neighbour_send_periodic").c_str(), NC_INT, 1, - &corner_dimids_p[corner], &cornerSend_vid_p[corner])); - } - // Write metadata to file NC_CHECK(nc_enddef(nc_id)); @@ -541,10 +440,6 @@ void Partitioner::saveMetadata(const std::string& filename) const size_t start = _rank; NC_CHECK(nc_var_par_access(connectivity_gid, num_vid[edge], NC_COLLECTIVE)); NC_CHECK(nc_put_var1_int(connectivity_gid, num_vid[edge], &start, &num_neighbours[edge])); - // Numbers of neighbours for periodic dimensions - NC_CHECK(nc_var_par_access(connectivity_gid, num_vid_p[edge], NC_COLLECTIVE)); - NC_CHECK( - nc_put_var1_int(connectivity_gid, num_vid_p[edge], &start, &num_neighbours_p[edge])); // IDs and halos start = offsets[edge]; size_t count = num_neighbours[edge]; @@ -559,19 +454,6 @@ void Partitioner::saveMetadata(const std::string& filename) const connectivity_gid, haloSend_vid[edge], &start, &count, haloSend[edge].data())); NC_CHECK(nc_put_vara_int( connectivity_gid, haloRecv_vid[edge], &start, &count, haloRecv[edge].data())); - // IDs and halos for periodic dimensions - start = offsets_p[edge]; - count = num_neighbours_p[edge]; - NC_CHECK(nc_var_par_access(connectivity_gid, ids_vid_p[edge], NC_COLLECTIVE)); - NC_CHECK( - nc_put_vara_int(connectivity_gid, ids_vid_p[edge], &start, &count, ids_p[edge].data())); - NC_CHECK(nc_var_par_access(connectivity_gid, halos_vid_p[edge], NC_COLLECTIVE)); - NC_CHECK(nc_put_vara_int( - connectivity_gid, halos_vid_p[edge], &start, &count, halos_p[edge].data())); - NC_CHECK(nc_put_vara_int( - connectivity_gid, haloSend_vid_p[edge], &start, &count, haloSend_p[edge].data())); - NC_CHECK(nc_put_vara_int( - connectivity_gid, haloRecv_vid_p[edge], &start, &count, haloRecv_p[edge].data())); } for (auto corner : corners) { @@ -580,10 +462,6 @@ void Partitioner::saveMetadata(const std::string& filename) const NC_CHECK(nc_var_par_access(connectivity_gid, num_corner_vid[corner], NC_COLLECTIVE)); NC_CHECK(nc_put_var1_int( connectivity_gid, num_corner_vid[corner], &start, &numCornerNeighbours[corner])); - // Numbers of corner neighbours for periodic dimensions - NC_CHECK(nc_var_par_access(connectivity_gid, num_corner_vid_p[corner], NC_COLLECTIVE)); - NC_CHECK(nc_put_var1_int( - connectivity_gid, num_corner_vid_p[corner], &start, &numCornerNeighbours_p[corner])); // "Corner" IDs and halos start = corner_offsets[corner]; @@ -594,16 +472,6 @@ void Partitioner::saveMetadata(const std::string& filename) const NC_CHECK(nc_var_par_access(connectivity_gid, cornerSend_vid[corner], NC_COLLECTIVE)); NC_CHECK(nc_put_vara_int( connectivity_gid, cornerSend_vid[corner], &start, &count, cornerSend[corner].data())); - - // "Corner" IDs and halos for periodic dimensions - start = corner_offsets_p[corner]; - count = numCornerNeighbours_p[corner]; - NC_CHECK(nc_var_par_access(connectivity_gid, ids_corner_vid_p[corner], NC_COLLECTIVE)); - NC_CHECK(nc_put_vara_int(connectivity_gid, ids_corner_vid_p[corner], &start, &count, - corner_ids_p[corner].data())); - NC_CHECK(nc_var_par_access(connectivity_gid, halos_vid_p[corner], NC_COLLECTIVE)); - NC_CHECK(nc_put_vara_int(connectivity_gid, cornerSend_vid_p[corner], &start, &count, - cornerSend_p[corner].data())); } NC_CHECK(nc_close(nc_id)); @@ -725,34 +593,30 @@ void Partitioner::discover_neighbours() } } - // When finding neighours *across periodic boundaries*, we need to check against the + // When finding neighbours *across periodic boundaries*, we need to check against the // current rank, too, because a subdomain can be a periodic neighbour of itself. - // check periodic edge neighours + // check periodic edge neighbours for (auto edge : edges) { if (isNeighbour(domains[_rank], domains[p], edge, _px, _py)) { int haloSize = domainOverlap(domains[_rank], domains[p], edge); if (haloSize > 0) { - _neighbours_p[edge].insert(std::pair(p, haloSize)); + _neighbours[edge].insert(std::pair(p, haloSize)); int sendPos = 0; int recvPos = 0; haloEdgeBufferPositions(domains[_rank], domains[p], edge, sendPos, recvPos); - _sendPos_p[edge].insert(std::pair(p, sendPos)); - _recvPos_p[edge].insert(std::pair(p, recvPos)); + _sendPos[edge].insert(std::pair(p, sendPos)); + _recvPos[edge].insert(std::pair(p, recvPos)); } } } - // check periodic corner neighours + // check periodic corner neighbours for (auto corner : corners) { if (isCornerNeighbour(domains[_rank], domains[p], corner, _px, _py)) { - if (_cornerNeighbours[corner].size() > 0) { - // skip if we have already counted as a non-periodic neighbour - continue; - } - _cornerNeighbours_p[corner].insert(std::pair(p, 1)); + _cornerNeighbours[corner].insert(std::pair(p, 1)); int sendPos = 0; haloCornerBufferPositions(domains[_rank], domains[p], corner, sendPos); - _cornerSendPos_p[corner].insert(std::pair(p, sendPos)); + _cornerSendPos[corner].insert(std::pair(p, sendPos)); } } } diff --git a/Partitioner.hpp b/Partitioner.hpp index 7d8f7dc..ced5303 100644 --- a/Partitioner.hpp +++ b/Partitioner.hpp @@ -74,23 +74,6 @@ class LIB_EXPORT Partitioner { std::array, N_EDGE>& haloRecv, std::array, N_CORNER>& cornerIds, std::array, N_CORNER>& cornerSend) const; - /*! - * @brief Returns vectors containing the MPI ranks, halo sizes and halo starting indices of - * the neighbours of this process across periodic boundaries after partitioning. The - * neighbours are ordered left, right, bottom, top. - * - * @param ids MPI ranks of the periodic neighbours for each direction - * @param haloSizes Halo sizes of the periodic neighbours for each direction - * @param haloSend index in send buffer to get halo data - * @param haloRecv index in recv buffer to put halo data - */ - void getNeighbourInfoPeriodic(std::array, N_EDGE>& ids, - std::array, N_EDGE>& haloSizes, - std::array, N_EDGE>& haloSend, - std::array, N_EDGE>& haloRecv, - std::array, N_CORNER>& cornerIds, - std::array, N_CORNER>& cornerSend) const; - /*! * @brief Saves the partition IDs of the latest 2D domain decomposition in a * NetCDF file. @@ -199,23 +182,6 @@ class LIB_EXPORT Partitioner { // Vector of maps of "corner" neighbours to their halo start indices after partitioning std::vector> _cornerSendPos = std::vector>(NNBRS); - // Vector of maps of periodic neighbours to their halo sizes after partitioning - std::vector> _neighbours_p = std::vector>(NNBRS); - - // Vector of maps of periodic neighbours to their send buffer indices - index of data to fetch - // from send buffer - std::vector> _sendPos_p = std::vector>(NNBRS); - - // Vector of maps of periodic neighbours to their recv (receive) buffer indices - index where - // data will be stored in the recv buffer - std::vector> _recvPos_p = std::vector>(NNBRS); - - // Vector of maps of "corner" neighbours to their halo sizes after partitioning - std::vector> _cornerNeighbours_p = std::vector>(NNBRS); - - // Vector of maps of "corner" neighbours to their halo start indices after partitioning - std::vector> _cornerSendPos_p = std::vector>(NNBRS); - public: struct LIB_EXPORT Factory { /*! From b154a94218f75885c23f23e4267cd8c91738b6a1 Mon Sep 17 00:00:00 2001 From: Tom Meltzer Date: Fri, 19 Jun 2026 14:17:20 +0100 Subject: [PATCH 2/3] refactor(partitioner): refactor saveMetadata to use templates and lambdas - Replace duplicated edge/corner loop logic with templated compute_dims() - Introduce DimInfo struct to group neighbour dimension data - Add lambda helpers (def_dim, write_array, write_scalar) to reduce repetitive nc_* calls --- Partitioner.cpp | 225 +++++++++--------- test/test_1/ref_partition_metadata_3.cdl | 56 ----- test/test_1_px/ref_partition_metadata_3.cdl | 116 ++------- .../test_1_px_py/ref_partition_metadata_3.cdl | 176 ++++---------- test/test_1_py/ref_partition_metadata_3.cdl | 136 +++-------- test/test_2/ref_partition_metadata_3.cdl | 56 ----- test/test_3/ref_partition_metadata_4.cdl | 56 ----- test/test_4/ref_partition_metadata_4.cdl | 56 ----- test/test_4_px/ref_partition_metadata_4.cdl | 144 +++-------- .../test_4_px_py/ref_partition_metadata_4.cdl | 184 ++++---------- test/test_4_py/ref_partition_metadata_4.cdl | 144 +++-------- 11 files changed, 301 insertions(+), 1048 deletions(-) diff --git a/Partitioner.cpp b/Partitioner.cpp index ce25d39..556ce3e 100644 --- a/Partitioner.cpp +++ b/Partitioner.cpp @@ -321,6 +321,28 @@ void Partitioner::saveMask(const std::string& filename) const NC_CHECK(nc_close(nc_id)); } +struct DimInfo { + std::vector numNeighbours; + std::vector dims; + std::vector offsets; +}; + +template +static DimInfo compute_dims( + const std::array& items, const std::array, N>& data, MPI_Comm comm) +{ + DimInfo info; + info.numNeighbours.resize(items.size()); + info.dims.resize(items.size(), 0); + info.offsets.resize(items.size(), 0); + for (std::size_t i = 0; i < items.size(); i++) { + info.numNeighbours[i] = (int)data[items[i]].size(); + CHECK_MPI(MPI_Allreduce(&info.numNeighbours[i], &info.dims[i], 1, MPI_INT, MPI_SUM, comm)); + CHECK_MPI(MPI_Exscan(&info.numNeighbours[i], &info.offsets[i], 1, MPI_INT, MPI_SUM, comm)); + } + return info; +} + void Partitioner::saveMetadata(const std::string& filename) const { // Use C API for parallel I/O @@ -328,150 +350,133 @@ void Partitioner::saveMetadata(const std::string& filename) const nc_mode = NC_MPIIO | NC_NETCDF4; NC_CHECK(nc_create_par(filename.c_str(), nc_mode, _comm, MPI_INFO_NULL, &nc_id)); - // Create 2 dimensions - // The values to be written are associated with the netCDF variable by - // assuming that the last dimension of the netCDF variable varies fastest in - // the C interface - const int NDIMS = 2; // TODO: Why redeclared? - int dimid_global[NDIMS]; - for (int idx = 0; idx < NDIMS; idx++) { - NC_CHECK( - nc_def_dim(nc_id, globalExtentNames[idx].c_str(), _globalExt[idx], &dimid_global[idx])); + // utility lambdas for netcdf operations + // define a new dimension + auto def_dim = [&](const std::string& name, int len, int& dimid) { + NC_CHECK(nc_def_dim(nc_id, name.c_str(), len, &dimid)); + }; + + // write an array to netcdf file + auto write_array = [&](int gid, int vid, size_t start, size_t count, const int* data) { + NC_CHECK(nc_var_par_access(gid, vid, NC_COLLECTIVE)); + NC_CHECK(nc_put_vara_int(gid, vid, &start, &count, data)); + }; + + // write a scalar to netcdf file + auto write_scalar = [&](int gid, int vid, size_t start, int val) { + NC_CHECK(nc_var_par_access(gid, vid, NC_COLLECTIVE)); + NC_CHECK(nc_put_var1_int(gid, vid, &start, &val)); + }; + + // ---- Global dimensions (NX, NY) ---- + int dimid_global[Partitioner::NDIMS]; + for (int idx = 0; idx < Partitioner::NDIMS; idx++) { + def_dim(globalExtentNames[idx], _globalExt[idx], dimid_global[idx]); } - // Prepare neighbour data + // ---- Prepare neighbour data ---- std::array, N_EDGE> ids, halos, haloSend, haloRecv; std::array, N_CORNER> corner_ids, cornerSend; getNeighbourInfo(ids, halos, haloSend, haloRecv, corner_ids, cornerSend); - std::vector num_neighbours(N_EDGE), dims(N_EDGE, 0), offsets(N_EDGE, 0); - for (auto edge : edges) { - num_neighbours[edge] = (int)ids[edge].size(); - CHECK_MPI(MPI_Allreduce(&num_neighbours[edge], &dims[edge], 1, MPI_INT, MPI_SUM, _comm)); - CHECK_MPI(MPI_Exscan(&num_neighbours[edge], &offsets[edge], 1, MPI_INT, MPI_SUM, _comm)); - } - - std::vector numCornerNeighbours(N_CORNER), corner_dims(N_CORNER, 0), - corner_offsets(N_CORNER, 0); - for (auto corner : corners) { - numCornerNeighbours[corner] = (int)corner_ids[corner].size(); - CHECK_MPI(MPI_Allreduce( - &numCornerNeighbours[corner], &corner_dims[corner], 1, MPI_INT, MPI_SUM, _comm)); - CHECK_MPI(MPI_Exscan( - &numCornerNeighbours[corner], &corner_offsets[corner], 1, MPI_INT, MPI_SUM, _comm)); - } + // ---- Compute edge dimensions (MPI Allreduce/Exscan) ---- + DimInfo edge_info = compute_dims(edges, ids, _comm); + DimInfo corner_info = compute_dims(corners, corner_ids, _comm); - // Define dimensions in netCDF file + // ---- Define netCDF dimensions ---- int dimid; - std::vector dimids(N_EDGE); + std::vector edge_dimids(N_EDGE); + std::vector corner_dimids(N_CORNER); + NC_CHECK(nc_def_dim(nc_id, "P", _totalNumProcs, &dimid)); for (auto edge : edges) { - NC_CHECK(nc_def_dim(nc_id, dir_chars[edge].c_str(), dims[edge], &dimids[edge])); + def_dim(dir_chars[edge], edge_info.dims[edge], edge_dimids[edge]); } - - std::vector corner_dimids(N_CORNER); for (auto corner : corners) { - NC_CHECK(nc_def_dim( - nc_id, corner_dir_chars[corner].c_str(), corner_dims[corner], &corner_dimids[corner])); + def_dim(corner_dir_chars[corner], corner_info.dims[corner], corner_dimids[corner]); } - // Define groups in netCDF file - int bbox_gid, connectivity_gid; + // ---- Define groups ---- + int bbox_gid, c_grid; NC_CHECK(nc_def_grp(nc_id, "bounding_boxes", &bbox_gid)); - NC_CHECK(nc_def_grp(nc_id, "connectivity", &connectivity_gid)); - - // Define variables in netCDF file - int top_vid[NDIMS]; - int cnt_vid[NDIMS]; - int num_vid[N_EDGE]; - for (int idx = 0; idx < NDIMS; idx++) { - // Bounding boxes group - NC_CHECK(nc_def_var( - bbox_gid, ("domain_" + dim_chars[idx]).c_str(), NC_INT, 1, &dimid, &top_vid[idx])); - NC_CHECK(nc_def_var(bbox_gid, ("domain_extent_" + dim_chars[idx]).c_str(), NC_INT, 1, - &dimid, &cnt_vid[idx])); + NC_CHECK(nc_def_grp(nc_id, "connectivity", &c_grid)); + + // ---- Define variables: bounding boxes ---- + int top_vid[Partitioner::NDIMS]; + int cnt_vid[Partitioner::NDIMS]; + auto def_bbox_var = [&](const std::string& prefix, const auto idx, int& vid) { + NC_CHECK(nc_def_var(bbox_gid, (prefix + dim_chars[idx]).c_str(), NC_INT, 1, &dimid, &vid)); + }; + for (int idx = 0; idx < Partitioner::NDIMS; idx++) { + def_bbox_var("domain_", idx, top_vid[idx]); + def_bbox_var("domain_extent_", idx, cnt_vid[idx]); } - int ids_vid[N_EDGE]; - int halos_vid[N_EDGE]; - int haloSend_vid[N_EDGE]; - int haloRecv_vid[N_EDGE]; + // ---- Define variables: edge connectivity ---- + int num_vid[N_EDGE], ids_vid[N_EDGE], halos_vid[N_EDGE]; + int haloSend_vid[N_EDGE], haloRecv_vid[N_EDGE]; + + // lambda to define netcdf variables (for edges) + auto def_edge_var = [&](const auto edge, const std::string& name, const int* dimArr, + auto* vid) { + NC_CHECK( + nc_def_var(c_grid, (dir_names[edge] + name).c_str(), NC_INT, 1, dimArr, &vid[edge])); + }; for (auto edge : edges) { - // Connectivity group - NC_CHECK(nc_def_var(connectivity_gid, (dir_names[edge] + "_neighbours").c_str(), NC_INT, 1, - &dimid, &num_vid[edge])); - NC_CHECK(nc_def_var(connectivity_gid, (dir_names[edge] + "_neighbour_ids").c_str(), NC_INT, - 1, &dimids[edge], &ids_vid[edge])); - NC_CHECK(nc_def_var(connectivity_gid, (dir_names[edge] + "_neighbour_halos").c_str(), - NC_INT, 1, &dimids[edge], &halos_vid[edge])); - NC_CHECK(nc_def_var(connectivity_gid, (dir_names[edge] + "_neighbour_halo_send").c_str(), - NC_INT, 1, &dimids[edge], &haloSend_vid[edge])); - NC_CHECK(nc_def_var(connectivity_gid, (dir_names[edge] + "_neighbour_halo_recv").c_str(), - NC_INT, 1, &dimids[edge], &haloRecv_vid[edge])); + def_edge_var(edge, "_neighbours", &dimid, num_vid); + const auto* edgeDimId = &edge_dimids[edge]; + def_edge_var(edge, "_neighbour_ids", edgeDimId, ids_vid); + def_edge_var(edge, "_neighbour_halos", edgeDimId, halos_vid); + def_edge_var(edge, "_neighbour_halo_send", edgeDimId, haloSend_vid); + def_edge_var(edge, "_neighbour_halo_recv", edgeDimId, haloRecv_vid); } - int num_corner_vid[N_CORNER]; - int ids_corner_vid[N_CORNER]; + // ---- Define variables: corner connectivity ---- + int num_corner_vid[N_CORNER], ids_corner_vid[N_CORNER]; int cornerSend_vid[N_CORNER]; + // lambda to define netcdf variables (for corners) + auto def_corner_var + = [&](const auto corner, const std::string& name, const int* dimArr, int* vid) { + NC_CHECK(nc_def_var(c_grid, (corner_dir_names[corner] + name).c_str(), NC_INT, 1, + dimArr, &vid[corner])); + }; for (auto corner : corners) { - // Connectivity group - NC_CHECK(nc_def_var(connectivity_gid, (corner_dir_names[corner] + "_neighbours").c_str(), - NC_INT, 1, &dimid, &num_corner_vid[corner])); - NC_CHECK(nc_def_var(connectivity_gid, (corner_dir_names[corner] + "_neighbour_ids").c_str(), - NC_INT, 1, &corner_dimids[corner], &ids_corner_vid[corner])); - NC_CHECK( - nc_def_var(connectivity_gid, (corner_dir_names[corner] + "_neighbour_send").c_str(), - NC_INT, 1, &corner_dimids[corner], &cornerSend_vid[corner])); + def_corner_var(corner, "_neighbours", &dimid, num_corner_vid); + const auto* cornerDimId = &corner_dimids[corner]; + def_corner_var(corner, "_neighbour_ids", cornerDimId, ids_corner_vid); + def_corner_var(corner, "_neighbour_send", cornerDimId, cornerSend_vid); } - // Write metadata to file + // ---- Write ---- NC_CHECK(nc_enddef(nc_id)); - // Store data - for (int idx = 0; idx < NDIMS; idx++) { + // Bounding boxes: one value per process + for (int idx = 0; idx < Partitioner::NDIMS; idx++) { size_t start = _rank; - NC_CHECK(nc_var_par_access(bbox_gid, top_vid[idx], NC_COLLECTIVE)); - NC_CHECK(nc_put_var1_int(bbox_gid, top_vid[idx], &start, &_globalNew[idx])); - NC_CHECK(nc_var_par_access(bbox_gid, cnt_vid[idx], NC_COLLECTIVE)); - NC_CHECK(nc_put_var1_int(bbox_gid, cnt_vid[idx], &start, &_localExtNew[idx])); + write_scalar(bbox_gid, top_vid[idx], start, _globalNew[idx]); + write_scalar(bbox_gid, cnt_vid[idx], start, _localExtNew[idx]); } + + // Edge connectivity for (auto edge : edges) { - // Numbers of neighbours size_t start = _rank; - NC_CHECK(nc_var_par_access(connectivity_gid, num_vid[edge], NC_COLLECTIVE)); - NC_CHECK(nc_put_var1_int(connectivity_gid, num_vid[edge], &start, &num_neighbours[edge])); - // IDs and halos - start = offsets[edge]; - size_t count = num_neighbours[edge]; - NC_CHECK(nc_var_par_access(connectivity_gid, ids_vid[edge], NC_COLLECTIVE)); - NC_CHECK( - nc_put_vara_int(connectivity_gid, ids_vid[edge], &start, &count, ids[edge].data())); - NC_CHECK(nc_var_par_access(connectivity_gid, halos_vid[edge], NC_COLLECTIVE)); - NC_CHECK( - nc_put_vara_int(connectivity_gid, halos_vid[edge], &start, &count, halos[edge].data())); - NC_CHECK(nc_var_par_access(connectivity_gid, haloSend_vid[edge], NC_COLLECTIVE)); - NC_CHECK(nc_put_vara_int( - connectivity_gid, haloSend_vid[edge], &start, &count, haloSend[edge].data())); - NC_CHECK(nc_put_vara_int( - connectivity_gid, haloRecv_vid[edge], &start, &count, haloRecv[edge].data())); + size_t count = edge_info.numNeighbours[edge]; + write_scalar(c_grid, num_vid[edge], start, edge_info.numNeighbours[edge]); + start = edge_info.offsets[edge]; + write_array(c_grid, ids_vid[edge], start, count, ids[edge].data()); + write_array(c_grid, halos_vid[edge], start, count, halos[edge].data()); + write_array(c_grid, haloSend_vid[edge], start, count, haloSend[edge].data()); + write_array(c_grid, haloRecv_vid[edge], start, count, haloRecv[edge].data()); } + // Corner connectivity for (auto corner : corners) { - // Numbers of "corner" neighbours size_t start = _rank; - NC_CHECK(nc_var_par_access(connectivity_gid, num_corner_vid[corner], NC_COLLECTIVE)); - NC_CHECK(nc_put_var1_int( - connectivity_gid, num_corner_vid[corner], &start, &numCornerNeighbours[corner])); - - // "Corner" IDs and halos - start = corner_offsets[corner]; - size_t count = numCornerNeighbours[corner]; - NC_CHECK(nc_var_par_access(connectivity_gid, ids_corner_vid[corner], NC_COLLECTIVE)); - NC_CHECK(nc_put_vara_int( - connectivity_gid, ids_corner_vid[corner], &start, &count, corner_ids[corner].data())); - NC_CHECK(nc_var_par_access(connectivity_gid, cornerSend_vid[corner], NC_COLLECTIVE)); - NC_CHECK(nc_put_vara_int( - connectivity_gid, cornerSend_vid[corner], &start, &count, cornerSend[corner].data())); + size_t count = corner_info.numNeighbours[corner]; + write_scalar(c_grid, num_corner_vid[corner], start, corner_info.numNeighbours[corner]); + start = corner_info.offsets[corner]; + write_array(c_grid, ids_corner_vid[corner], start, count, corner_ids[corner].data()); + write_array(c_grid, cornerSend_vid[corner], start, count, cornerSend[corner].data()); } NC_CHECK(nc_close(nc_id)); diff --git a/test/test_1/ref_partition_metadata_3.cdl b/test/test_1/ref_partition_metadata_3.cdl index 5e4ce32..cd39bc4 100644 --- a/test/test_1/ref_partition_metadata_3.cdl +++ b/test/test_1/ref_partition_metadata_3.cdl @@ -11,14 +11,6 @@ dimensions: TR = 1 ; BR = 1 ; BL = UNLIMITED ; // (0 currently) - L_periodic = UNLIMITED ; // (0 currently) - R_periodic = UNLIMITED ; // (0 currently) - B_periodic = UNLIMITED ; // (0 currently) - T_periodic = UNLIMITED ; // (0 currently) - TL_periodic = UNLIMITED ; // (0 currently) - TR_periodic = UNLIMITED ; // (0 currently) - BR_periodic = UNLIMITED ; // (0 currently) - BL_periodic = UNLIMITED ; // (0 currently) group: bounding_boxes { variables: @@ -71,38 +63,6 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: left_neighbours = 0, 0, 2 ; @@ -160,21 +120,5 @@ group: connectivity { bottom_right_neighbour_send = 9 ; bottom_left_neighbours = 0, 0, 0 ; - - left_neighbours_periodic = 0, 0, 0 ; - - right_neighbours_periodic = 0, 0, 0 ; - - bottom_neighbours_periodic = 0, 0, 0 ; - - top_neighbours_periodic = 0, 0, 0 ; - - top_left_neighbours_periodic = 0, 0, 0 ; - - top_right_neighbours_periodic = 0, 0, 0 ; - - bottom_right_neighbours_periodic = 0, 0, 0 ; - - bottom_left_neighbours_periodic = 0, 0, 0 ; } // group connectivity } diff --git a/test/test_1_px/ref_partition_metadata_3.cdl b/test/test_1_px/ref_partition_metadata_3.cdl index 7560911..e96e16d 100644 --- a/test/test_1_px/ref_partition_metadata_3.cdl +++ b/test/test_1_px/ref_partition_metadata_3.cdl @@ -3,22 +3,14 @@ dimensions: NX = 6 ; NY = 4 ; P = 3 ; - L = 2 ; - R = 2 ; + L = 4 ; + R = 4 ; B = 1 ; T = 1 ; - TL = UNLIMITED ; // (0 currently) + TL = 1 ; TR = 1 ; BR = 1 ; - BL = UNLIMITED ; // (0 currently) - L_periodic = 2 ; - R_periodic = 2 ; - B_periodic = UNLIMITED ; // (0 currently) - T_periodic = UNLIMITED ; // (0 currently) - TL_periodic = 1 ; - TR_periodic = UNLIMITED ; // (0 currently) - BR_periodic = UNLIMITED ; // (0 currently) - BL_periodic = 1 ; + BL = 1 ; group: bounding_boxes { variables: @@ -71,59 +63,27 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: - left_neighbours = 0, 0, 2 ; + left_neighbours = 1, 1, 2 ; - left_neighbour_ids = 0, 1 ; + left_neighbour_ids = 2, 2, 0, 1 ; - left_neighbour_halos = 2, 2 ; + left_neighbour_halos = 2, 2, 2, 2 ; - left_neighbour_halo_send = 4, 4 ; + left_neighbour_halo_send = 2, 4, 4, 4 ; - left_neighbour_halo_recv = 8, 10 ; + left_neighbour_halo_recv = 10, 10, 8, 10 ; - right_neighbours = 1, 1, 0 ; + right_neighbours = 1, 1, 2 ; - right_neighbour_ids = 2, 2 ; + right_neighbour_ids = 2, 2, 0, 1 ; - right_neighbour_halos = 2, 2 ; + right_neighbour_halos = 2, 2, 2, 2 ; - right_neighbour_halo_send = 8, 10 ; + right_neighbour_halo_send = 8, 10, 10, 10 ; - right_neighbour_halo_recv = 4, 4 ; + right_neighbour_halo_recv = 4, 4, 2, 4 ; bottom_neighbours = 0, 1, 0 ; @@ -145,7 +105,11 @@ group: connectivity { top_neighbour_halo_recv = 6 ; - top_left_neighbours = 0, 0, 0 ; + top_left_neighbours = 1, 0, 0 ; + + top_left_neighbour_ids = 2 ; + + top_left_neighbour_send = 4 ; top_right_neighbours = 1, 0, 0 ; @@ -159,46 +123,10 @@ group: connectivity { bottom_right_neighbour_send = 9 ; - bottom_left_neighbours = 0, 0, 0 ; - - left_neighbours_periodic = 1, 1, 0 ; - - left_neighbour_ids_periodic = 2, 2 ; - - left_neighbour_halos_periodic = 2, 2 ; - - left_neighbour_halo_send_periodic = 2, 4 ; - - left_neighbour_halo_recv_periodic = 10, 10 ; - - right_neighbours_periodic = 0, 0, 2 ; - - right_neighbour_ids_periodic = 0, 1 ; - - right_neighbour_halos_periodic = 2, 2 ; - - right_neighbour_halo_send_periodic = 10, 10 ; - - right_neighbour_halo_recv_periodic = 2, 4 ; - - bottom_neighbours_periodic = 0, 0, 0 ; - - top_neighbours_periodic = 0, 0, 0 ; - - top_left_neighbours_periodic = 1, 0, 0 ; - - top_left_neighbour_ids_periodic = 2 ; - - top_left_neighbour_send_periodic = 4 ; - - top_right_neighbours_periodic = 0, 0, 0 ; - - bottom_right_neighbours_periodic = 0, 0, 0 ; - - bottom_left_neighbours_periodic = 0, 1, 0 ; + bottom_left_neighbours = 0, 1, 0 ; - bottom_left_neighbour_ids_periodic = 2 ; + bottom_left_neighbour_ids = 2 ; - bottom_left_neighbour_send_periodic = 3 ; + bottom_left_neighbour_send = 3 ; } // group connectivity } diff --git a/test/test_1_px_py/ref_partition_metadata_3.cdl b/test/test_1_px_py/ref_partition_metadata_3.cdl index 0c2ea2f..5018fd3 100644 --- a/test/test_1_px_py/ref_partition_metadata_3.cdl +++ b/test/test_1_px_py/ref_partition_metadata_3.cdl @@ -3,22 +3,14 @@ dimensions: NX = 6 ; NY = 4 ; P = 3 ; - L = 2 ; - R = 2 ; - B = 1 ; - T = 1 ; - TL = UNLIMITED ; // (0 currently) - TR = 1 ; - BR = 1 ; - BL = UNLIMITED ; // (0 currently) - L_periodic = 2 ; - R_periodic = 2 ; - B_periodic = 2 ; - T_periodic = 2 ; - TL_periodic = 3 ; - TR_periodic = 2 ; - BR_periodic = 2 ; - BL_periodic = 3 ; + L = 4 ; + R = 4 ; + B = 3 ; + T = 3 ; + TL = 3 ; + TR = 3 ; + BR = 3 ; + BL = 3 ; group: bounding_boxes { variables: @@ -71,158 +63,70 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: - left_neighbours = 0, 0, 2 ; + left_neighbours = 1, 1, 2 ; - left_neighbour_ids = 0, 1 ; + left_neighbour_ids = 2, 2, 0, 1 ; - left_neighbour_halos = 2, 2 ; + left_neighbour_halos = 2, 2, 2, 2 ; - left_neighbour_halo_send = 4, 4 ; + left_neighbour_halo_send = 2, 4, 4, 4 ; - left_neighbour_halo_recv = 8, 10 ; + left_neighbour_halo_recv = 10, 10, 8, 10 ; - right_neighbours = 1, 1, 0 ; + right_neighbours = 1, 1, 2 ; - right_neighbour_ids = 2, 2 ; + right_neighbour_ids = 2, 2, 0, 1 ; - right_neighbour_halos = 2, 2 ; + right_neighbour_halos = 2, 2, 2, 2 ; - right_neighbour_halo_send = 8, 10 ; + right_neighbour_halo_send = 8, 10, 10, 10 ; - right_neighbour_halo_recv = 4, 4 ; + right_neighbour_halo_recv = 4, 4, 2, 4 ; - bottom_neighbours = 0, 1, 0 ; + bottom_neighbours = 1, 1, 1 ; - bottom_neighbour_ids = 0 ; + bottom_neighbour_ids = 1, 0, 2 ; - bottom_neighbour_halos = 4 ; + bottom_neighbour_halos = 4, 4, 2 ; - bottom_neighbour_halo_send = 6 ; + bottom_neighbour_halo_send = 6, 6, 6 ; - bottom_neighbour_halo_recv = 0 ; + bottom_neighbour_halo_recv = 0, 0, 0 ; - top_neighbours = 1, 0, 0 ; + top_neighbours = 1, 1, 1 ; - top_neighbour_ids = 1 ; + top_neighbour_ids = 1, 0, 2 ; - top_neighbour_halos = 4 ; + top_neighbour_halos = 4, 4, 2 ; - top_neighbour_halo_send = 0 ; + top_neighbour_halo_send = 0, 0, 0 ; - top_neighbour_halo_recv = 6 ; + top_neighbour_halo_recv = 6, 6, 6 ; - top_left_neighbours = 0, 0, 0 ; + top_left_neighbours = 1, 1, 1 ; - top_right_neighbours = 1, 0, 0 ; + top_left_neighbour_ids = 2, 2, 0 ; - top_right_neighbour_ids = 2 ; + top_left_neighbour_send = 4, 2, 4 ; - top_right_neighbour_send = 10 ; + top_right_neighbours = 1, 1, 1 ; - bottom_right_neighbours = 0, 1, 0 ; + top_right_neighbour_ids = 2, 2, 0 ; - bottom_right_neighbour_ids = 2 ; + top_right_neighbour_send = 10, 8, 10 ; - bottom_right_neighbour_send = 9 ; + bottom_right_neighbours = 1, 1, 1 ; - bottom_left_neighbours = 0, 0, 0 ; + bottom_right_neighbour_ids = 2, 2, 1 ; - left_neighbours_periodic = 1, 1, 0 ; + bottom_right_neighbour_send = 11, 9, 11 ; - left_neighbour_ids_periodic = 2, 2 ; + bottom_left_neighbours = 1, 1, 1 ; - left_neighbour_halos_periodic = 2, 2 ; + bottom_left_neighbour_ids = 2, 2, 1 ; - left_neighbour_halo_send_periodic = 2, 4 ; - - left_neighbour_halo_recv_periodic = 10, 10 ; - - right_neighbours_periodic = 0, 0, 2 ; - - right_neighbour_ids_periodic = 0, 1 ; - - right_neighbour_halos_periodic = 2, 2 ; - - right_neighbour_halo_send_periodic = 10, 10 ; - - right_neighbour_halo_recv_periodic = 2, 4 ; - - bottom_neighbours_periodic = 1, 0, 1 ; - - bottom_neighbour_ids_periodic = 1, 2 ; - - bottom_neighbour_halos_periodic = 4, 2 ; - - bottom_neighbour_halo_send_periodic = 6, 6 ; - - bottom_neighbour_halo_recv_periodic = 0, 0 ; - - top_neighbours_periodic = 0, 1, 1 ; - - top_neighbour_ids_periodic = 0, 2 ; - - top_neighbour_halos_periodic = 4, 2 ; - - top_neighbour_halo_send_periodic = 0, 0 ; - - top_neighbour_halo_recv_periodic = 6, 6 ; - - top_left_neighbours_periodic = 1, 1, 1 ; - - top_left_neighbour_ids_periodic = 2, 2, 0 ; - - top_left_neighbour_send_periodic = 4, 2, 4 ; - - top_right_neighbours_periodic = 0, 1, 1 ; - - top_right_neighbour_ids_periodic = 2, 0 ; - - top_right_neighbour_send_periodic = 8, 10 ; - - bottom_right_neighbours_periodic = 1, 0, 1 ; - - bottom_right_neighbour_ids_periodic = 2, 1 ; - - bottom_right_neighbour_send_periodic = 11, 11 ; - - bottom_left_neighbours_periodic = 1, 1, 1 ; - - bottom_left_neighbour_ids_periodic = 2, 2, 1 ; - - bottom_left_neighbour_send_periodic = 5, 3, 5 ; + bottom_left_neighbour_send = 5, 3, 5 ; } // group connectivity } diff --git a/test/test_1_py/ref_partition_metadata_3.cdl b/test/test_1_py/ref_partition_metadata_3.cdl index 5140a4e..cefcee1 100644 --- a/test/test_1_py/ref_partition_metadata_3.cdl +++ b/test/test_1_py/ref_partition_metadata_3.cdl @@ -5,20 +5,12 @@ dimensions: P = 3 ; L = 2 ; R = 2 ; - B = 1 ; - T = 1 ; - TL = UNLIMITED ; // (0 currently) - TR = 1 ; - BR = 1 ; - BL = UNLIMITED ; // (0 currently) - L_periodic = UNLIMITED ; // (0 currently) - R_periodic = UNLIMITED ; // (0 currently) - B_periodic = 2 ; - T_periodic = 2 ; - TL_periodic = 1 ; - TR_periodic = 1 ; - BR_periodic = 1 ; - BL_periodic = 1 ; + B = 3 ; + T = 3 ; + TL = 1 ; + TR = 2 ; + BR = 2 ; + BL = 1 ; group: bounding_boxes { variables: @@ -71,38 +63,6 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: left_neighbours = 0, 0, 2 ; @@ -125,88 +85,48 @@ group: connectivity { right_neighbour_halo_recv = 4, 4 ; - bottom_neighbours = 0, 1, 0 ; + bottom_neighbours = 1, 1, 1 ; - bottom_neighbour_ids = 0 ; + bottom_neighbour_ids = 1, 0, 2 ; - bottom_neighbour_halos = 4 ; + bottom_neighbour_halos = 4, 4, 2 ; - bottom_neighbour_halo_send = 6 ; + bottom_neighbour_halo_send = 6, 6, 6 ; - bottom_neighbour_halo_recv = 0 ; + bottom_neighbour_halo_recv = 0, 0, 0 ; - top_neighbours = 1, 0, 0 ; + top_neighbours = 1, 1, 1 ; - top_neighbour_ids = 1 ; + top_neighbour_ids = 1, 0, 2 ; - top_neighbour_halos = 4 ; + top_neighbour_halos = 4, 4, 2 ; - top_neighbour_halo_send = 0 ; + top_neighbour_halo_send = 0, 0, 0 ; - top_neighbour_halo_recv = 6 ; + top_neighbour_halo_recv = 6, 6, 6 ; - top_left_neighbours = 0, 0, 0 ; + top_left_neighbours = 0, 0, 1 ; - top_right_neighbours = 1, 0, 0 ; + top_left_neighbour_ids = 0 ; - top_right_neighbour_ids = 2 ; + top_left_neighbour_send = 4 ; - top_right_neighbour_send = 10 ; + top_right_neighbours = 1, 1, 0 ; - bottom_right_neighbours = 0, 1, 0 ; + top_right_neighbour_ids = 2, 2 ; - bottom_right_neighbour_ids = 2 ; + top_right_neighbour_send = 10, 8 ; - bottom_right_neighbour_send = 9 ; + bottom_right_neighbours = 1, 1, 0 ; - bottom_left_neighbours = 0, 0, 0 ; + bottom_right_neighbour_ids = 2, 2 ; - left_neighbours_periodic = 0, 0, 0 ; + bottom_right_neighbour_send = 11, 9 ; - right_neighbours_periodic = 0, 0, 0 ; + bottom_left_neighbours = 0, 0, 1 ; - bottom_neighbours_periodic = 1, 0, 1 ; + bottom_left_neighbour_ids = 1 ; - bottom_neighbour_ids_periodic = 1, 2 ; - - bottom_neighbour_halos_periodic = 4, 2 ; - - bottom_neighbour_halo_send_periodic = 6, 6 ; - - bottom_neighbour_halo_recv_periodic = 0, 0 ; - - top_neighbours_periodic = 0, 1, 1 ; - - top_neighbour_ids_periodic = 0, 2 ; - - top_neighbour_halos_periodic = 4, 2 ; - - top_neighbour_halo_send_periodic = 0, 0 ; - - top_neighbour_halo_recv_periodic = 6, 6 ; - - top_left_neighbours_periodic = 0, 0, 1 ; - - top_left_neighbour_ids_periodic = 0 ; - - top_left_neighbour_send_periodic = 4 ; - - top_right_neighbours_periodic = 0, 1, 0 ; - - top_right_neighbour_ids_periodic = 2 ; - - top_right_neighbour_send_periodic = 8 ; - - bottom_right_neighbours_periodic = 1, 0, 0 ; - - bottom_right_neighbour_ids_periodic = 2 ; - - bottom_right_neighbour_send_periodic = 11 ; - - bottom_left_neighbours_periodic = 0, 0, 1 ; - - bottom_left_neighbour_ids_periodic = 1 ; - - bottom_left_neighbour_send_periodic = 5 ; + bottom_left_neighbour_send = 5 ; } // group connectivity } diff --git a/test/test_2/ref_partition_metadata_3.cdl b/test/test_2/ref_partition_metadata_3.cdl index 966d389..a4bf12c 100644 --- a/test/test_2/ref_partition_metadata_3.cdl +++ b/test/test_2/ref_partition_metadata_3.cdl @@ -11,14 +11,6 @@ dimensions: TR = UNLIMITED ; // (0 currently) BR = UNLIMITED ; // (0 currently) BL = UNLIMITED ; // (0 currently) - L_periodic = UNLIMITED ; // (0 currently) - R_periodic = UNLIMITED ; // (0 currently) - B_periodic = UNLIMITED ; // (0 currently) - T_periodic = UNLIMITED ; // (0 currently) - TL_periodic = UNLIMITED ; // (0 currently) - TR_periodic = UNLIMITED ; // (0 currently) - BR_periodic = UNLIMITED ; // (0 currently) - BL_periodic = UNLIMITED ; // (0 currently) group: bounding_boxes { variables: @@ -71,38 +63,6 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: left_neighbours = 0, 1, 1 ; @@ -136,21 +96,5 @@ group: connectivity { bottom_right_neighbours = 0, 0, 0 ; bottom_left_neighbours = 0, 0, 0 ; - - left_neighbours_periodic = 0, 0, 0 ; - - right_neighbours_periodic = 0, 0, 0 ; - - bottom_neighbours_periodic = 0, 0, 0 ; - - top_neighbours_periodic = 0, 0, 0 ; - - top_left_neighbours_periodic = 0, 0, 0 ; - - top_right_neighbours_periodic = 0, 0, 0 ; - - bottom_right_neighbours_periodic = 0, 0, 0 ; - - bottom_left_neighbours_periodic = 0, 0, 0 ; } // group connectivity } diff --git a/test/test_3/ref_partition_metadata_4.cdl b/test/test_3/ref_partition_metadata_4.cdl index 7ea93cf..f128ec4 100644 --- a/test/test_3/ref_partition_metadata_4.cdl +++ b/test/test_3/ref_partition_metadata_4.cdl @@ -11,14 +11,6 @@ dimensions: TR = 1 ; BR = 1 ; BL = 1 ; - L_periodic = UNLIMITED ; // (0 currently) - R_periodic = UNLIMITED ; // (0 currently) - B_periodic = UNLIMITED ; // (0 currently) - T_periodic = UNLIMITED ; // (0 currently) - TL_periodic = UNLIMITED ; // (0 currently) - TR_periodic = UNLIMITED ; // (0 currently) - BR_periodic = UNLIMITED ; // (0 currently) - BL_periodic = UNLIMITED ; // (0 currently) group: bounding_boxes { variables: @@ -71,38 +63,6 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: left_neighbours = 0, 0, 1, 1 ; @@ -168,21 +128,5 @@ group: connectivity { bottom_left_neighbour_ids = 0 ; bottom_left_neighbour_send = 5 ; - - left_neighbours_periodic = 0, 0, 0, 0 ; - - right_neighbours_periodic = 0, 0, 0, 0 ; - - bottom_neighbours_periodic = 0, 0, 0, 0 ; - - top_neighbours_periodic = 0, 0, 0, 0 ; - - top_left_neighbours_periodic = 0, 0, 0, 0 ; - - top_right_neighbours_periodic = 0, 0, 0, 0 ; - - bottom_right_neighbours_periodic = 0, 0, 0, 0 ; - - bottom_left_neighbours_periodic = 0, 0, 0, 0 ; } // group connectivity } diff --git a/test/test_4/ref_partition_metadata_4.cdl b/test/test_4/ref_partition_metadata_4.cdl index cad47ba..4236db2 100644 --- a/test/test_4/ref_partition_metadata_4.cdl +++ b/test/test_4/ref_partition_metadata_4.cdl @@ -11,14 +11,6 @@ dimensions: TR = 1 ; BR = 1 ; BL = 1 ; - L_periodic = UNLIMITED ; // (0 currently) - R_periodic = UNLIMITED ; // (0 currently) - B_periodic = UNLIMITED ; // (0 currently) - T_periodic = UNLIMITED ; // (0 currently) - TL_periodic = UNLIMITED ; // (0 currently) - TR_periodic = UNLIMITED ; // (0 currently) - BR_periodic = UNLIMITED ; // (0 currently) - BL_periodic = UNLIMITED ; // (0 currently) group: bounding_boxes { variables: @@ -71,38 +63,6 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: left_neighbours = 0, 0, 2, 1 ; @@ -168,21 +128,5 @@ group: connectivity { bottom_left_neighbour_ids = 1 ; bottom_left_neighbour_send = 5 ; - - left_neighbours_periodic = 0, 0, 0, 0 ; - - right_neighbours_periodic = 0, 0, 0, 0 ; - - bottom_neighbours_periodic = 0, 0, 0, 0 ; - - top_neighbours_periodic = 0, 0, 0, 0 ; - - top_left_neighbours_periodic = 0, 0, 0, 0 ; - - top_right_neighbours_periodic = 0, 0, 0, 0 ; - - bottom_right_neighbours_periodic = 0, 0, 0, 0 ; - - bottom_left_neighbours_periodic = 0, 0, 0, 0 ; } // group connectivity } diff --git a/test/test_4_px/ref_partition_metadata_4.cdl b/test/test_4_px/ref_partition_metadata_4.cdl index 5fb00cc..33530be 100644 --- a/test/test_4_px/ref_partition_metadata_4.cdl +++ b/test/test_4_px/ref_partition_metadata_4.cdl @@ -3,22 +3,14 @@ dimensions: NX = 7 ; NY = 7 ; P = 4 ; - L = 3 ; - R = 3 ; + L = 6 ; + R = 6 ; B = 2 ; T = 2 ; - TL = 1 ; - TR = 1 ; - BR = 1 ; - BL = 1 ; - L_periodic = 3 ; - R_periodic = 3 ; - B_periodic = UNLIMITED ; // (0 currently) - T_periodic = UNLIMITED ; // (0 currently) - TL_periodic = 1 ; - TR_periodic = 1 ; - BR_periodic = 1 ; - BL_periodic = 1 ; + TL = 2 ; + TR = 2 ; + BR = 2 ; + BL = 2 ; group: bounding_boxes { variables: @@ -71,59 +63,27 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: - left_neighbours = 0, 0, 2, 1 ; + left_neighbours = 1, 2, 2, 1 ; - left_neighbour_ids = 0, 1, 1 ; + left_neighbour_ids = 2, 2, 3, 0, 1, 1 ; - left_neighbour_halos = 2, 3, 2 ; + left_neighbour_halos = 2, 3, 2, 2, 3, 2 ; - left_neighbour_halo_send = 3, 3, 6 ; + left_neighbour_halo_send = 4, 6, 4, 3, 3, 6 ; - left_neighbour_halo_recv = 13, 15, 10 ; + left_neighbour_halo_recv = 8, 11, 14, 13, 15, 10 ; - right_neighbours = 1, 2, 0, 0 ; + right_neighbours = 1, 2, 2, 1 ; - right_neighbour_ids = 2, 2, 3 ; + right_neighbour_ids = 2, 2, 3, 0, 1, 1 ; - right_neighbour_halos = 2, 3, 2 ; + right_neighbour_halos = 2, 3, 2, 2, 3, 2 ; - right_neighbour_halo_send = 13, 15, 10 ; + right_neighbour_halo_send = 13, 15, 10, 8, 11, 14 ; - right_neighbour_halo_recv = 3, 3, 6 ; + right_neighbour_halo_recv = 3, 3, 6, 4, 6, 4 ; bottom_neighbours = 0, 1, 0, 1 ; @@ -145,76 +105,28 @@ group: connectivity { top_neighbour_halo_recv = 5, 9 ; - top_left_neighbours = 0, 0, 1, 0 ; + top_left_neighbours = 1, 0, 1, 0 ; - top_left_neighbour_ids = 1 ; + top_left_neighbour_ids = 2, 1 ; - top_left_neighbour_send = 6 ; + top_left_neighbour_send = 6, 6 ; - top_right_neighbours = 1, 0, 0, 0 ; + top_right_neighbours = 1, 0, 1, 0 ; - top_right_neighbour_ids = 2 ; + top_right_neighbour_ids = 2, 1 ; - top_right_neighbour_send = 15 ; + top_right_neighbour_send = 15, 14 ; - bottom_right_neighbours = 0, 1, 0, 0 ; + bottom_right_neighbours = 0, 1, 0, 1 ; - bottom_right_neighbour_ids = 2 ; + bottom_right_neighbour_ids = 2, 1 ; - bottom_right_neighbour_send = 14 ; + bottom_right_neighbour_send = 14, 13 ; - bottom_left_neighbours = 0, 0, 0, 1 ; + bottom_left_neighbours = 0, 1, 0, 1 ; - bottom_left_neighbour_ids = 1 ; + bottom_left_neighbour_ids = 2, 1 ; - bottom_left_neighbour_send = 5 ; - - left_neighbours_periodic = 1, 2, 0, 0 ; - - left_neighbour_ids_periodic = 2, 2, 3 ; - - left_neighbour_halos_periodic = 2, 3, 2 ; - - left_neighbour_halo_send_periodic = 4, 6, 4 ; - - left_neighbour_halo_recv_periodic = 8, 11, 14 ; - - right_neighbours_periodic = 0, 0, 2, 1 ; - - right_neighbour_ids_periodic = 0, 1, 1 ; - - right_neighbour_halos_periodic = 2, 3, 2 ; - - right_neighbour_halo_send_periodic = 8, 11, 14 ; - - right_neighbour_halo_recv_periodic = 4, 6, 4 ; - - bottom_neighbours_periodic = 0, 0, 0, 0 ; - - top_neighbours_periodic = 0, 0, 0, 0 ; - - top_left_neighbours_periodic = 1, 0, 0, 0 ; - - top_left_neighbour_ids_periodic = 2 ; - - top_left_neighbour_send_periodic = 6 ; - - top_right_neighbours_periodic = 0, 0, 1, 0 ; - - top_right_neighbour_ids_periodic = 1 ; - - top_right_neighbour_send_periodic = 14 ; - - bottom_right_neighbours_periodic = 0, 0, 0, 1 ; - - bottom_right_neighbour_ids_periodic = 1 ; - - bottom_right_neighbour_send_periodic = 13 ; - - bottom_left_neighbours_periodic = 0, 1, 0, 0 ; - - bottom_left_neighbour_ids_periodic = 2 ; - - bottom_left_neighbour_send_periodic = 5 ; + bottom_left_neighbour_send = 5, 5 ; } // group connectivity } diff --git a/test/test_4_px_py/ref_partition_metadata_4.cdl b/test/test_4_px_py/ref_partition_metadata_4.cdl index 0fae92e..037a24f 100644 --- a/test/test_4_px_py/ref_partition_metadata_4.cdl +++ b/test/test_4_px_py/ref_partition_metadata_4.cdl @@ -3,22 +3,14 @@ dimensions: NX = 7 ; NY = 7 ; P = 4 ; - L = 3 ; - R = 3 ; - B = 2 ; - T = 2 ; - TL = 1 ; - TR = 1 ; - BR = 1 ; - BL = 1 ; - L_periodic = 3 ; - R_periodic = 3 ; - B_periodic = 2 ; - T_periodic = 2 ; - TL_periodic = 3 ; - TR_periodic = 3 ; - BR_periodic = 3 ; - BL_periodic = 3 ; + L = 6 ; + R = 6 ; + B = 4 ; + T = 4 ; + TL = 4 ; + TR = 4 ; + BR = 4 ; + BL = 4 ; group: bounding_boxes { variables: @@ -71,166 +63,70 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: - left_neighbours = 0, 0, 2, 1 ; + left_neighbours = 1, 2, 2, 1 ; - left_neighbour_ids = 0, 1, 1 ; + left_neighbour_ids = 2, 2, 3, 0, 1, 1 ; - left_neighbour_halos = 2, 3, 2 ; + left_neighbour_halos = 2, 3, 2, 2, 3, 2 ; - left_neighbour_halo_send = 3, 3, 6 ; + left_neighbour_halo_send = 4, 6, 4, 3, 3, 6 ; - left_neighbour_halo_recv = 13, 15, 10 ; + left_neighbour_halo_recv = 8, 11, 14, 13, 15, 10 ; - right_neighbours = 1, 2, 0, 0 ; + right_neighbours = 1, 2, 2, 1 ; - right_neighbour_ids = 2, 2, 3 ; + right_neighbour_ids = 2, 2, 3, 0, 1, 1 ; - right_neighbour_halos = 2, 3, 2 ; + right_neighbour_halos = 2, 3, 2, 2, 3, 2 ; - right_neighbour_halo_send = 13, 15, 10 ; + right_neighbour_halo_send = 13, 15, 10, 8, 11, 14 ; - right_neighbour_halo_recv = 3, 3, 6 ; + right_neighbour_halo_recv = 3, 3, 6, 4, 6, 4 ; - bottom_neighbours = 0, 1, 0, 1 ; + bottom_neighbours = 1, 1, 1, 1 ; - bottom_neighbour_ids = 0, 2 ; + bottom_neighbour_ids = 1, 0, 3, 2 ; - bottom_neighbour_halos = 3, 4 ; + bottom_neighbour_halos = 3, 3, 4, 4 ; - bottom_neighbour_halo_send = 5, 9 ; + bottom_neighbour_halo_send = 8, 5, 6, 9 ; - bottom_neighbour_halo_recv = 0, 0 ; + bottom_neighbour_halo_recv = 0, 0, 0, 0 ; - top_neighbours = 1, 0, 1, 0 ; + top_neighbours = 1, 1, 1, 1 ; - top_neighbour_ids = 1, 3 ; + top_neighbour_ids = 1, 0, 3, 2 ; - top_neighbour_halos = 3, 4 ; + top_neighbour_halos = 3, 3, 4, 4 ; - top_neighbour_halo_send = 0, 0 ; + top_neighbour_halo_send = 0, 0, 0, 0 ; - top_neighbour_halo_recv = 5, 9 ; + top_neighbour_halo_recv = 5, 8, 9, 6 ; - top_left_neighbours = 0, 0, 1, 0 ; + top_left_neighbours = 1, 1, 1, 1 ; - top_left_neighbour_ids = 1 ; + top_left_neighbour_ids = 2, 2, 1, 0 ; - top_left_neighbour_send = 6 ; + top_left_neighbour_send = 6, 4, 6, 3 ; - top_right_neighbours = 1, 0, 0, 0 ; + top_right_neighbours = 1, 1, 1, 1 ; - top_right_neighbour_ids = 2 ; + top_right_neighbour_ids = 2, 2, 1, 0 ; - top_right_neighbour_send = 15 ; + top_right_neighbour_send = 15, 13, 14, 8 ; - bottom_right_neighbours = 0, 1, 0, 0 ; + bottom_right_neighbours = 1, 1, 1, 1 ; - bottom_right_neighbour_ids = 2 ; + bottom_right_neighbour_ids = 3, 2, 1, 1 ; - bottom_right_neighbour_send = 14 ; + bottom_right_neighbour_send = 11, 14, 15, 13 ; - bottom_left_neighbours = 0, 0, 0, 1 ; + bottom_left_neighbours = 1, 1, 1, 1 ; - bottom_left_neighbour_ids = 1 ; + bottom_left_neighbour_ids = 3, 2, 1, 1 ; - bottom_left_neighbour_send = 5 ; - - left_neighbours_periodic = 1, 2, 0, 0 ; - - left_neighbour_ids_periodic = 2, 2, 3 ; - - left_neighbour_halos_periodic = 2, 3, 2 ; - - left_neighbour_halo_send_periodic = 4, 6, 4 ; - - left_neighbour_halo_recv_periodic = 8, 11, 14 ; - - right_neighbours_periodic = 0, 0, 2, 1 ; - - right_neighbour_ids_periodic = 0, 1, 1 ; - - right_neighbour_halos_periodic = 2, 3, 2 ; - - right_neighbour_halo_send_periodic = 8, 11, 14 ; - - right_neighbour_halo_recv_periodic = 4, 6, 4 ; - - bottom_neighbours_periodic = 1, 0, 1, 0 ; - - bottom_neighbour_ids_periodic = 1, 3 ; - - bottom_neighbour_halos_periodic = 3, 4 ; - - bottom_neighbour_halo_send_periodic = 8, 6 ; - - bottom_neighbour_halo_recv_periodic = 0, 0 ; - - top_neighbours_periodic = 0, 1, 0, 1 ; - - top_neighbour_ids_periodic = 0, 2 ; - - top_neighbour_halos_periodic = 3, 4 ; - - top_neighbour_halo_send_periodic = 0, 0 ; - - top_neighbour_halo_recv_periodic = 8, 6 ; - - top_left_neighbours_periodic = 1, 1, 0, 1 ; - - top_left_neighbour_ids_periodic = 2, 2, 0 ; - - top_left_neighbour_send_periodic = 6, 4, 3 ; - - top_right_neighbours_periodic = 0, 1, 1, 1 ; - - top_right_neighbour_ids_periodic = 2, 1, 0 ; - - top_right_neighbour_send_periodic = 13, 14, 8 ; - - bottom_right_neighbours_periodic = 1, 0, 1, 1 ; - - bottom_right_neighbour_ids_periodic = 3, 1, 1 ; - - bottom_right_neighbour_send_periodic = 11, 15, 13 ; - - bottom_left_neighbours_periodic = 1, 1, 1, 0 ; - - bottom_left_neighbour_ids_periodic = 3, 2, 1 ; - - bottom_left_neighbour_send_periodic = 5, 5, 7 ; + bottom_left_neighbour_send = 5, 5, 7, 5 ; } // group connectivity } diff --git a/test/test_4_py/ref_partition_metadata_4.cdl b/test/test_4_py/ref_partition_metadata_4.cdl index da226a3..fe52a73 100644 --- a/test/test_4_py/ref_partition_metadata_4.cdl +++ b/test/test_4_py/ref_partition_metadata_4.cdl @@ -5,20 +5,12 @@ dimensions: P = 4 ; L = 3 ; R = 3 ; - B = 2 ; - T = 2 ; - TL = 1 ; - TR = 1 ; - BR = 1 ; - BL = 1 ; - L_periodic = UNLIMITED ; // (0 currently) - R_periodic = UNLIMITED ; // (0 currently) - B_periodic = 2 ; - T_periodic = 2 ; - TL_periodic = 1 ; - TR_periodic = 1 ; - BR_periodic = 1 ; - BL_periodic = 1 ; + B = 4 ; + T = 4 ; + TL = 2 ; + TR = 2 ; + BR = 2 ; + BL = 2 ; group: bounding_boxes { variables: @@ -71,38 +63,6 @@ group: connectivity { int bottom_left_neighbours(P) ; int bottom_left_neighbour_ids(BL) ; int bottom_left_neighbour_send(BL) ; - int left_neighbours_periodic(P) ; - int left_neighbour_ids_periodic(L_periodic) ; - int left_neighbour_halos_periodic(L_periodic) ; - int left_neighbour_halo_send_periodic(L_periodic) ; - int left_neighbour_halo_recv_periodic(L_periodic) ; - int right_neighbours_periodic(P) ; - int right_neighbour_ids_periodic(R_periodic) ; - int right_neighbour_halos_periodic(R_periodic) ; - int right_neighbour_halo_send_periodic(R_periodic) ; - int right_neighbour_halo_recv_periodic(R_periodic) ; - int bottom_neighbours_periodic(P) ; - int bottom_neighbour_ids_periodic(B_periodic) ; - int bottom_neighbour_halos_periodic(B_periodic) ; - int bottom_neighbour_halo_send_periodic(B_periodic) ; - int bottom_neighbour_halo_recv_periodic(B_periodic) ; - int top_neighbours_periodic(P) ; - int top_neighbour_ids_periodic(T_periodic) ; - int top_neighbour_halos_periodic(T_periodic) ; - int top_neighbour_halo_send_periodic(T_periodic) ; - int top_neighbour_halo_recv_periodic(T_periodic) ; - int top_left_neighbours_periodic(P) ; - int top_left_neighbour_ids_periodic(TL_periodic) ; - int top_left_neighbour_send_periodic(TL_periodic) ; - int top_right_neighbours_periodic(P) ; - int top_right_neighbour_ids_periodic(TR_periodic) ; - int top_right_neighbour_send_periodic(TR_periodic) ; - int bottom_right_neighbours_periodic(P) ; - int bottom_right_neighbour_ids_periodic(BR_periodic) ; - int bottom_right_neighbour_send_periodic(BR_periodic) ; - int bottom_left_neighbours_periodic(P) ; - int bottom_left_neighbour_ids_periodic(BL_periodic) ; - int bottom_left_neighbour_send_periodic(BL_periodic) ; data: left_neighbours = 0, 0, 2, 1 ; @@ -125,96 +85,48 @@ group: connectivity { right_neighbour_halo_recv = 3, 3, 6 ; - bottom_neighbours = 0, 1, 0, 1 ; + bottom_neighbours = 1, 1, 1, 1 ; - bottom_neighbour_ids = 0, 2 ; + bottom_neighbour_ids = 1, 0, 3, 2 ; - bottom_neighbour_halos = 3, 4 ; + bottom_neighbour_halos = 3, 3, 4, 4 ; - bottom_neighbour_halo_send = 5, 9 ; + bottom_neighbour_halo_send = 8, 5, 6, 9 ; - bottom_neighbour_halo_recv = 0, 0 ; + bottom_neighbour_halo_recv = 0, 0, 0, 0 ; - top_neighbours = 1, 0, 1, 0 ; + top_neighbours = 1, 1, 1, 1 ; - top_neighbour_ids = 1, 3 ; + top_neighbour_ids = 1, 0, 3, 2 ; - top_neighbour_halos = 3, 4 ; + top_neighbour_halos = 3, 3, 4, 4 ; - top_neighbour_halo_send = 0, 0 ; + top_neighbour_halo_send = 0, 0, 0, 0 ; - top_neighbour_halo_recv = 5, 9 ; + top_neighbour_halo_recv = 5, 8, 9, 6 ; - top_left_neighbours = 0, 0, 1, 0 ; + top_left_neighbours = 0, 0, 1, 1 ; - top_left_neighbour_ids = 1 ; + top_left_neighbour_ids = 1, 0 ; - top_left_neighbour_send = 6 ; + top_left_neighbour_send = 6, 3 ; - top_right_neighbours = 1, 0, 0, 0 ; + top_right_neighbours = 1, 1, 0, 0 ; - top_right_neighbour_ids = 2 ; + top_right_neighbour_ids = 2, 2 ; - top_right_neighbour_send = 15 ; + top_right_neighbour_send = 15, 13 ; - bottom_right_neighbours = 0, 1, 0, 0 ; + bottom_right_neighbours = 1, 1, 0, 0 ; - bottom_right_neighbour_ids = 2 ; + bottom_right_neighbour_ids = 3, 2 ; - bottom_right_neighbour_send = 14 ; + bottom_right_neighbour_send = 11, 14 ; - bottom_left_neighbours = 0, 0, 0, 1 ; + bottom_left_neighbours = 0, 0, 1, 1 ; - bottom_left_neighbour_ids = 1 ; + bottom_left_neighbour_ids = 1, 1 ; - bottom_left_neighbour_send = 5 ; - - left_neighbours_periodic = 0, 0, 0, 0 ; - - right_neighbours_periodic = 0, 0, 0, 0 ; - - bottom_neighbours_periodic = 1, 0, 1, 0 ; - - bottom_neighbour_ids_periodic = 1, 3 ; - - bottom_neighbour_halos_periodic = 3, 4 ; - - bottom_neighbour_halo_send_periodic = 8, 6 ; - - bottom_neighbour_halo_recv_periodic = 0, 0 ; - - top_neighbours_periodic = 0, 1, 0, 1 ; - - top_neighbour_ids_periodic = 0, 2 ; - - top_neighbour_halos_periodic = 3, 4 ; - - top_neighbour_halo_send_periodic = 0, 0 ; - - top_neighbour_halo_recv_periodic = 8, 6 ; - - top_left_neighbours_periodic = 0, 0, 0, 1 ; - - top_left_neighbour_ids_periodic = 0 ; - - top_left_neighbour_send_periodic = 3 ; - - top_right_neighbours_periodic = 0, 1, 0, 0 ; - - top_right_neighbour_ids_periodic = 2 ; - - top_right_neighbour_send_periodic = 13 ; - - bottom_right_neighbours_periodic = 1, 0, 0, 0 ; - - bottom_right_neighbour_ids_periodic = 3 ; - - bottom_right_neighbour_send_periodic = 11 ; - - bottom_left_neighbours_periodic = 0, 0, 1, 0 ; - - bottom_left_neighbour_ids_periodic = 1 ; - - bottom_left_neighbour_send_periodic = 7 ; + bottom_left_neighbour_send = 7, 5 ; } // group connectivity } From 0f0d7fb0947978560aa9c2ad67784aa12a8827c1 Mon Sep 17 00:00:00 2001 From: "M. A. Kowalski" Date: Tue, 21 Jul 2026 17:36:02 +0100 Subject: [PATCH 3/3] doc: add specification for the partition NetCDF files The text is intended to specify what each entry means and hence establish the interface through which we communicate partition information. We use pseudo-CDL and comments to document the entries. It is judged to be more readable (especially when comparing against CDL dump of the partition outputs) --- interface.md | 212 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 interface.md diff --git a/interface.md b/interface.md new file mode 100644 index 0000000..96c8298 --- /dev/null +++ b/interface.md @@ -0,0 +1,212 @@ +# Description of the NetCDF-4 partition data format + +In typical use `domain_decomp` produces two NetCDF-4 files that describe the +partition of the grid across the processors. These are: + +- 'mask' file +- 'metadata' file + +The purpose of this document is to give a brief specification of the format of +these files and to identify invariants that can be used for validation of the data. + +## Abstract grid + +The NextSimDG uses a 2D quadrilateral grid to represent the domain. +As a result, we can represent the grid as a 2D array of entries, where +each entry corresponds to a cell. + +Every cell has exactly 4 neighbours (where the outer edge of a domain may be considered +a neighbour). + +## Mask file + +The mask file is currently not used to pass information to the NextSimDG and is +intended for debugging purposes. + +The contents are as described below in the pseudo-CDL: +``` +netcdf partition_mask { +dimensions: + y = ... ; // Grid size in the y direction + x = ... ; // Grid size in the x direction +variables: + // Mask that assigns each grid point to a specific rank + // + // For: value = pid[i,j] + // + // - value in [0; ...) -> value is the rank of the processor in [0; num_processes-1) + // - value in {-1} -> point is inactive ("Land") + // - otherwise value is invalid + // + int pid(y, x) ; + + // Total number of ranks + :num_processes = 3; +} +``` + +## Metadata file + +### Assumptions about partition + +The metadata NetCDF-4 file is the intended mechanism to communicate the partition +information from `domain_decomp` to the NextSimDG. + +The partition will always have the following properties: + +- Each rank is assigned a rectangular patch of the domain. + (i.e., can be represented by integer coordinates of the lower left and upper right + points) +- Domain edges may be periodic. If it is the case, neighbour information across + the periodic edge is stored. For non-periodic edges, it is empty. +- Connectivity information is via: + - 4 edges named (`left`, `right`, `top`, `bottom`) + - 4 corners named (`top_left`, `top_right`, `bottom_left`, `bottom_right`) + + +### Dimensions + +The NetCDF-4 file contains the following dimensions. +Note that in the description we reference necessary *variables* defined in the +file which will be described later: +``` +dimensions: + NX = ... ; // Number of grid points in the x direction + NY = ... ; // Number of grid points in the y direction + P = ... ; // Number of partitions (ranks) + + // Sum of the number of 'left' neighbours across all partitions + // Neighbour information is appended across all ranks into a single list + // `L` is the size of this list. + // + // Satisfies: L == sum(left_neighbours) + // + // The index for each rank is computed by `cumsum`, e.g. first entry + // for rank 3, in L-dimensioned variable `var(L)` by: + // var[cumsum(left_neighbours)[3]] + // + L = [sum(left_neighbours)] ; + + // Same as `L` but for right edge + R = [sum(right_neighbours)] ; + + // Same as `L` but for bottom edge + B = [sum(bottom_neighbours)] ; + + // Same as `L` but for top edge + T = [sum(top_neighbours)] ; + + // Same as `L` but for the top left corner + // Since each rank can have at most 1 corner, the following invariant + // holds: TL <= P + TL = ...; + + // Other corners are similar to TL + TR = ...; // Top right corner + BR = ...; // Bottom right corner + BL = ...; // Bottom left corner +``` + +Note that NetCDF does not allow to define ordinary dimensions with size 0. +We do need to transfer empty lists in case, e.g., no partition has a left neighbour +(L == 0). In this case, the dimension will be defined as `UNLIMITED`, but will +never hold any entries. + +We use the `UNLIMITED` dimension to represent dimensions with size 0 and empty lists. + + +### 'bounding_boxes' group + +This group contains information about the patch assigned to each rank. + +All entries for all variables in this group must be non-negative integers. + +Each patch is stored by integer coordinates `(i_x, i_y)` in the grid and the +extent of the patch in the positive direction `(extent_x, extent_y)`. + +``` +group: bounding_boxes { + variables: + + // 'i_x' for each rank + int domain_x(P) ; + + // 'extent_x' for each rank + int domain_extent_x(P) ; + + // 'i_y' for each rank + int domain_y(P) ; + + // 'extent_y' for each rank + int domain_extent_y(P) ; +} +``` + +The following invariants should hold: + + - `NX * NY == sum(domain_extent_x * domain_extent_y)` (area consistency) + - `domain_x + domain_extent_x <= NX` (patch fits in the domain) + - `domain_y + domain_extent_y <= NY` (patch fits in the domain) + + +### 'connectivity' group + +This group contains information about the topology of the partition. + +All entries for all variables in this group must be non-negative integers. + +For conciseness, we will only describe the variables for the 'left' edge. +The other edges {'right', 'top', 'bottom' } and corners +{'top_left', 'top_right', 'bottom_left', 'bottom_right'} follow the same format: + +The concept of a `perimeter buffer` is important. In our communication model, +each rank establishes two buffers, to be populated with the cell data 'send' and +'recv' (receive). Send is for *other ranks* to read from, recv is for *this rank* +to write to when fetching the data. + +The buffers are addressed from the *lower left* in the anti-clockwise direction +(i.e., 'bottom' -> 'right' -> 'top' -> 'left'). + +When establishing the `recv` buffer, the corners are ignored. + + +``` +group: connectivity { + variables: + + // Number of left neighbours for each rank + // i.e., number of ranks reachable through the left edge + int left_neighbours(P) ; + + // List of left neighbour ids + // Appended across all ranks + int left_neighbour_ids(L) ; + + // For each left neighbour, number of cells through which the neighbour + // is connected + // Must satisfy: + // sum(left_neighbour_halos) == domain_extent_y[rank] + // + int left_neighbour_halos(L) ; + + // For each left neighbour, coordinate of the first point + // in the target `send` buffer + // + int left_neighbour_halo_send(L) ; + + + // For each left neighbour, coordinate of the first point + // in the 'own' `recv` buffer + // + // This variable is **not** defined for the corners! + // + int left_neighbour_halo_recv(L) ; + + + ... +} +``` + +Of course connectivity is bi-directional. If `n` is connected to `m` through +the left edge, then `m` is connected to `n` through the right edge. +