From 6689013845a037f281eaa0303b33855632a163d8 Mon Sep 17 00:00:00 2001 From: Istvan Reguly Date: Fri, 4 Sep 2026 12:02:36 +0200 Subject: [PATCH 1/2] type conversions moved outside of HDF5 to improve performance --- ops/c/include/ops_hdf5_common.h | 12 ++ ops/c/src/externlib/ops_hdf5.cpp | 30 +++-- ops/c/src/externlib/ops_hdf5_common.cpp | 159 ++++++++++++++++++++---- ops/c/src/mpi/ops_mpi_hdf5.cpp | 61 ++++++--- 4 files changed, 211 insertions(+), 51 deletions(-) diff --git a/ops/c/include/ops_hdf5_common.h b/ops/c/include/ops_hdf5_common.h index 1d38fa30d4..a4e15da9fd 100644 --- a/ops/c/include/ops_hdf5_common.h +++ b/ops/c/include/ops_hdf5_common.h @@ -49,6 +49,18 @@ hid_t h5_type(const char *type); +hid_t h5_storage_type(const char *data_type, REAL_PRECISION real_precision); + +/* H5Dwrite after converting buf to the dataset type when it differs from + * mem_type. Needed so parallel HDF5 can use collective MPI-IO. */ +herr_t H5Dwrite_matching_types(hid_t dataset_id, hid_t mem_type, hid_t mem_space, + hid_t file_space, hid_t xfer_plist, + const void *buf, size_t nelem); + +herr_t H5Dread_matching_types(hid_t dataset_id, hid_t mem_type, hid_t mem_space, + hid_t file_space, hid_t xfer_plist, void *buf, + size_t nelem); + void split_h5_name(const char *data_name, std::vector &h5_name_list); diff --git a/ops/c/src/externlib/ops_hdf5.cpp b/ops/c/src/externlib/ops_hdf5.cpp index 79215f1100..9990bf4622 100755 --- a/ops/c/src/externlib/ops_hdf5.cpp +++ b/ops/c/src/externlib/ops_hdf5.cpp @@ -608,7 +608,8 @@ void ops_fetch_dat_hdf5_file(ops_dat dat, char const *file_name) { hid_t dset_id = H5Dopen(group_id, dat->name, H5P_DEFAULT); hid_t dataspace = H5Dget_space(dset_id); hid_t datatype = h5_type(dat->type); - H5Dwrite(dset_id, datatype, H5S_ALL, dataspace, H5P_DEFAULT, data); + H5Dwrite_matching_types(dset_id, datatype, H5S_ALL, dataspace, H5P_DEFAULT, + data, (size_t)t_size * (size_t)dat->dim); } free(data); H5Gclose(group_id); @@ -1226,7 +1227,8 @@ void ops_get_const_hdf5(char const *name, int dim, char const *type, size_t bytesize = H5Tget_size(datatype); char *data = (char *)xmalloc(bytesize * const_dim); - H5Dread(dset_id, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + H5Dread_matching_types(dset_id, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data, + (size_t)const_dim); memcpy((void *)const_data, (void *)data, bytesize * const_dim); ops_free(typ); @@ -1307,10 +1309,10 @@ void ops_write_const_hdf5(char const *name, int dim, char const *type, OPS_instance::getOPSInstance()->ostream() << " overwriting" << "\n"; dataspace = H5Dget_space(dset_id); - hid_t datatype = h5_type(typ); + hid_t datatype = h5_type(type); // write to the existing dataset with default properties - H5Dwrite(dset_id, datatype, H5S_ALL, dataspace, H5P_DEFAULT, - const_data); + H5Dwrite_matching_types(dset_id, datatype, H5S_ALL, dataspace, H5P_DEFAULT, + const_data, (size_t)dim); ops_free(typ); H5Dclose(dset_id); H5Fclose(file_id); @@ -1329,8 +1331,8 @@ void ops_write_const_hdf5(char const *name, int dim, char const *type, dset_id = H5Dcreate(file_id, name, datatype, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); // write data - H5Dwrite(dset_id, datatype, H5S_ALL, dataspace, H5P_DEFAULT, - const_data); + H5Dwrite_matching_types(dset_id, datatype, H5S_ALL, dataspace, H5P_DEFAULT, + const_data, (size_t)dim); H5Dclose(dset_id); H5Sclose(dataspace); @@ -1488,8 +1490,11 @@ void write_buf_hdf5(char const *file_name, const char *data_name, H5_dataset_space(file_id, dims, size_f, h5_name_list, dat->type, groupid_list, dataset_id, file_space); - H5Dwrite(dataset_id, h5_type(dat->type), H5S_ALL, file_space, H5P_DEFAULT, - buf); + size_t nelem = 1; + for (int d = 0; d < dims; d++) + nelem *= (size_t)size[d]; + H5Dwrite_matching_types(dataset_id, h5_type(dat->type), H5S_ALL, file_space, + H5P_DEFAULT, buf, nelem); H5Sclose(file_space); H5Dclose(dataset_id); for (int grp = groupid_list.size() - 1; grp >= 0; grp--) { @@ -1517,8 +1522,11 @@ void write_buf_hdf5(char const *file_name, const char *data_name, H5_dataset_space(file_id, dims, size_f, h5_name_list, dat->type, float_precision, groupid_list, dataset_id, file_space); - H5Dwrite(dataset_id, h5_type(dat->type), H5S_ALL, file_space, H5P_DEFAULT, - buf); + size_t nelem = 1; + for (int d = 0; d < dims; d++) + nelem *= (size_t)size[d]; + H5Dwrite_matching_types(dataset_id, h5_type(dat->type), H5S_ALL, file_space, + H5P_DEFAULT, buf, nelem); H5Sclose(file_space); H5Dclose(dataset_id); for (int grp = groupid_list.size() - 1; grp >= 0; grp--) { diff --git a/ops/c/src/externlib/ops_hdf5_common.cpp b/ops/c/src/externlib/ops_hdf5_common.cpp index db5edf5b2b..9ba0ac390e 100644 --- a/ops/c/src/externlib/ops_hdf5_common.cpp +++ b/ops/c/src/externlib/ops_hdf5_common.cpp @@ -46,6 +46,8 @@ // hdf5 header #include "ops_hdf5_common.h" #include +#include +#include int half_type_init = 0; hid_t H5T_IEEE_FP16; @@ -161,6 +163,136 @@ hid_t h5_type(const char *type) { return h5t; } +hid_t h5_storage_type(const char *data_type, REAL_PRECISION real_precision) { + hid_t type = h5_type(data_type); + if (real_precision == REAL_PRECISION::Undefined) + return type; + + if (type == H5T_NATIVE_DOUBLE) { + if (real_precision == REAL_PRECISION::Single) + return H5T_NATIVE_FLOAT; + if (real_precision == REAL_PRECISION::Half) + return H5T_IEEE_FP16; + return H5T_NATIVE_DOUBLE; + } + if (type == H5T_NATIVE_FLOAT) { + if (real_precision == REAL_PRECISION::Half) + return H5T_IEEE_FP16; + return H5T_NATIVE_FLOAT; + } + return type; +} + +/* Convert buf to the dataset's type before H5Dwrite. Parallel HDF5 disables + * collective MPI-IO when H5Dwrite itself has to convert + * (H5D_MPIO_DATATYPE_CONVERSION). */ +herr_t H5Dwrite_matching_types(hid_t dataset_id, hid_t mem_type, hid_t mem_space, + hid_t file_space, hid_t xfer_plist, + const void *buf, size_t nelem) { + hid_t dset_type = H5Dget_type(dataset_id); + if (dset_type < 0) { + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: H5Dget_type failed"; + throw ex; + } + + char *converted = nullptr; + const void *write_buf = buf; + hid_t write_type = mem_type; + htri_t equal = H5Tequal(mem_type, dset_type); + if (equal < 0) { + H5Tclose(dset_type); + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: H5Tequal failed comparing memory and dataset types"; + throw ex; + } + if (equal == 0 && nelem > 0 && buf != nullptr) { + size_t src_sz = H5Tget_size(mem_type); + size_t dst_sz = H5Tget_size(dset_type); + size_t nbytes = nelem * (src_sz > dst_sz ? src_sz : dst_sz); + converted = (char *)malloc(nbytes); + if (converted == nullptr) { + H5Tclose(dset_type); + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: out of memory converting HDF5 write buffer"; + throw ex; + } + memcpy(converted, buf, nelem * src_sz); + if (H5Tconvert(mem_type, dset_type, nelem, converted, NULL, H5P_DEFAULT) < + 0) { + free(converted); + H5Tclose(dset_type); + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: H5Tconvert failed for HDF5 write buffer"; + throw ex; + } + write_buf = converted; + write_type = dset_type; + } + + herr_t err = + H5Dwrite(dataset_id, write_type, mem_space, file_space, xfer_plist, + write_buf); + free(converted); + H5Tclose(dset_type); + return err; +} + +/* Read with the dataset's type, then convert into mem_type. Same collective + * MPI-IO constraint as the write path. */ +herr_t H5Dread_matching_types(hid_t dataset_id, hid_t mem_type, hid_t mem_space, + hid_t file_space, hid_t xfer_plist, void *buf, + size_t nelem) { + hid_t dset_type = H5Dget_type(dataset_id); + if (dset_type < 0) { + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: H5Dget_type failed"; + throw ex; + } + + htri_t equal = H5Tequal(mem_type, dset_type); + if (equal < 0) { + H5Tclose(dset_type); + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: H5Tequal failed comparing memory and dataset types"; + throw ex; + } + if (equal > 0) { + herr_t err = + H5Dread(dataset_id, mem_type, mem_space, file_space, xfer_plist, buf); + H5Tclose(dset_type); + return err; + } + + size_t src_sz = H5Tget_size(dset_type); + size_t dst_sz = H5Tget_size(mem_type); + size_t nbytes = nelem * (src_sz > dst_sz ? src_sz : dst_sz); + if (nbytes == 0) + nbytes = 1; + char *tmp = (char *)malloc(nbytes); + if (tmp == nullptr) { + H5Tclose(dset_type); + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: out of memory converting HDF5 read buffer"; + throw ex; + } + herr_t err = + H5Dread(dataset_id, dset_type, mem_space, file_space, xfer_plist, tmp); + if (err >= 0 && nelem > 0 && buf != nullptr) { + if (H5Tconvert(dset_type, mem_type, nelem, tmp, NULL, H5P_DEFAULT) < 0) { + free(tmp); + H5Tclose(dset_type); + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: H5Tconvert failed for HDF5 read buffer"; + throw ex; + } + memcpy(buf, tmp, nelem * dst_sz); + } + free(tmp); + H5Tclose(dset_type); + return err; +} + void split_h5_name(const char *data_name, std::vector &h5_name_list) { std::stringstream name_stream(data_name); @@ -197,32 +329,7 @@ void H5_dataset_space(const hid_t file_id, const int data_dims, hid_t type = h5_type(data_type); if (type == H5T_NATIVE_FLOAT || type == H5T_NATIVE_DOUBLE || type == H5T_IEEE_FP16) { - hid_t real_type; - if (type == H5T_NATIVE_DOUBLE) { - if (real_precision == REAL_PRECISION::Double) { - real_type = H5T_NATIVE_DOUBLE; - } - if (real_precision == REAL_PRECISION::Single) { - real_type = H5T_NATIVE_FLOAT; - } - if (real_precision == REAL_PRECISION::Half) { - real_type = H5T_IEEE_FP16; - } - } - - if (type == H5T_NATIVE_FLOAT) { - if (real_precision == REAL_PRECISION::Single) { - real_type = H5T_NATIVE_FLOAT; - } - if (real_precision == REAL_PRECISION::Half) { - real_type = H5T_IEEE_FP16; - } - } - - if (type == H5T_IEEE_FP16) { - real_type = H5T_IEEE_FP16; - } - + hid_t real_type = h5_storage_type(data_type, real_precision); dataset_id = H5Dcreate(parent_group, data_name, real_type, file_space, H5P_DEFAULT, data_plist_id, H5P_DEFAULT); } else { diff --git a/ops/c/src/mpi/ops_mpi_hdf5.cpp b/ops/c/src/mpi/ops_mpi_hdf5.cpp index 29fe317115..8e48942c6e 100755 --- a/ops/c/src/mpi/ops_mpi_hdf5.cpp +++ b/ops/c/src/mpi/ops_mpi_hdf5.cpp @@ -37,6 +37,7 @@ */ #include +#include #include #include #include @@ -62,6 +63,23 @@ static char *copy_str(char const *src) { return strncpy(dest, src, len); } +/* Report whether HDF5 honoured the collective MPI-IO request. Datatype + * conversion (mem type != file type) is a documented reason for falling + * back to independent I/O (H5D_MPIO_DATATYPE_CONVERSION = 0x02). */ +static void report_mpio_io_mode(hid_t xfer_plist, const char *tag) { + if (OPS_instance::getOPSInstance()->OPS_diags <= 3) + return; + H5D_mpio_actual_io_mode_t mode = H5D_MPIO_NO_COLLECTIVE; + herr_t e1 = H5Pget_mpio_actual_io_mode(xfer_plist, &mode); + uint32_t local_cause = 0, global_cause = 0; + herr_t e2 = + H5Pget_mpio_no_collective_cause(xfer_plist, &local_cause, &global_cause); + ops_printf("HDF5 %s actual_io_mode=%d (4=contiguous collective, 0=none) " + "no_collective_cause local=0x%x global=0x%x " + "(0x2=DATATYPE_CONVERSION) get_status=%d/%d\n", + tag, (int)mode, local_cause, global_cause, (int)e1, (int)e2); +} + // // MPI Communicator for parallel I/O // @@ -860,7 +878,11 @@ void ops_fetch_dat_hdf5_file(ops_dat dat, char const *file_name) { // write data hid_t datatype = h5_type(dat->type); - H5Dwrite(dset_id, datatype, memspace, filespace, plist_id, data); + size_t nelem = 1; + for (int d = 0; d < block->dims; d++) + nelem *= (size_t)SIZE[d]; + H5Dwrite_matching_types(dset_id, datatype, memspace, filespace, plist_id, + data, nelem); MPI_Barrier(OPS_MPI_HDF5_BLOCK_WORLD); free(data); @@ -1567,7 +1589,11 @@ void ops_read_dat_hdf5(ops_dat dat) { // read data hid_t datatype = h5_type(dat->type); - H5Dread(dset_id, datatype, memspace, filespace, plist_id, data); + size_t nelem = 1; + for (int d = 0; d < block->dims; d++) + nelem *= (size_t)size[d]; + H5Dread_matching_types(dset_id, datatype, memspace, filespace, plist_id, + data, nelem); ops_dat_set_data(dat, 0, data); @@ -1789,7 +1815,6 @@ void ops_get_const_hdf5(char const *name, int dim, char const *type, "type of constant %s in file %s and requested type %s do not match, " "performing automatic type conversion\n", typ, file_name, type); - typ = type; } H5Dclose(dset_id); @@ -1797,10 +1822,11 @@ void ops_get_const_hdf5(char const *name, int dim, char const *type, dset_id = H5Dopen(file_id, name, H5P_DEFAULT); char *data = nullptr; - hid_t datatype = h5_type(typ); + hid_t datatype = h5_type(type); size_t type_size = H5Tget_size(datatype); data = (char *)xmalloc(type_size * const_dim); - H5Dread(dset_id, datatype, H5S_ALL, H5S_ALL, plist_id, data); + H5Dread_matching_types(dset_id, datatype, H5S_ALL, H5S_ALL, plist_id, data, + (size_t)const_dim); memcpy((void *)const_data, (void *)data, type_size * const_dim); free(data); @@ -1895,7 +1921,6 @@ void ops_write_const_hdf5(char const *name, int dim, char const *type, ops_printf("type of constant %s in file %s and requested type %s do " "not match, performing automatic type conversion\n", typ, file_name, type); - typ = type; } // existing const attributes matches with const to be written .. overwriting @@ -1904,8 +1929,8 @@ void ops_write_const_hdf5(char const *name, int dim, char const *type, dataspace = H5Dget_space(dset_id); hid_t datatype = h5_type(type); - H5Dwrite(dset_id, datatype, H5S_ALL, dataspace, plist_id, - const_data); + H5Dwrite_matching_types(dset_id, datatype, H5S_ALL, dataspace, plist_id, + const_data, (size_t)dim); H5Pclose(plist_id); H5Sclose(dataspace); @@ -1932,8 +1957,8 @@ void ops_write_const_hdf5(char const *name, int dim, char const *type, dset_id = H5Dcreate(file_id, name, datatype, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); // write data - H5Dwrite(dset_id, datatype, H5S_ALL, dataspace, plist_id, - const_data); + H5Dwrite_matching_types(dset_id, datatype, H5S_ALL, dataspace, plist_id, + const_data, (size_t)dim); H5Dclose(dset_id); H5Pclose(plist_id); @@ -2127,8 +2152,12 @@ void write_plane_buf_hdf5(const char *file_name, const char *data_name, hid_t xfer_data_plist_id = H5Pcreate(H5P_DATASET_XFER); H5Pset_dxpl_mpio(xfer_data_plist_id, H5FD_MPIO_COLLECTIVE); - H5Dwrite(dataset_id, h5_type(dat->type), memspace, file_space, - xfer_data_plist_id, buf); + size_t nelem = 1; + for (int d = 0; d < data_dims; d++) + nelem *= (size_t)local_data_size_c[d]; + H5Dwrite_matching_types(dataset_id, h5_type(dat->type), memspace, file_space, + xfer_data_plist_id, buf, nelem); + report_mpio_io_mode(xfer_data_plist_id, "plane write"); H5Pclose(xfer_data_plist_id); H5Sclose(file_space); H5Sclose(memspace); @@ -2250,8 +2279,12 @@ void write_slab_buf_hdf5(const char *file_name, const char *data_name, hid_t xfer_data_plist_id = H5Pcreate(H5P_DATASET_XFER); H5Pset_dxpl_mpio(xfer_data_plist_id, H5FD_MPIO_COLLECTIVE); - H5Dwrite(dataset_id, h5_type(dat->type), memspace, file_space, - xfer_data_plist_id, buf); + size_t nelem = 1; + for (int d = 0; d < space_dim; d++) + nelem *= (size_t)local_data_size_c[d]; + H5Dwrite_matching_types(dataset_id, h5_type(dat->type), memspace, file_space, + xfer_data_plist_id, buf, nelem); + report_mpio_io_mode(xfer_data_plist_id, "slab write"); H5Pclose(xfer_data_plist_id); H5Sclose(file_space); H5Sclose(memspace); From dac3412b63e4f5825b16daeb784995056f8cbdc1 Mon Sep 17 00:00:00 2001 From: Istvan Reguly Date: Mon, 7 Sep 2026 10:58:37 +0100 Subject: [PATCH 2/2] generalized precision conversion --- apps/c/hdf5_slice/hdf5_slice.cpp | 69 ++++++++++--------------- ops/c/src/externlib/ops_hdf5_common.cpp | 48 +++++++++-------- ops/c/src/mpi/ops_mpi_hdf5.cpp | 11 ++-- 3 files changed, 62 insertions(+), 66 deletions(-) diff --git a/apps/c/hdf5_slice/hdf5_slice.cpp b/apps/c/hdf5_slice/hdf5_slice.cpp index b1fb3e6b2f..fe2f20513e 100644 --- a/apps/c/hdf5_slice/hdf5_slice.cpp +++ b/apps/c/hdf5_slice/hdf5_slice.cpp @@ -54,9 +54,10 @@ // Defining the computational problem domain. As a test, we use the // simplest grid See the document for the meaning of variables r1 and r2. double xyzRange[2]{0, 1}; -int nx{64}; -int ny{64}; -int nz{64}; +/* Larger default so MPI single-vs-double slab conversion cost is visible. */ +int nx{512}; +int ny{512}; +int nz{512}; double h{(xyzRange[1] - xyzRange[0]) / (nx - 1)}; void copy_double_single(const ops_dat src_double, ops_dat desc_single) { @@ -151,52 +152,36 @@ int main(int argc, char *argv[]) { ops_par_loop(initKernelV, "initKernelV", slice3Dv, 3, iterRange, ops_arg_dat(v, 1, S3D_000, "int", OPS_WRITE), ops_arg_idx()); + /* Partial slab: many ranks contribute empty hyperslabs (OpenSBLI-like). */ + int slab_range[]{0, nx, 0, ny, nz / 4, nz / 4 + 32}; double ct0, ct1, et0, et1; - double total1{0}, total2{0}, total3{0}, total4{0}; - ops_timers(&ct0, &et0); - ops_write_plane_group_hdf5({{1, 16}, {0, 1}, {2, 16}}, "1", - {{u, buffer_single, v}, {u, v}, {u, v}}); - ops_timers(&ct1, &et1); - total1 += et1 - et0; +#ifdef OPS_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif ops_timers(&ct0, &et0); - copy_double_single(u, buffer_single); - std::string file_name_single{"single.h5"}; - std::string file_name_double{"double.h5"}; - std::string file_name_half{"half.h5"}; - std::string dataset_name_u{"slice3Du/0/u"}; - ops_write_plane_hdf5(u, 1, 16, file_name_double.c_str(), - dataset_name_u.c_str()); - ops_write_plane_hdf5(u, 1, 16, file_name_single.c_str(), - dataset_name_u.c_str(),REAL_PRECISION::Single); - ops_write_plane_hdf5(u, 1, 16, file_name_half.c_str(), - dataset_name_u.c_str(),REAL_PRECISION::Half); - - ops_write_plane_group_hdf5({{1, 16}, {0, 1}, {2, 16}}, "2", - {{u, v}, {u, v}, {u, v}}); + ops_write_data_slab_hdf5(u, slab_range, "slab_output_double.h5", "u", + REAL_PRECISION::Double); +#ifdef OPS_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif ops_timers(&ct1, &et1); - total2 += et1 - et0; + ops_printf("Double: time taken to write dataset onto slab_output.h5: %f sec\n", + et1 - et0); +#ifdef OPS_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif ops_timers(&ct0, &et0); - ops_write_plane_group_hdf5({{1, 8}, {0, 4}, {2, 15}}, "0", - {{velo}, {velo}, {velo}}); - ops_timers(&ct1, &et1); - total3 += et1 - et0; - ops_printf("The time write 1 series is %f\n", total1); - ops_printf("The time write 2 series is %f\n", total2); - ops_printf("The time write velo series is %f\n", total3); - int range[]{-1, 16, -1, 32, 32, 64}; - ops_timers(&ct0, &et0); - ops_write_data_slab_hdf5(v, range, "slab.h5", "vslab"); + ops_write_data_slab_hdf5(u, slab_range, "slab_output_single.h5", "u", + REAL_PRECISION::Single); +#ifdef OPS_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif ops_timers(&ct1, &et1); - total4 += et1 - et0; - ops_printf("The time write slab is %f\n", total4); - ops_fetch_block_hdf5_file(slice3Du, "slice3Du.h5"); - ops_fetch_dat_hdf5_file(u, "slice3Du.h5"); - ops_fetch_dat_hdf5_file(buffer_single, "slice3Du.h5"); - ops_fetch_dat_hdf5_file(velo, "slice3Du.h5"); - ops_fetch_block_hdf5_file(slice3Dv, "slice3Dv.h5"); - ops_fetch_dat_hdf5_file(v, "slice3Dv.h5"); + ops_printf("Single: time taken to write dataset onto slab_output.h5: %f sec\n", + et1 - et0); + ops_printf("\nSuccessful exit from OPS!\n"); ops_exit(); return 0; diff --git a/ops/c/src/externlib/ops_hdf5_common.cpp b/ops/c/src/externlib/ops_hdf5_common.cpp index 9ba0ac390e..081afe0a58 100644 --- a/ops/c/src/externlib/ops_hdf5_common.cpp +++ b/ops/c/src/externlib/ops_hdf5_common.cpp @@ -206,28 +206,34 @@ herr_t H5Dwrite_matching_types(hid_t dataset_id, hid_t mem_type, hid_t mem_space ex << "Error: H5Tequal failed comparing memory and dataset types"; throw ex; } - if (equal == 0 && nelem > 0 && buf != nullptr) { - size_t src_sz = H5Tget_size(mem_type); - size_t dst_sz = H5Tget_size(dset_type); - size_t nbytes = nelem * (src_sz > dst_sz ? src_sz : dst_sz); - converted = (char *)malloc(nbytes); - if (converted == nullptr) { - H5Tclose(dset_type); - OPSException ex(OPS_HDF5_ERROR); - ex << "Error: out of memory converting HDF5 write buffer"; - throw ex; - } - memcpy(converted, buf, nelem * src_sz); - if (H5Tconvert(mem_type, dset_type, nelem, converted, NULL, H5P_DEFAULT) < - 0) { - free(converted); - H5Tclose(dset_type); - OPSException ex(OPS_HDF5_ERROR); - ex << "Error: H5Tconvert failed for HDF5 write buffer"; - throw ex; - } - write_buf = converted; + /* Always present a matching mem type to H5Dwrite when types differ — even + * for empty writes (nelem==0). A single rank still using the native type + * triggers H5D_MPIO_DATATYPE_CONVERSION and disables collective MPI-IO + * for the whole communicator. */ + if (equal == 0) { write_type = dset_type; + if (nelem > 0 && buf != nullptr) { + size_t src_sz = H5Tget_size(mem_type); + size_t dst_sz = H5Tget_size(dset_type); + size_t nbytes = nelem * (src_sz > dst_sz ? src_sz : dst_sz); + converted = (char *)malloc(nbytes); + if (converted == nullptr) { + H5Tclose(dset_type); + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: out of memory converting HDF5 write buffer"; + throw ex; + } + memcpy(converted, buf, nelem * src_sz); + if (H5Tconvert(mem_type, dset_type, nelem, converted, NULL, + H5P_DEFAULT) < 0) { + free(converted); + H5Tclose(dset_type); + OPSException ex(OPS_HDF5_ERROR); + ex << "Error: H5Tconvert failed for HDF5 write buffer"; + throw ex; + } + write_buf = converted; + } } herr_t err = diff --git a/ops/c/src/mpi/ops_mpi_hdf5.cpp b/ops/c/src/mpi/ops_mpi_hdf5.cpp index 8e48942c6e..a09051df7a 100755 --- a/ops/c/src/mpi/ops_mpi_hdf5.cpp +++ b/ops/c/src/mpi/ops_mpi_hdf5.cpp @@ -2197,10 +2197,15 @@ void write_slab_buf_hdf5(const char *file_name, const char *data_name, MPI_Comm BLOCK_WORLD{sb->comm1}; MPI_Comm_rank(BLOCK_WORLD, &my_block_rank); int color{0}; - bool no_data{true}; + /* A hyperslab is empty if any dimension has non-positive extent. The old + * "all dims zero" check left ranks with e.g. (nx,ny,0) in the writer + * communicator; with a type mismatch those empty writes still disable + * collective MPI-IO (H5D_MPIO_DATATYPE_CONVERSION) for everyone. */ + bool no_data{false}; for (int d = 0; d < space_dim; d++) { - if ((local_range[2 * d + 1] - local_range[2 * d]) != 0) { - no_data = false; + if ((local_range[2 * d + 1] - local_range[2 * d]) <= 0) { + no_data = true; + break; } } if (no_data){