HDF5 Output slow over multiple nodes

I have a centrally installed version of HDF5 which is reasonably performant writing to file on a single node of a GPFS HPC cluster, but when I try to run it on multiple nodes the performance is significantly worse.

It is used in a package I am installing (mpb:https://github.com/NanoComp/mpb https://github.com/NanoComp/mpb/issues/171 ) and I am struggling to get a performant build. It outputs chemical band structures to HDF5. on a single node (192 cores) it will write around 30 or so 125MB files per minute, when I try to run it on multiple nodes the number of files it writes reduces to 2/3 125MB files per minute.

If anyone has any suggestions as to how to speed the output up I would be keen to hear them.

The same problem is exhibited with the following code:

```
#include
#include
#include
#include "hdf5.h"

#define FILE_NAME "parallel_test.h5"
#define DATASET_NAME "data"
#define NFILES 100
#define DIM0 9216 // divisible by 576
#define DIM1 3556
int main(int argc, char **argv) {
MPI_Init(&argc, &argv);

int mpi_rank, mpi_size;
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);

// Make sure we can split DIM0 evenly
if (DIM0 % mpi_size != 0) {
if (mpi_rank == 0)
printf("Error: DIM0 (%d) must be divisible by number of MPI processes (%d)\n", DIM0, mpi_size);
MPI_Finalize();
return 1;
}

hsize_t dims[2] = {DIM0, DIM1}; // global dataset size
hsize_t count[2] = {DIM0 / mpi_size, DIM1}; // local dataset size
hsize_t offset[2]; // per-rank offset

// Local buffer
int (*data)[DIM1] = malloc(count[0] * DIM1 * sizeof(int));

for (int f = 0; f < NFILES; f++) {
// Fill buffer with something
for (int i = 0; i < count[0]; i++)
for (int j = 0; j < DIM1; j++)
data[i][j] = mpi_rank + f;

offset[0] = mpi_rank * count[0];
offset[1] = 0;

// File name
char fname[256];
sprintf(fname, "output_%03d.h5", f);

// File access property list
hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL);

// Create file collectively
hid_t file_id = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
H5Pclose(plist_id);

// Create dataspace and dataset
hid_t filespace = H5Screate_simple(2, dims, NULL);
hid_t dset_id = H5Dcreate(file_id, "dataset", H5T_NATIVE_INT, filespace,
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Sclose(filespace);

// Select hyperslab in file
filespace = H5Dget_space(dset_id);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL);

// Memory dataspace
hid_t memspace = H5Screate_simple(2, count, NULL);

// Collective write
plist_id = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);
H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, data[0]);
H5Pclose(plist_id);

// Close
H5Sclose(memspace);
H5Sclose(filespace);
H5Dclose(dset_id);
H5Fclose(file_id);

if (mpi_rank == 0)
printf("File %s written (%.2f MB)\n", fname,
(dims[0] * dims[1] * sizeof(int)) / (1024.0 * 1024.0));
}

free(data);

MPI_Finalize();
return 0;
}

```

Is this likely down to a problem with how the code is written itself, or is it a problem with the underlying HDF5 build? Are there some flags that can be passed to HDF5 that can speed things up on multiple nodes?