Problem with MPI-IO's atomic file I/O

Nobody has claimed this yet.

I observed errors in the results for one of the ROMIO tests, atomicity.c, running under Unify. This original example is posted at

https://github.com/pmodels/mpich/blob/master/src/mpi/romio/test/atomicity.c

This test checks "whether atomicity semantics are satisfied for overlapping accesses in atomic mode."

The original program has two phases. I divided it into two programs, one for the first phase (which works fine) and one for the second phase (where I observe errors). This second phase is reproduced in the example below.

In the example, rank=0 first writes "0" to the entire file. Next, rank=0 writes all elements as "10" while the other ranks read the file, and check the values read. This check is such that all elements are compared to the first element read. The example uses non-contiguous data, created with a new datayype and a file "view" for that. It must be noticed that this example uses a different MPI_Info, which sets the sizes for read and write operations in ROMIO. Thus, although the main piece of the code has only one MPI_File_write (by rank=0) and one MPI_File_read (by all other ranks), in fact there are many write/read operations.

Running this on 8 processors of the same node of Quartz@LLNL, I obtain this output:

Process 3: readbuf[139] is 0, should be 10
[3] At Exit, i=139
[0] At Exit, i=10000
Process 1: readbuf[139] is 0, should be 10
[1] At Exit, i=139
Process 2: readbuf[139] is 0, should be 10
[2] At Exit, i=139
Process 5: readbuf[9972] is 0, should be 10
[5] At Exit, i=9972
Process 4: readbuf[9972] is 0, should be 10
[4] At Exit, i=9972
Process 6: readbuf[9972] is 0, should be 10
[6] At Exit, i=9972
Process 7: readbuf[9972] is 0, should be 10
[7] At Exit, i=9972
Found 7 errors

Thus, all seven ranks detected an error at some point. Running this same program multiple times results in errors at different positions.

This is the test code:

```
#include "mpi.h"
#include
#include
#include
#include

/* tests whether atomicity semantics are satisfied for overlapping accesses
in atomic mode. The probability of detecting errors is higher if you run
it on 8 or more processes. */

static void handle_error(int errcode, const char *str)
{
char msg[MPI_MAX_ERROR_STRING];
int resultlen;
MPI_Error_string(errcode, msg, &resultlen);
fprintf(stderr, "%s: %s\n", str, msg);
MPI_Abort(MPI_COMM_WORLD, 1);
}

#define MPI_CHECK(fn) { int errcode; errcode = (fn); if (errcode != MPI_SUCCESS) handle_error(errcode, #fn); }

#define BUFSIZE 10000 /* no. of integers */
#define VERBOSE 0
int main(int argc, char **argv)
{
int *writebuf, *readbuf, i, mynod, nprocs, len, err;
int errs = 0, toterrs;
MPI_Datatype newtype;
MPI_File fh;
MPI_Status status;
MPI_Info info;

MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
err = unifyfs_mount("/unifyfs", mynod, nprocs, 0);
if (err) {
printf("[%d] unifyfs_mount failed (return = %d)\n", mynod, err);
exit(-1);
}

writebuf = (int *) malloc(BUFSIZE * sizeof(int));
readbuf = (int *) malloc(BUFSIZE * sizeof(int));

/* repeat the same test with a noncontiguous filetype */

MPI_Type_vector(BUFSIZE, 1, 2, MPI_INT, &newtype);
MPI_Type_commit(&newtype);

MPI_Info_create(&info);
/* I am setting these info values for testing purposes only. It is
* better to use the default values in practice. */
MPI_Info_set(info, "ind_rd_buffer_size", "1209");
MPI_Info_set(info, "ind_wr_buffer_size", "1107");

if (!mynod) {
MPI_CHECK(MPI_File_open(MPI_COMM_SELF, "ufs:/unifyfs/ofile",
MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh));
for (i = 0; i < BUFSIZE; i++)
writebuf[i] = 0;
MPI_CHECK(MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info));
MPI_CHECK(MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status));
MPI_File_close(&fh);
#if VERBOSE
fprintf(stderr, "\ntesting noncontiguous accesses\n");
#endif
}
MPI_Barrier(MPI_COMM_WORLD);

for (i = 0; i < BUFSIZE; i++)
writebuf[i] = 10;
for (i = 0; i < BUFSIZE; i++)
readbuf[i] = 20;

MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, "ufs:/unifyfs/ofile", MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh));
MPI_CHECK(MPI_File_set_atomicity(fh, 1));
MPI_CHECK(MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info));
MPI_Barrier(MPI_COMM_WORLD);

if (!mynod) {
MPI_CHECK(MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status));
} else {
err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status);
if (err == MPI_SUCCESS) {
if (readbuf[0] == 0) {
for (i = 1; i < BUFSIZE; i++)
if (readbuf[i] != 0) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 0\n", mynod, i,
readbuf[i]);
goto fn_exit;
}
} else if (readbuf[0] == 10) {
for (i = 1; i < BUFSIZE; i++)
if (readbuf[i] != 10) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 10\n", mynod, i,
readbuf[i]);
goto fn_exit;
}
} else {
errs++;
fprintf(stderr, "Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod,
readbuf[0]);
}
}
}

MPI_Type_free(&newtype);
MPI_Info_free(&info);

fn_exit:
fprintf(stderr,"[%d] At Exit, i=%d\n",mynod,i);
MPI_File_close(&fh);

MPI_Barrier(MPI_COMM_WORLD);

MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (mynod == 0) {
if (toterrs > 0) {
fprintf(stderr, "Found %d errors\n", toterrs);
} else {
fprintf(stdout, " No Errors\n");
}
}
free(writebuf);
free(readbuf);

unifyfs_unmount();
MPI_Finalize();
return 0;
}
```

Start with src/mpi/romio/test/atomicity.c and the supplied MPI-IO reproducer, focusing on MPI_File_set_atomicity, the noncontiguous file view, and the ind_rd_buffer_size and ind_wr_buffer_size settings. Reproduce the second phase under UnifyFS and investigate why readers observe mixed values; done means the atomicity check completes without inconsistent reads.

Written by the indexing model from the issue text.