Handle EFAULT faithfully

Hey,

I have a target program containing a vulnerability due to a buffer overflow in a call to recvfrom. I also have a custom memory allocator that places guard pages, mprotect'd to PROT_NONE after each buffer that it allocates. If I run the target application, with the custom memory allocator, on the command line (or under gdb) with a length such that the destination buffer overflows and the guard page is hit then the kernel generates an EFAULT and recvfrom returns -1, as expected.

However, when running the above (application with vulnerability + custom memory allocator with guard pages) under rr the kernel does not generate the EFAULT. In fact, it completes the write as if the guard page was writable: the guard page is modified by the write, and the application continues.

Needless to say, this confused me. I wrote the following test application to that attempts to use recvfrom to write to a PROT_NONE memory region. If you run it normally and send data via netcat/whatever it will print recvfrom failed: Bad address, as expected. If you run it under rr it will simply hang. A hang isn't the same behaviour I was seeing in the full application, but there's a lot more going on in that I have yet to isolate (could be a different socket type causing a difference?). I am hoping this test case will be sufficient to tip you off as to what the problem might be.

#include <netdb.h>
#include <stdio.h>
#include <stddef.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/types.h>

int main(int argc, char *argv[])
{
  struct sockaddr_in myaddr;
  struct sockaddr_in remaddr;
  socklen_t addrlen = sizeof(remaddr);
  int recvlen;
  int fd;

  void* buf = mmap(NULL, 4096, PROT_READ | PROT_WRITE,
      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

  if (!buf) {
    perror("mmap failed\n");
  }
  if (mprotect(buf, 4096, PROT_NONE)) {
    perror("mprotect failed\n");
  }


  if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
    perror("cannot create socket\n");
    return 0;
  }

  memset((char *)&myaddr, 0, sizeof(myaddr));
  myaddr.sin_family = AF_INET;
  myaddr.sin_addr.s_addr = htonl(INADDR_ANY);
  myaddr.sin_port = htons(6666);

  if (bind(fd, (struct sockaddr *)&myaddr, sizeof(myaddr)) < 0) {
    perror("bind failed");
    return 0;
  }

  for (;;) {
    printf("waiting on port %d\n", 6666);
    recvlen = recvfrom(fd, buf, 4, 0, (struct sockaddr *)&remaddr, &addrlen);
    if (recvlen == -1) {
      perror("recvfrom failed");
      continue;
    }

    printf("received %d bytes\n", recvlen);
    if (recvlen > 0) {
      ((char *)buf)[recvlen] = 0;
      printf("received message: \"%s\"\n", buf);
    }
  }
}

Build: gcc -o test test.c
Run: ./test
Trigger: echo "AAAAAAAAAAAAAAAAAAAAAA" | nc -u 127.0.0.1 6666

Begin with the supplied test.c reproducer using recvfrom and mprotect, and compare its normal behavior with recording and replay under rr. Trace rr's recvfrom and EFAULT handling, then verify that the replayed program reports the same Bad address failure and does not modify the protected page or hang.