cygwin 3.5.4-1: lockf() aborts on overlap and does not fail on overflow

[Christian Franke]

Two possibly independent bugs found by 'stress-ng --lockf ...':

1) lockf() may abort the process with api_fatal() if the new range 
partly overlaps with two ranges previously locked by the same process.

2) lockf() prints a warning on too many locks and returns success. It 
should not print a warning and fail with ENOLCK instead.

Testcase for both:

$ uname -r
3.5.4-1.x86_64

$ cat locktest.c
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>

static int lock_at(int fd, int off, int size)
{
   if (lseek(fd, off, SEEK_SET) < 0) {
     perror("lseek"); return -1;
   }
   printf("\rlock %d-%d\n", off, off + size - 1); fflush(stdout);
   if (lockf(fd, F_LOCK, size) < 0) {
     perror("lock"); return -1;
   }
   return 0;
}

int main(int argc, char **argv)
{
   int fd = open("locktest.tmp", O_RDWR|O_CREAT, 0666);
   if (fd < 0) {
     perror("open"); return 1;
   }

   if (argc == 1) {
     lock_at(fd, 0, 2);
     lock_at(fd, 2, 2);
     lock_at(fd, 1, 2);
   }
   else {
     for (int i = 0; i < 914; i++)
       if (lock_at(fd, i, 1))
         return 1;
   }
   printf("\rdone\n");
   return 0;
}

$ gcc -o locktest locktest.c

$ ./locktest
lock 0-1
lock 2-3
lock 1-2
      1 [main] locktest 44864 C:\cygwin64\tmp\locktest.exe: \
        *** fatal error - NtCreateEvent(lock): 0xC0000035\
        Hangup

$ ./locktest loop
lock 0-0
lock 1-1
lock 2-2
lock 3-3
...
lock 909-909
lock 910-910
lock 911-911
       0 [main] locktest 44865 inode_t::get_all_locks_list: \
         Warning, can't handle more than 910 locks per file.
lock 912-912
     727 [main] locktest 44865 inode_t::get_all_locks_list: \
         Warning, can't handle more than 910 locks per file.
lock 913-913
    1329 [main] locktest 44865 inode_t::get_all_locks_list: \
         Warning, can't handle more than 910 locks per file.
done

There is possibly also an off-by-one error as the 912'th lockf() prints 
the first warning.

-- 
Regards,
Christian