[newlib-cygwin/main] Cygwin: file locking: always use and expect explicit lock variant

[Corinna Vinschen]

https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;h=63f8b31bf51cae9bd948702ae1ed7e0118fdd211

commit 63f8b31bf51cae9bd948702ae1ed7e0118fdd211
Author:     Corinna Vinschen <corinna@vinschen.de>
AuthorDate: Mon Aug 4 20:45:38 2025 +0200
Commit:     Corinna Vinschen <corinna@vinschen.de>
CommitDate: Mon Aug 4 20:51:03 2025 +0200

    Cygwin: file locking: always use and expect explicit lock variant
    
    So far, not setting a lock variant (F_POSIX/F_FLOCK) defaulted to
    F_POSIX locks.  Adding OFD locks in a followup patch may lead to
    confusion, so make sure that the lock variant is always set in the
    calling functions (fcntl, flock, lockf).  Bail out with EINVAL if
    no lock variant is set.
    
    Signed-off-by: Corinna Vinschen <corinna@vinschen.de>

Diff:
---
 winsup/cygwin/fhandler/base.cc |  1 +
 winsup/cygwin/flock.cc         | 13 +++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/winsup/cygwin/fhandler/base.cc b/winsup/cygwin/fhandler/base.cc
index beebd710c5d9..09b0a35e614f 100644
--- a/winsup/cygwin/fhandler/base.cc
+++ b/winsup/cygwin/fhandler/base.cc
@@ -1499,6 +1499,7 @@ int fhandler_base::fcntl (int cmd, intptr_t arg)
 	{
 	  struct flock *fl = (struct flock *) arg;
 	  fl->l_type &= F_RDLCK | F_WRLCK | F_UNLCK;
+	  fl->l_type |= F_POSIX;
 	  res = mandatory_locking () ? mand_lock (cmd, fl) : lock (cmd, fl);
 	}
       break;
diff --git a/winsup/cygwin/flock.cc b/winsup/cygwin/flock.cc
index b41cba5c7036..0eb908fd32a6 100644
--- a/winsup/cygwin/flock.cc
+++ b/winsup/cygwin/flock.cc
@@ -951,7 +951,10 @@ fhandler_base::lock (int a_op, struct flock *fl)
   short type = fl->l_type & (F_RDLCK | F_WRLCK | F_UNLCK);
 
   if (!a_flags)
-    a_flags = F_POSIX; /* default */
+    {
+      set_errno (EINVAL);
+      return -1;
+    }
 
   /* FIXME: For BSD flock(2) we need a valid, per file table entry OS handle.
      Therefore we can't allow using flock(2) on nohandle devices. */
@@ -1859,8 +1862,7 @@ flock (int fd, int operation)
 	  set_errno (EINVAL);
 	  __leave;
 	}
-      if (!cfd->mandatory_locking ())
-	fl.l_type |= F_FLOCK;
+      fl.l_type |= F_FLOCK;
       res = cfd->mandatory_locking () ? cfd->mand_lock (cmd, &fl)
 				      : cfd->lock (cmd, &fl);
       if ((res == -1) && ((get_errno () == EAGAIN) || (get_errno () == EACCES)))
@@ -1906,7 +1908,7 @@ lockf (int filedes, int function, off_t size)
 	  fl.l_type = F_WRLCK;
 	  break;
 	case F_TEST:
-	  fl.l_type = F_WRLCK;
+	  fl.l_type = F_POSIX | F_WRLCK;
 	  if (cfd->lock (F_GETLK, &fl) == -1)
 	    __leave;
 	  if (fl.l_type == F_UNLCK || fl.l_pid == getpid ())
@@ -1920,6 +1922,7 @@ lockf (int filedes, int function, off_t size)
 	  __leave;
 	  /* NOTREACHED */
 	}
+      fl.l_type |= F_POSIX;
       res = cfd->mandatory_locking () ? cfd->mand_lock (cmd, &fl)
 				      : cfd->lock (cmd, &fl);
     }
@@ -2022,6 +2025,8 @@ fhandler_disk_file::mand_lock (int a_op, struct flock *fl)
      the entire file, even when file grows later. */
   if (length.QuadPart == 0)
     length.QuadPart = UINT64_MAX;
+  /* Filter lock types */
+  fl->l_type &= (F_RDLCK | F_WRLCK | F_UNLCK);
   /* Action! */
   if (fl->l_type == F_UNLCK)
     {