[PATCH 2/2] Cygwin: lockf: Make lockf() return ENOLCK when too many locks

[Takashi Yano]

Previously, lockf() printed a warning message when the number of locks
per file exceeds the limit (MAX_LOCKF_CNT). This patch makes lockf()
return ENOLCK in that case rather than printing the warning message.

Addresses: https://cygwin.com/pipermail/cygwin/2024-October/256528.html
Fixes: 31390e4ca643 ("(inode_t::get_all_locks_list): Use pre-allocated buffer in i_all_lf instead of allocating every lock.  Return pointer to start of linked list of locks.")
Reported-by: Christian Franke <Christian.Franke@t-online.de>
Reviewed-by:
Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>
---
 winsup/cygwin/flock.cc | 46 ++++++++++++++++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/winsup/cygwin/flock.cc b/winsup/cygwin/flock.cc
index 5550b3a5b..3b8475c18 100644
--- a/winsup/cygwin/flock.cc
+++ b/winsup/cygwin/flock.cc
@@ -297,6 +297,7 @@ class inode_t
     HANDLE		 i_dir;
     HANDLE		 i_mtx;
     uint32_t		 i_cnt;    /* # of threads referencing this instance. */
+    uint32_t		 i_lock_cnt; /* # of locks for this file */
 
   public:
     inode_t (dev_t dev, ino_t ino);
@@ -321,6 +322,8 @@ class inode_t
     void unlock_and_remove_if_unused ();
 
     lockf_t *get_all_locks_list ();
+    uint32_t get_lock_count () /* needs get_all_locks_list() */
+    { return i_lock_cnt; }
 
     bool del_my_locks (long long id, HANDLE fhdl);
 };
@@ -503,7 +506,8 @@ inode_t::get (dev_t dev, ino_t ino, bool create_if_missing, bool lock)
 }
 
 inode_t::inode_t (dev_t dev, ino_t ino)
-: i_lockf (NULL), i_all_lf (NULL), i_dev (dev), i_ino (ino), i_cnt (0L)
+: i_lockf (NULL), i_all_lf (NULL), i_dev (dev), i_ino (ino), i_cnt (0L),
+  i_lock_cnt (0)
 {
   HANDLE parent_dir;
   WCHAR name[48];
@@ -610,17 +614,15 @@ inode_t::get_all_locks_list ()
 	  dbi->ObjectName.Buffer[LOCK_OBJ_NAME_LEN] = L'\0';
 	  if (!newlock.from_obj_name (this, &i_all_lf, dbi->ObjectName.Buffer))
 	    continue;
-	  if (lock - i_all_lf >= MAX_LOCKF_CNT)
-	    {
-	      system_printf ("Warning, can't handle more than %d locks per file.",
-			     MAX_LOCKF_CNT);
-	      break;
-	    }
+	  if (lock - i_all_lf > MAX_LOCKF_CNT)
+	    api_fatal ("Can't handle more than %d locks per file.",
+		       MAX_LOCKF_CNT);
 	  if (lock > i_all_lf)
 	    lock[-1].lf_next = lock;
 	  new (lock++) lockf_t (newlock);
 	}
     }
+  i_lock_cnt = lock - i_all_lf;
   /* If no lock has been found, return NULL. */
   if (lock == i_all_lf)
     return NULL;
@@ -1346,6 +1348,8 @@ lf_setlock (lockf_t *lock, inode_t *node, lockf_t **clean, HANDLE fhdl)
    *
    * Handle any locks that overlap.
    */
+  node->get_all_locks_list (); /* Update lock count */
+  const uint32_t lock_cnt = node->get_lock_count ();
   prev = head;
   block = *head;
   needtolink = 1;
@@ -1368,6 +1372,8 @@ lf_setlock (lockf_t *lock, inode_t *node, lockf_t **clean, HANDLE fhdl)
 	case 0: /* no overlap */
 	  if (needtolink)
 	    {
+	      if (lock_cnt + 1 > MAX_LOCKF_CNT)
+		return ENOLCK;
 	      *prev = lock;
 	      lock->lf_next = overlap;
 	      lock->create_lock_obj ();
@@ -1399,12 +1405,20 @@ lf_setlock (lockf_t *lock, inode_t *node, lockf_t **clean, HANDLE fhdl)
 	    }
 	  if (overlap->lf_start == lock->lf_start)
 	    {
+	      if (lock_cnt + 1 > MAX_LOCKF_CNT)
+		return ENOLCK;
 	      *prev = lock;
 	      lock->lf_next = overlap;
 	      overlap->lf_start = lock->lf_end + 1;
 	    }
 	  else
-	    lf_split (overlap, lock, clean);
+	    {
+	      if ((overlap->lf_end > lock->lf_end
+		   && lock_cnt + 2 > MAX_LOCKF_CNT)
+		  || lock_cnt + 1 > MAX_LOCKF_CNT)
+		return ENOLCK;
+	      lf_split (overlap, lock, clean);
+	    }
 	  lf_wakelock (overlap, fhdl);
 	  overlap->create_lock_obj ();
 	  lock->create_lock_obj ();
@@ -1440,6 +1454,8 @@ lf_setlock (lockf_t *lock, inode_t *node, lockf_t **clean, HANDLE fhdl)
 	  /*
 	   * Add lock after overlap on the list.
 	   */
+	  if (lock_cnt + 1 > MAX_LOCKF_CNT)
+	    return ENOLCK;
 	  lock->lf_next = overlap->lf_next;
 	  overlap->lf_next = lock;
 	  overlap->lf_end = lock->lf_start - 1;
@@ -1456,6 +1472,8 @@ lf_setlock (lockf_t *lock, inode_t *node, lockf_t **clean, HANDLE fhdl)
 	   */
 	  if (needtolink)
 	    {
+	      if (lock_cnt + 1 > MAX_LOCKF_CNT)
+		return ENOLCK;
 	      *prev = lock;
 	      lock->lf_next = overlap;
 	      lock->create_lock_obj ();
@@ -1483,12 +1501,24 @@ lf_clearlock (lockf_t *unlock, lockf_t **clean, HANDLE fhdl)
   lockf_t *lf = *head;
   lockf_t *overlap, **prev;
   int ovcase;
+  inode_t *node = lf->lf_inode;
 
   if (lf == NOLOCKF)
     return 0;
   prev = head;
   while ((ovcase = lf_findoverlap (lf, unlock, SELF, &prev, &overlap)))
     {
+      /* Check if # of locks will be increased. */
+      if (ovcase == 2 /* overlap contains lock */
+	  && overlap->lf_start < unlock->lf_start
+	  && overlap->lf_end > unlock->lf_end)
+	{
+	  tmp_pathbuf tp;
+	  node->i_all_lf = (lockf_t *) tp.w_get ();
+	  node->get_all_locks_list (); /* Update lock count */
+	  if (node->get_lock_count () + 1 > MAX_LOCKF_CNT)
+	    return ENOLCK;
+	}
       /*
        * Wakeup the list of locks to be retried.
        */
-- 
2.45.1