[newlib-cygwin/cygwin-3_5-branch] Cygwin: console: Fix a bug that 64th console cannot be handled.

[Takashi Yano]

https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;h=d8f6db2fffaed7e0faaa58e17dd089b5d9bbd3e2

commit d8f6db2fffaed7e0faaa58e17dd089b5d9bbd3e2
Author: Takashi Yano <takashi.yano@nifty.ne.jp>
Date:   Fri Feb 16 03:35:07 2024 +0900

    Cygwin: console: Fix a bug that 64th console cannot be handled.
    
    Fixes: 930e553da8e9 ("Cygwin: console: Unify EnumWindows() callback functions.");
    Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>

Diff:
---
 winsup/cygwin/devices.cc                | 2 +-
 winsup/cygwin/devices.in                | 2 +-
 winsup/cygwin/fhandler/console.cc       | 8 +++-----
 winsup/cygwin/local_includes/fhandler.h | 5 +++--
 4 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/winsup/cygwin/devices.cc b/winsup/cygwin/devices.cc
index 5b67fd1da..d4c003bac 100644
--- a/winsup/cygwin/devices.cc
+++ b/winsup/cygwin/devices.cc
@@ -83,7 +83,7 @@ exists_console (const device& dev)
     default:
       if (dev.get_minor () < MAX_CONS_DEV)
 	{
-	  unsigned long bitmask = fhandler_console::console_unit (-1);
+	  unsigned long bitmask = fhandler_console::console_unit (CONS_LIST_USED);
 	  return !!(bitmask & (1UL << dev.get_minor ()));
 	}
       return false;
diff --git a/winsup/cygwin/devices.in b/winsup/cygwin/devices.in
index be54346fb..a86e5015f 100644
--- a/winsup/cygwin/devices.in
+++ b/winsup/cygwin/devices.in
@@ -79,7 +79,7 @@ exists_console (const device& dev)
     default:
       if (dev.get_minor () < MAX_CONS_DEV)
 	{
-	  unsigned long bitmask = fhandler_console::console_unit (-1);
+	  unsigned long bitmask = fhandler_console::console_unit (CONS_LIST_USED);
 	  return !!(bitmask & (1UL << dev.get_minor ()));
 	}
       return false;
diff --git a/winsup/cygwin/fhandler/console.cc b/winsup/cygwin/fhandler/console.cc
index 66b4905f4..c16ca3962 100644
--- a/winsup/cygwin/fhandler/console.cc
+++ b/winsup/cygwin/fhandler/console.cc
@@ -253,12 +253,10 @@ fhandler_console::enum_windows (HWND hw, LPARAM lp)
 }
 
 fhandler_console::console_unit::console_unit (int n0):
-  n (n0), bitmask (0)
+  n (n0), bitmask (0), shared_console_info (NULL)
 {
   EnumWindows (fhandler_console::enum_windows, (LPARAM) this);
-  if (n < 0)
-    n = (_minor_t) ffsl (~bitmask) - 1;
-  if (n < 0)
+  if (n0 == CONS_SCAN_UNUSED && (n = ffsl (~bitmask) - 1) < 0)
     api_fatal (sizeof (bitmask) == 8 ?
 	       "console device allocation failure - "
 	       "too many consoles in use, max consoles is 64" :
@@ -674,7 +672,7 @@ fhandler_console::set_unit ()
 	      ProtectHandleINH (cygheap->console_h);
 	      if (created)
 		{
-		  unit = console_unit (-1);
+		  unit = console_unit (CONS_SCAN_UNUSED);
 		  cs->tty_min_state.setntty (DEV_CONS_MAJOR, unit);
 		}
 	      else
diff --git a/winsup/cygwin/local_includes/fhandler.h b/winsup/cygwin/local_includes/fhandler.h
index 15ea3f89f..a21aeb7d8 100644
--- a/winsup/cygwin/local_includes/fhandler.h
+++ b/winsup/cygwin/local_includes/fhandler.h
@@ -2178,6 +2178,8 @@ class dev_console
 };
 
 #define MAX_CONS_DEV (sizeof (unsigned long) * 8)
+#define CONS_SCAN_UNUSED (-1)
+#define CONS_LIST_USED (-2)
 
 /* This is a input and output console handle */
 class fhandler_console: public fhandler_termios
@@ -2374,9 +2376,8 @@ private:
     unsigned long bitmask;
     console_state *shared_console_info;
   public:
-    operator _minor_t () const {return n;}
     operator console_state * () const {return shared_console_info;}
-    operator unsigned long () const {return bitmask;}
+    operator unsigned long () const {return n == CONS_LIST_USED ? bitmask : n;}
     console_unit (int);
     friend BOOL CALLBACK fhandler_console::enum_windows (HWND, LPARAM);
   };