[PATCH v7 3/3] Cygwin: console: Fix master thread for OpenConsole.exe

[Takashi Yano]

If the console is originating from a pseudo console, current master
thread code does not work as expected if ENABLE_VIRTUAL_TERMINAL_INPUT
flag is set, particularly when OpenConsole.exe is used. This is because
the pseudo console does not preserve all the key event as is.
All bKeyDown == 0 events will be omitted from the input record written
by WriteConsoleInput() and events regarding pressing shift/control/alt
keys will be dropped as well.

This patch adds strip_inrec() function to remove all the key events
of bKeyDown == 0 or UnicodeChar == 0 before comparing/writing input
record. This function is called only when the console is originating
from a pseudo console and ENABLE_VIRTUAL_TERMINAL_INPUT flag is set.

Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>
Reviewed-by: Johannes Schindelin <Johannes.Schindelin@gmx.de>
---
 winsup/cygwin/fhandler/console.cc | 42 +++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/winsup/cygwin/fhandler/console.cc b/winsup/cygwin/fhandler/console.cc
index 39c4f6ff0..3da3a80db 100644
--- a/winsup/cygwin/fhandler/console.cc
+++ b/winsup/cygwin/fhandler/console.cc
@@ -305,6 +305,23 @@ cons_master_thread (VOID *arg)
   return 0;
 }
 
+static inline DWORD
+strip_inrec (INPUT_RECORD *r, DWORD n)
+{
+  /* Pseudo console with OpenConsole.exe removes the events
+     whose bKeyDown is 0 as well as ones whose charcode is 0. */
+  DWORD j = 0;
+  for (DWORD i = 0; i < n; i++)
+    {
+      if (r[i].EventType != KEY_EVENT)
+	r[j++] = r[i];
+      else if (r[i].Event.KeyEvent.bKeyDown
+	       && r[i].Event.KeyEvent.uChar.UnicodeChar)
+	r[j++] = r[i];
+    }
+  return j;
+}
+
 /* Compare two INPUT_RECORD sequences */
 static inline bool
 inrec_eq (const INPUT_RECORD *a, const INPUT_RECORD *b, DWORD n)
@@ -417,6 +434,8 @@ fhandler_console::cons_master_thread (handle_set_t *p, tty *ttyp)
   while (con.owner == GetCurrentProcessId ())
     {
       DWORD total_read, n, i;
+      DWORD mode;
+      bool need_strip = false;
 
       if (con.disable_master_thread)
 	{
@@ -472,6 +491,23 @@ fhandler_console::cons_master_thread (handle_set_t *p, tty *ttyp)
 	{
 	case WAIT_OBJECT_0:
 	  acquire_attach_mutex (mutex_timeout);
+	  /* When ENABLE_VIRTUAL_TERMINAL_INPUT is set, the key events
+	     are not preserved as is. Particularly, when OpenConsole.exe
+	     is used, the following key events are simlified so much.
+	     Writing the events:
+	         press shift key -> press 'A' key ->
+	         release 'A' key -> release shift key
+	     results in only one key event whth:
+	         uChar.UnicodeChar = 0x41,
+		 wVirtualKeyCode = 0,
+		 wVirtualScanCode = 0,
+		 dwControlKeyState = 0,
+		 bKeyDown = 1
+	     Therefore, we need fixup the input record by calling
+	     strip_inrec() if the ENABLE_VIRTUAL_TERMINAL_INPUT flag is
+	     set, so that the input records are compared as expected. */
+	  GetConsoleMode (p->input_handle, &mode);
+	  need_strip = inside_pcon && (mode & ENABLE_VIRTUAL_TERMINAL_INPUT);
 	  total_read = 0;
 	  while (cygwait (p->input_handle, (DWORD) 0) == WAIT_OBJECT_0
 		 && total_read < inrec_size)
@@ -483,6 +519,8 @@ fhandler_console::cons_master_thread (handle_set_t *p, tty *ttyp)
 	      total_read += len;
 	    }
 	  release_attach_mutex ();
+	  if (need_strip)
+	    total_read = strip_inrec (input_rec, total_read);
 	  break;
 	case WAIT_TIMEOUT:
 	  con.num_processed = 0;
@@ -607,6 +645,8 @@ remove_record:
 	      acquire_attach_mutex (mutex_timeout);
 	      PeekConsoleInputW (p->input_handle, input_tmp, inrec_size, &n);
 	      release_attach_mutex ();
+	      if (need_strip)
+		n = strip_inrec (input_tmp, n);
 	      if (n < min (total_read, inrec_size))
 		break; /* Someone has read input without acquiring
 			  input_mutex. ConEmu cygwin-connector? */
@@ -625,6 +665,8 @@ remove_record:
 		  n += len;
 		}
 	      release_attach_mutex ();
+	      if (need_strip)
+		n = strip_inrec (input_tmp, n);
 	      bool fixed = false;
 	      for (DWORD ofs = n - total_read; ofs > 0; ofs--)
 		{
-- 
2.51.0