[PATCH v3 2/6] Cygwin: pty: Add workaround for handling of backspace when pcon enabled

[Takashi Yano]

In Windows 11, pseudo console has a weird behaviour that the Ctrl-H
is translated into Ctrl-Backspace (not Backspace). Similary, Backspace
(0x7f) is translated into Ctrl-H. Due to this behaviour, inrec_eq()
in cons_master_thread() fails to compare backspace/Ctrl-H events in
the input record sequence. This patch is a workaround for the issue
that replaces Ctrl-H with backspace (0x7f), which will be translated
into Ctrl-H in pseudo console.

Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>
Reviewed-by: Johannes Schindelin <Johannes.Schindelin@gmx.de>
---
 winsup/cygwin/fhandler/console.cc | 12 ++++---
 winsup/cygwin/fhandler/pty.cc     | 57 ++++++++++++++++++++++++++-----
 2 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/winsup/cygwin/fhandler/console.cc b/winsup/cygwin/fhandler/console.cc
index a36bbd0e2..2b1b50f0a 100644
--- a/winsup/cygwin/fhandler/console.cc
+++ b/winsup/cygwin/fhandler/console.cc
@@ -324,10 +324,14 @@ inrec_eq (const INPUT_RECORD *a, const INPUT_RECORD *b, DWORD n1, DWORD n2)
 	{
 	  WCHAR c1 = a[i].Event.KeyEvent.uChar.UnicodeChar;
 	  WCHAR c2 = b[j].Event.KeyEvent.uChar.UnicodeChar;
-	  if (c1 == 127) /* Backspace */
-	    c1 = 8; /* Ctrl-H */
-	  if (c2 == 127) /* Backspace */
-	    c2 = 8; /* Ctrl-H */
+	  if (inside_pcon)
+	    {
+	      /* Workaround for backspace behaviour in Windows 11 */
+	      if (c1 == 8)
+		c1 = 127;
+	      if (c2 == 8)
+		c2 = 127;
+	    }
 	  if (c1 != c2)
 	    return false;
 	}
diff --git a/winsup/cygwin/fhandler/pty.cc b/winsup/cygwin/fhandler/pty.cc
index 371e67103..72a8ba140 100644
--- a/winsup/cygwin/fhandler/pty.cc
+++ b/winsup/cygwin/fhandler/pty.cc
@@ -2266,28 +2266,65 @@ fhandler_pty_master::write (const void *ptr, size_t len)
     { /* Reaches here when non-cygwin app is foreground and pseudo console
 	 is activated. */
       tmp_pathbuf tp;
-      char *buf = (char *) ptr;
+      char *buf = tp.c_get ();
       size_t nlen = len;
       if (get_ttyp ()->term_code_page != CP_UTF8)
 	{
 	  static mbstate_t mbp;
-	  buf = tp.c_get ();
 	  nlen = NT_MAX_PATH;
 	  convert_mb_str (CP_UTF8, buf, &nlen,
 			  get_ttyp ()->term_code_page, (const char *) ptr, len,
 			  &mbp);
 	}
+      else
+	memcpy (buf, ptr, nlen);
+
+      /* Retrieve console mode */
+      HANDLE h_pcon_in = get_ttyp ()->h_pcon_in;
+      DWORD cons_mode;
+      if (!nat_pipe_owner_self (get_ttyp ()->nat_pipe_owner_pid))
+	{
+	  HANDLE pcon_owner = OpenProcess (PROCESS_DUP_HANDLE, FALSE,
+					   get_ttyp ()->nat_pipe_owner_pid);
+	  DuplicateHandle (pcon_owner, h_pcon_in,
+			   GetCurrentProcess (), &h_pcon_in,
+			   0, FALSE, DUPLICATE_SAME_ACCESS);
+	  CloseHandle(pcon_owner);
+	  DWORD resume_pid =
+	    attach_console_temporarily (get_ttyp()->nat_pipe_owner_pid);
+	  GetConsoleMode (h_pcon_in, &cons_mode);
+	  resume_from_temporarily_attach (resume_pid);
+	  CloseHandle (h_pcon_in);
+	}
+      else
+	GetConsoleMode (h_pcon_in, &cons_mode);
 
-      for (size_t i = 0; i < nlen; i++)
+      for (size_t i = 0, j = 0; i < nlen; i++)
 	{
 	  process_sig_state r = process_sigs (buf[i], get_ttyp (), this);
-	  if (r == done_with_debugger)
+	  if (r != done_with_debugger)
 	    {
-	      for (size_t j = i; j < nlen - 1; j++)
-		buf[j] = buf[j + 1];
-	      nlen--;
-	      i--;
+	      char c = buf[i];
+	      if (!(cons_mode & ENABLE_VIRTUAL_TERMINAL_INPUT))
+		/* Workaround for pseudo console in Windows 11 */
+		/* Undesired backspace conversion in pseudo console does
+		   not happen if ENABLE_VIRTUAL_TERMINAL_INPUT is set. */
+		switch (c)
+		  {
+		  case '\010': /* Ctrl-H */
+		    c = '\177'; /* Backspace */
+		    break;
+		  case '\177': /* Backspace */
+#if 0 /* Unfortunately, Ctrl-H will be translated into Ctrl-Backspace
+	 (not Backspace) */
+		    c = '\010'; /* Ctrl-H */
+#endif
+		    break;
+		  }
+	      buf[j++] = c;
 	    }
+	  else
+	    nlen--;
 	}
 
       DWORD n;
@@ -4031,6 +4068,10 @@ fhandler_pty_slave::transfer_input (tty::xfer_dir dir, HANDLE from, tty *ttyp,
 	    if (r[i].EventType == KEY_EVENT && r[i].Event.KeyEvent.bKeyDown)
 	      {
 		DWORD ctrl_key_state = r[i].Event.KeyEvent.dwControlKeyState;
+		if (r[i].Event.KeyEvent.uChar.AsciiChar == '\010' /* Ctrl-H */
+		    && !(ctrl_key_state & ALT_PRESSED))
+		  /* Workaround for pseudo console in Windows 11 */
+		  r[i].Event.KeyEvent.uChar.AsciiChar = '\177'; /* Backspace */
 		if (r[i].Event.KeyEvent.uChar.AsciiChar)
 		  {
 		    if ((ctrl_key_state & ALT_PRESSED)
-- 
2.51.0