[PATCH v4] Cygwin: pty: Fix handling of data after CSI6n response

[Takashi Yano]

Previously, CSI6n was not handled correctly if the some sequences
are appended after the response for CSI6n. Especially, if the
appended sequence is a ESC sequence, which is longer than the
expected maximum length of the CSI6n response, the sequence will
not be written atomically.

Moreover, when the terminal's CSI 6n response and subsequent data
(e.g. keystrokes) arrive in the same write buffer, master::write()
processes all of it inside the pcon_start loop and returns early.
Bytes after the 'R' terminator go through per-byte line_edit() in
that loop instead of falling through to the `nat` pipe fast path
or the normal bulk `line_edit()` call. Due to this behaviour,
the chance of code conversion to the terminal code page for the
subsequent data in `to_be_read_from_nat_pipe()` case, will be lost.

Fix this by breaking out of the loop when 'R' is found and letting
the remaining data fall through to the normal write paths, which
are now reachable because `pcon_start` has been cleared.

Fixes: f20641789427 ("Cygwin: pty: Reduce unecessary input transfer.")
Signed-off-by: Takashi Yano <takashi.yano@nifty.ne.jp>
Co-authored-by: Johannes Schindelin <Johannes.Schindelin@gmx.de>
Reviewed-by: Johannes Schindelin <Johannes.Schindelin@gmx.de>
---
 winsup/cygwin/fhandler/pty.cc | 42 ++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/winsup/cygwin/fhandler/pty.cc b/winsup/cygwin/fhandler/pty.cc
index b59f54096..3717542e6 100644
--- a/winsup/cygwin/fhandler/pty.cc
+++ b/winsup/cygwin/fhandler/pty.cc
@@ -2137,6 +2137,8 @@ fhandler_pty_master::close (int flag)
 ssize_t
 fhandler_pty_master::write (const void *ptr, size_t len)
 {
+  size_t orig_len = len;
+
   ssize_t ret;
   char *p = (char *) ptr;
   termios &ti = tc ()->ti;
@@ -2160,7 +2162,8 @@ fhandler_pty_master::write (const void *ptr, size_t len)
 
       DWORD n;
       WaitForSingleObject (input_mutex, mutex_timeout);
-      for (size_t i = 0; i < len; i++)
+      len = 0;
+      for (size_t i = 0; i < orig_len; i++)
 	{
 	  if (p[i] == '\033')
 	    {
@@ -2185,18 +2188,21 @@ fhandler_pty_master::write (const void *ptr, size_t len)
 	    line_edit (p + i, 1, ti, &ret);
 	  if (state == 1 && p[i] == 'R')
 	    state = 2;
-	}
-      if (state == 2)
-	{
-	  /* req_xfer_input is true if "ESC[6n" was sent just for
-	     triggering transfer_input() in master. In this case,
-	     the responce sequence should not be written. */
-	  if (!get_ttyp ()->req_xfer_input)
-	    WriteFile (to_slave_nat, wpbuf, ixput, &n, NULL);
-	  ixput = 0;
-	  state = 0;
-	  get_ttyp ()->req_xfer_input = false;
-	  get_ttyp ()->pcon_start = false;
+	  if (state == 2)
+	    {
+	      /* req_xfer_input is true if "ESC[6n" was sent just for
+		 triggering transfer_input() in master. In this case,
+		 the response sequence should not be written. */
+	      if (!get_ttyp ()->req_xfer_input)
+		WriteFile (to_slave_nat, wpbuf, ixput, &n, NULL);
+	      len = orig_len - i - 1;
+	      ptr = p + i + 1;
+	      ixput = 0;
+	      state = 0;
+	      get_ttyp ()->req_xfer_input = false;
+	      get_ttyp ()->pcon_start = false;
+	      break;
+	    }
 	}
       ReleaseMutex (input_mutex);
 
@@ -2220,8 +2226,8 @@ fhandler_pty_master::write (const void *ptr, size_t len)
 	    }
 	  get_ttyp ()->pcon_start_pid = 0;
 	}
-
-      return len;
+      if (len == 0)
+	return orig_len;
     }
 
   /* Write terminal input to to_slave_nat pipe instead of output_handle
@@ -2261,7 +2267,7 @@ fhandler_pty_master::write (const void *ptr, size_t len)
 	WriteFile (to_slave_nat, buf, nlen, &n, NULL);
       ReleaseMutex (input_mutex);
 
-      return len;
+      return orig_len;
     }
 
   /* The code path reaches here when pseudo console is not activated
@@ -2283,8 +2289,8 @@ fhandler_pty_master::write (const void *ptr, size_t len)
   ReleaseMutex (input_mutex);
 
   if (status > line_edit_signalled && status != line_edit_pipe_full)
-    ret = -1;
-  return ret;
+    return -1;
+  return orig_len - len + ret;
 }
 
 void
-- 
2.51.0