Cygwin Filesystem Performance degradation 1.7.5 vs 1.7.7, and methods for improving performance

Corinna Vinschen corinna-cygwin@cygwin.com
Thu Oct 7 11:50:00 GMT 2010


On Oct  7 12:57, Corinna Vinschen wrote:
> On Oct  7 08:51, Yoni Londner wrote:
> > This improved the performance of stat significally, without changing
> > the behaviour.
> > 
> > FROM:
> > /bin$ time ls -l > /dev/null
> > real    0m3.875s
> > user    0m0.108s
> > sys     0m0.452s
> > 
> > TO:
> > /bin$ time ls -l > /dev/null
> > real    0m0.562s
> > user    0m0.171s
> > sys     0m0.296s
> 
> I still suspect that you have some different trouble on your machine.
> 
> Yes, opening files without FILE_READ_DATA (or GENERIC_READ) might be
> actually faster and opening files always with GENERIC_READ in the first
> place might have been a bad idea.
> 
> But a slow down by a factor of 6 is something I can't reproduce at all.
> Neither on XP nor on W7.

Actually, I can't reproduce it at all.  I just created a patch which
drops the GENERIC_READ and only opens the symlinks with it, just as
proposed.  The results between the two versions are statistically
identical.

What's really strange is the fact that the non-GENERIC_READ version is
noticable *slower* when accessing a Samba share.  I checked the access
with sysinternal's Procmon, but apart from the different acesss flags,
eveything else is identical.  Very strange, that.

I didn't apply the patch for now, just attached it here.  Maybe I did
something wrong?!?


Corinna

	* fhandler.h (fhandler_base::get_stat_access): Delete.
	* fhandler_disk_file.cc (fhandler_base::fstat_helper): Always check
	executable suffix to get x-bits for .exe files also in notexec case.
	Always reopen file when checking for file header.
	* ntdll.h (wait_pending): Delete.
	* path.cc (symlink_info::check_shortcut): Drop call to wait_pending
	since file is always opened for sync IO.
	(symlink_info::check_sysfile): Ditto.
	(MIN_STAT_ACCESS): Remove.
	(FULL_STAT_ACCESS): Remove.
	(symlink_info::check): Drop access flag.  Revert to open file with
	just read attributes access.  Reorder symlink check to check for
	reparse points first.  Don't check reparse points for anything else,
	even on remote drives.  Open file for GENERIC_READ when trying to
	read shortcuts or system-bit symlinks. Accommodate dropped access
	flag in call to path_conv_handle::set.
	* path.h (class path_conv_handle): Drop access flag and accommodate
	all related methods.


Index: fhandler.h
===================================================================
RCS file: /cvs/src/src/winsup/cygwin/fhandler.h,v
retrieving revision 1.406
diff -u -p -r1.406 fhandler.h
--- fhandler.h	24 Sep 2010 16:22:53 -0000	1.406
+++ fhandler.h	7 Oct 2010 11:42:15 -0000
@@ -198,7 +198,6 @@ class fhandler_base
 
   int get_access () const { return access; }
   void set_access (int x) { access = x; }
-  int get_stat_access () const { return pc.handle () ? pc.access () : access; }
 
   int get_flags () { return openflags; }
   void set_flags (int x, int supplied_bin = 0);
Index: fhandler_disk_file.cc
===================================================================
RCS file: /cvs/src/src/winsup/cygwin/fhandler_disk_file.cc,v
retrieving revision 1.344
diff -u -p -r1.344 fhandler_disk_file.cc
--- fhandler_disk_file.cc	2 Oct 2010 19:03:44 -0000	1.344
+++ fhandler_disk_file.cc	7 Oct 2010 11:42:15 -0000
@@ -591,7 +591,7 @@ fhandler_base::fstat_helper (struct __st
 	{
 	  buf->st_mode |= S_IFREG;
 	  /* Check suffix for executable file. */
-	  if (pc.exec_state () == dont_know_if_executable)
+	  if (pc.exec_state () != is_executable)
 	    {
 	      PUNICODE_STRING path = pc.get_nt_native_path ();
 
@@ -604,34 +604,28 @@ fhandler_base::fstat_helper (struct __st
 	     shebang scripts. */
 	  if (pc.exec_state () == dont_know_if_executable)
 	    {
-	      LARGE_INTEGER off = { QuadPart:0LL };
-	      char magic[3];
+	      OBJECT_ATTRIBUTES attr;
 	      NTSTATUS status = 0;
 	      IO_STATUS_BLOCK io;
-	      bool opened = false;
 
-	      if (h == get_handle ())
-		{
-		  /* We have been opened via fstat.  We have to re-open the
-		     file.  Either the file is not opened for reading, or the
-		     read will change the file position. */
-		  OBJECT_ATTRIBUTES attr;
-		  pc.init_reopen_attr (&attr, h);
-		  status = NtOpenFile (&h, SYNCHRONIZE | FILE_READ_DATA,
-				       &attr, &io, FILE_SHARE_VALID_FLAGS,
-				       FILE_OPEN_FOR_BACKUP_INTENT
-				       | FILE_OPEN_REPARSE_POINT);
-		  if (!NT_SUCCESS (status))
-		    debug_printf ("%p = NtOpenFile(%S)", status,
-				  pc.get_nt_native_path ());
-		  else
-		    opened = true;
-		}
-	      if (NT_SUCCESS (status))
+	      /* We have to re-open the file.  Either the file is not opened
+	      	 for reading, or the read will change the file position of the
+		 original handle. */
+	      pc.init_reopen_attr (&attr, h);
+	      status = NtOpenFile (&h, SYNCHRONIZE | FILE_READ_DATA,
+				   &attr, &io, FILE_SHARE_VALID_FLAGS,
+				   FILE_OPEN_FOR_BACKUP_INTENT
+				   | FILE_SYNCHRONOUS_IO_NONALERT);
+	      if (!NT_SUCCESS (status))
+		debug_printf ("%p = NtOpenFile(%S)", status,
+			      pc.get_nt_native_path ());
+	      else
 		{
+		  LARGE_INTEGER off = { QuadPart:0LL };
+		  char magic[3];
+
 		  status = NtReadFile (h, NULL, NULL, NULL,
 				       &io, magic, 3, &off, NULL);
-		  status = wait_pending (status, h, io);
 		  if (!NT_SUCCESS (status))
 		    debug_printf ("%p = NtReadFile(%S)", status,
 				  pc.get_nt_native_path ());
@@ -641,9 +635,8 @@ fhandler_base::fstat_helper (struct __st
 		      pc.set_exec ();
 		      buf->st_mode |= STD_XBITS;
 		    }
+		  NtClose (h);
 		}
-	      if (opened)
-		NtClose (h);
 	    }
 	}
       if (pc.exec_state () == is_executable)
Index: ntdll.h
===================================================================
RCS file: /cvs/src/src/winsup/cygwin/ntdll.h,v
retrieving revision 1.104
diff -u -p -r1.104 ntdll.h
--- ntdll.h	24 Sep 2010 12:41:33 -0000	1.104
+++ ntdll.h	7 Oct 2010 11:42:16 -0000
@@ -879,16 +879,6 @@ typedef enum _EVENT_INFORMATION_CLASS
 #define NtCurrentProcess() ((HANDLE) 0xffffffff)
 #define NtCurrentThread()  ((HANDLE) 0xfffffffe)
 
-/* Helper macro for sync I/O with async handle. */
-inline NTSTATUS
-wait_pending (NTSTATUS status, HANDLE h, IO_STATUS_BLOCK &io)
-{
-  if (status != STATUS_PENDING)
-    return status;
-  WaitForSingleObject (h, INFINITE);
-  return io.Status;
-}
-
 extern "C"
 {
   NTSTATUS NTAPI NtAdjustPrivilegesToken (HANDLE, BOOLEAN, PTOKEN_PRIVILEGES,
Index: path.cc
===================================================================
RCS file: /cvs/src/src/winsup/cygwin/path.cc,v
retrieving revision 1.615
diff -u -p -r1.615 path.cc
--- path.cc	2 Oct 2010 19:03:44 -0000	1.615
+++ path.cc	7 Oct 2010 11:42:16 -0000
@@ -1752,7 +1752,6 @@ symlink_info::check_shortcut (HANDLE h)
     buf = (char *) alloca (fsi.EndOfFile.LowPart + 1);
   status = NtReadFile (h, NULL, NULL, NULL, &io, buf, fsi.EndOfFile.LowPart,
 		       &off, NULL);
-  status = wait_pending (status, h, io);
   if (!NT_SUCCESS (status))
     {
       if (status != STATUS_END_OF_FILE)
@@ -1817,7 +1816,6 @@ symlink_info::check_sysfile (HANDLE h)
 
   status = NtReadFile (h, NULL, NULL, NULL, &io, cookie_buf,
 		       sizeof (cookie_buf), &off, NULL);
-  status = wait_pending (status, h, io);
   if (!NT_SUCCESS (status))
     {
       debug_printf ("ReadFile1 failed %p", status);
@@ -1851,7 +1849,6 @@ symlink_info::check_sysfile (HANDLE h)
     {
       status = NtReadFile (h, NULL, NULL, NULL, &io, srcbuf,
 			   NT_MAX_PATH, &off, NULL);
-      status = wait_pending (status, h, io);
       if (!NT_SUCCESS (status))
 	{
 	  debug_printf ("ReadFile2 failed");
@@ -2257,10 +2254,6 @@ restart:
   PVOID eabuf = &nfs_aol_ffei;
   ULONG easize = sizeof nfs_aol_ffei;
 
-# define MIN_STAT_ACCESS	(READ_CONTROL | FILE_READ_ATTRIBUTES)
-# define FULL_STAT_ACCESS	(SYNCHRONIZE | GENERIC_READ)
-  ACCESS_MASK access = 0;
-
   bool had_ext = !!*ext_here;
   while (suffix.next ())
     {
@@ -2278,23 +2271,14 @@ restart:
 	 symlink (which would spoil the task of this method quite a bit).
 	 Fortunately it's ignored on most other file systems so we don't have
 	 to special case NFS too much. */
-      status = NtCreateFile (&h, access = FULL_STAT_ACCESS, &attr, &io, NULL,
-			     0, FILE_SHARE_VALID_FLAGS, FILE_OPEN,
+      status = NtCreateFile (&h,
+			     READ_CONTROL | FILE_READ_ATTRIBUTES | FILE_READ_EA,
+			     &attr, &io, NULL, 0, FILE_SHARE_VALID_FLAGS,
+			     FILE_OPEN,
 			     FILE_OPEN_REPARSE_POINT
 			     | FILE_OPEN_FOR_BACKUP_INTENT,
 			     eabuf, easize);
-      if (status == STATUS_ACCESS_DENIED && eabuf)
-	{
-	  status = NtCreateFile (&h, access = MIN_STAT_ACCESS | FILE_READ_EA,
-				 &attr, &io, NULL, 0, FILE_SHARE_VALID_FLAGS,
-				 FILE_OPEN,
-				 FILE_OPEN_REPARSE_POINT
-				 | FILE_OPEN_FOR_BACKUP_INTENT,
-				 eabuf, easize);
-	  debug_printf ("%p = NtCreateFile (2:%S)", status, &upath);
-	}
-      else
-	debug_printf ("%p = NtCreateFile (1:%S)", status, &upath);
+      debug_printf ("%p = NtCreateFile (%S)", status, &upath);
       /* No right to access EAs or EAs not supported? */
       if (!NT_SUCCESS (status)
 	  && (status == STATUS_ACCESS_DENIED
@@ -2314,20 +2298,11 @@ restart:
 	      eabuf = NULL;
 	      easize = 0;
 	    }
-	  status = NtOpenFile (&h, access = FULL_STAT_ACCESS, &attr, &io,
-			       FILE_SHARE_VALID_FLAGS,
+	  status = NtOpenFile (&h, READ_CONTROL | FILE_READ_ATTRIBUTES,
+			       &attr, &io, FILE_SHARE_VALID_FLAGS,
 			       FILE_OPEN_REPARSE_POINT
 			       | FILE_OPEN_FOR_BACKUP_INTENT);
-	  if (status == STATUS_ACCESS_DENIED)
-	    {
-	      status = NtOpenFile (&h, access = MIN_STAT_ACCESS, &attr, &io,
-				   FILE_SHARE_VALID_FLAGS,
-				   FILE_OPEN_REPARSE_POINT
-				   | FILE_OPEN_FOR_BACKUP_INTENT);
-	      debug_printf ("%p = NtOpenFile (no-EAs 2:%S)", status, &upath);
-	    }
-	  else
-	    debug_printf ("%p = NtOpenFile (no-EA 1:%S)", status, &upath);
+	  debug_printf ("%p = NtOpenFile (no-EAs %S)", status, &upath);
 	}
       if (status == STATUS_OBJECT_NAME_NOT_FOUND)
 	{
@@ -2559,19 +2534,66 @@ restart:
 
       res = -1;
 
+      /* Reparse points are potentially symlinks.  This check must be
+	 performed before checking the SYSTEM attribute for sysfile
+	 symlinks, since reparse points can have this flag set, too.
+	 For instance, Vista starts to create a couple of reparse points
+	 with SYSTEM and HIDDEN flags set. */
+      if ((fileattr & FILE_ATTRIBUTE_REPARSE_POINT))
+	{
+	  /* Don't check reparse points on remote filesystems.  A reparse point
+	     pointing to another file on the remote system will be mistreated
+	     as pointing to a local file on the local system.  This breaks the
+	     way reparse points are transparently handled on remote systems. */
+	  if (fs.is_remote_drive())
+	    res = 0;
+	  else
+	    res = check_reparse_point (h);
+	  if (res == -1)
+	    {
+	      /* Volume mount point.  The filesystem information for the top
+		 level directory should be for the volume top level directory,
+		 rather than for the reparse point itself.  So we fetch the
+		 filesystem information again, but with a NULL handle.
+		 This does what we want because fs_info::update opens the
+		 handle without FILE_OPEN_REPARSE_POINT. */
+	      fs.update (&upath, NULL);
+	      /* Make sure the open handle is not used in later stat calls.
+	         The handle has been opened with the FILE_OPEN_REPARSE_POINT
+		 flag, so it's a handle to the reparse point, not a handle
+		 to the volumes root dir. */
+	      pflags &= ~PC_KEEP_HANDLE;
+	    }
+	  else if (res)
+	    {
+	      /* A symlink is never a directory. */
+	      conv_hdl.fnoi ()->FileAttributes &= ~FILE_ATTRIBUTE_DIRECTORY;
+	      break;
+	    }
+	}
+
       /* Windows shortcuts are potentially treated as symlinks.  Valid Cygwin
 	 & U/WIN shortcuts are R/O, but definitely not directories. */
-      if ((fileattr & (FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_DIRECTORY))
+      else if ((fileattr & (FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_DIRECTORY))
 	  == FILE_ATTRIBUTE_READONLY && suffix.lnk_match ())
 	{
-	  if (!(access & GENERIC_READ))
+	  HANDLE sym_h;
+
+	  status = NtOpenFile (&sym_h, SYNCHRONIZE | GENERIC_READ, &attr, &io,
+			       FILE_SHARE_VALID_FLAGS,
+			       FILE_OPEN_FOR_BACKUP_INTENT
+			       | FILE_SYNCHRONOUS_IO_NONALERT);
+	  if (!NT_SUCCESS (status))
 	    res = 0;
 	  else
-	    res = check_shortcut (h);
+	    {
+	      res = check_shortcut (sym_h);
+	      NtClose (sym_h);
+	    }
 	  if (!res)
 	    {
-	      /* If searching for `foo' and then finding a `foo.lnk' which is
-		 no shortcut, return the same as if file not found. */
+	      /* If searching for `foo' and then finding a `foo.lnk' which
+		 is no shortcut, return the same as if file not found. */
 	      if (ext_tacked_on)
 		{
 		  fileattr = INVALID_FILE_ATTRIBUTES;
@@ -2593,53 +2615,26 @@ restart:
 	  continue;
 	}
 
-      /* Reparse points are potentially symlinks.  This check must be
-	 performed before checking the SYSTEM attribute for sysfile
-	 symlinks, since reparse points can have this flag set, too.
-	 For instance, Vista starts to create a couple of reparse points
-	 with SYSTEM and HIDDEN flags set.
-	 Also don't check reparse points on remote filesystems.
-	 A reparse point pointing to another file on the remote system will be
-	 mistreated as pointing to a local file on the local system.  This
-	 breaks the way reparse points are transparently handled on remote
-	 systems. */
-      else if ((fileattr & FILE_ATTRIBUTE_REPARSE_POINT)
-	       && !fs.is_remote_drive())
-	{
-	  res = check_reparse_point (h);
-	  if (res == -1)
-	    {
-	      /* Volume mount point.  The filesystem information for the top
-		 level directory should be for the volume top level directory,
-		 rather than for the reparse point itself.  So we fetch the
-		 filesystem information again, but with a NULL handle.
-		 This does what we want because fs_info::update opens the
-		 handle without FILE_OPEN_REPARSE_POINT. */
-	      fs.update (&upath, NULL);
-	      /* Make sure the open handle is not used in later stat calls.
-	         The handle has been opened with the FILE_OPEN_REPARSE_POINT
-		 flag, so it's a handle to the reparse point, not a handle
-		 to the volumes root dir. */
-	      pflags &= ~PC_KEEP_HANDLE;
-	    }
-	  else if (res)
-	    {
-	      /* A symlink is never a directory. */
-	      conv_hdl.fnoi ()->FileAttributes &= ~FILE_ATTRIBUTE_DIRECTORY;
-	      break;
-	    }
-	}
-
       /* This is the old Cygwin method creating symlinks.  A symlink will
 	 have the `system' file attribute.  Only files can be symlinks
 	 (which can be symlinks to directories). */
       else if ((fileattr & (FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_DIRECTORY))
 	       == FILE_ATTRIBUTE_SYSTEM)
 	{
-	  if (!(access & GENERIC_READ))
+	  HANDLE sym_h;
+
+	  status = NtOpenFile (&sym_h, SYNCHRONIZE | GENERIC_READ, &attr, &io,
+			       FILE_SHARE_VALID_FLAGS,
+			       FILE_OPEN_FOR_BACKUP_INTENT
+			       | FILE_SYNCHRONOUS_IO_NONALERT);
+
+	  if (!NT_SUCCESS (status))
 	    res = 0;
 	  else
-	    res = check_sysfile (h);
+	    {
+	      res = check_sysfile (sym_h);
+	      NtClose (sym_h);
+	    }
 	  if (res)
 	    break;
 	}
@@ -2665,7 +2660,7 @@ restart:
   if (h)
     {
       if (pflags & PC_KEEP_HANDLE)
-	conv_hdl.set (h, access);
+	conv_hdl.set (h);
       else
 	NtClose (h);
     }
Index: path.h
===================================================================
RCS file: /cvs/src/src/winsup/cygwin/path.h,v
retrieving revision 1.151
diff -u -p -r1.151 path.h
--- path.h	5 Oct 2010 14:19:17 -0000	1.151
+++ path.h	7 Oct 2010 11:42:16 -0000
@@ -95,7 +95,6 @@ struct _FILE_NETWORK_OPEN_INFORMATION;
 class path_conv_handle
 {
   HANDLE      hdl;
-  ACCESS_MASK acc;
   union {
     /* Identical to FILE_NETWORK_OPEN_INFORMATION.  We don't want to pull in
        ntdll.h here, though. */
@@ -112,26 +111,22 @@ class path_conv_handle
     fattr3 _fattr3;
   } attribs;
 public:
-  path_conv_handle () : hdl (NULL), acc (0) {}
-  inline void set (HANDLE h, ACCESS_MASK a) { hdl = h; acc = a; }
+  path_conv_handle () : hdl (NULL) {}
+  inline void set (HANDLE h) { hdl = h; }
   inline void close ()
   {
     if (hdl)
       CloseHandle (hdl);
-    set (NULL, 0);
+    set (NULL);
   }
   inline void dup (path_conv_handle &pch)
   {
     if (!DuplicateHandle (GetCurrentProcess (), pch.handle (),
 			  GetCurrentProcess (), &hdl,
 			  0, TRUE, DUPLICATE_SAME_ACCESS))
-      {
-	hdl = NULL;
-	acc = 0;
-      }
+      hdl = NULL;
   }
   inline HANDLE handle () const { return hdl; }
-  inline ACCESS_MASK access () const { return acc; }
   inline struct _FILE_NETWORK_OPEN_INFORMATION *fnoi ()
   { return (struct _FILE_NETWORK_OPEN_INFORMATION *) &attribs._fnoi; }
   inline struct fattr3 *nfsattr ()
@@ -325,10 +320,9 @@ class path_conv
   bool is_binary ();
 
   HANDLE handle () const { return conv_handle.handle (); }
-  ACCESS_MASK access () const { return conv_handle.access (); }
   struct _FILE_NETWORK_OPEN_INFORMATION *fnoi () { return conv_handle.fnoi (); }
   struct fattr3 *nfsattr () { return conv_handle.nfsattr (); }
-  void reset_conv_handle () { conv_handle.set (NULL, 0); }
+  void reset_conv_handle () { conv_handle.set (NULL); }
   void close_conv_handle () { conv_handle.close (); }
 
   __ino64_t get_ino_by_handle (HANDLE h);

-- 
Corinna Vinschen                  Please, send mails regarding Cygwin to
Cygwin Project Co-Leader          cygwin AT cygwin DOT com
Red Hat



More information about the Cygwin-developers mailing list