Excessive thrashing when popen()ing after a large malloc()

[Loic Grenie <grenie at matapp dot unimib dot it>]

   The subjects says it all: when a process has a large memory space,
  a popen() triggers a long disk thrashing. The result can clearly be
  seen iwth the allegated cygtharsh program (running with 1GB of memory,
  change the size of the malloc to roughly half your memory size):

% ./cygthrash
  0.000 Before std
 48.318 After std
 67.654 Result: Tue Nov 14 11:29:03     2006
 67.700 After close std
  0.000 Before new
  0.720 After new
  1.106 Result: Tue Nov 14 11:29:04     2006
  1.107 After close new
  0.000 Before std
122.989 After std
131.007 Result: Tue Nov 14 11:31:15     2006
131.008 After close std

  Namely:  48 seconds for the first  popen
           19                        fgets
          123                 second popen
            8                        fgets

   The _popen function is a proposition for popen. Namely, I suggest to
  change in the popen function the lines:


        switch (pid = vfork()) {
        case -1:                        /* Error. */
                (void)close(pdes[0]);
                (void)close(pdes[1]);
                free(cur);
                return (NULL);
                /* NOTREACHED */
        case 0:                         /* Child. */
                if (*type == 'r') {
                        if (pdes[1] != STDOUT_FILENO) {
                                (void)dup2(pdes[1], STDOUT_FILENO);
                                (void)close(pdes[1]);
                        }
                        if (pdes[0] != STDOUT_FILENO) {
                                (void) close(pdes[0]);
                        }
                } else {
                        if (pdes[0] != STDIN_FILENO) {
                                (void)dup2(pdes[0], STDIN_FILENO);
                                (void)close(pdes[0]);
                        }
                        (void)close(pdes[1]);
                }
                execl(_PATH_BSHELL, "sh", "-c", program, NULL);
#ifdef __CYGWIN__
                /* On cygwin32, we may not have /bin/sh.  In that
                   case, try to find sh on PATH.  */
                execlp("sh", "sh", "-c", program, NULL);
#endif
                _exit(127);
                /* NOTREACHED */
        }


  to something similar to (warning: untested, needs a char *cmd; at the
  beggining):


  if (cmd = malloc(strlen(program)+64)) == NULL) {
    (void)close(pdes[1]);
    (void)close(pdes[2]);
    free(cur);
    return (NULL);
  }
  if (*type == 'r') {
    if (pdes[1] != STDOUT_FILENO)
      sprintf(cmd, "sh -c '%s' >&%d %d>&-", program, pdes[1], pdes[1]);
    else
      sprintf(cmd, "sh -c '%s'", program);
    if (pdes[0] != STDOUT_FILENO)
      sprintf(cmd+strlen(cmd), " %d>&-", pdes[0]);
  }
  else {
    if (pdes[0] != STDIN_FILENO)
      sprintf(cmd, "sh -c '%s' <&%d %d>&- %d>&-", program,
              pdes[0], pdes[0], pdes[1]);
    else
      sprintf(cmd, "sh -c '%s' %d>&-", program, pdes[1])
  }
  pid = spawnl(_P_NOWAIT, _PATH_BSHELL, "sh", "-c", cmd, NULL);


    With lots of thanks from the community who made that happen,

        Loïc

#include <stdlib.h>
#include <stdio.h>
#include <signal.h>
#include <time.h>
#include <sys/time.h>


/*******START*******/
#include <process.h>
#include <string.h>
#include <unistd.h>
static int newpid;
static FILE *
_popen(char *cmd, char *mode)
{
  int fd[2];

  if (!pipe(fd)) {
    char *str;

    str = malloc(strlen(cmd)+64);
    if (*mode == 'r')
      sprintf(str, "%s >&%d %d>&-", cmd, fd[1], fd[1]);
    else
      sprintf(str, "%s <&%d %d>&-", cmd, fd[0], fd[0]);
    newpid=spawnlp(_P_NOWAIT, "sh", "sh", "-c", str, NULL);
    free(str);
    if (*mode == 'r') {
        close(fd[1]);
        return fdopen(fd[0], "r");
    }
    else {
        close(fd[0]);
        return fdopen(fd[1], "w");
    }
  }
  return NULL;
}
/******* END *******/

#define GET(tc)		gettimeofday(&tc, NULL);	\
			if (tc.tv_usec < ts.tv_usec) {	\
			    tc.tv_usec += 1000000;	\
			    tc.tv_sec -= 1;		\
			}				\
			tc.tv_sec  -= ts.tv_sec;	\
			tc.tv_usec -= ts.tv_usec

int
main()
{
    FILE *f;
    char buf[1024];
    struct timeval ts, tc;

    malloc(1<<29);	/* 512 MB */

    gettimeofday(&ts, NULL);
    printf("%3d.%03d Before std\n", 0, 0);
    f = popen("date", "r");
    GET(tc);
    printf("%3d.%03d After std\n", (int)tc.tv_sec, (int)tc.tv_usec/1000);
    fgets(buf, sizeof buf, f);
    GET(tc);
    printf("%3d.%03d Result: %s", (int)tc.tv_sec, (int)tc.tv_usec/1000, buf);
    pclose(f);
    GET(tc);
    printf("%3d.%03d After close std\n", (int)tc.tv_sec, (int)tc.tv_usec/1000);

    gettimeofday(&ts, NULL);
    printf("%3d.%03d Before new\n", 0, 0);
    f = _popen("date", "r");
    GET(tc);
    printf("%3d.%03d After new\n", (int)tc.tv_sec, (int)tc.tv_usec/1000);
    fgets(buf, sizeof buf, f);
    GET(tc);
    printf("%3d.%03d Result: %s", (int)tc.tv_sec, (int)tc.tv_usec/1000, buf);
    kill(newpid, SIGTERM);
    fclose(f);
    kill(newpid, SIGKILL);
    GET(tc);
    printf("%3d.%03d After close new\n", (int)tc.tv_sec, (int)tc.tv_usec/1000);

    gettimeofday(&ts, NULL);
    printf("%3d.%03d Before std\n", 0, 0);
    f = popen("date", "r");
    GET(tc);
    printf("%3d.%03d After std\n", (int)tc.tv_sec, (int)tc.tv_usec/1000);
    fgets(buf, sizeof buf, f);
    GET(tc);
    printf("%3d.%03d Result: %s", (int)tc.tv_sec, (int)tc.tv_usec/1000, buf);
    pclose(f);
    GET(tc);
    printf("%3d.%03d After close std\n", (int)tc.tv_sec, (int)tc.tv_usec/1000);
    exit(0);
}

Attachment: cygcheck.out
Description: Text document

--
Unsubscribe info:      http://cygwin.com/ml/#unsubscribe-simple
Problem reports:       http://cygwin.com/problems.html
Documentation:         http://cygwin.com/docs.html
FAQ:                   http://cygwin.com/faq/