[PATCH] Fix memusage

[Jakub Jelinek <jakub at redhat dot com>]

Hi!

memusage does not work with any threaded program.
The problem is that malloc or calloc call dlsym, which in turn (through
_dlerror_run) call calloc and thus end up in endless
recursion.
Fortunately, in _dlerror_run nothing bad happens if calloc returns NULL,
because 1) dlsym (RTLD_NEXT, "") will succeed, there are at least the alloc
routines in glibc and 2) no threads are running yet (because
pthread_initialize_manager calls malloc) and thus the static area can be
used without problems.
I tried to use a static variable for the first thread instead in
dlerror.c (init), but it did not help - because pthread_setspecific calls
calloc as well.

2001-08-07  Jakub Jelinek  <jakub@redhat.com>

	* malloc/memusage.c (initializing): New variable.
	(malloc): Set/clear it. Initialize mallocp and callocp together.
	(calloc): Likewise. If initializing, return NULL.

--- libc/malloc/memusage.c.jj	Fri Jul  6 00:55:35 2001
+++ libc/malloc/memusage.c	Tue Aug  7 08:40:35 2001
@@ -82,6 +82,7 @@ static size_t buffer_size;
 static int fd = -1;
 
 static int not_me;
+static int initializing;
 extern const char *__progname;
 
 struct entry
@@ -264,7 +265,10 @@ malloc (size_t len)
   if (mallocp == NULL)
     {
       me ();
+      initializing = 1;
       mallocp = (void *(*) (size_t)) dlsym (RTLD_NEXT, "malloc");
+      callocp = (void *(*) (size_t, size_t)) dlsym (RTLD_NEXT, "calloc");
+      initializing = 0;
     }
 
   /* If this is not the correct program just use the normal function.  */
@@ -385,17 +389,23 @@ calloc (size_t n, size_t len)
      room to the allocation to put the header in.  */
   if (mallocp == NULL)
     {
+      /* dlsym may call calloc, in which case we'd end up in endless
+	 recursion. No threads should be running at this point, since
+	 pthread_initialize_manager calls malloc. If calloc returns
+	 NULL in _dlerror_run, a static buffer will be used which is
+	 fine, since no threads are running yet.  */
+      if (initializing)
+	return NULL;
       me ();
+      initializing = 1;
       mallocp = (void *(*) (size_t)) dlsym (RTLD_NEXT, "malloc");
+      callocp = (void *(*) (size_t, size_t)) dlsym (RTLD_NEXT, "calloc");
+      initializing = 0;
     }
 
   /* If this is not the correct program just use the normal function.  */
   if (not_me)
-    {
-      callocp = (void *(*) (size_t, size_t)) dlsym (RTLD_NEXT, "calloc");
-
-      return (*callocp) (n, len);
-    }
+    return (*callocp) (n, len);
 
   /* Keep track of number of calls.  */
   ++calls[idx_calloc];

	Jakub