This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[RFC PATCH] Add ix86 support for REQUEUE PI
- From: Dinakar Guniguntala <dino at in dot ibm dot com>
- To: libc-alpha at sources dot redhat dot com
- Cc: Ulrich Drepper <drepper at redhat dot com>, Darren Hart <dvhltc at us dot ibm dot com>, Thomas Gleixner <tglx at linutronix dot de>
- Date: Fri, 14 Aug 2009 19:16:32 +0530
- Subject: [RFC PATCH] Add ix86 support for REQUEUE PI
- Reply-to: dino at in dot ibm dot com
The following patch adds requeue pi support for i386 arch. It closely follows
the x86_64 implementation in current git. This is an RFC as it has atleast one
bug that is yet to be resolved, details below. Appreciate the review !
One important change (it needs to be changed in x86_64 as well) though,
specifically check the return code from the futex requeue pi call for
ENOSYS and only then use the non requeue pi functionality. For other
errors (such as EAGAIN etc), just return back to the application.
This is required as the kernel now treats mixing requeue pi and non requeue
pi APIs as an EINVAL (and rightly so). I'll be sending a separate patch
to modify the x86_64 implementation
The patch is against latest glibc git (13 Aug 2009)
Testing: prio-wake testcase (Part of the LTP realtime testcases) passes
with this patch. However some long running java tests randomly hang.
Attaching and detaching gdb to the hung process results in the process
continuing. I'm suspecting that the internal counters are getting
messed up in an error path somewhere and we are missing a wake,
still debugging this
-Dinakar
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S 2009-08-13 04:13:10.000000000 -0700
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S 2009-08-13 23:47:08.000000000 -0700
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2004, 2006, 2007, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -90,12 +90,14 @@ __pthread_cond_broadcast:
8: cmpl $-1, %edi
je 9f
- /* XXX: The kernel so far doesn't support requeue to PI futex. */
- /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
- type of futex (private resp. shared). */
- testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
+ /* Do not use requeue for pshared condvars. */
+ testl $PS_BIT, MUTEX_KIND(%edi)
jne 9f
+ /* REQUEUE to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edi)
+ jne 81f
+
/* Wake up all threads. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
@@ -140,6 +142,25 @@ __pthread_cond_broadcast:
cfi_offset(%esi, -12)
cfi_offset(%edi, -16)
cfi_offset(%ebp, -20)
+
+81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $0x7fffffff, %esi
+ movl $1, %edx
+ /* Get the address of the futex involved. */
+# if MUTEX_FUTEX != 0
+ addl $MUTEX_FUTEX, %edi
+# endif
+ /* 6 arg syscalls are broken for sysenter. */
+ int $0x80
+
+ /* For any kind of error, which mainly is EAGAIN, we try again
+ with WAKE. The general test also covers running on old
+ kernels. */
+ cmpl $0xfffff001, %eax
+ jb 10b
+ jmp 9f
+
.align 16
/* Unlock. */
4: LOCK
@@ -166,6 +187,7 @@ __pthread_cond_broadcast:
cfi_offset(%esi, -12)
cfi_offset(%edi, -16)
cfi_offset(%ebp, -20)
+
/* Initial locking failed. */
1:
#if cond_lock == 0
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S 2009-08-13 04:13:10.000000000 -0700
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S 2009-08-13 23:47:09.000000000 -0700
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <kernel-features.h>
+#include <pthread-pi-defines.h>
#include <pthread-errnos.h>
@@ -85,7 +86,14 @@ __pthread_cond_signal:
#endif
cmpl $-1, dep_mutex-cond_futex(%ebx)
sete %cl
- subl $1, %ecx
+ je 8f
+
+ movl dep_mutex-cond_futex(%ebx), %edx
+ /* REQUEUE to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edx)
+ jne 9f
+
+8: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -125,9 +133,37 @@ __pthread_cond_signal:
cfi_offset(%ebx, -8)
cfi_offset(%edi, -12)
-7: /* %ecx should be either FUTEX_WAKE_OP or
- FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
- xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
+9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $1, %edx
+ xorl %esi, %esi
+ movl dep_mutex-cond_futex(%ebx), %edi
+ movl (%ebx), %ebp
+ /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
+ sysenter.
+ ENTER_KERNEL */
+ int $0x80
+ popl %ebp
+ cfi_adjust_cfa_offset(-4)
+ cfi_restore(%ebp)
+ popl %esi
+ cfi_adjust_cfa_offset(-4)
+ cfi_restore(%esi)
+
+ leal -cond_futex(%ebx), %edi
+
+ /* For any kind of error, we try again with WAKE.
+ The general test also covers running on old kernels. */
+ cmpl $-4095, %eax
+ jb 4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+ andl $FUTEX_PRIVATE_FLAG, %ecx
+#else
+ andl %gs:PRIVATE_FUTEX, %ecx
+#endif
+ orl $FUTEX_WAKE, %ecx
movl $SYS_futex, %eax
/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
movl $1, %edx */
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S 2009-08-13 04:13:10.000000000 -0700
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S 2009-08-13 23:47:11.000000000 -0700
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2004, 2006, 2007, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -80,7 +81,7 @@ __pthread_cond_timedwait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 24
+#define FRAME_SIZE 32
subl $FRAME_SIZE, %esp
.Lsubl:
@@ -143,6 +144,7 @@ __pthread_cond_timedwait:
movl %edx, 8(%esp)
movl cond_futex(%ebx), %edi
+ movl %edi, 28(%esp)
/* Unlock. */
LOCK
@@ -157,13 +159,48 @@ __pthread_cond_timedwait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
- leal 4(%esp), %esi
+ movl $0, 24(%esp)
+
#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 18f
+
+ movl dep_mutex(%ebx), %edi
+ /* REQUEUE to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edi)
+ je 18f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ /* The following only works like this because we only support
+ two clocks, represented using a single bit. */
+ testl $1, cond_nwaiters(%ebx)
+ /* XXX Need to implement using sete instead of a jump. */
+ jne 44f
+ orl $FUTEX_CLOCK_REALTIME, %ecx
+
+ /* REQUEUE-PI uses absolute timeout. */
+44: leal (%ebp), %esi
+ movl 28(%esp), %edx
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ movl %eax, %esi
+ cmpl $0, %eax
+ /* If REQUEUE-PI succeeded, kernel holds the mutex lock. */
+ sete 24(%esp)
+ je 19f
+
+ /* Try with FUTEX_WAIT if kernel does not support
+ REQUEUE_PI. */
+ cmpl $-ENOSYS, %eax
+ jne 19f
+ xorl %ecx, %ecx
+
+18: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -172,7 +209,8 @@ __pthread_cond_timedwait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ leal 4(%esp), %esi
+ movl 28(%esp), %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -181,7 +219,7 @@ __pthread_cond_timedwait:
.Lsub_cond_futex:
movl %eax, %esi
- movl (%esp), %eax
+19: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -269,10 +307,16 @@ __pthread_cond_timedwait:
#endif
jne 10f
+11: xorl %eax, %eax
+ /* With REQUEUE PI, the mutex lock is held in the kernel. */
+ movl 24(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 26f
+
/* Remove cancellation handler. */
-11: movl 24+FRAME_SIZE(%esp), %eax
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+26: addl $FRAME_SIZE, %esp
.Laddl:
/* We return the result of the mutex_lock operation if it failed. */
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S 2009-08-13 04:13:10.000000000 -0700
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S 2009-08-13 23:47:12.000000000 -0700
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2004, 2006, 2007, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -22,6 +22,8 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <tcb-offsets.h>
+#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -34,6 +36,8 @@
__pthread_cond_wait:
.LSTARTCODE:
+ pushl %ebp
+.Lpush_ebp:
pushl %edi
.Lpush_edi:
pushl %esi
@@ -42,7 +46,7 @@ __pthread_cond_wait:
.Lpush_ebx:
xorl %esi, %esi
- movl 16(%esp), %ebx
+ movl 20(%esp), %ebx
/* Get internal lock. */
movl $1, %edx
@@ -58,7 +62,7 @@ __pthread_cond_wait:
/* Store the reference to the mutex. If there is already a
different value in there this is a bad user bug. */
2: cmpl $-1, dep_mutex(%ebx)
- movl 20(%esp), %eax
+ movl 24(%esp), %eax
je 15f
movl %eax, dep_mutex(%ebx)
@@ -74,7 +78,7 @@ __pthread_cond_wait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 16
+#define FRAME_SIZE 20
subl $FRAME_SIZE, %esp
.Lsubl:
@@ -86,8 +90,8 @@ __pthread_cond_wait:
movl %edx, 8(%esp)
movl %eax, 12(%esp)
-8: movl cond_futex(%ebx), %edi
-
+8: movl cond_futex(%ebx), %ebp
+
/* Unlock. */
LOCK
#if cond_lock == 0
@@ -101,12 +105,38 @@ __pthread_cond_wait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
+ movl $0, 16(%esp)
#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 18f
+
+ movl dep_mutex(%ebx), %edi
+ /* REQUEUE to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edi)
+ je 18f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl %ebp, %edx
+ xorl %esi, %esi
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ cmpl $0, %eax
+ /* If REQUEUE-PI succeeded, kernel holds the mutex lock. */
+ sete 16(%esp)
+ je 19f
+
+ /* Try with FUTEX_WAIT if kernel does not support
+ REQUEUE_PI. */
+ cmpl $-ENOSYS, %eax
+ jne 19f
+ xorl %ecx, %ecx
+
+18: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -115,7 +145,7 @@ __pthread_cond_wait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ movl %ebp, %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -123,7 +153,7 @@ __pthread_cond_wait:
subl $cond_futex, %ebx
.Lsub_cond_futex:
- movl (%esp), %eax
+19: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -199,9 +229,15 @@ __pthread_cond_wait:
#endif
jne 10f
-11: movl 20+FRAME_SIZE(%esp), %eax
+ /* With requeue_pi, the mutex lock is held in the kernel */
+11: xorl %eax, %eax
+ movl 16(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 20f
+
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+20: addl $FRAME_SIZE, %esp
.Laddl:
14: popl %ebx
@@ -210,6 +246,8 @@ __pthread_cond_wait:
.Lpop_esi:
popl %edi
.Lpop_edi:
+ popl %ebp
+.Lpop_ebp:
/* We return the result of the mutex_lock operation. */
ret
@@ -561,7 +599,12 @@ __condvar_w_cleanup:
#else
.long .LexceptSTART
#endif
- .byte 0x40+.Lpush_edi-.LSTARTCODE # DW_CFA_advance_loc+N
+ .byte 0x40+.Lpush_ebp-.LSTARTCODE # DW_CFA_advance_loc+N
+ .byte 14 # DW_CFA_def_cfa_offset
+ .uleb128 8
+ .byte 0x85 # DW_CFA_offset %ebp
+ .uleb128 2
+ .byte 0x40+.Lpush_edi-.Lpush_ebp # DW_CFA_advance_loc+N
.byte 14 # DW_CFA_def_cfa_offset
.uleb128 8
.byte 0x87 # DW_CFA_offset %edi
@@ -596,9 +639,15 @@ __condvar_w_cleanup:
.byte 14 # DW_CFA_def_cfa_offset
.uleb128 4
.byte 0xc7 # DW_CFA_restore %edi
- .byte 0x40+.LSbl1-.Lpop_edi # DW_CFA_advance_loc+N
+ .byte 0x40+.Lpop_ebp-.Lpop_edi # DW_CFA_advance_loc+N
.byte 14 # DW_CFA_def_cfa_offset
- .uleb128 16
+ .uleb128 4
+ .byte 0xc5 # DW_CFA_restore %ebp
+ .byte 0x40+.LSbl1-.Lpop_ebp # DW_CFA_advance_loc+N
+ .byte 14 # DW_CFA_def_cfa_offset
+ .uleb128 20
+ .byte 0x85 # DW_CFA_offset %ebp
+ .uleb128 2
.byte 0x87 # DW_CFA_offset %edi
.uleb128 2
.byte 0x86 # DW_CFA_offset %esi
diff -X ignore -Nurp glibc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
--- glibc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h 2009-08-13 04:13:10.000000000 -0700
+++ glibc.mod/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h 2009-08-13 05:00:32.000000000 -0700
@@ -54,6 +54,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
+#define FUTEX_WAIT_REQUEUE_PI 11
+#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256