[PATCH 3/4] s390: Reorganize assembly

[Richard Henderson <rth at twiddle dot net>]

From: Richard Henderson <rth@redhat.com>

Avoid using ffi_prep_args as a callback; do all the work setting
up the frame within ffi_call_int directly.  Save fewer registers
in ffi_closure_SYSV.
---
 src/s390/ffi.c      | 153 +++++++-------
 src/s390/internal.h |  11 +
 src/s390/sysv.S     | 578 ++++++++++++++++++----------------------------------
 3 files changed, 286 insertions(+), 456 deletions(-)
 create mode 100644 src/s390/internal.h

diff --git a/src/s390/ffi.c b/src/s390/ffi.c
index c06b79e..1189f7b 100644
--- a/src/s390/ffi.c
+++ b/src/s390/ffi.c
@@ -30,9 +30,7 @@
 
 #include <ffi.h>
 #include <ffi_common.h>
-
-#include <stdlib.h>
-#include <stdio.h>
+#include "internal.h"
 
 /*====================== End of Includes =============================*/
 
@@ -54,14 +52,6 @@
 /* Round to multiple of 16.  */
 #define ROUND_SIZE(size) (((size) + 15) & ~15)
 
-/* If these values change, sysv.S must be adapted!  */
-#define FFI390_RET_VOID		0
-#define FFI390_RET_STRUCT	1
-#define FFI390_RET_FLOAT	2
-#define FFI390_RET_DOUBLE	3
-#define FFI390_RET_INT32	4
-#define FFI390_RET_INT64	5
-
 /*===================== End of Defines ===============================*/
 
 /*====================================================================*/
@@ -69,12 +59,17 @@
 /*                          ---------                                 */
 /*====================================================================*/
 
-extern void ffi_call_SYSV(unsigned,
-			  extended_cif *,
-			  void (*)(unsigned char *, extended_cif *),
-			  unsigned,
-			  void *,
-			  void (*fn)(void), void *);
+struct call_frame
+{
+  void *back_chain;
+  void *eos;
+  unsigned long gpr_args[5];
+  unsigned long gpr_save[9];
+  unsigned long long fpr_args[4];
+};
+
+extern void FFI_HIDDEN ffi_call_SYSV(struct call_frame *, unsigned, void *,
+			             void (*fn)(void), void *);
 
 extern void ffi_closure_SYSV(void);
 extern void ffi_go_closure_SYSV(void);
@@ -145,54 +140,29 @@ ffi_check_struct_type (ffi_type *arg)
 /*====================================================================*/
 
 static void
-ffi_prep_args (unsigned char *stack, extended_cif *ecif)
+ffi_prep_args (ffi_cif *cif, void *rvalue, void **p_argv,
+	       unsigned long *p_ov, struct call_frame *p_frame)
 {
-  /* The stack space will be filled with those areas:
-
-	FPR argument register save area     (highest addresses)
-	GPR argument register save area
-	temporary struct copies
-	overflow argument area              (lowest addresses)
-
-     We set up the following pointers:
-
-        p_fpr: bottom of the FPR area (growing upwards)
-	p_gpr: bottom of the GPR area (growing upwards)
-	p_ov: bottom of the overflow area (growing upwards)
-	p_struct: top of the struct copy area (growing downwards)
-
-     All areas are kept aligned to twice the word size.  */
-
-  int gpr_off = ecif->cif->bytes;
-  int fpr_off = gpr_off + ROUND_SIZE (MAX_GPRARGS * sizeof (long));
-
-  unsigned long long *p_fpr = (unsigned long long *)(stack + fpr_off);
-  unsigned long *p_gpr = (unsigned long *)(stack + gpr_off);
-  unsigned char *p_struct = (unsigned char *)p_gpr;
-  unsigned long *p_ov = (unsigned long *)stack;
-
+  unsigned char *p_struct = (unsigned char *)p_frame;
+  unsigned long *p_gpr = p_frame->gpr_args;
+  unsigned long long *p_fpr = p_frame->fpr_args;
   int n_fpr = 0;
   int n_gpr = 0;
   int n_ov = 0;
-
   ffi_type **ptr;
-  void **p_argv = ecif->avalue;
   int i;
 
   /* If we returning a structure then we set the first parameter register
      to the address of where we are returning this structure.  */
-
-  if (ecif->cif->flags == FFI390_RET_STRUCT)
-    p_gpr[n_gpr++] = (unsigned long) ecif->rvalue;
+  if (cif->flags & FFI390_RET_IN_MEM)
+    p_gpr[n_gpr++] = (unsigned long) rvalue;
 
   /* Now for the arguments.  */
-
-  for (ptr = ecif->cif->arg_types, i = ecif->cif->nargs;
-       i > 0;
-       i--, ptr++, p_argv++)
+  for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++, p_argv++)
     {
+      ffi_type *ty = *ptr;
       void *arg = *p_argv;
-      int type = (*ptr)->type;
+      int type = ty->type;
 
 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
       /* 16-byte long double is passed like a struct.  */
@@ -206,13 +176,13 @@ ffi_prep_args (unsigned char *stack, extended_cif *ecif)
 	  if (type == FFI_TYPE_COMPLEX)
 	    type = FFI_TYPE_POINTER;
 	  else
-	    type = ffi_check_struct_type (*ptr);
+	    type = ffi_check_struct_type (ty);
 
 	  /* If we pass the struct via pointer, copy the data.  */
 	  if (type == FFI_TYPE_POINTER)
 	    {
-	      p_struct -= ROUND_SIZE ((*ptr)->size);
-	      memcpy (p_struct, (char *)arg, (*ptr)->size);
+	      p_struct -= ROUND_SIZE (ty->size);
+	      memcpy (p_struct, (char *)arg, ty->size);
 	      arg = &p_struct;
 	    }
 	}
@@ -234,7 +204,7 @@ ffi_prep_args (unsigned char *stack, extended_cif *ecif)
 
 	  case FFI_TYPE_FLOAT:
 	    if (n_fpr < MAX_FPRARGS)
-	      p_fpr[n_fpr++] = (long long) *(unsigned int *) arg << 32;
+	      p_fpr[n_fpr++] = (unsigned long long)*(unsigned int *) arg << 32;
 	    else
 	      p_ov[n_ov++] = *(unsigned int *) arg;
 	    break;
@@ -325,7 +295,7 @@ ffi_prep_args (unsigned char *stack, extended_cif *ecif)
 /*                                                                    */
 /*====================================================================*/
 
-ffi_status
+ffi_status FFI_HIDDEN
 ffi_prep_cif_machdep(ffi_cif *cif)
 {
   size_t struct_size = 0;
@@ -498,32 +468,55 @@ ffi_call_int(ffi_cif *cif,
 	     void *closure)
 {
   int ret_type = cif->flags;
-  extended_cif ecif;
+  size_t rsize = 0, bytes = cif->bytes;
+  unsigned char *stack;
+  struct call_frame *frame;
 
-  ecif.cif    = cif;
-  ecif.avalue = avalue;
-  ecif.rvalue = rvalue;
+  FFI_ASSERT (cif->abi == FFI_SYSV);
 
   /* If we don't have a return value, we need to fake one.  */
   if (rvalue == NULL)
     {
-      if (ret_type == FFI390_RET_STRUCT)
-	ecif.rvalue = alloca (cif->rtype->size);
+      if (ret_type & FFI390_RET_IN_MEM)
+	rsize = cif->rtype->size;
       else
 	ret_type = FFI390_RET_VOID;
     }
 
-  switch (cif->abi)
-    {
-      case FFI_SYSV:
-        ffi_call_SYSV (cif->bytes, &ecif, ffi_prep_args,
-		       ret_type, ecif.rvalue, fn, closure);
-        break;
+  /* The stack space will be filled with those areas:
 
-      default:
-        FFI_ASSERT (0);
-        break;
-    }
+	dummy structure return		    (highest addresses)
+	  FPR argument register save area
+	  GPR argument register save area
+	stack frame for ffi_call_SYSV
+	temporary struct copies
+	overflow argument area              (lowest addresses)
+
+     We set up the following pointers:
+
+        p_fpr: bottom of the FPR area (growing upwards)
+	p_gpr: bottom of the GPR area (growing upwards)
+	p_ov: bottom of the overflow area (growing upwards)
+	p_struct: top of the struct copy area (growing downwards)
+
+     All areas are kept aligned to twice the word size.  */
+
+  stack = alloca (bytes + sizeof(struct call_frame) + rsize);
+  frame = (struct call_frame *)(stack + bytes);
+  if (rsize)
+    rvalue = frame + 1;
+
+  /* Assuming that the current function has the standard call frame,
+     we can maintain the linked list like so.  */
+  frame->back_chain = __builtin_dwarf_cfa() - sizeof(struct call_frame);
+
+  /* Pass the outgoing stack frame in the r15 save slot.  */
+  frame->gpr_save[8] = (unsigned long)(stack - sizeof(struct call_frame));
+
+  /* Fill in all of the argument stuff.  */
+  ffi_prep_args (cif, rvalue, avalue, (unsigned long *)stack, frame);
+
+  ffi_call_SYSV (frame, ret_type & FFI360_RET_MASK, rvalue, fn, closure);
 }
 
 void
@@ -549,8 +542,7 @@ ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
 /*                                                                    */
 /*====================================================================*/
 
-FFI_HIDDEN
-void
+void FFI_HIDDEN
 ffi_closure_helper_SYSV (ffi_cif *cif,
 			 void (*fun)(ffi_cif*,void*,void**,void*),
 			 void *user_data,
@@ -572,18 +564,15 @@ ffi_closure_helper_SYSV (ffi_cif *cif,
   int i;
 
   /* Allocate buffer for argument list pointers.  */
-
   p_arg = avalue = alloca (cif->nargs * sizeof (void *));
 
   /* If we returning a structure, pass the structure address
      directly to the target function.  Otherwise, have the target
      function store the return value to the GPR save area.  */
-
-  if (cif->flags == FFI390_RET_STRUCT)
+  if (cif->flags & FFI390_RET_IN_MEM)
     rvalue = (void *) p_gpr[n_gpr++];
 
   /* Now for the arguments.  */
-
   for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, p_arg++, ptr++)
     {
       int deref_struct_pointer = 0;
@@ -611,11 +600,13 @@ ffi_closure_helper_SYSV (ffi_cif *cif,
 
       /* Pointers are passed like UINTs of the same size.  */
       if (type == FFI_TYPE_POINTER)
+	{
 #ifdef __s390x__
-	type = FFI_TYPE_UINT64;
+	  type = FFI_TYPE_UINT64;
 #else
-	type = FFI_TYPE_UINT32;
+	  type = FFI_TYPE_UINT32;
 #endif
+	}
 
       /* Now handle all primitive int/float data types.  */
       switch (type)
diff --git a/src/s390/internal.h b/src/s390/internal.h
new file mode 100644
index 0000000..b875578
--- /dev/null
+++ b/src/s390/internal.h
@@ -0,0 +1,11 @@
+/* If these values change, sysv.S must be adapted!  */
+#define FFI390_RET_DOUBLE	0
+#define FFI390_RET_FLOAT	1
+#define FFI390_RET_INT64	2
+#define FFI390_RET_INT32	3
+#define FFI390_RET_VOID		4
+
+#define FFI360_RET_MASK		7
+#define FFI390_RET_IN_MEM	8
+
+#define FFI390_RET_STRUCT	(FFI390_RET_VOID | FFI390_RET_IN_MEM)
diff --git a/src/s390/sysv.S b/src/s390/sysv.S
index b01a7eb..df9083e 100644
--- a/src/s390/sysv.S
+++ b/src/s390/sysv.S
@@ -29,221 +29,140 @@
 #include <fficonfig.h>
 #include <ffi.h>
 
-#ifndef __s390x__
+	.text
 
-.text
+#ifndef __s390x__
 
-	# r2:	cif->bytes
-	# r3:	&ecif
-	# r4:	ffi_prep_args
-	# r5:	ret_type
-	# r6:	ecif.rvalue
-	# ov:	fn
-	# ov+8:	closure
+	# r2:	frame
+	# r3:	ret_type
+	# r4:	ret_addr
+	# r5:	fun
+	# r6:	closure
 
 	# This assumes we are using gas.
+	.balign	8
 	.globl	ffi_call_SYSV
 	FFI_HIDDEN(ffi_call_SYSV)
 	.type	ffi_call_SYSV,%function
 ffi_call_SYSV:
 	.cfi_startproc
-	stm	%r6,%r15,24(%r15)		# Save registers
-	.cfi_offset r6, -72
-	.cfi_offset r7, -68
-	.cfi_offset r8, -64
-	.cfi_offset r9, -60
-	.cfi_offset r10, -56
-	.cfi_offset r11, -52
-	.cfi_offset r12, -48
-	.cfi_offset r13, -44
-	.cfi_offset r14, -40
-	.cfi_offset r15, -36
-	basr	%r13,0				# Set up base register
+	st	%r6,44(%r2)			# Save registers
+	stm	%r12,%r14,48(%r2)
+	lr	%r13,%r2			# Install frame pointer
+	.cfi_rel_offset r6, 44
+	.cfi_rel_offset r12, 48
+	.cfi_rel_offset r13, 52
+	.cfi_rel_offset r14, 56
+	.cfi_def_cfa_register r13
+	l	%r15,60(%r2)			# Set up outgoing stack
+	basr	%r14,0				# Set up base register
 .Lbase:
-	lr	%r11,%r15			# Set up frame pointer
-	.cfi_def_cfa_register r11
-	sr	%r15,%r2
-	ahi	%r15,-96-48			# Allocate stack
-	lr	%r8,%r6				# Save ecif.rvalue
-	sr	%r9,%r9
-	ic	%r9,.Ltable-.Lbase(%r13,%r5)	# Load epilog address
-	l	%r7,96(%r11)			# Load function address
-	st	%r11,0(%r15)			# Set up back chain
-	ahi	%r11,-48			# Register save area
-	.cfi_adjust_cfa_offset 48
-
-	la	%r2,96(%r15)			# Save area
-						# r3 already holds &ecif
-	basr	%r14,%r4			# Call ffi_prep_args
-
-	l	%r0,96+48+4(%r11)		# Go closure -> static chain
-	lm	%r2,%r6,0(%r11)			# Load arguments
-	ld	%f0,32(%r11)
-	ld	%f2,40(%r11)
-	la	%r14,0(%r13,%r9)		# Set return address
-	br	%r7				# ... and call function
-
-.LretNone:					# Return void
-	l	%r4,48+56(%r11)
-	lm	%r6,%r15,48+24(%r11)
-	.cfi_remember_state
-	.cfi_restore 15
-	.cfi_restore 14
-	.cfi_restore 13
-	.cfi_restore 12
-	.cfi_restore 11
-	.cfi_restore 10
-	.cfi_restore 9
-	.cfi_restore 8
-	.cfi_restore 7
-	.cfi_restore 6
-	.cfi_def_cfa r15, 96
-	br	%r4
-	.cfi_restore_state
-	# This nopr is necessary so that the .cfi instructions between the br
-	# above and the label below get executed.  See execute_cfa_program() in
-	# the Gcc source code, libgcc/unwind-dw2.c.
-	nopr
-
-.LretFloat:
-	l	%r4,48+56(%r11)
-	ste	%f0,0(%r8)			# Return float
-	lm	%r6,%r15,48+24(%r11)
-	.cfi_remember_state
-	.cfi_restore 15
-	.cfi_restore 14
-	.cfi_restore 13
-	.cfi_restore 12
-	.cfi_restore 11
-	.cfi_restore 10
-	.cfi_restore 9
-	.cfi_restore 8
-	.cfi_restore 7
-	.cfi_restore 6
-	.cfi_def_cfa r15, 96
-	br	%r4
-	.cfi_restore_state
-	# See comment on the nopr above.
-	nopr
-
-.LretDouble:
-	l	%r4,48+56(%r11)
-	std	%f0,0(%r8)			# Return double
-	lm	%r6,%r15,48+24(%r11)
-	.cfi_remember_state
-	.cfi_restore 15
-	.cfi_restore 14
-	.cfi_restore 13
-	.cfi_restore 12
-	.cfi_restore 11
-	.cfi_restore 10
-	.cfi_restore 9
-	.cfi_restore 8
-	.cfi_restore 7
-	.cfi_restore 6
-	.cfi_def_cfa r15, 96
-	br	%r4
-	.cfi_restore_state
-	# See comment on the nopr above.
-	nopr
-
-.LretInt32:
-	l	%r4,48+56(%r11)
-	st	%r2,0(%r8)			# Return int
-	lm	%r6,%r15,48+24(%r11)
-	.cfi_remember_state
-	.cfi_restore 15
-	.cfi_restore 14
-	.cfi_restore 13
-	.cfi_restore 12
-	.cfi_restore 11
-	.cfi_restore 10
-	.cfi_restore 9
-	.cfi_restore 8
-	.cfi_restore 7
-	.cfi_restore 6
-	.cfi_def_cfa r15, 96
-	br	%r4
-	.cfi_restore_state
-	# See comment on the nopr above.
-	nopr
-
-.LretInt64:
-	l	%r4,48+56(%r11)
-	stm	%r2,%r3,0(%r8)			# Return long long
-	lm	%r6,%r15,48+24(%r11)
-	.cfi_remember_state
-	.cfi_restore 15
+	sla	%r3,3				# ret_type *= 8
+	lr	%r12,%r4			# Save ret_addr
+	lr	%r1,%r5				# Save fun
+	lr	%r0,%r6				# Install static chain
+	la	%r14,.Ltable-.Lbase(%r14,%r3)	# Set return address
+	lm	%r2,%r6,8(%r13)			# Load arguments
+	ld	%f0,64(%r13)
+	ld	%f2,72(%r13)
+	st	%r13,0(%r15)			# Set up back chain
+	br	%r1				# ... and call function
+
+	.balign	8
+.Ltable:
+# FFI390_RET_DOUBLE
+	std	%f0,0(%r12)
+	j	.Ldone
+
+	.balign	8
+# FFI390_RET_FLOAT
+	ste	%f0,0(%r12)
+	j	.Ldone
+
+	.balign	8
+# FFI390_RET_INT64
+	st	%r3,4(%r12)
+	nop
+	# fallthru
+
+	.balign	8
+# FFI390_RET_INT32
+	st	%r2,0(%r12)
+	nop
+	# fallthru
+
+	.balign	8
+# FFI390_RET_VOID
+.Ldone:
+	l	%r14,56(%r13)
+	l	%r12,48(%r13)
+	l	%r6,44(%r13)
+	l	%r13,52(%r13)
 	.cfi_restore 14
 	.cfi_restore 13
 	.cfi_restore 12
-	.cfi_restore 11
-	.cfi_restore 10
-	.cfi_restore 9
-	.cfi_restore 8
-	.cfi_restore 7
 	.cfi_restore 6
 	.cfi_def_cfa r15, 96
-	br	%r4
+	br	%r14
 	.cfi_endproc
+	.size	 ffi_call_SYSV,.-ffi_call_SYSV
 
-.Ltable:
-	.byte	.LretNone-.Lbase		# FFI390_RET_VOID
-	.byte	.LretNone-.Lbase		# FFI390_RET_STRUCT
-	.byte	.LretFloat-.Lbase		# FFI390_RET_FLOAT
-	.byte	.LretDouble-.Lbase		# FFI390_RET_DOUBLE
-	.byte	.LretInt32-.Lbase		# FFI390_RET_INT32
-	.byte	.LretInt64-.Lbase		# FFI390_RET_INT64
-
-.ffi_call_SYSV_end:
-	.size	 ffi_call_SYSV,.ffi_call_SYSV_end-ffi_call_SYSV
 
+	.balign	8
+	.globl	ffi_go_closure_SYSV
+	FFI_HIDDEN(ffi_go_closure_SYSV)
+	.type	ffi_go_closure_SYSV,%function
+ffi_go_closure_SYSV:
+	.cfi_startproc
+	stm	%r2,%r6,8(%r15)			# Save arguments
+	lr	%r4,%r0				# Load closure -> user_data
+	l	%r2,4(%r4)			#   ->cif
+	l	%r3,8(%r4)			#   ->fun
+	j	.Ldoclosure
+	.cfi_endproc
 
+	.balign	8
 	.globl	ffi_closure_SYSV
 	FFI_HIDDEN(ffi_closure_SYSV)
 	.type	ffi_closure_SYSV,%function
 ffi_closure_SYSV:
 	.cfi_startproc
 	stm	%r2,%r6,8(%r15)			# Save arguments
-	.cfi_offset r6, -72
 	lr	%r4,%r0				# Closure
 	l	%r2,16(%r4)			#   ->cif
 	l	%r3,20(%r4)			#   ->fun
 	l	%r4,24(%r4)			#   ->user_data
 .Ldoclosure:
 	stm	%r12,%r15,48(%r15)		# Save registers
-	.cfi_offset r12, -48
-	.cfi_offset r13, -44
-	.cfi_offset r14, -40
-	.cfi_offset r15, -36
+	lr	%r12,%r15
+	.cfi_def_cfa_register r12
+	.cfi_rel_offset r6, 24
+	.cfi_rel_offset r12, 48
+	.cfi_rel_offset r13, 52
+	.cfi_rel_offset r14, 56
+	.cfi_rel_offset r15, 60
 	basr	%r13,0				# Set up base register
 .Lcbase:
-	std	%f0,64(%r15)
-	std	%f2,72(%r15)
-	lr	%r1,%r15			# Set up stack frame
-	ahi	%r15,-104
-	.cfi_adjust_cfa_offset 104
-	l	%r12,.Lchelper-.Lcbase(%r13)	# Get helper function
-	la	%r5,96(%r1)
-	st	%r5,96(%r15)			# Overflow
-	la	%r5,8(%r1)			# GPRs
-	la	%r6,64(%r1)			# FPRs
-	st	%r1,0(%r15)			# Set up back chain
-
-	bas	%r14,0(%r12,%r13)		# Call helper
-
-	l	%r4,104+56(%r15)
-	ld	%f0,104+64(%r15)		# Load return registers
-	lm	%r2,%r3,104+8(%r15)
-	l	%r6,104+24(%r15)		# Restore saved registers
-	.cfi_restore r6
-	lm	%r12,%r15,104+48(%r15)
-	.cfi_adjust_cfa_offset -104
-	.cfi_restore r12
-	.cfi_restore r13
-	.cfi_restore r14
-	.cfi_restore r15
-	br	%r4
+	ahi	%r15,-96-8			# Set up stack frame
+	l	%r1,.Lchelper-.Lcbase(%r13)	# Get helper function
+	st	%r12,0(%r15)			# Set up back chain
+
+	std	%f0,64(%r12)			# Save fp arguments
+	std	%f2,72(%r12)
+
+	la	%r5,96(%r12)			# Overflow
+	st	%r5,96(%r15)
+	la	%r6,64(%r12)			# FPRs
+	la	%r5,8(%r12)			# GPRs
+	bas	%r14,0(%r1,%r13)		# Call helper
+
+	lr	%r15,%r12
+	.cfi_def_cfa_register r15
+	lm	%r12,%r14,48(%r12)		# Restore saved registers
+	l	%r6,24(%r15)
+	ld	%f0,64(%r15)			# Load return registers
+	lm	%r2,%r3,8(%r15)
+	br	%r14
 	.cfi_endproc
 
 	.align 4
@@ -251,234 +170,143 @@ ffi_closure_SYSV:
 	.long	ffi_closure_helper_SYSV-.Lcbase
 
 
-.ffi_closure_SYSV_end:
-	.size	 ffi_closure_SYSV,.ffi_closure_SYSV_end-ffi_closure_SYSV
-
-
-	.globl	ffi_go_closure_SYSV
-	FFI_HIDDEN(ffi_go_closure_SYSV)
-	.type	ffi_go_closure_SYSV,%function
-ffi_go_closure_SYSV:
-	.cfi_startproc
-	stm	%r2,%r6,8(%r15)			# Save arguments
-	.cfi_offset r6, -72
-	lr	%r4,%r0				# Load closure -> user_data
-	l	%r2,4(%r4)			#   ->cif
-	l	%r3,8(%r4)			#   ->fun
-	j	.Ldoclosure
-	.cfi_endproc
+	.size	 ffi_closure_SYSV,.-ffi_closure_SYSV
 
 #else
 
-.text
-
-	# r2:	cif->bytes
-	# r3:	&ecif
-	# r4:	ffi_prep_args
-	# r5:	ret_type
-	# r6:	ecif.rvalue
-	# ov:	fn
-	# ov+8:	closure
+	# r2:	frame
+	# r3:	ret_type
+	# r4:	ret_addr
+	# r5:	fun
+	# r6:	closure
 
 	# This assumes we are using gas.
+	.balign	8
 	.globl	ffi_call_SYSV
 	FFI_HIDDEN(ffi_call_SYSV)
 	.type	ffi_call_SYSV,%function
 ffi_call_SYSV:
 	.cfi_startproc
-	stmg	%r6,%r15,48(%r15)		# Save registers
-	.cfi_offset r6, -112
-	.cfi_offset r7, -104
-	.cfi_offset r8, -96
-	.cfi_offset r9, -88
-	.cfi_offset r10, -80
-	.cfi_offset r11, -72
-	.cfi_offset r12, -64
-	.cfi_offset r13, -56
-	.cfi_offset r14, -48
-	.cfi_offset r15, -40
-	larl	%r13,.Lbase			# Set up base register
-	lgr	%r11,%r15			# Set up frame pointer
-	.cfi_def_cfa_register r11
-	sgr	%r15,%r2
-	aghi	%r15,-160-80			# Allocate stack
-	lgr	%r8,%r6				# Save ecif.rvalue
-	llgc	%r9,.Ltable-.Lbase(%r13,%r5)	# Load epilog address
-	lg	%r7,160(%r11)			# Load function address
-	stg	%r11,0(%r15)			# Set up back chain
-	aghi	%r11,-80			# Register save area
-	.cfi_adjust_cfa_offset 80
-
-	la	%r2,160(%r15)			# Save area
-						# r3 already holds &ecif
-	basr	%r14,%r4			# Call ffi_prep_args
-
-	lg	%r0,160+80+8(%r11)		# Go closure -> static chain
-	lmg	%r2,%r6,0(%r11)			# Load arguments
-	ld	%f0,48(%r11)
-	ld	%f2,56(%r11)
-	ld	%f4,64(%r11)
-	ld	%f6,72(%r11)
-	la	%r14,0(%r13,%r9)		# Set return address
-	br	%r7				# ... and call function
-
-.Lbase:
-.LretNone:					# Return void
-	lg	%r4,80+112(%r11)
-	lmg	%r6,%r15,80+48(%r11)
-	.cfi_remember_state
-	.cfi_restore r15
-	.cfi_restore r14
-	.cfi_restore r13
-	.cfi_restore r12
-	.cfi_restore r11
-	.cfi_restore r10
-	.cfi_restore r9
-	.cfi_restore r8
-	.cfi_restore r7
-	.cfi_restore r6
-	.cfi_def_cfa r15, 160
-	br	%r4
-	.cfi_restore_state
-	# This nopr is necessary so that the .cfi instructions between the br
-	# above and the label below get executed.  See execute_cfa_program() in
-	# the Gcc source code, libgcc/unwind-dw2.c.
-	nopr
-
-.LretFloat:
-	lg	%r4,80+112(%r11)
-	ste	%f0,0(%r8)			# Return float
-	lmg	%r6,%r15,80+48(%r11)
-	.cfi_remember_state
-	.cfi_restore r6
-	.cfi_restore r7
-	.cfi_restore r8
-	.cfi_restore r9
-	.cfi_restore r10
-	.cfi_restore r11
-	.cfi_restore r12
-	.cfi_restore r13
-	.cfi_restore r14
-	.cfi_restore r15
-	.cfi_def_cfa r15, 160
-	br	%r4
-	.cfi_restore_state
-	# See comment on the nopr above.
-	nopr
-
-.LretDouble:
-	lg	%r4,80+112(%r11)
-	std	%f0,0(%r8)			# Return double
-	lmg	%r6,%r15,80+48(%r11)
-	.cfi_remember_state
-	.cfi_restore r15
-	.cfi_restore r14
-	.cfi_restore r13
-	.cfi_restore r12
-	.cfi_restore r11
-	.cfi_restore r10
-	.cfi_restore r9
-	.cfi_restore r8
-	.cfi_restore r7
-	.cfi_restore r6
-	.cfi_def_cfa r15, 160
-	br	%r4
-	.cfi_restore_state
-	# See comment on the nopr above.
-	nopr
-
-.LretInt64:
-	lg	%r4,80+112(%r11)
-	stg	%r2,0(%r8)			# Return long
-	lmg	%r6,%r15,80+48(%r11)
-	.cfi_restore r15
+	stg	%r6,88(%r2)			# Save registers
+	stmg	%r12,%r14,96(%r2)
+	lgr	%r13,%r2			# Install frame pointer
+	.cfi_rel_offset r6, 88
+	.cfi_rel_offset r12, 96
+	.cfi_rel_offset r13, 104
+	.cfi_rel_offset r14, 112
+	.cfi_def_cfa_register r13
+	lg	%r15,120(%r2)			# Set up outgoing stack
+	larl	%r14,.Ltable			# Set up return address
+	slag	%r3,%r3,3			# ret_type *= 8
+	lgr	%r12,%r4			# Save ret_addr
+	lgr	%r1,%r5				# Save fun
+	lgr	%r0,%r6				# Install static chain
+	agr	%r14,%r3
+	lmg	%r2,%r6,16(%r13)		# Load arguments
+	ld	%f0,128(%r13)
+	ld	%f2,136(%r13)
+	ld	%f4,144(%r13)
+	ld	%f6,152(%r13)
+	stg	%r13,0(%r15)			# Set up back chain
+	br	%r1				# ... and call function
+
+	.balign	8
+.Ltable:
+# FFI390_RET_DOUBLE
+	std	%f0,0(%r12)
+	j	.Ldone
+
+	.balign	8
+# FFI390_RET_DOUBLE
+	ste	%f0,0(%r12)
+	j	.Ldone
+
+	.balign	8
+# FFI390_RET_INT64
+	stg	%r2,0(%r12)
+
+	.balign	8
+# FFI390_RET_INT32
+	# Never used, as we always store type ffi_arg.
+	# But the stg above is 6 bytes and we cannot
+	# jump around this case, so fall through.
+	nop
+	nop
+
+	.balign	8
+# FFI390_RET_VOID
+.Ldone:
+	lg	%r14,112(%r13)
+	lg	%r12,96(%r13)
+	lg	%r6,88(%r13)
+	lg	%r13,104(%r13)
 	.cfi_restore r14
 	.cfi_restore r13
 	.cfi_restore r12
-	.cfi_restore r11
-	.cfi_restore r10
-	.cfi_restore r9
-	.cfi_restore r8
-	.cfi_restore r7
 	.cfi_restore r6
 	.cfi_def_cfa r15, 160
-	br	%r4
+	br	%r14
 	.cfi_endproc
+	.size	 ffi_call_SYSV,.-ffi_call_SYSV
 
-.Ltable:
-	.byte	.LretNone-.Lbase		# FFI390_RET_VOID
-	.byte	.LretNone-.Lbase		# FFI390_RET_STRUCT
-	.byte	.LretFloat-.Lbase		# FFI390_RET_FLOAT
-	.byte	.LretDouble-.Lbase		# FFI390_RET_DOUBLE
-	.byte	0				# int32 retval not supported
-	.byte	.LretInt64-.Lbase		# FFI390_RET_INT64
 
-.ffi_call_SYSV_end:
-	.size	 ffi_call_SYSV,.ffi_call_SYSV_end-ffi_call_SYSV
+	.balign	8
+	.globl	ffi_go_closure_SYSV
+	FFI_HIDDEN(ffi_go_closure_SYSV)
+	.type	ffi_go_closure_SYSV,%function
+ffi_go_closure_SYSV:
+	.cfi_startproc
+	stmg	%r2,%r6,16(%r15)		# Save arguments
+	lgr	%r4,%r0				# Load closure -> user_data
+	lg	%r2,8(%r4)			#   ->cif
+	lg	%r3,16(%r4)			#   ->fun
+	j	.Ldoclosure
+	.cfi_endproc
+	.size	 ffi_go_closure_SYSV,.-ffi_go_closure_SYSV
 
 
+	.balign	8
 	.globl	ffi_closure_SYSV
 	FFI_HIDDEN(ffi_closure_SYSV)
 	.type	ffi_closure_SYSV,%function
 ffi_closure_SYSV:
 	.cfi_startproc
 	stmg	%r2,%r6,16(%r15)		# Save arguments
-	.cfi_offset r6, -112
 	lgr	%r4,%r0				# Load closure
 	lg	%r2,32(%r4)			#   ->cif
 	lg	%r3,40(%r4)			#   ->fun
 	lg	%r4,48(%r4)			#   ->user_data
 .Ldoclosure:
-	stmg	%r14,%r15,112(%r15)		# Save registers
-	.cfi_offset r14, -48
-	.cfi_offset r15, -40
-	std	%f0,128(%r15)			# Save arguments
-	std	%f2,136(%r15)
-	std	%f4,144(%r15)
-	std	%f6,152(%r15)
-	lgr	%r1,%r15			# Set up stack frame
-	aghi	%r15,-168
-	.cfi_adjust_cfa_offset 168
-	la	%r5,160(%r1)
-	stg	%r5,160(%r15)			# Overflow
-	la	%r5,16(%r1)			# GPRs
-	la	%r6,128(%r1)			# FPRs
-	stg	%r1,0(%r15)			# Set up back chain
-
+	stmg	%r13,%r15,104(%r15)		# Save registers
+	lgr	%r13,%r15
+	.cfi_def_cfa_register r13
+	.cfi_rel_offset r6, 48
+	.cfi_rel_offset r13, 104
+	.cfi_rel_offset r14, 112
+	.cfi_rel_offset r15, 120
+	aghi	%r15,-160-16			# Set up stack frame
+	stg	%r13,0(%r15)			# Set up back chain
+
+	std	%f0,128(%r13)			# Save fp arguments
+	std	%f2,136(%r13)
+	std	%f4,144(%r13)
+	std	%f6,152(%r13)
+	la	%r5,160(%r13)			# Overflow
+	stg	%r5,160(%r15)
+	la	%r6,128(%r13)			# FPRs
+	la	%r5,16(%r13)			# GPRs
 	brasl	%r14,ffi_closure_helper_SYSV	# Call helper
 
-	ld	%f0,168+128(%r15)		# Load return registers
-	lg	%r2,168+16(%r15)
-	lg	%r6,168+48(%r15)		# Restore saved registers
-	.cfi_restore r6
-	lmg	%r14,%r15,168+112(%r15)
-	.cfi_restore r14
-	.cfi_restore r15
-	.cfi_adjust_cfa_offset -168
+	lgr	%r15,%r13
+	.cfi_def_cfa_register r15
+	lmg	%r13,%r14,104(%r13)		# Restore saved registers
+	lg	%r6,48(%r15)
+	ld	%f0,128(%r15)			# Load return registers
+	lg	%r2,16(%r15)
 	br	%r14
 	.cfi_endproc
-
-.ffi_closure_SYSV_end:
-	.size	 ffi_closure_SYSV,.ffi_closure_SYSV_end-ffi_closure_SYSV
-
-
-	.globl	ffi_go_closure_SYSV
-	FFI_HIDDEN(ffi_go_closure_SYSV)
-	.type	ffi_go_closure_SYSV,%function
-ffi_go_closure_SYSV:
-	.cfi_startproc
-	stmg	%r2,%r6,16(%r15)		# Save arguments
-	.cfi_offset r6, -112
-	lgr	%r4,%r0				# Load closure -> user_data
-	lg	%r2,8(%r4)			#   ->cif
-	lg	%r3,16(%r4)			#   ->fun
-	j	.Ldoclosure
-	.cfi_endproc
-
-.ffi_go_closure_SYSV_end:
-	.size	 ffi_go_closure_SYSV,.ffi_go_closure_SYSV_end-ffi_go_closure_SYSV
-
-#endif
+	.size	 ffi_closure_SYSV,.-ffi_closure_SYSV
+#endif /* !s390x */
 
 #if defined __ELF__ && defined __linux__
 	.section	.note.GNU-stack,"",@progbits
-- 
2.1.0