File 0008-S390-Add-z13-memmove-ifunc-variant.patch of Package glibc.15734
From 9dcf8c206f6bbc79c52913904503a1b81e193b5a Mon Sep 17 00:00:00 2001
From: Stefan Liebler <stli@linux.ibm.com>
Date: Tue, 18 Dec 2018 13:57:08 +0100
Subject: [PATCH 08/18] S390: Add z13 memmove ifunc variant.
This patch introduces a z13 specific ifunc variant for memmove.
As the common code implementation, it checks if we can copy from
the beginning to the end - with z196 memcpy implementation - or
if we have to copy from the end to the beginning.
The latter case is done by using vector load/store instructions.
If vector instructions are not available, the common-code is
used as fallback. Therefore it is implemented in memmove-c with
a different name.
Furthermore the ifunc logic decides if we need the common-code
implementation at all. If vector instructions are supported
due to the minimum architecture level set we can skip the
common-code ifunc variant.
ChangeLog:
* sysdeps/s390/Makefile (sysdep_routines): Add memmove-c.
* sysdeps/s390/ifunc-memcpy.h (HAVE_MEMMOVE_IFUNC,
HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT, MEMMOVE_DEFAULT,
HAVE_MEMMOVE_C, MEMMOVE_C, HAVE_MEMMOVE_Z13, MEMMOVE_Z13):
New defines.
* sysdeps/s390/memcpy-z900.S: Add z13 memmove implementation.
* sysdeps/s390/memmove-c.c: New file.
* sysdeps/s390/memmove.c: Likewise.
* sysdeps/s390/multiarch/ifunc-impl-list.c
(__libc_ifunc_impl_list): Add ifunc variants for memmove.
(cherry picked from commit cdd927d98cc38acf55e1c6594b5c9451df8f239f)
---
sysdeps/s390/Makefile | 3 +-
sysdeps/s390/ifunc-memcpy.h | 35 +++++++++++++
sysdeps/s390/memcpy-z900.S | 87 ++++++++++++++++++++++++++++++++
sysdeps/s390/memmove-c.c | 37 ++++++++++++++
sysdeps/s390/memmove.c | 44 ++++++++++++++++
sysdeps/s390/multiarch/ifunc-impl-list.c | 12 +++++
6 files changed, 217 insertions(+), 1 deletion(-)
create mode 100644 sysdeps/s390/memmove-c.c
create mode 100644 sysdeps/s390/memmove.c
Index: glibc-2.26/sysdeps/s390/Makefile
===================================================================
--- glibc-2.26.orig/sysdeps/s390/Makefile
+++ glibc-2.26/sysdeps/s390/Makefile
@@ -31,5 +31,6 @@ sysdeps-gconv-modules = ../sysdeps/s390/
endif
ifeq ($(subdir),string)
-sysdep_routines += mempcpy memcpy memcpy-z900
+sysdep_routines += mempcpy memcpy memcpy-z900 \
+ memmove memmove-c
endif
Index: glibc-2.26/sysdeps/s390/ifunc-memcpy.h
===================================================================
--- glibc-2.26.orig/sysdeps/s390/ifunc-memcpy.h
+++ glibc-2.26/sysdeps/s390/ifunc-memcpy.h
@@ -43,6 +43,29 @@
# define HAVE_MEMCPY_Z196 HAVE_MEMCPY_IFUNC
#endif
+#if defined SHARED && defined USE_MULTIARCH && IS_IN (libc) \
+ && ! defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
+# define HAVE_MEMMOVE_IFUNC 1
+#else
+# define HAVE_MEMMOVE_IFUNC 0
+#endif
+
+#ifdef HAVE_S390_VX_ASM_SUPPORT
+# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT HAVE_MEMMOVE_IFUNC
+#else
+# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT 0
+#endif
+
+#if defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
+# define MEMMOVE_DEFAULT MEMMOVE_Z13
+# define HAVE_MEMMOVE_C 0
+# define HAVE_MEMMOVE_Z13 1
+#else
+# define MEMMOVE_DEFAULT MEMMOVE_C
+# define HAVE_MEMMOVE_C 1
+# define HAVE_MEMMOVE_Z13 HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT
+#endif
+
#if HAVE_MEMCPY_Z900_G5
# define MEMCPY_Z900_G5 __memcpy_default
# define MEMPCPY_Z900_G5 __mempcpy_default
@@ -66,3 +89,15 @@
# define MEMCPY_Z196 NULL
# define MEMPCPY_Z196 NULL
#endif
+
+#if HAVE_MEMMOVE_C
+# define MEMMOVE_C __memmove_c
+#else
+# define MEMMOVE_C NULL
+#endif
+
+#if HAVE_MEMMOVE_Z13
+# define MEMMOVE_Z13 __memmove_z13
+#else
+# define MEMMOVE_Z13 NULL
+#endif
Index: glibc-2.26/sysdeps/s390/memcpy-z900.S
===================================================================
--- glibc-2.26.orig/sysdeps/s390/memcpy-z900.S
+++ glibc-2.26/sysdeps/s390/memcpy-z900.S
@@ -182,6 +182,7 @@ ENTRY(MEMCPY_Z196)
# endif /* !defined __s390x__ */
ltgr %r4,%r4
je .L_Z196_4
+.L_Z196_start2:
aghi %r4,-1
srlg %r5,%r4,8
ltgr %r5,%r5
@@ -207,6 +208,75 @@ ENTRY(MEMCPY_Z196)
END(MEMCPY_Z196)
#endif /* HAVE_MEMCPY_Z196 */
+#if HAVE_MEMMOVE_Z13
+ENTRY(MEMMOVE_Z13)
+ .machine "z13"
+ .machinemode "zarch_nohighgprs"
+# if !defined __s390x__
+ /* Note: The 31bit dst and src pointers are prefixed with zeroes. */
+ llgfr %r4,%r4
+ llgfr %r3,%r3
+ llgfr %r2,%r2
+# endif /* !defined __s390x__ */
+ sgrk %r0,%r2,%r3
+ clgijh %r4,16,.L_MEMMOVE_Z13_LARGE
+ aghik %r5,%r4,-1
+.L_MEMMOVE_Z13_SMALL:
+ jl .L_MEMMOVE_Z13_END /* Jump away if len was zero. */
+ /* Store up to 16 bytes with vll/vstl which needs the index
+ instead of lengths. */
+ vll %v16,%r5,0(%r3)
+ vstl %v16,%r5,0(%r2)
+.L_MEMMOVE_Z13_END:
+ br %r14
+.L_MEMMOVE_Z13_LARGE:
+ lgr %r1,%r2 /* For memcpy: r1: Use as dest ;
+ r2: Return dest */
+ /* The unsigned comparison (dst - src >= len) determines if we can
+ execute the forward case with memcpy. */
+#if ! HAVE_MEMCPY_Z196
+# error The z13 variant of memmove needs the z196 variant of memcpy!
+#endif
+ clgrjhe %r0,%r4,.L_Z196_start2
+ risbgn %r5,%r4,4,128+63,60 /* r5 = r4 / 16 */
+ aghi %r4,-16
+ clgijhe %r5,8,.L_MEMMOVE_Z13_LARGE_64B
+.L_MEMMOVE_Z13_LARGE_16B_LOOP:
+ /* Store at least 16 bytes with vl/vst. The number of 16byte blocks
+ is stored in r5. */
+ vl %v16,0(%r4,%r3)
+ vst %v16,0(%r4,%r2)
+ aghi %r4,-16
+ brctg %r5,.L_MEMMOVE_Z13_LARGE_16B_LOOP
+ aghik %r5,%r4,15
+ j .L_MEMMOVE_Z13_SMALL
+.L_MEMMOVE_Z13_LARGE_64B:
+ /* Store at least 128 bytes with 4x vl/vst. The number of 64byte blocks
+ will be stored in r0. */
+ aghi %r4,-48
+ srlg %r0,%r5,2 /* r5 = %r0 / 4
+ => Number of 64byte blocks. */
+.L_MEMMOVE_Z13_LARGE_64B_LOOP:
+ vl %v20,48(%r4,%r3)
+ vl %v19,32(%r4,%r3)
+ vl %v18,16(%r4,%r3)
+ vl %v17,0(%r4,%r3)
+ vst %v20,48(%r4,%r2)
+ vst %v19,32(%r4,%r2)
+ vst %v18,16(%r4,%r2)
+ vst %v17,0(%r4,%r2)
+ aghi %r4,-64
+ brctg %r0,.L_MEMMOVE_Z13_LARGE_64B_LOOP
+ aghi %r4,48
+ /* Recalculate the number of 16byte blocks. */
+ risbg %r5,%r5,62,128+63,0 /* r5 = r5 & 3
+ => Remaining 16byte blocks. */
+ jne .L_MEMMOVE_Z13_LARGE_16B_LOOP
+ aghik %r5,%r4,15
+ j .L_MEMMOVE_Z13_SMALL
+END(MEMMOVE_Z13)
+#endif /* HAVE_MEMMOVE_Z13 */
+
#if ! HAVE_MEMCPY_IFUNC
/* If we don't use ifunc, define an alias for mem[p]cpy here.
Otherwise see sysdeps/s390/mem[p]cpy.c. */
@@ -215,10 +285,27 @@ strong_alias (MEMPCPY_DEFAULT, __mempcpy
weak_alias (__mempcpy, mempcpy)
#endif
+#if ! HAVE_MEMMOVE_IFUNC
+/* If we don't use ifunc, define an alias for memmove here.
+ Otherwise see sysdeps/s390/memmove.c. */
+# if ! HAVE_MEMMOVE_C
+/* If the c variant is needed, then sysdeps/s390/memmove-c.c
+ defines memmove.
+ Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
+strong_alias (MEMMOVE_DEFAULT, memmove)
+# endif
+#endif
+
#if defined SHARED && IS_IN (libc)
/* Defines the internal symbols.
Compare to libc_hidden_[builtin_]def (mem[p]cpy) in string/mem[p]cpy.c. */
strong_alias (MEMCPY_DEFAULT, __GI_memcpy)
strong_alias (MEMPCPY_DEFAULT, __GI_mempcpy)
strong_alias (MEMPCPY_DEFAULT, __GI___mempcpy)
+# if ! HAVE_MEMMOVE_C
+/* If the c variant is needed, then sysdeps/s390/memmove-c.c
+ defines the internal symbol.
+ Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
+strong_alias (MEMMOVE_DEFAULT, __GI_memmove)
+# endif
#endif
Index: glibc-2.26/sysdeps/s390/memmove-c.c
===================================================================
--- /dev/null
+++ glibc-2.26/sysdeps/s390/memmove-c.c
@@ -0,0 +1,37 @@
+/* Fallback C version of memmove.
+ Copyright (C) 2018 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <ifunc-memcpy.h>
+
+#if HAVE_MEMMOVE_C
+# if HAVE_MEMMOVE_IFUNC
+/* If we use ifunc, then the memmove symbol is defined
+ in sysdeps/s390/memmove.c and we use a different name here.
+ Otherwise, we have to define memmove here or in
+ sysdeps/s390/memcpy.S depending on the used default implementation. */
+# define MEMMOVE MEMMOVE_C
+# if defined SHARED && IS_IN (libc)
+/* Define the internal symbol. */
+# undef libc_hidden_builtin_def
+# define libc_hidden_builtin_def(name) \
+ __hidden_ver1 (__memmove_c, __GI_memmove, __memmove_c);
+# endif
+# endif
+
+# include <string/memmove.c>
+#endif
Index: glibc-2.26/sysdeps/s390/memmove.c
===================================================================
--- /dev/null
+++ glibc-2.26/sysdeps/s390/memmove.c
@@ -0,0 +1,44 @@
+/* Multiple versions of memmove.
+ Copyright (C) 2016-2018 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <ifunc-memcpy.h>
+
+#if HAVE_MEMMOVE_IFUNC
+/* If we don't use ifunc, an alias is defined for memmove
+ in sysdeps/s390/memmove-c.c or sysdeps/s390/memcpy.S
+ depending on the used default implementation. */
+# undef memmove
+# define memmove __redirect_memmove
+# include <string.h>
+# include <ifunc-resolve.h>
+# undef memmove
+
+# if HAVE_MEMMOVE_C
+extern __typeof (__redirect_memmove) MEMMOVE_C attribute_hidden;
+# endif
+
+# if HAVE_MEMMOVE_Z13
+extern __typeof (__redirect_memmove) MEMMOVE_Z13 attribute_hidden;
+# endif
+
+s390_libc_ifunc_expr (__redirect_memmove, memmove,
+ (HAVE_MEMMOVE_Z13 && (hwcap & HWCAP_S390_VX))
+ ? MEMMOVE_Z13
+ : MEMMOVE_DEFAULT
+ )
+#endif
Index: glibc-2.26/sysdeps/s390/multiarch/ifunc-impl-list.c
===================================================================
--- glibc-2.26.orig/sysdeps/s390/multiarch/ifunc-impl-list.c
+++ glibc-2.26/sysdeps/s390/multiarch/ifunc-impl-list.c
@@ -91,6 +91,18 @@ __libc_ifunc_impl_list (const char *name
)
#endif /* HAVE_MEMCPY_IFUNC */
+#if HAVE_MEMMOVE_IFUNC
+ IFUNC_IMPL (i, name, memmove,
+# if HAVE_MEMMOVE_Z13
+ IFUNC_IMPL_ADD (array, i, memmove,
+ dl_hwcap & HWCAP_S390_VX, MEMMOVE_Z13)
+# endif
+# if HAVE_MEMMOVE_C
+ IFUNC_IMPL_ADD (array, i, memmove, 1, MEMMOVE_C)
+# endif
+ )
+#endif /* HAVE_MEMMOVE_IFUNC */
+
#ifdef HAVE_S390_VX_ASM_SUPPORT
# define IFUNC_VX_IMPL(FUNC) \
