diff options
| author | Stefan Liebler <stli@linux.ibm.com> | 2025-10-28 15:21:18 +0100 |
|---|---|---|
| committer | Stefan Liebler <stli@linux.ibm.com> | 2025-11-18 14:21:13 +0100 |
| commit | b9579342c68baf0beacfb4e93b9a6a325d3f9527 (patch) | |
| tree | b2912881442af3f3ad2378633f0bb3e4a970284d /sysdeps/unix/sysv/linux | |
| parent | 5029b63280b505467d3452e91a1c51705e34ed35 (diff) | |
Remove support for lock elision.
The support for lock elision was already deprecated with glibc 2.42:
commit 77438db8cfa6ee66b3906230156bdae11c49a195
"Mark support for lock elision as deprecated."
See also discussions:
https://sourceware.org/pipermail/libc-alpha/2025-July/168492.html
This patch removes the architecture specific support for lock elision
for x86, powerpc and s390 by removing the elision-conf.h, elision-conf.c,
elision-lock.c, elision-timed.c, elision-unlock.c, elide.h, htm.h/hle.h files.
Those generic files are also removed.
The architecture specific structures are adjusted and the elision fields are
marked as unused. See struct_mutex.h files.
Furthermore in struct_rwlock.h, the leftover __rwelision was also removed.
Those were originally removed with commit 0377a7fde6dfcc078dda29a1225d7720a0931357
"nptl: Remove rwlock elision definitions"
and by chance reintroduced with commit 7df8af43ad1cd8ce527444de50bee6f35eebe071
"nptl: Add struct_rwlock.h"
The common code (e.g. the pthread_mutex-files) are changed back to the time
before lock elision was introduced with the x86-support:
- commit 1cdbe579482c07e9f4bb3baa4864da2d3e7eb837
"Add the low level infrastructure for pthreads lock elision with TSX"
- commit b023e4ca99f5e81f90d87d23cd267ef2abd2388c
"Add new internal mutex type flags for elision."
- commit 68cc29355f3334c7ad18f648ff9a6383a0916d23
"Add minimal test suite changes for elision enabled kernels"
- commit e8c659d74e011346785355eeef03b7fb6f533c61
"Add elision to pthread_mutex_{try,timed,un}lock"
- commit 49186d21ef2d87986bccaf0a7c45c48c91b265f3
"Disable elision for any pthread_mutexattr_settype call"
- commit 1717da59aed9612becd56aaa1249aac695af4c8a
"Add a configure option to enable lock elision and disable by default"
Elision is removed also from the tunables, the initialization part, the
pretty-printers and the manual.
Some extra handling in the testsuite is removed as well as the full tst-mutex10
testcase, which tested a race while enabling lock elision.
I've also searched the code for "elision", "elide", "transaction" and e.g.
cleaned some comments.
I've run the testsuite on x86_64 and s390x and run the build-many-glibcs.py
script.
Thanks to Sachin Monga, this patch is also tested on powerpc.
A NEWS entry also mentions the removal.
Reviewed-by: Wilco Dijkstra <Wilco.Dijkstra@arm.com>
Diffstat (limited to 'sysdeps/unix/sysv/linux')
23 files changed, 0 insertions, 1638 deletions
diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-conf.c b/sysdeps/unix/sysv/linux/powerpc/elision-conf.c deleted file mode 100644 index 451897bd3d..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-conf.c +++ /dev/null @@ -1,138 +0,0 @@ -/* elision-conf.c: Lock elision tunable parameters. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include "config.h" -#include <pthreadP.h> -#include <elision-conf.h> -#include <unistd.h> -#include <ldsodefs.h> - -#define TUNABLE_NAMESPACE elision -#include <elf/dl-tunables.h> - -/* Reasonable initial tuning values, may be revised in the future. - This is a conservative initial value. */ - -struct elision_config __elision_aconf = - { - /* How many times to use a non-transactional lock after a transactional - failure has occurred because the lock is already acquired. Expressed - in number of lock acquisition attempts. */ - .skip_lock_busy = 3, - /* How often to not attempt to use elision if a transaction aborted due - to reasons other than other threads' memory accesses. Expressed in - number of lock acquisition attempts. */ - .skip_lock_internal_abort = 3, - /* How often to not attempt to use elision if a lock used up all retries - without success. Expressed in number of lock acquisition attempts. */ - .skip_lock_out_of_tbegin_retries = 3, - /* How often we retry using elision if there is chance for the transaction - to finish execution (e.g., it wasn't aborted due to the lock being - already acquired. */ - .try_tbegin = 3, - /* Same as SKIP_LOCK_INTERNAL_ABORT but for trylock. */ - .skip_trylock_internal_abort = 3, - }; - -static inline void -__always_inline -do_set_elision_enable (int32_t elision_enable) -{ - /* Enable elision if it's available in hardware. It's not necessary to check - if __libc_enable_secure isn't enabled since elision_enable will be set - according to the default, which is disabled. */ - if (elision_enable == 1) - __pthread_force_elision = (GLRO (dl_hwcap2) - & PPC_FEATURE2_HAS_HTM) ? 1 : 0; -} - -/* The pthread->elision_enable tunable is 0 or 1 indicating that elision - should be disabled or enabled respectively. The feature will only be used - if it's supported by the hardware. */ - -void -TUNABLE_CALLBACK (set_elision_enable) (tunable_val_t *valp) -{ - int32_t elision_enable = (int32_t) valp->numval; - do_set_elision_enable (elision_enable); -} - -#define TUNABLE_CALLBACK_FNDECL(__name, __type) \ -static inline void \ -__always_inline \ -do_set_elision_ ## __name (__type value) \ -{ \ - __elision_aconf.__name = value; \ -} \ -void \ -TUNABLE_CALLBACK (set_elision_ ## __name) (tunable_val_t *valp) \ -{ \ - __type value = (__type) (valp)->numval; \ - do_set_elision_ ## __name (value); \ -} - -TUNABLE_CALLBACK_FNDECL (skip_lock_busy, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_internal_abort, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_out_of_tbegin_retries, int32_t); -TUNABLE_CALLBACK_FNDECL (try_tbegin, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_trylock_internal_abort, int32_t); - -/* Initialize elision. */ - -void -__lll_elision_init (void) -{ - /* Elision depends on tunables and must be explicitly turned on by setting - the appropriate tunable on a supported platform. */ - - TUNABLE_GET (enable, int32_t, - TUNABLE_CALLBACK (set_elision_enable)); - TUNABLE_GET (skip_lock_busy, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_busy)); - TUNABLE_GET (skip_lock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_internal_abort)); - TUNABLE_GET (skip_lock_after_retries, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_out_of_tbegin_retries)); - TUNABLE_GET (tries, int32_t, - TUNABLE_CALLBACK (set_elision_try_tbegin)); - TUNABLE_GET (skip_trylock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_trylock_internal_abort)); - - /* Linux from 3.9 through 4.2 do not abort HTM transaction on syscalls, - instead it suspends the transaction and resumes it when returning to - usercode. The side-effects of the syscall will always remain visible, - even if the transaction is aborted. This is an issue when a transaction - is used along with futex syscall, on pthread_cond_wait for instance, - where futex might succeed but the transaction is rolled back leading - the condition variable object in an inconsistent state. - - Glibc used to prevent it by always aborting a transaction before issuing - a syscall. Linux 4.2 also decided to abort active transaction in - syscalls which makes the glibc workaround superflours. Worse, glibc - transaction abortions leads to a performance issues on recent kernels. - - So Lock Elision is just enabled when it has been explicitly set (either - by tunables of by a configure switch) and if kernel aborts HTM - transactions on syscalls (PPC_FEATURE2_HTM_NOSC) */ - - __pthread_force_elision = (__pthread_force_elision - && GLRO (dl_hwcap2) & PPC_FEATURE2_HTM_NOSC); - - if (!__pthread_force_elision) - __elision_aconf.try_tbegin = 0; /* Disable elision on rwlocks. */ -} diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-conf.h b/sysdeps/unix/sysv/linux/powerpc/elision-conf.h deleted file mode 100644 index 11bc7edaff..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-conf.h +++ /dev/null @@ -1,40 +0,0 @@ -/* elision-conf.h: Lock elision tunable parameters. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#ifndef _ELISION_CONF_H -#define _ELISION_CONF_H 1 - -#include <pthread.h> -#include <time.h> - -#define ENABLE_ELISION_SUPPORT 1 -#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 1 - -/* Should make sure there is no false sharing on this. */ -struct elision_config -{ - int skip_lock_busy; - int skip_lock_internal_abort; - int skip_lock_out_of_tbegin_retries; - int try_tbegin; - int skip_trylock_internal_abort; -} __attribute__ ((__aligned__ (128))); - -extern struct elision_config __elision_aconf attribute_hidden; - -#endif diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-lock.c b/sysdeps/unix/sysv/linux/powerpc/elision-lock.c deleted file mode 100644 index 8eb7c3e547..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-lock.c +++ /dev/null @@ -1,81 +0,0 @@ -/* elision-lock.c: Elided pthread mutex lock. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <stdio.h> -#include <pthread.h> -#include <pthreadP.h> -#include <lowlevellock.h> -#include <elision-conf.h> -#include "htm.h" - -#ifndef EXTRAARG -# define EXTRAARG -#endif -#ifndef LLL_LOCK -# define LLL_LOCK(a,b) lll_lock(a,b), 0 -#endif - -#define aconf __elision_aconf - -/* Adaptive lock using transactions. - By default the lock region is run as a transaction, and when it - aborts or the lock is busy the lock adapts itself. */ - -int -__lll_lock_elision (int *lock, short *adapt_count, EXTRAARG int pshared) -{ - /* adapt_count is accessed concurrently but is just a hint. Thus, - use atomic accesses but relaxed MO is sufficient. */ - if (atomic_load_relaxed (adapt_count) > 0) - { - goto use_lock; - } - - for (int i = aconf.try_tbegin; i > 0; i--) - { - if (__libc_tbegin (0)) - { - if (*lock == 0) - return 0; - /* Lock was busy. Fall back to normal locking. */ - __libc_tabort (_ABORT_LOCK_BUSY); - } - else - { - /* A persistent failure indicates that a retry will probably - result in another failure. Use normal locking now and - for the next couple of calls. */ - if (_TEXASRU_FAILURE_PERSISTENT (__builtin_get_texasru ())) - { - if (aconf.skip_lock_internal_abort > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_internal_abort); - goto use_lock; - } - } - } - - /* Fall back to locks for a bit if retries have been exhausted */ - if (aconf.try_tbegin > 0 && aconf.skip_lock_out_of_tbegin_retries > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_out_of_tbegin_retries); - -use_lock: - return LLL_LOCK ((*lock), pshared); -} -libc_hidden_def (__lll_lock_elision) diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-timed.c b/sysdeps/unix/sysv/linux/powerpc/elision-timed.c deleted file mode 100644 index 6442408a33..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-timed.c +++ /dev/null @@ -1,29 +0,0 @@ -/* elision-timed.c: Lock elision timed lock. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <time.h> -#include <elision-conf.h> -#include "lowlevellock.h" -#include "futex-internal.h" - -#define __lll_lock_elision __lll_clocklock_elision -#define EXTRAARG clockid_t clockid, const struct __timespec64 *t, -#undef LLL_LOCK -#define LLL_LOCK(a, b) __futex_clocklock64 (&(a), clockid, t, b) - -#include "elision-lock.c" diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-trylock.c b/sysdeps/unix/sysv/linux/powerpc/elision-trylock.c deleted file mode 100644 index d252799c46..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-trylock.c +++ /dev/null @@ -1,70 +0,0 @@ -/* elision-trylock.c: Lock eliding trylock for pthreads. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <pthread.h> -#include <pthreadP.h> -#include <lowlevellock.h> -#include <elision-conf.h> -#include "htm.h" - -#define aconf __elision_aconf - -/* Try to elide a futex trylock. FUTEX is the futex variable. ADAPT_COUNT is - the adaptation counter in the mutex. */ - -int -__lll_trylock_elision (int *futex, short *adapt_count) -{ - /* Implement POSIX semantics by forbiding nesting elided trylocks. */ - __libc_tabort (_ABORT_NESTED_TRYLOCK); - - /* Only try a transaction if it's worth it. */ - if (atomic_load_relaxed (adapt_count) > 0) - { - goto use_lock; - } - - if (__libc_tbegin (0)) - { - if (*futex == 0) - return 0; - - /* Lock was busy. This is never a nested transaction. - End it, and set the adapt count. */ - __libc_tend (0); - - if (aconf.skip_lock_busy > 0) - atomic_store_relaxed (adapt_count, aconf.skip_lock_busy); - } - else - { - if (_TEXASRU_FAILURE_PERSISTENT (__builtin_get_texasru ())) - { - /* A persistent failure indicates that a retry will probably - result in another failure. Use normal locking now and - for the next couple of calls. */ - if (aconf.skip_trylock_internal_abort > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_trylock_internal_abort); - } - } - -use_lock: - return lll_trylock (*futex); -} -libc_hidden_def (__lll_trylock_elision) diff --git a/sysdeps/unix/sysv/linux/powerpc/elision-unlock.c b/sysdeps/unix/sysv/linux/powerpc/elision-unlock.c deleted file mode 100644 index 774151474e..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/elision-unlock.c +++ /dev/null @@ -1,44 +0,0 @@ -/* elision-unlock.c: Commit an elided pthread lock. - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include "pthreadP.h" -#include "lowlevellock.h" -#include "htm.h" - -int -__lll_unlock_elision (int *lock, short *adapt_count, int pshared) -{ - /* When the lock was free we're in a transaction. */ - if (*lock == 0) - __libc_tend (0); - else - { - /* Update adapt_count in the critical section to prevent a - write-after-destroy error as mentioned in BZ 20822. The - following update of adapt_count has to be contained within - the critical region of the fall-back lock in order to not violate - the mutex destruction requirements. */ - short __tmp = atomic_load_relaxed (adapt_count); - if (__tmp > 0) - atomic_store_relaxed (adapt_count, __tmp - 1); - - lll_unlock ((*lock), pshared); - } - return 0; -} -libc_hidden_def (__lll_unlock_elision) diff --git a/sysdeps/unix/sysv/linux/powerpc/htm.h b/sysdeps/unix/sysv/linux/powerpc/htm.h deleted file mode 100644 index 672b197a2c..0000000000 --- a/sysdeps/unix/sysv/linux/powerpc/htm.h +++ /dev/null @@ -1,171 +0,0 @@ -/* Shared HTM header. Emulate transactional execution facility intrinsics for - compilers and assemblers that do not support the intrinsics and instructions - yet. - - Copyright (C) 2015-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#ifndef _HTM_H -#define _HTM_H 1 - -#ifdef __ASSEMBLER__ - -/* tbegin. */ -.macro TBEGIN - .long 0x7c00051d -.endm - -/* tend. 0 */ -.macro TEND - .long 0x7c00055d -.endm - -/* tabort. code */ -.macro TABORT code - .byte 0x7c - .byte \code - .byte 0x07 - .byte 0x1d -.endm - -/*"TEXASR - Transaction EXception And Summary Register" - mfspr %dst,130 */ -.macro TEXASR dst - mfspr \dst,130 -.endm - -#else - -#include <bits/endian.h> - -/* Official HTM intrinsics interface matching GCC, but works - on older GCC compatible compilers and binutils. - We should somehow detect if the compiler supports it, because - it may be able to generate slightly better code. */ - -#define TBEGIN ".long 0x7c00051d" -#define TEND ".long 0x7c00055d" -#if __BYTE_ORDER == __LITTLE_ENDIAN -# define TABORT ".byte 0x1d,0x07,%1,0x7c" -#else -# define TABORT ".byte 0x7c,%1,0x07,0x1d" -#endif - -#define __force_inline inline __attribute__((__always_inline__)) - -#ifndef __HTM__ - -#define _TEXASRU_EXTRACT_BITS(TEXASR,BITNUM,SIZE) \ - (((TEXASR) >> (31-(BITNUM))) & ((1<<(SIZE))-1)) -#define _TEXASRU_FAILURE_PERSISTENT(TEXASRU) \ - _TEXASRU_EXTRACT_BITS(TEXASRU, 7, 1) - -#define _tbegin() \ - ({ unsigned int __ret; \ - asm volatile ( \ - TBEGIN "\t\n" \ - "mfcr %0\t\n" \ - "rlwinm %0,%0,3,1\t\n" \ - "xori %0,%0,1\t\n" \ - : "=r" (__ret) : \ - : "cr0", "memory"); \ - __ret; \ - }) - -#define _tend() \ - ({ unsigned int __ret; \ - asm volatile ( \ - TEND "\t\n" \ - "mfcr %0\t\n" \ - "rlwinm %0,%0,3,1\t\n" \ - "xori %0,%0,1\t\n" \ - : "=r" (__ret) : \ - : "cr0", "memory"); \ - __ret; \ - }) - -#define _tabort(__code) \ - ({ unsigned int __ret; \ - asm volatile ( \ - TABORT "\t\n" \ - "mfcr %0\t\n" \ - "rlwinm %0,%0,3,1\t\n" \ - "xori %0,%0,1\t\n" \ - : "=r" (__ret) : "r" (__code) \ - : "cr0", "memory"); \ - __ret; \ - }) - -#define _texasru() \ - ({ unsigned long __ret; \ - asm volatile ( \ - "mfspr %0,131\t\n" \ - : "=r" (__ret)); \ - __ret; \ - }) - -#define __libc_tbegin(tdb) _tbegin () -#define __libc_tend(nested) _tend () -#define __libc_tabort(abortcode) _tabort (abortcode) -#define __builtin_get_texasru() _texasru () - -#else -# include <htmintrin.h> - -# ifdef __TM_FENCE__ - /* New GCC behavior. */ -# define __libc_tbegin(R) __builtin_tbegin (R) -# define __libc_tend(R) __builtin_tend (R) -# define __libc_tabort(R) __builtin_tabort (R) -# else - /* Workaround an old GCC behavior. Earlier releases of GCC 4.9 and 5.0, - didn't use to treat __builtin_tbegin, __builtin_tend and - __builtin_tabort as compiler barriers, moving instructions into and - out the transaction. - Remove this when glibc drops support for GCC 5.0. */ -# define __libc_tbegin(R) \ - ({ __asm__ volatile("" ::: "memory"); \ - unsigned int __ret = __builtin_tbegin (R); \ - __asm__ volatile("" ::: "memory"); \ - __ret; \ - }) -# define __libc_tabort(R) \ - ({ __asm__ volatile("" ::: "memory"); \ - unsigned int __ret = __builtin_tabort (R); \ - __asm__ volatile("" ::: "memory"); \ - __ret; \ - }) -# define __libc_tend(R) \ - ({ __asm__ volatile("" ::: "memory"); \ - unsigned int __ret = __builtin_tend (R); \ - __asm__ volatile("" ::: "memory"); \ - __ret; \ - }) -# endif /* __TM_FENCE__ */ -#endif /* __HTM__ */ - -#endif /* __ASSEMBLER__ */ - -/* Definitions used for TEXASR Failure code (bits 0:7). If the failure - should be persistent, the abort code must be odd. 0xd0 through 0xff - are reserved for the kernel and potential hypervisor. */ -#define _ABORT_PERSISTENT 0x01 /* An unspecified persistent abort. */ -#define _ABORT_LOCK_BUSY 0x34 /* Busy lock, not persistent. */ -#define _ABORT_NESTED_TRYLOCK (0x32 | _ABORT_PERSISTENT) -#define _ABORT_SYSCALL (0x30 | _ABORT_PERSISTENT) - -#endif diff --git a/sysdeps/unix/sysv/linux/s390/Makefile b/sysdeps/unix/sysv/linux/s390/Makefile index 3de9579e23..d5433f1dd9 100644 --- a/sysdeps/unix/sysv/linux/s390/Makefile +++ b/sysdeps/unix/sysv/linux/s390/Makefile @@ -11,14 +11,6 @@ ifeq ($(subdir),stdlib) gen-as-const-headers += ucontext_i.sym endif -ifeq ($(subdir),nptl) -elision-CFLAGS = -mhtm -msoft-float -CFLAGS-elision-lock.c = $(elision-CFLAGS) -CFLAGS-elision-timed.c = $(elision-CFLAGS) -CFLAGS-elision-trylock.c = $(elision-CFLAGS) -CFLAGS-elision-unlock.c = $(elision-CFLAGS) -endif - ifeq ($(subdir),misc) tests += tst-ptrace-singleblock endif diff --git a/sysdeps/unix/sysv/linux/s390/elision-conf.c b/sysdeps/unix/sysv/linux/s390/elision-conf.c deleted file mode 100644 index 32b9b3a17d..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-conf.c +++ /dev/null @@ -1,118 +0,0 @@ -/* Lock elision tunable parameters. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <config.h> -#include <pthreadP.h> -#include <elision-conf.h> -#include <unistd.h> -#include <ldsodefs.h> -#include <sys/auxv.h> - -#define TUNABLE_NAMESPACE elision -#include <elf/dl-tunables.h> - -/* Reasonable initial tuning values, may be revised in the future. - This is a conservative initial value. */ - -struct elision_config __elision_aconf = - { - /* How often to not attempt to use elision if a transaction aborted - because the lock is already acquired. Expressed in number of lock - acquisition attempts. */ - .skip_lock_busy = 3, - /* How often to not attempt to use elision if a transaction aborted due - to reasons other than other threads' memory accesses. Expressed in - number of lock acquisition attempts. */ - .skip_lock_internal_abort = 3, - /* How often to not attempt to use elision if a lock used up all retries - without success. Expressed in number of lock acquisition attempts. */ - .skip_lock_out_of_tbegin_retries = 3, - /* How often we try using elision if there is chance for the transaction - to finish execution (e.g., it wasn't aborted due to the lock being - already acquired. */ - .try_tbegin = 3, - /* Same as SKIP_LOCK_INTERNAL_ABORT but for trylock. */ - .skip_trylock_internal_abort = 3, - }; - -static inline void -__always_inline -do_set_elision_enable (int32_t elision_enable) -{ - /* Enable elision if it's available in hardware. It's not necessary to check - if __libc_enable_secure isn't enabled since elision_enable will be set - according to the default, which is disabled. */ - if (elision_enable == 1) - __pthread_force_elision = (GLRO (dl_hwcap) & HWCAP_S390_TE) ? 1 : 0; -} - -/* The pthread->elision_enable tunable is 0 or 1 indicating that elision - should be disabled or enabled respectively. The feature will only be used - if it's supported by the hardware. */ - -void -TUNABLE_CALLBACK (set_elision_enable) (tunable_val_t *valp) -{ - int32_t elision_enable = (int32_t) valp->numval; - do_set_elision_enable (elision_enable); -} - -#define TUNABLE_CALLBACK_FNDECL(__name, __type) \ -static inline void \ -__always_inline \ -do_set_elision_ ## __name (__type value) \ -{ \ - __elision_aconf.__name = value; \ -} \ -void \ -TUNABLE_CALLBACK (set_elision_ ## __name) (tunable_val_t *valp) \ -{ \ - __type value = (__type) (valp)->numval; \ - do_set_elision_ ## __name (value); \ -} - -TUNABLE_CALLBACK_FNDECL (skip_lock_busy, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_internal_abort, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_out_of_tbegin_retries, int32_t); -TUNABLE_CALLBACK_FNDECL (try_tbegin, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_trylock_internal_abort, int32_t); - -/* Initialize elison. */ - -void -__lll_elision_init (void) -{ - /* Elision depends on tunables and must be explicitly turned on by setting - the appropriate tunable on a supported platform. */ - - TUNABLE_GET (enable, int32_t, - TUNABLE_CALLBACK (set_elision_enable)); - TUNABLE_GET (skip_lock_busy, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_busy)); - TUNABLE_GET (skip_lock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_internal_abort)); - TUNABLE_GET (skip_lock_after_retries, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_out_of_tbegin_retries)); - TUNABLE_GET (tries, int32_t, - TUNABLE_CALLBACK (set_elision_try_tbegin)); - TUNABLE_GET (skip_trylock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_trylock_internal_abort)); - - if (!__pthread_force_elision) - __elision_aconf.try_tbegin = 0; /* Disable elision on rwlocks. */ -} diff --git a/sysdeps/unix/sysv/linux/s390/elision-conf.h b/sysdeps/unix/sysv/linux/s390/elision-conf.h deleted file mode 100644 index 90845de01e..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-conf.h +++ /dev/null @@ -1,40 +0,0 @@ -/* Lock elision tunable parameters. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ -#ifndef _ELISION_CONF_H -#define _ELISION_CONF_H 1 - -#include <pthread.h> -#include <time.h> - -#define ENABLE_ELISION_SUPPORT 1 -#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 1 - -/* Should make sure there is no false sharing on this. */ - -struct elision_config -{ - int skip_lock_busy; - int skip_lock_internal_abort; - int skip_lock_out_of_tbegin_retries; - int try_tbegin; - int skip_trylock_internal_abort; -}; - -extern struct elision_config __elision_aconf attribute_hidden; - -#endif diff --git a/sysdeps/unix/sysv/linux/s390/elision-lock.c b/sysdeps/unix/sysv/linux/s390/elision-lock.c deleted file mode 100644 index b22e6cd938..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-lock.c +++ /dev/null @@ -1,120 +0,0 @@ -/* Elided pthread mutex lock. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <pthread.h> -#include <pthreadP.h> -#include <lowlevellock.h> -#include <htm.h> -#include <elision-conf.h> -#include <stdint.h> - -#ifndef EXTRAARG -#define EXTRAARG -#endif -#ifndef LLL_LOCK -#define LLL_LOCK(a,b) lll_lock(a,b), 0 -#endif - -#define aconf __elision_aconf - -/* Adaptive lock using transactions. - By default the lock region is run as a transaction, and when it - aborts or the lock is busy the lock adapts itself. */ - -int -__lll_lock_elision (int *futex, short *adapt_count, EXTRAARG int private) -{ - /* adapt_count can be accessed concurrently; these accesses can be both - inside of transactions (if critical sections are nested and the outer - critical section uses lock elision) and outside of transactions. Thus, - we need to use atomic accesses to avoid data races. However, the - value of adapt_count is just a hint, so relaxed MO accesses are - sufficient. */ - if (atomic_load_relaxed (adapt_count) <= 0 && aconf.try_tbegin > 0) - { - /* Start a transaction and retry it automatically if it aborts with - _HTM_TBEGIN_TRANSIENT. This macro calls tbegin at most retry_cnt - + 1 times. The second argument is considered as retry_cnt. */ - int status = __libc_tbegin_retry ((void *) 0, aconf.try_tbegin - 1); - if (__glibc_likely (status == _HTM_TBEGIN_STARTED)) - { - /* Check the futex to make sure nobody has touched it in the - mean time. This forces the futex into the cache and makes - sure the transaction aborts if another thread acquires the lock - concurrently. */ - if (__glibc_likely (atomic_load_relaxed (futex) == 0)) - /* Lock was free. Return to user code in a transaction. */ - return 0; - - /* Lock was busy. Fall back to normal locking. - This can be the case if e.g. adapt_count was decremented to zero - by a former release and another thread has been waken up and - acquired it. */ - if (__glibc_likely (__libc_tx_nesting_depth () <= 1)) - { - /* In a non-nested transaction there is no need to abort, - which is expensive. Simply end the started transaction. */ - __libc_tend (); - /* Don't try to use transactions for the next couple of times. - See above for why relaxed MO is sufficient. */ - if (aconf.skip_lock_busy > 0) - atomic_store_relaxed (adapt_count, aconf.skip_lock_busy); - } - else /* nesting depth is > 1 */ - { - /* A nested transaction will abort eventually because it - cannot make any progress before *futex changes back to 0. - So we may as well abort immediately. - This persistently aborts the outer transaction to force - the outer mutex use the default lock instead of retrying - with transactions until the try_tbegin of the outer mutex - is zero. - The adapt_count of this inner mutex is not changed, - because using the default lock with the inner mutex - would abort the outer transaction. */ - __libc_tabort (_HTM_FIRST_USER_ABORT_CODE | 1); - __builtin_unreachable (); - } - } - else if (status != _HTM_TBEGIN_TRANSIENT) - { - /* A persistent abort (cc 1 or 3) indicates that a retry is - probably futile. Use the normal locking now and for the - next couple of calls. - Be careful to avoid writing to the lock. See above for why - relaxed MO is sufficient. */ - if (aconf.skip_lock_internal_abort > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_internal_abort); - } - else - { - /* The transaction failed for some retries with - _HTM_TBEGIN_TRANSIENT. Use the normal locking now and for the - next couple of calls. */ - if (aconf.skip_lock_out_of_tbegin_retries > 0) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_out_of_tbegin_retries); - } - } - - /* Use normal locking as fallback path if the transaction does not - succeed. */ - return LLL_LOCK ((*futex), private); -} -libc_hidden_def (__lll_lock_elision) diff --git a/sysdeps/unix/sysv/linux/s390/elision-timed.c b/sysdeps/unix/sysv/linux/s390/elision-timed.c deleted file mode 100644 index 2611e9cf9b..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-timed.c +++ /dev/null @@ -1,27 +0,0 @@ -/* Lock elision timed lock. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <time.h> -#include <elision-conf.h> -#include <lowlevellock.h> -#include "futex-internal.h" -#define __lll_lock_elision __lll_clocklock_elision -#define EXTRAARG clockid_t clockid, const struct __timespec64 *t, -#undef LLL_LOCK -#define LLL_LOCK(a, b) __futex_clocklock64 (&(a), clockid, t, b) -#include "elision-lock.c" diff --git a/sysdeps/unix/sysv/linux/s390/elision-trylock.c b/sysdeps/unix/sysv/linux/s390/elision-trylock.c deleted file mode 100644 index 1aa99d806c..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-trylock.c +++ /dev/null @@ -1,98 +0,0 @@ -/* Elided pthread mutex trylock. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <pthread.h> -#include <pthreadP.h> -#include <lowlevellock.h> -#include <htm.h> -#include <elision-conf.h> - -#define aconf __elision_aconf - -/* Try to elide a futex trylock. FUTEX is the futex variable. ADAPT_COUNT is - the adaptation counter in the mutex. */ - -int -__lll_trylock_elision (int *futex, short *adapt_count) -{ - /* Implement POSIX semantics by forbiding nesting elided trylocks. - Sorry. After the abort the code is re-executed - non transactional and if the lock was already locked - return an error. */ - if (__libc_tx_nesting_depth () > 0) - { - /* Note that this abort may terminate an outermost transaction that - was created outside glibc. - This persistently aborts the current transactions to force - them to use the default lock instead of retrying transactions - until their try_tbegin is zero. - */ - __libc_tabort (_HTM_FIRST_USER_ABORT_CODE | 1); - __builtin_unreachable (); - } - - /* adapt_count can be accessed concurrently; these accesses can be both - inside of transactions (if critical sections are nested and the outer - critical section uses lock elision) and outside of transactions. Thus, - we need to use atomic accesses to avoid data races. However, the - value of adapt_count is just a hint, so relaxed MO accesses are - sufficient. */ - if (atomic_load_relaxed (adapt_count) <= 0 && aconf.try_tbegin > 0) - { - int status = __libc_tbegin ((void *) 0); - if (__glibc_likely (status == _HTM_TBEGIN_STARTED)) - { - /* Check the futex to make sure nobody has touched it in the - mean time. This forces the futex into the cache and makes - sure the transaction aborts if another thread acquires the lock - concurrently. */ - if (__glibc_likely (atomic_load_relaxed (futex) == 0)) - /* Lock was free. Return to user code in a transaction. */ - return 0; - - /* Lock was busy. Fall back to normal locking. - This can be the case if e.g. adapt_count was decremented to zero - by a former release and another thread has been waken up and - acquired it. - Since we are in a non-nested transaction there is no need to abort, - which is expensive. Simply end the started transaction. */ - __libc_tend (); - /* Note: Changing the adapt_count here might abort a transaction on a - different CPU, but that could happen anyway when the futex is - acquired, so there's no need to check the nesting depth here. - See above for why relaxed MO is sufficient. */ - if (aconf.skip_lock_busy > 0) - atomic_store_relaxed (adapt_count, aconf.skip_lock_busy); - } - else if (status != _HTM_TBEGIN_TRANSIENT) - { - /* A persistent abort (cc 1 or 3) indicates that a retry is - probably futile. Use the normal locking now and for the - next couple of calls. - Be careful to avoid writing to the lock. */ - if (aconf.skip_trylock_internal_abort > 0) - *adapt_count = aconf.skip_trylock_internal_abort; - } - /* Could do some retries here. */ - } - - /* Use normal locking as fallback path if the transaction does not - succeed. */ - return lll_trylock (*futex); -} -libc_hidden_def (__lll_trylock_elision) diff --git a/sysdeps/unix/sysv/linux/s390/elision-unlock.c b/sysdeps/unix/sysv/linux/s390/elision-unlock.c deleted file mode 100644 index 11e96abd5f..0000000000 --- a/sysdeps/unix/sysv/linux/s390/elision-unlock.c +++ /dev/null @@ -1,62 +0,0 @@ -/* Commit an elided pthread lock. - Copyright (C) 2014-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <pthreadP.h> -#include <lowlevellock.h> -#include <htm.h> - -int -__lll_unlock_elision(int *futex, short *adapt_count, int private) -{ - /* If the lock is free, we elided the lock earlier. This does not - necessarily mean that we are in a transaction, because the user code may - have closed the transaction, but that is impossible to detect reliably. - Relaxed MO access to futex is sufficient because a correct program - will only release a lock it has acquired; therefore, it must either - changed the futex word's value to something !=0 or it must have used - elision; these are actions by the same thread, so these actions are - sequenced-before the relaxed load (and thus also happens-before the - relaxed load). Therefore, relaxed MO is sufficient. */ - if (atomic_load_relaxed (futex) == 0) - { - __libc_tend (); - } - else - { - /* Update the adapt_count while unlocking before completing the critical - section. adapt_count is accessed concurrently outside of a - transaction or a critical section (e.g. in elision-lock.c). So we need - to use atomic accesses. However, the value of adapt_count is just a - hint, so relaxed MO accesses are sufficient. - If adapt_count would be decremented while locking, multiple - CPUs, trying to lock the acquired mutex, will decrement adapt_count to - zero and another CPU will try to start a transaction, which will be - immediately aborted as the mutex is locked. - The update of adapt_count is done before releasing the lock as POSIX' - mutex destruction requirements disallow accesses to the mutex after it - has been released and thus could have been acquired or destroyed by - another thread. */ - short adapt_count_val = atomic_load_relaxed (adapt_count); - if (adapt_count_val > 0) - atomic_store_relaxed (adapt_count, adapt_count_val - 1); - - lll_unlock ((*futex), private); - } - return 0; -} -libc_hidden_def (__lll_unlock_elision) diff --git a/sysdeps/unix/sysv/linux/s390/htm.h b/sysdeps/unix/sysv/linux/s390/htm.h deleted file mode 100644 index bf1a89c1a5..0000000000 --- a/sysdeps/unix/sysv/linux/s390/htm.h +++ /dev/null @@ -1,187 +0,0 @@ -/* Shared HTM header. Work around false transactional execution facility - intrinsics. - - Copyright (C) 2016-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#ifndef _HTM_H -#define _HTM_H 1 - -#include <htmintrin.h> - -#ifdef __s390x__ -# define TX_FPRS_BYTES 64 -# define TX_SAVE_FPRS \ - " std %%f8, 0(%[R_FPRS])\n\t" \ - " std %%f9, 8(%[R_FPRS])\n\t" \ - " std %%f10, 16(%[R_FPRS])\n\t" \ - " std %%f11, 24(%[R_FPRS])\n\t" \ - " std %%f12, 32(%[R_FPRS])\n\t" \ - " std %%f13, 40(%[R_FPRS])\n\t" \ - " std %%f14, 48(%[R_FPRS])\n\t" \ - " std %%f15, 56(%[R_FPRS])\n\t" - -# define TX_RESTORE_FPRS \ - " ld %%f8, 0(%[R_FPRS])\n\t" \ - " ld %%f9, 8(%[R_FPRS])\n\t" \ - " ld %%f10, 16(%[R_FPRS])\n\t" \ - " ld %%f11, 24(%[R_FPRS])\n\t" \ - " ld %%f12, 32(%[R_FPRS])\n\t" \ - " ld %%f13, 40(%[R_FPRS])\n\t" \ - " ld %%f14, 48(%[R_FPRS])\n\t" \ - " ld %%f15, 56(%[R_FPRS])\n\t" - -#else - -# define TX_FPRS_BYTES 16 -# define TX_SAVE_FPRS \ - " std %%f4, 0(%[R_FPRS])\n\t" \ - " std %%f6, 8(%[R_FPRS])\n\t" - -# define TX_RESTORE_FPRS \ - " ld %%f4, 0(%[R_FPRS])\n\t" \ - " ld %%f6, 8(%[R_FPRS])\n\t" - -#endif /* ! __s390x__ */ - -/* Use own inline assembly instead of __builtin_tbegin, as tbegin - has to filter program interruptions which can't be done with the builtin. - Now the fprs have to be saved / restored here, too. - The fpc is also not saved / restored with the builtin. - The used inline assembly does not clobber the volatile fprs / vrs! - Clobbering the latter ones would force the compiler to save / restore - the call saved fprs as those overlap with the vrs, but they only need to be - restored if the transaction fails but not if the transaction is successfully - started. Thus the user of the tbegin macros in this header file has to - compile the file / function with -msoft-float. It prevents gcc from using - fprs / vrs. */ -#define __libc_tbegin(tdb) __libc_tbegin_base(tdb,,,) - -#define __libc_tbegin_retry_output_regs , [R_TX_CNT] "+&d" (__tx_cnt) -#define __libc_tbegin_retry_input_regs(retry_cnt) , [R_RETRY] "d" (retry_cnt) -#define __libc_tbegin_retry_abort_path_insn \ - /* If tbegin returned _HTM_TBEGIN_TRANSIENT, retry immediately so \ - that max tbegin_cnt transactions are tried. Otherwise return and \ - let the caller of this macro do the fallback path. */ \ - " jnh 1f\n\t" /* cc 1/3: jump to fallback path. */ \ - /* tbegin returned _HTM_TBEGIN_TRANSIENT: retry with transaction. */ \ - " crje %[R_TX_CNT], %[R_RETRY], 1f\n\t" /* Reached max retries? */ \ - " ahi %[R_TX_CNT], 1\n\t" \ - " ppa %[R_TX_CNT], 0, 1\n\t" /* Transaction-Abort Assist. */ \ - " j 2b\n\t" /* Loop to tbegin. */ - -/* Same as __libc_tbegin except if tbegin aborts with _HTM_TBEGIN_TRANSIENT. - Then this macros restores the fpc, fprs and automatically retries up to - retry_cnt tbegins. Further saving of the state is omitted as it is already - saved. This macro calls tbegin at most as retry_cnt + 1 times. */ -#define __libc_tbegin_retry(tdb, retry_cnt) \ - ({ int __ret; \ - int __tx_cnt = 0; \ - __ret = __libc_tbegin_base(tdb, \ - __libc_tbegin_retry_abort_path_insn, \ - __libc_tbegin_retry_output_regs, \ - __libc_tbegin_retry_input_regs(retry_cnt)); \ - __ret; \ - }) - -#define __libc_tbegin_base(tdb, abort_path_insn, output_regs, input_regs) \ - ({ int __ret; \ - int __fpc; \ - char __fprs[TX_FPRS_BYTES]; \ - __asm__ __volatile__ (".machine push\n\t" \ - ".machinemode \"zarch_nohighgprs\"\n\t" \ - ".machine \"all\"\n\t" \ - /* Save state at the outermost transaction. \ - As extracting nesting depth is expensive \ - on at least zEC12, save fprs at inner \ - transactions, too. \ - The fpc and fprs are saved here as they \ - are not saved by tbegin. There exist no \ - call-saved vrs, thus they are not saved \ - here. */ \ - " efpc %[R_FPC]\n\t" \ - TX_SAVE_FPRS \ - /* Begin transaction: save all gprs, allow \ - ar modification and fp operations. Some \ - program-interruptions (e.g. a null \ - pointer access) are filtered and the \ - transaction will abort. In this case \ - the normal lock path will execute it \ - again and result in a core dump which does \ - now show at tbegin but the real executed \ - instruction. \ - However it is not guaranteed that this \ - retry operate on the same data and thus \ - may not end in an program-interruption. \ - Note: This could also be used to probe \ - memory for being accessible! */ \ - "2: tbegin 0, 0xFF0E\n\t" \ - /* Branch away in abort case (this is the \ - preferred sequence. See PoP in chapter 5 \ - Transactional-Execution Facility \ - Operation). */ \ - " jnz 0f\n\t" \ - /* Transaction has successfully started. */ \ - " lhi %[R_RET], 0\n\t" \ - " j 1f\n\t" \ - /* Transaction has aborted. Now we are at \ - the outermost transaction. Restore fprs \ - and fpc. */ \ - "0: ipm %[R_RET]\n\t" \ - " srl %[R_RET], 28\n\t" \ - " sfpc %[R_FPC]\n\t" \ - TX_RESTORE_FPRS \ - abort_path_insn \ - "1:\n\t" \ - ".machine pop\n" \ - : [R_RET] "=&d" (__ret), \ - [R_FPC] "=&d" (__fpc) \ - output_regs \ - : [R_FPRS] "a" (__fprs) \ - input_regs \ - : "cc", "memory"); \ - __ret; \ - }) - -/* These builtins are usable in context of glibc lock elision code without any - changes. Use them. */ -#define __libc_tend() \ - ({ __asm__ __volatile__ (".machine push\n\t" \ - ".machinemode \"zarch_nohighgprs\"\n\t" \ - ".machine \"all\"\n\t"); \ - int __ret = __builtin_tend (); \ - __asm__ __volatile__ (".machine pop"); \ - __ret; \ - }) - -#define __libc_tabort(abortcode) \ - __asm__ __volatile__ (".machine push\n\t" \ - ".machinemode \"zarch_nohighgprs\"\n\t" \ - ".machine \"all\"\n\t"); \ - __builtin_tabort (abortcode); \ - __asm__ __volatile__ (".machine pop") - -#define __libc_tx_nesting_depth() \ - ({ __asm__ __volatile__ (".machine push\n\t" \ - ".machinemode \"zarch_nohighgprs\"\n\t" \ - ".machine \"all\"\n\t"); \ - int __ret = __builtin_tx_nesting_depth (); \ - __asm__ __volatile__ (".machine pop"); \ - __ret; \ - }) - -#endif diff --git a/sysdeps/unix/sysv/linux/x86/Makefile b/sysdeps/unix/sysv/linux/x86/Makefile index 743b633b65..92a5e03f64 100644 --- a/sysdeps/unix/sysv/linux/x86/Makefile +++ b/sysdeps/unix/sysv/linux/x86/Makefile @@ -11,13 +11,6 @@ ifeq ($(subdir),misc) sysdep_headers += sys/elf.h sys/perm.h sys/reg.h sys/vm86.h sys/debugreg.h sys/io.h endif -ifeq ($(subdir),nptl) -CFLAGS-elision-lock.c += -mrtm -CFLAGS-elision-unlock.c += -mrtm -CFLAGS-elision-timed.c += -mrtm -CFLAGS-elision-trylock.c += -mrtm -endif - ifeq ($(subdir),setjmp) tests += tst-saved_mask-1 endif diff --git a/sysdeps/unix/sysv/linux/x86/elision-conf.c b/sysdeps/unix/sysv/linux/x86/elision-conf.c deleted file mode 100644 index cf120a9ec6..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-conf.c +++ /dev/null @@ -1,109 +0,0 @@ -/* elision-conf.c: Lock elision tunable parameters. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include "config.h" -#include <pthreadP.h> -#include <init-arch.h> -#include <elision-conf.h> -#include <unistd.h> - -#define TUNABLE_NAMESPACE elision -#include <elf/dl-tunables.h> - -/* Reasonable initial tuning values, may be revised in the future. - This is a conservative initial value. */ - -struct elision_config __elision_aconf = - { - /* How often to not attempt to use elision if a transaction aborted - because the lock is already acquired. Expressed in number of lock - acquisition attempts. */ - .skip_lock_busy = 3, - /* How often to not attempt to use elision if a transaction aborted due - to reasons other than other threads' memory accesses. Expressed in - number of lock acquisition attempts. */ - .skip_lock_internal_abort = 3, - /* How often we retry using elision if there is chance for the transaction - to finish execution (e.g., it wasn't aborted due to the lock being - already acquired. */ - .retry_try_xbegin = 3, - /* Same as SKIP_LOCK_INTERNAL_ABORT but for trylock. */ - .skip_trylock_internal_abort = 3, - }; - -static __always_inline void -do_set_elision_enable (int32_t elision_enable) -{ - /* Enable elision if it's available in hardware. It's not necessary to check - if __libc_enable_secure isn't enabled since elision_enable will be set - according to the default, which is disabled. */ - if (elision_enable == 1) - __pthread_force_elision = CPU_FEATURE_USABLE (RTM) ? 1 : 0; -} - -/* The pthread->elision_enable tunable is 0 or 1 indicating that elision - should be disabled or enabled respectively. The feature will only be used - if it's supported by the hardware. */ - -void -TUNABLE_CALLBACK (set_elision_enable) (tunable_val_t *valp) -{ - int32_t elision_enable = (int32_t) valp->numval; - do_set_elision_enable (elision_enable); -} - -#define TUNABLE_CALLBACK_FNDECL(__name, __type) \ -static __always_inline void \ -do_set_elision_ ## __name (__type value) \ -{ \ - __elision_aconf.__name = value; \ -} \ -void \ -TUNABLE_CALLBACK (set_elision_ ## __name) (tunable_val_t *valp) \ -{ \ - __type value = (__type) (valp)->numval; \ - do_set_elision_ ## __name (value); \ -} - -TUNABLE_CALLBACK_FNDECL (skip_lock_busy, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_lock_internal_abort, int32_t); -TUNABLE_CALLBACK_FNDECL (retry_try_xbegin, int32_t); -TUNABLE_CALLBACK_FNDECL (skip_trylock_internal_abort, int32_t); - -/* Initialize elision. */ - -void -__lll_elision_init (void) -{ - /* Elision depends on tunables and must be explicitly turned on by setting - the appropriate tunable on a supported platform. */ - - TUNABLE_GET (enable, int32_t, - TUNABLE_CALLBACK (set_elision_enable)); - TUNABLE_GET (skip_lock_busy, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_busy)); - TUNABLE_GET (skip_lock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_lock_internal_abort)); - TUNABLE_GET (tries, int32_t, - TUNABLE_CALLBACK (set_elision_retry_try_xbegin)); - TUNABLE_GET (skip_trylock_internal_abort, int32_t, - TUNABLE_CALLBACK (set_elision_skip_trylock_internal_abort)); - - if (!__pthread_force_elision) - __elision_aconf.retry_try_xbegin = 0; /* Disable elision on rwlocks. */ -} diff --git a/sysdeps/unix/sysv/linux/x86/elision-conf.h b/sysdeps/unix/sysv/linux/x86/elision-conf.h deleted file mode 100644 index 6ba8810654..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-conf.h +++ /dev/null @@ -1,39 +0,0 @@ -/* elision-conf.h: Lock elision tunable parameters. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ -#ifndef _ELISION_CONF_H -#define _ELISION_CONF_H 1 - -#include <pthread.h> -#include <time.h> - -#define ENABLE_ELISION_SUPPORT 1 -#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 0 - -/* Should make sure there is no false sharing on this. */ - -struct elision_config -{ - int skip_lock_busy; - int skip_lock_internal_abort; - int retry_try_xbegin; - int skip_trylock_internal_abort; -}; - -extern struct elision_config __elision_aconf attribute_hidden; - -#endif diff --git a/sysdeps/unix/sysv/linux/x86/elision-lock.c b/sysdeps/unix/sysv/linux/x86/elision-lock.c deleted file mode 100644 index 74e748266d..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-lock.c +++ /dev/null @@ -1,102 +0,0 @@ -/* elision-lock.c: Elided pthread mutex lock. - Copyright (C) 2011-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <pthread.h> -#include "pthreadP.h" -#include "lowlevellock.h" -#include "hle.h" -#include <elision-conf.h> - -#ifndef EXTRAARG -#define EXTRAARG -#endif -#ifndef LLL_LOCK -#define LLL_LOCK(a,b) lll_lock(a,b), 0 -#endif - -#define aconf __elision_aconf - -/* Adaptive lock using transactions. - By default the lock region is run as a transaction, and when it - aborts or the lock is busy the lock adapts itself. */ - -int -__lll_lock_elision (int *futex, short *adapt_count, EXTRAARG int private) -{ - /* adapt_count can be accessed concurrently; these accesses can be both - inside of transactions (if critical sections are nested and the outer - critical section uses lock elision) and outside of transactions. Thus, - we need to use atomic accesses to avoid data races. However, the - value of adapt_count is just a hint, so relaxed MO accesses are - sufficient. */ - if (atomic_load_relaxed (adapt_count) <= 0) - { - unsigned status; - int try_xbegin; - - for (try_xbegin = aconf.retry_try_xbegin; - try_xbegin > 0; - try_xbegin--) - { - if ((status = _xbegin()) == _XBEGIN_STARTED) - { - if (*futex == 0) - return 0; - - /* Lock was busy. Fall back to normal locking. - Could also _xend here but xabort with 0xff code - is more visible in the profiler. */ - _xabort (_ABORT_LOCK_BUSY); - } - - if (!(status & _XABORT_RETRY)) - { - if ((status & _XABORT_EXPLICIT) - && _XABORT_CODE (status) == _ABORT_LOCK_BUSY) - { - /* Right now we skip here. Better would be to wait a bit - and retry. This likely needs some spinning. See - above for why relaxed MO is sufficient. */ - if (atomic_load_relaxed (adapt_count) - != aconf.skip_lock_busy) - atomic_store_relaxed (adapt_count, aconf.skip_lock_busy); - } - /* Internal abort. There is no chance for retry. - Use the normal locking and next time use lock. - Be careful to avoid writing to the lock. See above for why - relaxed MO is sufficient. */ - else if (atomic_load_relaxed (adapt_count) - != aconf.skip_lock_internal_abort) - atomic_store_relaxed (adapt_count, - aconf.skip_lock_internal_abort); - break; - } - } - } - else - { - /* Use a normal lock until the threshold counter runs out. - Lost updates possible. */ - atomic_store_relaxed (adapt_count, - atomic_load_relaxed (adapt_count) - 1); - } - - /* Use a normal lock here. */ - return LLL_LOCK ((*futex), private); -} -libc_hidden_def (__lll_lock_elision) diff --git a/sysdeps/unix/sysv/linux/x86/elision-timed.c b/sysdeps/unix/sysv/linux/x86/elision-timed.c deleted file mode 100644 index 50925f4a60..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-timed.c +++ /dev/null @@ -1,27 +0,0 @@ -/* elision-timed.c: Lock elision timed lock. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <time.h> -#include <elision-conf.h> -#include "lowlevellock.h" -#include "futex-internal.h" -#define __lll_lock_elision __lll_clocklock_elision -#define EXTRAARG clockid_t clockid, const struct __timespec64 *t, -#undef LLL_LOCK -#define LLL_LOCK(a, b) __futex_clocklock64 (&(a), clockid, t, b) -#include "elision-lock.c" diff --git a/sysdeps/unix/sysv/linux/x86/elision-trylock.c b/sysdeps/unix/sysv/linux/x86/elision-trylock.c deleted file mode 100644 index 60c4311798..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-trylock.c +++ /dev/null @@ -1,76 +0,0 @@ -/* elision-trylock.c: Lock eliding trylock for pthreads. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <pthread.h> -#include <pthreadP.h> -#include <lowlevellock.h> -#include "hle.h" -#include <elision-conf.h> - -#define aconf __elision_aconf - -/* Try to elide a futex trylock. FUTEX is the futex variable. ADAPT_COUNT is - the adaptation counter in the mutex. */ - -int -__lll_trylock_elision (int *futex, short *adapt_count) -{ - /* Implement POSIX semantics by forbiding nesting - trylock. Sorry. After the abort the code is re-executed - non transactional and if the lock was already locked - return an error. */ - _xabort (_ABORT_NESTED_TRYLOCK); - - /* Only try a transaction if it's worth it. See __lll_lock_elision for - why we need atomic accesses. Relaxed MO is sufficient because this is - just a hint. */ - if (atomic_load_relaxed (adapt_count) <= 0) - { - unsigned status; - - if ((status = _xbegin()) == _XBEGIN_STARTED) - { - if (*futex == 0) - return 0; - - /* Lock was busy. Fall back to normal locking. - Could also _xend here but xabort with 0xff code - is more visible in the profiler. */ - _xabort (_ABORT_LOCK_BUSY); - } - - if (!(status & _XABORT_RETRY)) - { - /* Internal abort. No chance for retry. For future - locks don't try speculation for some time. See above for MO. */ - if (atomic_load_relaxed (adapt_count) - != aconf.skip_lock_internal_abort) - atomic_store_relaxed (adapt_count, aconf.skip_lock_internal_abort); - } - /* Could do some retries here. */ - } - else - { - /* Lost updates are possible but harmless (see above). */ - atomic_store_relaxed (adapt_count, - atomic_load_relaxed (adapt_count) - 1); - } - - return lll_trylock (*futex); -} -libc_hidden_def (__lll_trylock_elision) diff --git a/sysdeps/unix/sysv/linux/x86/elision-unlock.c b/sysdeps/unix/sysv/linux/x86/elision-unlock.c deleted file mode 100644 index 67dd1e6949..0000000000 --- a/sysdeps/unix/sysv/linux/x86/elision-unlock.c +++ /dev/null @@ -1,34 +0,0 @@ -/* elision-unlock.c: Commit an elided pthread lock. - Copyright (C) 2013-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include "pthreadP.h" -#include "lowlevellock.h" -#include "hle.h" - -int -__lll_unlock_elision(int *lock, int private) -{ - /* When the lock was free we're in a transaction. - When you crash here you unlocked a free lock. */ - if (*lock == 0) - _xend(); - else - lll_unlock ((*lock), private); - return 0; -} -libc_hidden_def (__lll_unlock_elision) diff --git a/sysdeps/unix/sysv/linux/x86/hle.h b/sysdeps/unix/sysv/linux/x86/hle.h deleted file mode 100644 index 0449026839..0000000000 --- a/sysdeps/unix/sysv/linux/x86/hle.h +++ /dev/null @@ -1,11 +0,0 @@ -/* Shared RTM header. */ -#ifndef _HLE_H -#define _HLE_H 1 - -#include <x86intrin.h> - -#define _ABORT_LOCK_BUSY 0xff -#define _ABORT_LOCK_IS_LOCKED 0xfe -#define _ABORT_NESTED_TRYLOCK 0xfd - -#endif |
