diff options
Diffstat (limited to 'llvm/lib/Support/rpmalloc/malloc.c')
| -rw-r--r-- | llvm/lib/Support/rpmalloc/malloc.c | 724 |
1 files changed, 724 insertions, 0 deletions
diff --git a/llvm/lib/Support/rpmalloc/malloc.c b/llvm/lib/Support/rpmalloc/malloc.c new file mode 100644 index 000000000000..3fcfe848250c --- /dev/null +++ b/llvm/lib/Support/rpmalloc/malloc.c @@ -0,0 +1,724 @@ +//===------------------------ malloc.c ------------------*- C -*-=============//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This library provides a cross-platform lock free thread caching malloc
+// implementation in C11.
+//
+//
+// This file provides overrides for the standard library malloc entry points for
+// C and new/delete operators for C++ It also provides automatic
+// initialization/finalization of process and threads
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__TINYC__)
+#include <sys/types.h>
+#endif
+
+#ifndef ARCH_64BIT
+#if defined(__LLP64__) || defined(__LP64__) || defined(_WIN64)
+#define ARCH_64BIT 1
+_Static_assert(sizeof(size_t) == 8, "Data type size mismatch");
+_Static_assert(sizeof(void *) == 8, "Data type size mismatch");
+#else
+#define ARCH_64BIT 0
+_Static_assert(sizeof(size_t) == 4, "Data type size mismatch");
+_Static_assert(sizeof(void *) == 4, "Data type size mismatch");
+#endif
+#endif
+
+#if (defined(__GNUC__) || defined(__clang__))
+#pragma GCC visibility push(default)
+#endif
+
+#define USE_IMPLEMENT 1
+#define USE_INTERPOSE 0
+#define USE_ALIAS 0
+
+#if defined(__APPLE__)
+#undef USE_INTERPOSE
+#define USE_INTERPOSE 1
+
+typedef struct interpose_t {
+ void *new_func;
+ void *orig_func;
+} interpose_t;
+
+#define MAC_INTERPOSE_PAIR(newf, oldf) {(void *)newf, (void *)oldf}
+#define MAC_INTERPOSE_SINGLE(newf, oldf) \
+ __attribute__((used)) static const interpose_t macinterpose##newf##oldf \
+ __attribute__((section("__DATA, __interpose"))) = \
+ MAC_INTERPOSE_PAIR(newf, oldf)
+
+#endif
+
+#if !defined(_WIN32) && !defined(__APPLE__)
+#undef USE_IMPLEMENT
+#undef USE_ALIAS
+#define USE_IMPLEMENT 0
+#define USE_ALIAS 1
+#endif
+
+#ifdef _MSC_VER
+#pragma warning(disable : 4100)
+#undef malloc
+#undef free
+#undef calloc
+#define RPMALLOC_RESTRICT __declspec(restrict)
+#else
+#define RPMALLOC_RESTRICT
+#endif
+
+#if ENABLE_OVERRIDE
+
+typedef struct rp_nothrow_t {
+ int __dummy;
+} rp_nothrow_t;
+
+#if USE_IMPLEMENT
+
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL malloc(size_t size) {
+ return rpmalloc(size);
+}
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL calloc(size_t count,
+ size_t size) {
+ return rpcalloc(count, size);
+}
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL realloc(void *ptr,
+ size_t size) {
+ return rprealloc(ptr, size);
+}
+extern inline void *RPMALLOC_CDECL reallocf(void *ptr, size_t size) {
+ return rprealloc(ptr, size);
+}
+extern inline void *RPMALLOC_CDECL aligned_alloc(size_t alignment,
+ size_t size) {
+ return rpaligned_alloc(alignment, size);
+}
+extern inline void *RPMALLOC_CDECL memalign(size_t alignment, size_t size) {
+ return rpmemalign(alignment, size);
+}
+extern inline int RPMALLOC_CDECL posix_memalign(void **memptr, size_t alignment,
+ size_t size) {
+ return rpposix_memalign(memptr, alignment, size);
+}
+extern inline void RPMALLOC_CDECL free(void *ptr) { rpfree(ptr); }
+extern inline void RPMALLOC_CDECL cfree(void *ptr) { rpfree(ptr); }
+extern inline size_t RPMALLOC_CDECL malloc_usable_size(void *ptr) {
+ return rpmalloc_usable_size(ptr);
+}
+extern inline size_t RPMALLOC_CDECL malloc_size(void *ptr) {
+ return rpmalloc_usable_size(ptr);
+}
+
+#ifdef _WIN32
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _malloc_base(size_t size) {
+ return rpmalloc(size);
+}
+extern inline void RPMALLOC_CDECL _free_base(void *ptr) { rpfree(ptr); }
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _calloc_base(size_t count,
+ size_t size) {
+ return rpcalloc(count, size);
+}
+extern inline size_t RPMALLOC_CDECL _msize(void *ptr) {
+ return rpmalloc_usable_size(ptr);
+}
+extern inline size_t RPMALLOC_CDECL _msize_base(void *ptr) {
+ return rpmalloc_usable_size(ptr);
+}
+extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL
+_realloc_base(void *ptr, size_t size) {
+ return rprealloc(ptr, size);
+}
+#endif
+
+#ifdef _WIN32
+// For Windows, #include <rpnew.h> in one source file to get the C++ operator
+// overrides implemented in your module
+#else
+// Overload the C++ operators using the mangled names
+// (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling) operators
+// delete and delete[]
+#define RPDEFVIS __attribute__((visibility("default")))
+extern void _ZdlPv(void *p);
+void RPDEFVIS _ZdlPv(void *p) { rpfree(p); }
+extern void _ZdaPv(void *p);
+void RPDEFVIS _ZdaPv(void *p) { rpfree(p); }
+#if ARCH_64BIT
+// 64-bit operators new and new[], normal and aligned
+extern void *_Znwm(uint64_t size);
+void *RPDEFVIS _Znwm(uint64_t size) { return rpmalloc(size); }
+extern void *_Znam(uint64_t size);
+void *RPDEFVIS _Znam(uint64_t size) { return rpmalloc(size); }
+extern void *_Znwmm(uint64_t size, uint64_t align);
+void *RPDEFVIS _Znwmm(uint64_t size, uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_Znamm(uint64_t size, uint64_t align);
+void *RPDEFVIS _Znamm(uint64_t size, uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnwmSt11align_val_t(uint64_t size, uint64_t align);
+void *RPDEFVIS _ZnwmSt11align_val_t(uint64_t size, uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnamSt11align_val_t(uint64_t size, uint64_t align);
+void *RPDEFVIS _ZnamSt11align_val_t(uint64_t size, uint64_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
+void *RPDEFVIS _ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern void *_ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
+void *RPDEFVIS _ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern void *_ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t);
+void *RPDEFVIS _ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t);
+void *RPDEFVIS _ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+// 64-bit operators sized delete and delete[], normal and aligned
+extern void _ZdlPvm(void *p, uint64_t size);
+void RPDEFVIS _ZdlPvm(void *p, uint64_t size) {
+ rpfree(p);
+ (void)sizeof(size);
+}
+extern void _ZdaPvm(void *p, uint64_t size);
+void RPDEFVIS _ZdaPvm(void *p, uint64_t size) {
+ rpfree(p);
+ (void)sizeof(size);
+}
+extern void _ZdlPvSt11align_val_t(void *p, uint64_t align);
+void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(align);
+}
+extern void _ZdaPvSt11align_val_t(void *p, uint64_t align);
+void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(align);
+}
+extern void _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
+void RPDEFVIS _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(size);
+ (void)sizeof(align);
+}
+extern void _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
+void RPDEFVIS _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(size);
+ (void)sizeof(align);
+}
+#else
+// 32-bit operators new and new[], normal and aligned
+extern void *_Znwj(uint32_t size);
+void *RPDEFVIS _Znwj(uint32_t size) { return rpmalloc(size); }
+extern void *_Znaj(uint32_t size);
+void *RPDEFVIS _Znaj(uint32_t size) { return rpmalloc(size); }
+extern void *_Znwjj(uint32_t size, uint32_t align);
+void *RPDEFVIS _Znwjj(uint32_t size, uint32_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_Znajj(uint32_t size, uint32_t align);
+void *RPDEFVIS _Znajj(uint32_t size, uint32_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnwjSt11align_val_t(size_t size, size_t align);
+void *RPDEFVIS _ZnwjSt11align_val_t(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnajSt11align_val_t(size_t size, size_t align);
+void *RPDEFVIS _ZnajSt11align_val_t(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t);
+void *RPDEFVIS _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t);
+void *RPDEFVIS _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+extern void *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t);
+void *RPDEFVIS _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+extern void *_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t);
+void *RPDEFVIS _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+// 32-bit operators sized delete and delete[], normal and aligned
+extern void _ZdlPvj(void *p, uint64_t size);
+void RPDEFVIS _ZdlPvj(void *p, uint64_t size) {
+ rpfree(p);
+ (void)sizeof(size);
+}
+extern void _ZdaPvj(void *p, uint64_t size);
+void RPDEFVIS _ZdaPvj(void *p, uint64_t size) {
+ rpfree(p);
+ (void)sizeof(size);
+}
+extern void _ZdlPvSt11align_val_t(void *p, uint32_t align);
+void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t a) {
+ rpfree(p);
+ (void)sizeof(align);
+}
+extern void _ZdaPvSt11align_val_t(void *p, uint32_t align);
+void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t a) {
+ rpfree(p);
+ (void)sizeof(align);
+}
+extern void _ZdlPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
+void RPDEFVIS _ZdlPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(size);
+ (void)sizeof(a);
+}
+extern void _ZdaPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
+void RPDEFVIS _ZdaPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
+ rpfree(p);
+ (void)sizeof(size);
+ (void)sizeof(a);
+}
+#endif
+#endif
+#endif
+
+#if USE_INTERPOSE || USE_ALIAS
+
+static void *rpmalloc_nothrow(size_t size, rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpmalloc(size);
+}
+static void *rpaligned_alloc_reverse(size_t size, size_t align) {
+ return rpaligned_alloc(align, size);
+}
+static void *rpaligned_alloc_reverse_nothrow(size_t size, size_t align,
+ rp_nothrow_t t) {
+ (void)sizeof(t);
+ return rpaligned_alloc(align, size);
+}
+static void rpfree_size(void *p, size_t size) {
+ (void)sizeof(size);
+ rpfree(p);
+}
+static void rpfree_aligned(void *p, size_t align) {
+ (void)sizeof(align);
+ rpfree(p);
+}
+static void rpfree_size_aligned(void *p, size_t size, size_t align) {
+ (void)sizeof(size);
+ (void)sizeof(align);
+ rpfree(p);
+}
+
+#endif
+
+#if USE_INTERPOSE
+
+__attribute__((used)) static const interpose_t macinterpose_malloc[]
+ __attribute__((section("__DATA, __interpose"))) = {
+ // new and new[]
+ MAC_INTERPOSE_PAIR(rpmalloc, _Znwm),
+ MAC_INTERPOSE_PAIR(rpmalloc, _Znam),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znwmm),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znamm),
+ MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnwmRKSt9nothrow_t),
+ MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnamRKSt9nothrow_t),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnwmSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnamSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
+ _ZnwmSt11align_val_tRKSt9nothrow_t),
+ MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
+ _ZnamSt11align_val_tRKSt9nothrow_t),
+ // delete and delete[]
+ MAC_INTERPOSE_PAIR(rpfree, _ZdlPv), MAC_INTERPOSE_PAIR(rpfree, _ZdaPv),
+ MAC_INTERPOSE_PAIR(rpfree_size, _ZdlPvm),
+ MAC_INTERPOSE_PAIR(rpfree_size, _ZdaPvm),
+ MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdlPvSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdaPvSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdlPvmSt11align_val_t),
+ MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdaPvmSt11align_val_t),
+ // libc entry points
+ MAC_INTERPOSE_PAIR(rpmalloc, malloc),
+ MAC_INTERPOSE_PAIR(rpmalloc, calloc),
+ MAC_INTERPOSE_PAIR(rprealloc, realloc),
+ MAC_INTERPOSE_PAIR(rprealloc, reallocf),
+#if defined(__MAC_10_15) && __MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_15
+ MAC_INTERPOSE_PAIR(rpaligned_alloc, aligned_alloc),
+#endif
+ MAC_INTERPOSE_PAIR(rpmemalign, memalign),
+ MAC_INTERPOSE_PAIR(rpposix_memalign, posix_memalign),
+ MAC_INTERPOSE_PAIR(rpfree, free), MAC_INTERPOSE_PAIR(rpfree, cfree),
+ MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_usable_size),
+ MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_size)};
+
+#endif
+
+#if USE_ALIAS
+
+#define RPALIAS(fn) __attribute__((alias(#fn), used, visibility("default")));
+
+// Alias the C++ operators using the mangled names
+// (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
+
+// operators delete and delete[]
+void _ZdlPv(void *p) RPALIAS(rpfree) void _ZdaPv(void *p) RPALIAS(rpfree)
+
+#if ARCH_64BIT
+ // 64-bit operators new and new[], normal and aligned
+ void *_Znwm(uint64_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1)
+ RPALIAS(rpmalloc) void *_Znam(uint64_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwmm(uint64_t size,
+ uint64_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_Znamm(uint64_t size,
+ uint64_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnwmSt11align_val_t(
+ size_t size, size_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnamSt11align_val_t(
+ size_t size, size_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnwmRKSt9nothrow_t(
+ size_t size, rp_nothrow_t t)
+ RPALIAS(rpmalloc_nothrow) void *_ZnamRKSt9nothrow_t(
+ size_t size,
+ rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
+ *_ZnwmSt11align_val_tRKSt9nothrow_t(size_t size,
+ size_t align,
+ rp_nothrow_t t)
+ RPALIAS(rpaligned_alloc_reverse_nothrow) void
+ *_ZnamSt11align_val_tRKSt9nothrow_t(
+ size_t size, size_t align,
+ rp_nothrow_t t)
+ RPALIAS(rpaligned_alloc_reverse_nothrow)
+ // 64-bit operators delete and delete[], sized and aligned
+ void _ZdlPvm(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvm(void *p,
+ size_t n)
+ RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
+ RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
+ size_t a)
+ RPALIAS(rpfree_aligned) void _ZdlPvmSt11align_val_t(void *p,
+ size_t n,
+ size_t a)
+ RPALIAS(rpfree_size_aligned) void _ZdaPvmSt11align_val_t(
+ void *p, size_t n, size_t a)
+ RPALIAS(rpfree_size_aligned)
+#else
+ // 32-bit operators new and new[], normal and aligned
+ void *_Znwj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1)
+ RPALIAS(rpmalloc) void *_Znaj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwjj(uint32_t size,
+ uint32_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_Znajj(uint32_t size,
+ uint32_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnwjSt11align_val_t(
+ size_t size, size_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnajSt11align_val_t(
+ size_t size, size_t align)
+ RPALIAS(rpaligned_alloc_reverse) void *_ZnwjRKSt9nothrow_t(
+ size_t size, rp_nothrow_t t)
+ RPALIAS(rpmalloc_nothrow) void *_ZnajRKSt9nothrow_t(
+ size_t size,
+ rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
+ *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size,
+ size_t align,
+ rp_nothrow_t t)
+ RPALIAS(rpaligned_alloc_reverse_nothrow) void
+ *_ZnajSt11align_val_tRKSt9nothrow_t(
+ size_t size, size_t align,
+ rp_nothrow_t t)
+ RPALIAS(rpaligned_alloc_reverse_nothrow)
+ // 32-bit operators delete and delete[], sized and aligned
+ void _ZdlPvj(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvj(void *p,
+ size_t n)
+ RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
+ RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
+ size_t a)
+ RPALIAS(rpfree_aligned) void _ZdlPvjSt11align_val_t(void *p,
+ size_t n,
+ size_t a)
+ RPALIAS(rpfree_size_aligned) void _ZdaPvjSt11align_val_t(
+ void *p, size_t n, size_t a)
+ RPALIAS(rpfree_size_aligned)
+#endif
+
+ void *malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1)
+ RPALIAS(rpmalloc) void *calloc(size_t count, size_t size)
+ RPALIAS(rpcalloc) void *realloc(void *ptr, size_t size)
+ RPALIAS(rprealloc) void *reallocf(void *ptr, size_t size)
+ RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2)
+ RPALIAS(rprealloc) void *aligned_alloc(size_t alignment, size_t size)
+ RPALIAS(rpaligned_alloc) void *memalign(
+ size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2)
+ RPALIAS(rpmemalign) int posix_memalign(void **memptr, size_t alignment,
+ size_t size)
+ RPALIAS(rpposix_memalign) void free(void *ptr)
+ RPALIAS(rpfree) void cfree(void *ptr) RPALIAS(rpfree)
+#if defined(__ANDROID__) || defined(__FreeBSD__)
+ size_t
+ malloc_usable_size(const void *ptr) RPALIAS(rpmalloc_usable_size)
+#else
+ size_t
+ malloc_usable_size(void *ptr) RPALIAS(rpmalloc_usable_size)
+#endif
+ size_t malloc_size(void *ptr) RPALIAS(rpmalloc_usable_size)
+
+#endif
+
+ static inline size_t _rpmalloc_page_size(void) {
+ return _memory_page_size;
+}
+
+extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size);
+
+extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size) {
+ size_t total;
+#if ENABLE_VALIDATE_ARGS
+#ifdef _MSC_VER
+ int err = SizeTMult(count, size, &total);
+ if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#else
+ int err = __builtin_umull_overflow(count, size, &total);
+ if (err || (total >= MAX_ALLOC_SIZE)) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+#else
+ total = count * size;
+#endif
+ return realloc(ptr, total);
+}
+
+extern inline void *RPMALLOC_CDECL valloc(size_t size) {
+ get_thread_heap();
+ return rpaligned_alloc(_rpmalloc_page_size(), size);
+}
+
+extern inline void *RPMALLOC_CDECL pvalloc(size_t size) {
+ get_thread_heap();
+ const size_t page_size = _rpmalloc_page_size();
+ const size_t aligned_size = ((size + page_size - 1) / page_size) * page_size;
+#if ENABLE_VALIDATE_ARGS
+ if (aligned_size < size) {
+ errno = EINVAL;
+ return 0;
+ }
+#endif
+ return rpaligned_alloc(_rpmalloc_page_size(), aligned_size);
+}
+
+#endif // ENABLE_OVERRIDE
+
+#if ENABLE_PRELOAD
+
+#ifdef _WIN32
+
+#if defined(BUILD_DYNAMIC_LINK) && BUILD_DYNAMIC_LINK
+
+extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
+ DWORD reason, LPVOID reserved);
+
+extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
+ DWORD reason,
+ LPVOID reserved) {
+ (void)sizeof(reserved);
+ (void)sizeof(instance);
+ if (reason == DLL_PROCESS_ATTACH)
+ rpmalloc_initialize();
+ else if (reason == DLL_PROCESS_DETACH)
+ rpmalloc_finalize();
+ else if (reason == DLL_THREAD_ATTACH)
+ rpmalloc_thread_initialize();
+ else if (reason == DLL_THREAD_DETACH)
+ rpmalloc_thread_finalize(1);
+ return TRUE;
+}
+
+// end BUILD_DYNAMIC_LINK
+#else
+
+extern void _global_rpmalloc_init(void) {
+ rpmalloc_set_main_thread();
+ rpmalloc_initialize();
+}
+
+#if defined(__clang__) || defined(__GNUC__)
+
+static void __attribute__((constructor)) initializer(void) {
+ _global_rpmalloc_init();
+}
+
+#elif defined(_MSC_VER)
+
+static int _global_rpmalloc_xib(void) {
+ _global_rpmalloc_init();
+ return 0;
+}
+
+#pragma section(".CRT$XIB", read)
+__declspec(allocate(".CRT$XIB")) void (*_rpmalloc_module_init)(void) =
+ _global_rpmalloc_xib;
+#if defined(_M_IX86) || defined(__i386__)
+#pragma comment(linker, "/include:" \
+ "__rpmalloc_module_init")
+#else
+#pragma comment(linker, "/include:" \
+ "_rpmalloc_module_init")
+#endif
+
+#endif
+
+// end !BUILD_DYNAMIC_LINK
+#endif
+
+#else
+
+#include <pthread.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+extern void rpmalloc_set_main_thread(void);
+
+static pthread_key_t destructor_key;
+
+static void thread_destructor(void *);
+
+static void __attribute__((constructor)) initializer(void) {
+ rpmalloc_set_main_thread();
+ rpmalloc_initialize();
+ pthread_key_create(&destructor_key, thread_destructor);
+}
+
+static void __attribute__((destructor)) finalizer(void) { rpmalloc_finalize(); }
+
+typedef struct {
+ void *(*real_start)(void *);
+ void *real_arg;
+} thread_starter_arg;
+
+static void *thread_starter(void *argptr) {
+ thread_starter_arg *arg = argptr;
+ void *(*real_start)(void *) = arg->real_start;
+ void *real_arg = arg->real_arg;
+ rpmalloc_thread_initialize();
+ rpfree(argptr);
+ pthread_setspecific(destructor_key, (void *)1);
+ return (*real_start)(real_arg);
+}
+
+static void thread_destructor(void *value) {
+ (void)sizeof(value);
+ rpmalloc_thread_finalize(1);
+}
+
+#ifdef __APPLE__
+
+static int pthread_create_proxy(pthread_t *thread, const pthread_attr_t *attr,
+ void *(*start_routine)(void *), void *arg) {
+ rpmalloc_initialize();
+ thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
+ starter_arg->real_start = start_routine;
+ starter_arg->real_arg = arg;
+ return pthread_create(thread, attr, thread_starter, starter_arg);
+}
+
+MAC_INTERPOSE_SINGLE(pthread_create_proxy, pthread_create);
+
+#else
+
+#include <dlfcn.h>
+
+int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
+ void *(*start_routine)(void *), void *arg) {
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \
+ defined(__NetBSD__) || defined(__DragonFly__) || defined(__APPLE__) || \
+ defined(__HAIKU__)
+ char fname[] = "pthread_create";
+#else
+ char fname[] = "_pthread_create";
+#endif
+ void *real_pthread_create = dlsym(RTLD_NEXT, fname);
+ rpmalloc_thread_initialize();
+ thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
+ starter_arg->real_start = start_routine;
+ starter_arg->real_arg = arg;
+ return (*(int (*)(pthread_t *, const pthread_attr_t *, void *(*)(void *),
+ void *))real_pthread_create)(thread, attr, thread_starter,
+ starter_arg);
+}
+
+#endif
+
+#endif
+
+#endif
+
+#if ENABLE_OVERRIDE
+
+#if defined(__GLIBC__) && defined(__linux__)
+
+void *__libc_malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(1)
+ RPALIAS(rpmalloc) void *__libc_calloc(size_t count, size_t size)
+ RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2)
+ RPALIAS(rpcalloc) void *__libc_realloc(void *p, size_t size)
+ RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2) RPALIAS(rprealloc) void __libc_free(void *p)
+ RPALIAS(rpfree) void __libc_cfree(void *p)
+ RPALIAS(rpfree) void *__libc_memalign(size_t align, size_t size)
+ RPMALLOC_ATTRIB_MALLOC
+ RPMALLOC_ATTRIB_ALLOC_SIZE(2)
+ RPALIAS(rpmemalign) int __posix_memalign(void **p, size_t align,
+ size_t size)
+ RPALIAS(rpposix_memalign)
+
+ extern void *__libc_valloc(size_t size);
+extern void *__libc_pvalloc(size_t size);
+
+void *__libc_valloc(size_t size) { return valloc(size); }
+
+void *__libc_pvalloc(size_t size) { return pvalloc(size); }
+
+#endif
+
+#endif
+
+#if (defined(__GNUC__) || defined(__clang__))
+#pragma GCC visibility pop
+#endif
|