diff options
| author | Sterling Augustine <saugustine@google.com> | 2022-10-14 11:17:46 -0700 |
|---|---|---|
| committer | Sterling Augustine <saugustine@google.com> | 2022-10-14 12:32:20 -0700 |
| commit | d8415b02a519f222ecf71b069c96cc85ac635de3 (patch) | |
| tree | d9377fe945b92f3898c7f2f168a18169a62b18c4 /libc/src/string/memmove.cpp | |
| parent | ef774bec63b72a0e900b3ddf22da81efbecd99a9 (diff) | |
Revert "[libc] New version of the mem* framework"
This reverts commit https://reviews.llvm.org/D135134 (b3f1d58a131eb546aaf1ac165c77ccb89c40d758)
That revision appears to have broken Arm memcpy in some subtle
ways. Am communicating with the original author to get a
good reproduction.
Diffstat (limited to 'libc/src/string/memmove.cpp')
| -rw-r--r-- | libc/src/string/memmove.cpp | 100 |
1 files changed, 16 insertions, 84 deletions
diff --git a/libc/src/string/memmove.cpp b/libc/src/string/memmove.cpp index f40ab62d4b10..f24257893b20 100644 --- a/libc/src/string/memmove.cpp +++ b/libc/src/string/memmove.cpp @@ -9,104 +9,36 @@ #include "src/string/memmove.h" #include "src/__support/common.h" -#include "src/string/memory_utils/op_aarch64.h" -#include "src/string/memory_utils/op_builtin.h" -#include "src/string/memory_utils/op_generic.h" -#include "src/string/memory_utils/op_x86.h" +#include "src/__support/integer_operations.h" +#include "src/string/memory_utils/elements.h" #include <stddef.h> // size_t, ptrdiff_t -#include <stdio.h> - namespace __llvm_libc { -[[maybe_unused]] static inline void -inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) { - if ((count == 0) || (dst == src)) - return; - if (dst < src) { -#pragma nounroll - for (size_t offset = 0; offset < count; ++offset) - builtin::Memcpy<1>::block(dst + offset, src + offset); - } else { -#pragma nounroll - for (ptrdiff_t offset = count - 1; offset >= 0; --offset) - builtin::Memcpy<1>::block(dst + offset, src + offset); - } -} - -template <size_t MaxSize> -[[maybe_unused]] static inline void inline_memmove_generic(Ptr dst, CPtr src, - size_t count) { +static inline void inline_memmove(char *dst, const char *src, size_t count) { + using namespace __llvm_libc::scalar; if (count == 0) return; if (count == 1) - return generic::Memmove<1, MaxSize>::block(dst, src); + return move<_1>(dst, src); if (count <= 4) - return generic::Memmove<2, MaxSize>::head_tail(dst, src, count); + return move<HeadTail<_2>>(dst, src, count); if (count <= 8) - return generic::Memmove<4, MaxSize>::head_tail(dst, src, count); + return move<HeadTail<_4>>(dst, src, count); if (count <= 16) - return generic::Memmove<8, MaxSize>::head_tail(dst, src, count); + return move<HeadTail<_8>>(dst, src, count); if (count <= 32) - return generic::Memmove<16, MaxSize>::head_tail(dst, src, count); + return move<HeadTail<_16>>(dst, src, count); if (count <= 64) - return generic::Memmove<32, MaxSize>::head_tail(dst, src, count); + return move<HeadTail<_32>>(dst, src, count); if (count <= 128) - return generic::Memmove<64, MaxSize>::head_tail(dst, src, count); - if (dst < src) { - generic::Memmove<32, MaxSize>::template align_forward<Arg::Src>(dst, src, - count); - return generic::Memmove<64, MaxSize>::loop_and_tail_forward(dst, src, - count); - } else { - generic::Memmove<32, MaxSize>::template align_backward<Arg::Src>(dst, src, - count); - return generic::Memmove<64, MaxSize>::loop_and_tail_backward(dst, src, - count); - } -} + return move<HeadTail<_64>>(dst, src, count); -static inline void inline_memmove(Ptr dst, CPtr src, size_t count) { -#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) -#if defined(LLVM_LIBC_ARCH_X86) - static constexpr size_t kMaxSize = x86::kAvx512F ? 64 - : x86::kAvx ? 32 - : x86::kSse2 ? 16 - : 8; -#elif defined(LLVM_LIBC_ARCH_AARCH64) - static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8; -#endif - // return inline_memmove_generic<kMaxSize>(dst, src, count); - if (count == 0) - return; - if (count == 1) - return generic::Memmove<1, kMaxSize>::block(dst, src); - if (count <= 4) - return generic::Memmove<2, kMaxSize>::head_tail(dst, src, count); - if (count <= 8) - return generic::Memmove<4, kMaxSize>::head_tail(dst, src, count); - if (count <= 16) - return generic::Memmove<8, kMaxSize>::head_tail(dst, src, count); - if (count <= 32) - return generic::Memmove<16, kMaxSize>::head_tail(dst, src, count); - if (count <= 64) - return generic::Memmove<32, kMaxSize>::head_tail(dst, src, count); - if (count <= 128) - return generic::Memmove<64, kMaxSize>::head_tail(dst, src, count); - if (dst < src) { - generic::Memmove<32, kMaxSize>::align_forward<Arg::Src>(dst, src, count); - return generic::Memmove<64, kMaxSize>::loop_and_tail_forward(dst, src, - count); - } else { - generic::Memmove<32, kMaxSize>::align_backward<Arg::Src>(dst, src, count); - return generic::Memmove<64, kMaxSize>::loop_and_tail_backward(dst, src, - count); - } -#elif defined(LLVM_LIBC_ARCH_ARM) - return inline_memmove_embedded_tiny(dst, src, count); -#else -#error "Unsupported platform" -#endif + using AlignedMoveLoop = Align<_16, Arg::Src>::Then<Loop<_64>>; + if (dst < src) + return move<AlignedMoveLoop>(dst, src, count); + else if (dst > src) + return move_backward<AlignedMoveLoop>(dst, src, count); } LLVM_LIBC_FUNCTION(void *, memmove, |
