1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
|
//===-- Memcpy implementation for arm ---------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// The functions defined in this file give approximate code size. These sizes
// assume the following configuration options:
// - LIBC_CONF_KEEP_FRAME_POINTER = false
// - LIBC_CONF_ENABLE_STRONG_STACK_PROTECTOR = false
// - LIBC_ADD_NULL_CHECKS = false
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H
#include "src/__support/CPP/type_traits.h" // always_false
#include "src/__support/macros/attributes.h" // LIBC_INLINE
#include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL
#include "src/string/memory_utils/arm/common.h" // LIBC_ATTR_LIKELY, LIBC_ATTR_UNLIKELY
#include "src/string/memory_utils/utils.h" // memcpy_inline, distance_to_align
#include <stddef.h> // size_t
namespace LIBC_NAMESPACE_DECL {
namespace {
// Performs a copy of `bytes` byte from `src` to `dst`. This function has the
// semantics of `memcpy` where `src` and `dst` are `__restrict`. The compiler is
// free to use whatever instruction is best for the size and assumed access.
template <size_t bytes, AssumeAccess access>
LIBC_INLINE void copy(void *dst, const void *src) {
if constexpr (access == AssumeAccess::kAligned) {
constexpr size_t alignment = bytes > kWordSize ? kWordSize : bytes;
memcpy_inline<bytes>(assume_aligned<alignment>(dst),
assume_aligned<alignment>(src));
} else if constexpr (access == AssumeAccess::kUnknown) {
memcpy_inline<bytes>(dst, src);
} else {
static_assert(cpp::always_false<decltype(access)>, "Invalid AssumeAccess");
}
}
template <size_t bytes, BlockOp block_op = BlockOp::kFull,
AssumeAccess access = AssumeAccess::kUnknown>
LIBC_INLINE void copy_block_and_bump_pointers(Ptr &dst, CPtr &src) {
if constexpr (block_op == BlockOp::kFull) {
copy<bytes, access>(dst, src);
} else if constexpr (block_op == BlockOp::kByWord) {
// We restrict loads/stores to 4 byte to prevent the use of load/store
// multiple (LDM, STM) and load/store double (LDRD, STRD).
static_assert((bytes % kWordSize == 0) && (bytes >= kWordSize));
LIBC_LOOP_UNROLL
for (size_t offset = 0; offset < bytes; offset += kWordSize) {
copy<kWordSize, access>(dst + offset, src + offset);
}
} else {
static_assert(cpp::always_false<decltype(block_op)>, "Invalid BlockOp");
}
// In the 1, 2, 4 byte copy case, the compiler can fold pointer offsetting
// into the load/store instructions.
// e.g.,
// ldrb r3, [r1], #1
// strb r3, [r0], #1
dst += bytes;
src += bytes;
}
template <size_t bytes, BlockOp block_op, AssumeAccess access>
LIBC_INLINE void consume_by_block(Ptr &dst, CPtr &src, size_t &size) {
LIBC_LOOP_NOUNROLL
for (size_t i = 0; i < size / bytes; ++i)
copy_block_and_bump_pointers<bytes, block_op, access>(dst, src);
size %= bytes;
}
[[maybe_unused]] LIBC_INLINE void
copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src, size_t size) {
LIBC_LOOP_NOUNROLL
for (size_t i = 0; i < size; ++i)
*dst++ = *src++;
}
} // namespace
// Implementation for Cortex-M0, M0+, M1 cores that do not allow for unaligned
// loads/stores. It compiles down to 208 bytes when used through `memcpy` that
// also needs to return the `dst` ptr.
// Note:
// - When `src` and `dst` are coaligned, we start by aligning them and perform
// bulk copies. We let the compiler know the pointers are aligned so it can
// use load/store multiple (LDM, STM). This significantly increase throughput
// but it also requires more registers and push/pop instructions. This impacts
// latency for small size copies.
// - When `src` and `dst` are misaligned, we align `dst` and recompose words
// using multiple aligned loads. `load_aligned` takes care of endianness
// issues.
[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm_low_end(Ptr dst, CPtr src,
size_t size) {
if (size >= 8) {
if (const size_t offset = distance_to_align_up<kWordSize>(dst))
LIBC_ATTR_UNLIKELY {
copy_bytes_and_bump_pointers(dst, src, offset);
size -= offset;
}
constexpr AssumeAccess kAligned = AssumeAccess::kAligned;
const auto src_alignment = distance_to_align_down<kWordSize>(src);
if (src_alignment == 0)
LIBC_ATTR_LIKELY {
// Both `src` and `dst` are now word-aligned.
// We first copy by blocks of 64 bytes, the compiler will use 4
// load/store multiple (LDM, STM), each of 4 words. This requires more
// registers so additional push/pop are needed but the speedup is worth
// it.
consume_by_block<64, BlockOp::kFull, kAligned>(dst, src, size);
// Then we use blocks of 4 word load/store.
consume_by_block<16, BlockOp::kByWord, kAligned>(dst, src, size);
// Then we use word by word copy.
consume_by_block<4, BlockOp::kByWord, kAligned>(dst, src, size);
}
else {
// `dst` is aligned but `src` is not.
LIBC_LOOP_NOUNROLL
while (size >= kWordSize) {
// Recompose word from multiple loads depending on the alignment.
const uint32_t value =
src_alignment == 2
? load_aligned<uint32_t, uint16_t, uint16_t>(src)
: load_aligned<uint32_t, uint8_t, uint16_t, uint8_t>(src);
copy<kWordSize, kAligned>(dst, &value);
dst += kWordSize;
src += kWordSize;
size -= kWordSize;
}
}
// Up to 3 bytes may still need to be copied.
// Handling them with the slow loop below.
}
copy_bytes_and_bump_pointers(dst, src, size);
}
// Implementation for Cortex-M3, M4, M7, M23, M33, M35P, M52 with hardware
// support for unaligned loads and stores. It compiles down to 272 bytes when
// used through `memcpy` that also needs to return the `dst` ptr.
[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm_mid_end(Ptr dst, CPtr src,
size_t size) {
if (misaligned(bitwise_or(src, dst)))
LIBC_ATTR_UNLIKELY {
if (size < 8)
LIBC_ATTR_UNLIKELY {
if (size & 1)
copy_block_and_bump_pointers<1>(dst, src);
if (size & 2)
copy_block_and_bump_pointers<2>(dst, src);
if (size & 4)
copy_block_and_bump_pointers<4>(dst, src);
return;
}
if (misaligned(src))
LIBC_ATTR_UNLIKELY {
const size_t offset = distance_to_align_up<kWordSize>(dst);
if (offset & 1)
copy_block_and_bump_pointers<1>(dst, src);
if (offset & 2)
copy_block_and_bump_pointers<2>(dst, src);
size -= offset;
}
}
// `dst` and `src` are not necessarily both aligned at that point but this
// implementation assumes hardware support for unaligned loads and stores so
// it is still fast to perform unrolled word by word copy. Note that wider
// accesses through the use of load/store multiple (LDM, STM) and load/store
// double (LDRD, STRD) instructions are generally not supported and can fault.
// By forcing decomposition of 64 bytes copy into word by word copy, the
// compiler uses a load to prefetch the next cache line:
// ldr r3, [r1, #64]! <- prefetch next cache line
// str r3, [r0]
// ldr r3, [r1, #0x4]
// str r3, [r0, #0x4]
// ...
// ldr r3, [r1, #0x3c]
// str r3, [r0, #0x3c]
// This is a bit detrimental for sizes between 64 and 256 (less than 10%
// penalty) but the prefetch yields better throughput for larger copies.
constexpr AssumeAccess kUnknown = AssumeAccess::kUnknown;
consume_by_block<64, BlockOp::kByWord, kUnknown>(dst, src, size);
consume_by_block<16, BlockOp::kByWord, kUnknown>(dst, src, size);
consume_by_block<4, BlockOp::kByWord, kUnknown>(dst, src, size);
if (size & 1)
copy_block_and_bump_pointers<1>(dst, src);
if (size & 2)
copy_block_and_bump_pointers<2>(dst, src);
}
[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm(Ptr dst, CPtr src,
size_t size) {
// The compiler performs alias analysis and is able to prove that `dst` and
// `src` do not alias by propagating the `__restrict` keyword from the
// `memcpy` prototype. This allows the compiler to merge consecutive
// load/store (LDR, STR) instructions generated in
// `copy_block_and_bump_pointers` with `BlockOp::kByWord` into load/store
// double (LDRD, STRD) instructions, this is is undesirable so we prevent the
// compiler from inferring `__restrict` with the following line.
asm volatile("" : "+r"(dst), "+r"(src));
#ifdef __ARM_FEATURE_UNALIGNED
return inline_memcpy_arm_mid_end(dst, src, size);
#else
return inline_memcpy_arm_low_end(dst, src, size);
#endif
}
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H
|
