Skip to content

Commit 8768b8e

Browse files
flangletflanglet
committed
Add intrinsics based versions of memcmp and memcpy for several architectures
1 parent d4034d6 commit 8768b8e

1 file changed

Lines changed: 157 additions & 7 deletions

File tree

‎src/Memory.hpp‎

Lines changed: 157 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,14 @@ limitations under the License.
2323
#include <cstring>
2424
#include "types.hpp"
2525

26+
#if !defined(NO_INTRINSICS)
27+
#if defined(__ARM_NEON) || defined(__aarch64__)
28+
#include <arm_neon.h>
29+
#elif defined(__AVX512F__) || defined(__AVX2__) || defined(__SSE2__)
30+
#include <immintrin.h>
31+
#endif
32+
#endif
33+
2634

2735
namespace kanzi {
2836

@@ -93,16 +101,158 @@ static KANZI_ALWAYS_INLINE uint64 knz_bswap64(uint64 x) {
93101
#endif
94102
}
95103

96-
#ifdef AGGRESSIVE_OPTIMIZATION
97-
// There be dragons!
98-
// User assumes responsibility for alignment and aliasing constraints.
99-
#define KANZI_MEM_EQ4(x, y) (*(const uint32*)(x) == *(const uint32*)(y))
100-
#define KANZI_MEM_EQ8(x, y) (*(const uint64*)(x) == *(const uint64*)(y))
104+
#if !defined(NO_INTRINSICS) && (defined(__ARM_NEON) || defined(__aarch64__))
105+
106+
static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y)
107+
{
108+
const uint32x2_t a = vld1_dup_u32(reinterpret_cast<const uint32_t*>(x));
109+
const uint32x2_t b = vld1_dup_u32(reinterpret_cast<const uint32_t*>(y));
110+
return vget_lane_u32(vceq_u32(a, b), 0) != 0;
111+
}
112+
113+
static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y)
114+
{
115+
#if defined(__aarch64__)
116+
const uint64x1_t a = vld1_u64(reinterpret_cast<const uint64_t*>(x));
117+
const uint64x1_t b = vld1_u64(reinterpret_cast<const uint64_t*>(y));
118+
return vget_lane_u64(vceq_u64(a, b), 0) != 0;
119+
#else
120+
const uint32x2_t a = vld1_u32(reinterpret_cast<const uint32_t*>(x));
121+
const uint32x2_t b = vld1_u32(reinterpret_cast<const uint32_t*>(y));
122+
const uint32x2_t eq = vceq_u32(a, b);
123+
return (vget_lane_u32(eq, 0) != 0) && (vget_lane_u32(eq, 1) != 0);
124+
#endif
125+
}
126+
127+
static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src)
128+
{
129+
vst1_u8(reinterpret_cast<uint8_t*>(dst), vld1_u8(reinterpret_cast<const uint8_t*>(src)));
130+
}
131+
132+
static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src)
133+
{
134+
vst1q_u8(reinterpret_cast<uint8_t*>(dst), vld1q_u8(reinterpret_cast<const uint8_t*>(src)));
135+
}
136+
137+
#elif !defined(NO_INTRINSICS) && defined(__AVX512F__)
138+
139+
static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y)
140+
{
141+
const __mmask16 mask = 0x0001;
142+
const __m512i a = _mm512_maskz_loadu_epi32(mask, x);
143+
const __m512i b = _mm512_maskz_loadu_epi32(mask, y);
144+
return _mm512_mask_cmpeq_epi32_mask(mask, a, b) == mask;
145+
}
146+
147+
static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y)
148+
{
149+
const __mmask8 mask = 0x01;
150+
const __m512i a = _mm512_maskz_loadu_epi64(mask, x);
151+
const __m512i b = _mm512_maskz_loadu_epi64(mask, y);
152+
return _mm512_mask_cmpeq_epi64_mask(mask, a, b) == mask;
153+
}
154+
155+
static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src)
156+
{
157+
const __m512i value = _mm512_maskz_loadu_epi64(0x01, src);
158+
_mm512_mask_storeu_epi64(dst, 0x01, value);
159+
}
160+
161+
static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src)
162+
{
163+
const __m512i value = _mm512_maskz_loadu_epi64(0x03, src);
164+
_mm512_mask_storeu_epi64(dst, 0x03, value);
165+
}
166+
167+
#elif !defined(NO_INTRINSICS) && defined(__AVX2__)
168+
169+
static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y)
170+
{
171+
const __m256i mask = _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, -1);
172+
const __m256i a = _mm256_maskload_epi32(reinterpret_cast<const int*>(x), mask);
173+
const __m256i b = _mm256_maskload_epi32(reinterpret_cast<const int*>(y), mask);
174+
return (_mm256_movemask_epi8(_mm256_cmpeq_epi32(a, b)) & 0x0F) == 0x0F;
175+
}
176+
177+
static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y)
178+
{
179+
const __m256i mask = _mm256_set_epi64x(0, 0, 0, -1);
180+
const __m256i a = _mm256_maskload_epi64(reinterpret_cast<const long long*>(x), mask);
181+
const __m256i b = _mm256_maskload_epi64(reinterpret_cast<const long long*>(y), mask);
182+
return (_mm256_movemask_epi8(_mm256_cmpeq_epi64(a, b)) & 0xFF) == 0xFF;
183+
}
184+
185+
static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src)
186+
{
187+
const __m256i mask = _mm256_set_epi64x(0, 0, 0, -1);
188+
const __m256i value = _mm256_maskload_epi64(reinterpret_cast<const long long*>(src), mask);
189+
_mm256_maskstore_epi64(reinterpret_cast<long long*>(dst), mask, value);
190+
}
191+
192+
static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src)
193+
{
194+
const __m256i mask = _mm256_set_epi64x(0, 0, -1, -1);
195+
const __m256i value = _mm256_maskload_epi64(reinterpret_cast<const long long*>(src), mask);
196+
_mm256_maskstore_epi64(reinterpret_cast<long long*>(dst), mask, value);
197+
}
198+
199+
#elif !defined(NO_INTRINSICS) && defined(__SSE2__)
200+
201+
static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y)
202+
{
203+
const __m128i va = _mm_loadu_si32(x);
204+
const __m128i vb = _mm_loadu_si32(y);
205+
return _mm_cvtsi128_si32(_mm_cmpeq_epi32(va, vb)) != 0;
206+
}
207+
208+
static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y)
209+
{
210+
const __m128i a = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(x));
211+
const __m128i b = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(y));
212+
return _mm_movemask_epi8(_mm_cmpeq_epi8(a, b)) == 0xFF;
213+
}
214+
215+
static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src)
216+
{
217+
_mm_storel_epi64(reinterpret_cast<__m128i*>(dst),
218+
_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src)));
219+
}
220+
221+
static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src)
222+
{
223+
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst),
224+
_mm_loadu_si128(reinterpret_cast<const __m128i*>(src)));
225+
}
226+
101227
#else
102-
#define KANZI_MEM_EQ4(x, y) (std::memcmp((x), (y), 4) == 0)
103-
#define KANZI_MEM_EQ8(x, y) (std::memcmp((x), (y), 8) == 0)
228+
229+
static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y)
230+
{
231+
return std::memcmp(x, y, 4) == 0;
232+
}
233+
234+
static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y)
235+
{
236+
return std::memcmp(x, y, 8) == 0;
237+
}
238+
239+
static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src)
240+
{
241+
memcpy(dst, src, 8);
242+
}
243+
244+
static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src)
245+
{
246+
memcpy(dst, src, 16);
247+
}
248+
104249
#endif
105250

251+
#define KANZI_MEM_EQ4(x, y) (::kanzi::memEq4((x), (y)))
252+
#define KANZI_MEM_EQ8(x, y) (::kanzi::memEq8((x), (y)))
253+
#define KANZI_MEM_CP8(dst, src) (::kanzi::memCp8((dst), (src)))
254+
#define KANZI_MEM_CP16(dst, src) (::kanzi::memCp16((dst), (src)))
255+
106256
// Detect host endianness
107257

108258
#if __cplusplus >= 202002L

0 commit comments

Comments
 (0)