@@ -816,16 +816,16 @@ static bool gDecodeBytesGroupInitialized = decodeBytesGroupBuildTables();
816816
817817#ifdef SIMD_SSE
818818// sent mask, replicating shuffle, and two multipliers (even/odd) for multishift emulation
819- static const __m128i kDecodeBytesGroupConfig [9 ][4 ] = {
820- {_mm_set1_epi8 ( 1 ), _mm_set1_epi8 (- 128 ), _mm_setzero_si128 (), _mm_setzero_si128 () },
821- {_mm_set1_epi8 ( 3 ), _mm_setr_epi8 ( 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 , 2 , 2 , 2 , 2 , 3 , 3 , 3 , 3 ), _mm_setr_epi16 ( 4 , 64 , 4 , 64 , 4 , 64 , 4 , 64 ), _mm_setr_epi16 ( 16 , 256 , 16 , 256 , 16 , 256 , 16 , 256 ) },
822- {_mm_set1_epi8 ( 15 ), _mm_setr_epi8 ( 0 , 0 , 1 , 1 , 2 , 2 , 3 , 3 , 4 , 4 , 5 , 5 , 6 , 6 , 7 , 7 ), _mm_set1_epi16 ( 16 ), _mm_set1_epi16 ( 256 ) },
823- {_mm_setzero_si128 (), _mm_set1_epi8 (- 128 ), _mm_setzero_si128 (), _mm_setzero_si128 () },
824- {_mm_set1_epi8 ( 1 ), _mm_set1_epi8 (- 128 ), _mm_setzero_si128 (), _mm_setzero_si128 () },
825- {_mm_set1_epi8 ( 1 ), _mm_setr_epi8 ( 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ), _mm_setr_epi16 ( 256 , 64 , 16 , 4 , 256 , 64 , 16 , 4 ), _mm_setr_epi16 ( 128 , 32 , 8 , 2 , 128 , 32 , 8 , 2 ) },
826- {_mm_set1_epi8 ( 3 ), _mm_setr_epi8 ( 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 , 2 , 2 , 2 , 2 , 3 , 3 , 3 , 3 ), _mm_setr_epi16 ( 4 , 64 , 4 , 64 , 4 , 64 , 4 , 64 ), _mm_setr_epi16 ( 16 , 256 , 16 , 256 , 16 , 256 , 16 , 256 ) },
827- {_mm_set1_epi8 ( 15 ), _mm_setr_epi8 ( 0 , 0 , 1 , 1 , 2 , 2 , 3 , 3 , 4 , 4 , 5 , 5 , 6 , 6 , 7 , 7 ), _mm_set1_epi16 ( 16 ), _mm_set1_epi16 ( 256 ) },
828- {_mm_setzero_si128 (), _mm_set1_epi8 (- 128 ), _mm_setzero_si128 (), _mm_setzero_si128 () },
819+ static const unsigned char kDecodeBytesGroupConfig [9 ][4 ][ 16 ] = {
820+ {{ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 }, { 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 }, { 0 }, { 0 } },
821+ {{ 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 }, { 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 , 2 , 2 , 2 , 2 , 3 , 3 , 3 , 3 }, { 4 , 0 , 64 , 0 , 4 , 0 , 64 , 0 , 4 , 0 , 64 , 0 , 4 , 0 , 64 , 0 }, { 16 , 0 , 0 , 1 , 16 , 0 , 0 , 1 , 16 , 0 , 0 , 1 , 16 , 0 , 0 , 1 } },
822+ {{ 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 }, { 0 , 0 , 1 , 1 , 2 , 2 , 3 , 3 , 4 , 4 , 5 , 5 , 6 , 6 , 7 , 7 }, { 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 }, { 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 } },
823+ {{ 0 }, { 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 }, { 0 }, { 0 } },
824+ {{ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 }, { 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 }, { 0 }, { 0 } },
825+ {{ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 }, { 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 }, { 0 , 1 , 64 , 0 , 16 , 0 , 4 , 0 , 0 , 1 , 64 , 0 , 16 , 0 , 4 , 0 }, { 128 , 0 , 32 , 0 , 8 , 0 , 2 , 0 , 128 , 0 , 32 , 0 , 8 , 0 , 2 , 0 } },
826+ {{ 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 }, { 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 , 2 , 2 , 2 , 2 , 3 , 3 , 3 , 3 }, { 4 , 0 , 64 , 0 , 4 , 0 , 64 , 0 , 4 , 0 , 64 , 0 , 4 , 0 , 64 , 0 }, { 16 , 0 , 0 , 1 , 16 , 0 , 0 , 1 , 16 , 0 , 0 , 1 , 16 , 0 , 0 , 1 } },
827+ {{ 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 }, { 0 , 0 , 1 , 1 , 2 , 2 , 3 , 3 , 4 , 4 , 5 , 5 , 6 , 6 , 7 , 7 }, { 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 , 16 , 0 }, { 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 , 0 , 1 } },
828+ {{ 0 }, { 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 , 128 }, { 0 }, { 0 } },
829829};
830830
831831SIMD_TARGET
@@ -856,13 +856,13 @@ inline const unsigned char* decodeBytesGroupSimd(const unsigned char* data, unsi
856856
857857 // unpack 1, 2 or 4-bit values: shuffle replicates each source byte into both halves of a 16-bit lane
858858 // mulhi extracts even and odd fields into the low byte; the results are interleaved back with shift/or
859- __m128i selw = _mm_shuffle_epi8 (selb, kDecodeBytesGroupConfig [hbits][1 ]);
860- __m128i sel0 = _mm_mulhi_epu16 (selw, kDecodeBytesGroupConfig [hbits][2 ]);
861- __m128i sel1 = _mm_mulhi_epu16 (selw, kDecodeBytesGroupConfig [hbits][3 ]);
859+ __m128i selw = _mm_shuffle_epi8 (selb, _mm_loadu_si128 ( reinterpret_cast < const __m128i*>( kDecodeBytesGroupConfig [hbits][1 ])) );
860+ __m128i sel0 = _mm_mulhi_epu16 (selw, _mm_loadu_si128 ( reinterpret_cast < const __m128i*>( kDecodeBytesGroupConfig [hbits][2 ])) );
861+ __m128i sel1 = _mm_mulhi_epu16 (selw, _mm_loadu_si128 ( reinterpret_cast < const __m128i*>( kDecodeBytesGroupConfig [hbits][3 ])) );
862862 __m128i seli = _mm_or_si128 (sel0, _mm_slli_epi16 (sel1, 8 ));
863863
864864 // the interleaved fields are masked by the bit count (special handling: for 0/8-bit values, mul produces 0)
865- __m128i sent = kDecodeBytesGroupConfig [hbits][0 ];
865+ __m128i sent = _mm_loadu_si128 ( reinterpret_cast < const __m128i*>( kDecodeBytesGroupConfig [hbits][0 ])) ;
866866 __m128i sel = _mm_and_si128 (seli, sent);
867867
868868 // compare sel to sentinel; returns 0 for 0-bit (mul produces 0, sent is 1), 1 for 8-bit (mul produces 0, sent is 0)
@@ -896,16 +896,16 @@ inline const unsigned char* decodeBytesGroupSimd(const unsigned char* data, unsi
896896
897897#ifdef SIMD_AVX
898898// sent mask, multishift control
899- static const __m128i kDecodeBytesGroupConfig [9 ][2 ] = {
900- {_mm_setzero_si128 (), _mm_setzero_si128 () },
901- {_mm_set1_epi8 ( 3 ), _mm_setr_epi8 ( 6 , 4 , 2 , 0 , 14 , 12 , 10 , 8 , 22 , 20 , 18 , 16 , 30 , 28 , 26 , 24 ) },
902- {_mm_set1_epi8 ( 15 ), _mm_setr_epi8 ( 4 , 0 , 12 , 8 , 20 , 16 , 28 , 24 , 36 , 32 , 44 , 40 , 52 , 48 , 60 , 56 ) },
903- {_mm_setzero_si128 (), _mm_setzero_si128 () },
904- {_mm_setzero_si128 (), _mm_setzero_si128 () },
905- {_mm_set1_epi8 ( 1 ), _mm_setr_epi8 ( 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 ) },
906- {_mm_set1_epi8 ( 3 ), _mm_setr_epi8 ( 6 , 4 , 2 , 0 , 14 , 12 , 10 , 8 , 22 , 20 , 18 , 16 , 30 , 28 , 26 , 24 ) },
907- {_mm_set1_epi8 ( 15 ), _mm_setr_epi8 ( 4 , 0 , 12 , 8 , 20 , 16 , 28 , 24 , 36 , 32 , 44 , 40 , 52 , 48 , 60 , 56 ) },
908- {_mm_setzero_si128 (), _mm_setzero_si128 () },
899+ static const unsigned char kDecodeBytesGroupConfig [9 ][2 ][ 16 ] = {
900+ {{ 0 }, { 0 } },
901+ {{ 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 }, { 6 , 4 , 2 , 0 , 14 , 12 , 10 , 8 , 22 , 20 , 18 , 16 , 30 , 28 , 26 , 24 } },
902+ {{ 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 }, { 4 , 0 , 12 , 8 , 20 , 16 , 28 , 24 , 36 , 32 , 44 , 40 , 52 , 48 , 60 , 56 } },
903+ {{ 0 }, { 0 } },
904+ {{ 0 }, { 0 } },
905+ {{ 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 }, { 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 } },
906+ {{ 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 }, { 6 , 4 , 2 , 0 , 14 , 12 , 10 , 8 , 22 , 20 , 18 , 16 , 30 , 28 , 26 , 24 } },
907+ {{ 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 , 15 }, { 4 , 0 , 12 , 8 , 20 , 16 , 28 , 24 , 36 , 32 , 44 , 40 , 52 , 48 , 60 , 56 } },
908+ {{ 0 }, { 0 } },
909909};
910910
911911SIMD_TARGET
@@ -934,8 +934,8 @@ inline const unsigned char* decodeBytesGroupSimd(const unsigned char* data, unsi
934934 __m128i selb = _mm_loadl_epi64 (reinterpret_cast <const __m128i*>(data));
935935 __m128i rest = _mm_maskz_loadu_epi8 (__mmask16 ((n >> 2 ) - 1 ), skip);
936936
937- __m128i sent = kDecodeBytesGroupConfig [hbits][0 ];
938- __m128i ctrl = kDecodeBytesGroupConfig [hbits][1 ];
937+ __m128i sent = _mm_loadu_si128 ( reinterpret_cast < const __m128i*>( kDecodeBytesGroupConfig [hbits][0 ])) ;
938+ __m128i ctrl = _mm_loadu_si128 ( reinterpret_cast < const __m128i*>( kDecodeBytesGroupConfig [hbits][1 ])) ;
939939
940940 // unpack 1, 2 or 4-bit values using multishift and mask the result; for 0/8-bit values, sel is always 0
941941 __m128i selw = _mm_shuffle_epi32 (selb, 0x44 );
0 commit comments