API tools faq
paste
Advertisement
Tarferi

Inc for AVX2

Tarferi
Jul 5th, 2025 (edited)
505
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
C Intermediate Language 15.53 KB | Software | 0 0
raw download clone embed print report
  1. static const size_t _(AVX2_VALUES_PER_256) = (32 / sizeof(BASE_TYPE));
  2. static const size_t _(AVX2_VALUES_PER_64) = 8 / sizeof(BASE_TYPE);
  3.  
  4. QINT_ALIGN(32)
  5. #if defined(DEFAULT_BASE_1BYTE)
  6. const uint8_t _(AVX2_EXTRACTOR_MASK)[33][32] = {
  7.     { 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  8.     { 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  9.     { 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  10.     { 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  11.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  12.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  13.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  14.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  15.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  16.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  17.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  18.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  19.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  20.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  21.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  22.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  23.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  24.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  25.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  26.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  27.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  28.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  29.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  30.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  31.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  32.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  33.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00 },
  34.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00 },
  35.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00 },
  36.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00 },
  37.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00 },
  38.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
  39.     { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
  40. };
  41. #elif defined(DEFAULT_BASE_2BYTE)
  42. const uint16_t _(AVX2_EXTRACTOR_MASK)[17][16] = {
  43.     { 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  44.     { 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  45.     { 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  46.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  47.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  48.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  49.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  50.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  51.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  52.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  53.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 },
  54.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000 },
  55.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000 },
  56.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000 },
  57.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000 },
  58.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF },
  59.     { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF },
  60. };
  61. #elif defined(DEFAULT_BASE_4BYTE)
  62. const uint32_t _(AVX2_EXTRACTOR_MASK)[9][8] = {
  63.     { 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
  64.     { 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
  65.     { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
  66.     { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
  67.     { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000 },
  68.     { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 },
  69.     { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 },
  70.     { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF },
  71.     { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF },
  72. };
  73. #elif defined(DEFAULT_BASE_8BYTE)
  74. const uint64_t _(AVX2_EXTRACTOR_MASK)[5][4] = {
  75.     { 0xFFFFFFFFFFFFFFFF, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 },
  76.     { 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0x0000000000000000, 0x0000000000000000 },
  77.     { 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0x0000000000000000 },
  78.     { 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF },
  79.     { 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF }
  80. };
  81. #else
  82. #error No base defined
  83. #endif
  84.  
  85. // Intrisc version of arithmetics
  86.  
  87. static bool _(Inc_AVX2)(BASE_TYPE* vals, size_t valsCount, size_t* usedValues, size_t* usedBytes) {
  88.     const size_t total256Values = valsCount / _(AVX2_VALUES_PER_256);
  89.     const size_t remainingValues = valsCount % _(AVX2_VALUES_PER_256);
  90.     __m256i* values = (__m256i*)vals;
  91.  
  92. #if defined(DEFAULT_BASE_1BYTE)
  93.     __m256i one = _mm256_set1_epi8(1);
  94. #elif defined(DEFAULT_BASE_2BYTE)
  95.     __m256i one = _mm256_set1_epi16(1);
  96. #elif defined(DEFAULT_BASE_4BYTE)
  97.     __m256i one = _mm256_set1_epi32(1);
  98. #elif defined(DEFAULT_BASE_8BYTE)
  99.     __m256i one = _mm256_set1_epi64x(1);
  100. #else
  101. #error No base defined
  102. #endif
  103.  
  104.     for (size_t i = 0; i < total256Values; i++) {
  105.         __m256i val = _mm256_load_si256(&values[i]);
  106.        
  107.         // Add 1 to each value and compare with 0
  108. #if defined(DEFAULT_BASE_1BYTE)
  109.         __m256i added1 = _mm256_add_epi8(val, one);
  110.         __m256i cmp = _mm256_cmpeq_epi8(added1, _mm256_setzero_si256());
  111. #elif defined(DEFAULT_BASE_2BYTE)
  112.         __m256i added1 = _mm256_add_epi16(val, one);
  113.         __m256i cmp = _mm256_cmpeq_epi16(added1, _mm256_setzero_si256());
  114. #elif defined(DEFAULT_BASE_4BYTE)
  115.         __m256i added1 = _mm256_add_epi32(val, one);
  116.         __m256i cmp = _mm256_cmpeq_epi32(added1, _mm256_setzero_si256());
  117. #elif defined(DEFAULT_BASE_8BYTE)
  118.         __m256i added1 = _mm256_add_epi64(val, one);
  119.         __m256i cmp = _mm256_cmpeq_epi64(added1, _mm256_setzero_si256());
  120. #else
  121. #error No base defined
  122. #endif
  123.  
  124.         // Get MSB of bytes
  125.         unsigned int mask = ~_mm256_movemask_epi8(cmp);
  126.  
  127.         // Number of values (-1) that were actually incremented
  128.         int incCount = _(TrailingZeroBits_AVX2)((uint32_t)mask) / sizeof(BASE_TYPE);
  129.  
  130.         // Get extractor mask
  131.         __m256i* extractorMask = (__m256i*)_(AVX2_EXTRACTOR_MASK)[incCount];
  132.  
  133.         // Load extractor mask
  134.         __m256i vMask = _mm256_load_si256(extractorMask);
  135.  
  136.         // Store incremented using extractor mask
  137. #if defined(DEFAULT_BASE_1BYTE)
  138.         _mm256_storeu_si256(&values[i], _mm256_blendv_epi8(val, added1, vMask));
  139. #elif defined(DEFAULT_BASE_2BYTE)
  140.         _mm256_storeu_si256(&values[i], _mm256_blendv_epi8(val, added1, vMask));
  141. #elif defined(DEFAULT_BASE_4BYTE)
  142.         _mm256_maskstore_epi32((uint32_t*)&values[i], vMask, added1);
  143. #elif defined(DEFAULT_BASE_8BYTE)
  144.         _mm256_maskstore_epi64((uint64_t*)&values[i], vMask, added1);
  145. #else
  146. #error No base defined
  147. #endif
  148.  
  149.         if (incCount < _(AVX2_VALUES_PER_256)) { // Carry not propagated further
  150.             size_t nowUsedValues = (i * _(AVX2_VALUES_PER_256)) + ((incCount + 1) * _(AVX2_VALUES_PER_64));
  151.             if (usedValues[0] < nowUsedValues) {
  152.                 usedValues[0] = nowUsedValues;
  153.                 usedBytes[0] = nowUsedValues * sizeof(BASE_TYPE);
  154.             }
  155.             return false;
  156.         }
  157.     }
  158.     if (remainingValues > 0) {
  159.         // Increment by bytes. Slower, but only done once
  160.         __m256i lastValue;
  161.  
  162.         const size_t remainingBytes = remainingValues * sizeof(BASE_TYPE);
  163.         memset(&lastValue, 0, sizeof(__m256i));
  164.         memcpy(&lastValue, &vals[valsCount - remainingValues], remainingBytes);
  165.  
  166.         __m256i val = _mm256_loadu_si256(&lastValue);
  167.  
  168.         // Add 1 to each value and compare with 0
  169. #if defined(DEFAULT_BASE_1BYTE)
  170.         __m256i added1 = _mm256_add_epi8(val, _mm256_set1_epi8(1));
  171.         __m256i cmp = _mm256_cmpeq_epi8(added1, _mm256_setzero_si256());
  172. #elif defined(DEFAULT_BASE_2BYTE)
  173.         __m256i added1 = _mm256_add_epi16(val, _mm256_set1_epi16(1));
  174.         __m256i cmp = _mm256_cmpeq_epi16(added1, _mm256_setzero_si256());
  175. #elif defined(DEFAULT_BASE_4BYTE)
  176.         __m256i added1 = _mm256_add_epi32(val, _mm256_set1_epi32(1));
  177.         __m256i cmp = _mm256_cmpeq_epi32(added1, _mm256_setzero_si256());
  178. #elif defined(DEFAULT_BASE_8BYTE)
  179.         __m256i added1 = _mm256_add_epi64(val, one);
  180.         __m256i cmp = _mm256_cmpeq_epi64(added1, _mm256_setzero_si256());
  181. #else
  182. #error No base defined
  183. #endif
  184.  
  185.         // Get MSB of bytes
  186.         unsigned int mask = ~_mm256_movemask_epi8(cmp);
  187.  
  188.         // Number of values (-1) that were actually incremented
  189.         int incCount = _(TrailingZeroBits_AVX2)((uint32_t)mask) / sizeof(BASE_TYPE);
  190.  
  191.         // Get extractor mask
  192.         __m256i* extractorMask = (__m256i*)_(AVX2_EXTRACTOR_MASK)[incCount];
  193.  
  194.         // Load extractor mask
  195.         __m256i vMask = _mm256_load_si256(extractorMask);
  196.  
  197.         // Store incremented using extractor mask
  198. #if defined(DEFAULT_BASE_1BYTE)
  199.         _mm256_storeu_si256(&lastValue, _mm256_blendv_epi8(val, added1, vMask));
  200. #elif defined(DEFAULT_BASE_2BYTE)
  201.         _mm256_storeu_si256(&lastValue, _mm256_blendv_epi8(val, added1, vMask));
  202. #elif defined(DEFAULT_BASE_4BYTE)
  203.         _mm256_maskstore_epi32((uint32_t*)&lastValue, vMask, added1);
  204. #elif defined(DEFAULT_BASE_8BYTE)
  205.         _mm256_maskstore_epi64((uint64_t*)&lastValue, vMask, added1);
  206. #else
  207. #error No base defined
  208. #endif
  209.         memcpy(&vals[valsCount - remainingValues], &lastValue, remainingBytes);
  210.         size_t changedValues = incCount + 1;
  211.  
  212.         size_t nowUsedValues = (total256Values * _(AVX2_VALUES_PER_256)) + changedValues;
  213.         if (changedValues > remainingValues) {
  214.             return true;
  215.         } else {
  216.             if (usedValues[0] < nowUsedValues) {
  217.                 usedValues[0] = nowUsedValues;
  218.                 usedBytes[0] = nowUsedValues * sizeof(BASE_TYPE);
  219.             }
  220.             return false;
  221.         }
  222.     } else {
  223.         // Carry propagated, we don't have to set anything here
  224.         return true;
  225.     }
  226. }
  227.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!