From 019bfbb4a8e49c5ec2fb09c18af1d818ce30e2a4 Mon Sep 17 00:00:00 2001 From: Andrei Karas Date: Wed, 14 Nov 2018 01:16:36 +0300 Subject: Fix possible out of buffer access in simd dye code. --- src/resources/dye/dyepalette_replaceaoglcolor.cpp | 215 +++++++++++----------- src/resources/dye/dyepalette_replacesoglcolor.cpp | 206 ++++++++++----------- 2 files changed, 202 insertions(+), 219 deletions(-) (limited to 'src/resources') diff --git a/src/resources/dye/dyepalette_replaceaoglcolor.cpp b/src/resources/dye/dyepalette_replaceaoglcolor.cpp index e50136783..8631f3ebf 100644 --- a/src/resources/dye/dyepalette_replaceaoglcolor.cpp +++ b/src/resources/dye/dyepalette_replaceaoglcolor.cpp @@ -109,75 +109,71 @@ void DyePalette::replaceAOGLColorSse2(uint32_t *restrict pixels, if ((sz % 2) != 0U) -- it_end; - if (bufSize >= 8) + const int mod = bufSize % 4; + const int bufEnd = bufSize - mod; + + for (int ptr = 0; ptr < bufEnd; ptr += 4) { - for (int ptr = 0; ptr < bufSize; ptr += 4) +// __m128i base = _mm_load_si128(reinterpret_cast<__m128i*>( +// &pixels[ptr])); + __m128i base = _mm_loadu_si128(reinterpret_cast<__m128i*>( + &pixels[ptr])); + + STD_VECTOR::const_iterator it = mColors.begin(); + while (it != it_end) { -// __m128i base = _mm_load_si128(reinterpret_cast<__m128i*>( -// &pixels[ptr])); - __m128i base = _mm_loadu_si128(reinterpret_cast<__m128i*>( - &pixels[ptr])); + const DyeColor &col = *it; + ++ it; + const DyeColor &col2 = *it; - STD_VECTOR::const_iterator it = mColors.begin(); - while (it != it_end) - { - const DyeColor &col = *it; - ++ it; - const DyeColor &col2 = *it; - - __m128i newMask = _mm_set1_epi32(col2.valueAOgl); - __m128i cmpMask = _mm_set1_epi32(col.valueAOgl); - __m128i cmpRes = _mm_cmpeq_epi32(base, cmpMask); - __m128i srcAnd = _mm_andnot_si128(cmpRes, base); - __m128i dstAnd = _mm_and_si128(cmpRes, newMask); - base = _mm_or_si128(srcAnd, dstAnd); - - ++ it; - } -// _mm_store_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base); - _mm_storeu_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base); + __m128i newMask = _mm_set1_epi32(col2.valueAOgl); + __m128i cmpMask = _mm_set1_epi32(col.valueAOgl); + __m128i cmpRes = _mm_cmpeq_epi32(base, cmpMask); + __m128i srcAnd = _mm_andnot_si128(cmpRes, base); + __m128i dstAnd = _mm_and_si128(cmpRes, newMask); + base = _mm_or_si128(srcAnd, dstAnd); + + ++ it; } +// _mm_store_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base); + _mm_storeu_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base); } - else + + for (int ptr = bufSize - mod; ptr < bufSize; ptr ++) { - for (const uint32_t *const p_end = pixels + CAST_SIZE(bufSize); - pixels != p_end; - ++pixels) - { - uint8_t *const p = reinterpret_cast(pixels); - const unsigned int data = *pixels; + uint8_t *const p = reinterpret_cast(&pixels[ptr]); + const unsigned int data = pixels[ptr]; - STD_VECTOR::const_iterator it = mColors.begin(); - while (it != it_end) - { - const DyeColor &col = *it; - ++ it; - const DyeColor &col2 = *it; + STD_VECTOR::const_iterator it = mColors.begin(); + while (it != it_end) + { + const DyeColor &col = *it; + ++ it; + const DyeColor &col2 = *it; #if SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int coldata = (col.value[0] << 24U) - | (col.value[1] << 16U) - | (col.value[2] << 8U) - | col.value[3]; + const unsigned int coldata = (col.value[0] << 24U) + | (col.value[1] << 16U) + | (col.value[2] << 8U) + | col.value[3]; #else // SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int coldata = (col.value[0]) - | (col.value[1] << 8U) - | (col.value[2] << 16U) - | (col.value[3] << 24U); + const unsigned int coldata = (col.value[0]) + | (col.value[1] << 8U) + | (col.value[2] << 16U) + | (col.value[3] << 24U); #endif // SDL_BYTEORDER == SDL_BIG_ENDIAN - if (data == coldata) - { - p[0] = col2.value[0]; - p[1] = col2.value[1]; - p[2] = col2.value[2]; - p[3] = col2.value[3]; - break; - } - - ++ it; + if (data == coldata) + { + p[0] = col2.value[0]; + p[1] = col2.value[1]; + p[2] = col2.value[2]; + p[3] = col2.value[3]; + break; } + + ++ it; } } } @@ -193,77 +189,72 @@ void DyePalette::replaceAOGLColorAvx2(uint32_t *restrict pixels, if ((sz % 2) != 0U) -- it_end; - if (bufSize >= 8) + const int mod = bufSize % 8; + const int bufEnd = bufSize - mod; + + for (int ptr = 0; ptr < bufEnd; ptr += 8) { - for (int ptr = 0; ptr < bufSize; ptr += 8) +// __m256i base = _mm256_load_si256(reinterpret_cast<__m256i*>( +// &pixels[ptr])); + __m256i base = _mm256_loadu_si256(reinterpret_cast<__m256i*>( + &pixels[ptr])); + + STD_VECTOR::const_iterator it = mColors.begin(); + while (it != it_end) { -// __m256i base = _mm256_load_si256(reinterpret_cast<__m256i*>( -// &pixels[ptr])); - __m256i base = _mm256_loadu_si256(reinterpret_cast<__m256i*>( - &pixels[ptr])); + const DyeColor &col = *it; + ++ it; + const DyeColor &col2 = *it; - STD_VECTOR::const_iterator it = mColors.begin(); - while (it != it_end) - { - const DyeColor &col = *it; - ++ it; - const DyeColor &col2 = *it; - - __m256i newMask = _mm256_set1_epi32(col2.valueAOgl); - __m256i cmpMask = _mm256_set1_epi32(col.valueAOgl); - __m256i cmpRes = _mm256_cmpeq_epi32(base, cmpMask); - __m256i srcAnd = _mm256_andnot_si256(cmpRes, base); - __m256i dstAnd = _mm256_and_si256(cmpRes, newMask); - base = _mm256_or_si256(srcAnd, dstAnd); - - ++ it; - } -// _mm256_store_si256(reinterpret_cast<__m256i*>(&pixels[ptr]), -// base); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(&pixels[ptr]), - base); + __m256i newMask = _mm256_set1_epi32(col2.valueAOgl); + __m256i cmpMask = _mm256_set1_epi32(col.valueAOgl); + __m256i cmpRes = _mm256_cmpeq_epi32(base, cmpMask); + __m256i srcAnd = _mm256_andnot_si256(cmpRes, base); + __m256i dstAnd = _mm256_and_si256(cmpRes, newMask); + base = _mm256_or_si256(srcAnd, dstAnd); + + ++ it; } +// _mm256_store_si256(reinterpret_cast<__m256i*>(&pixels[ptr]), +// base); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(&pixels[ptr]), + base); } - else + + for (int ptr = bufSize - mod; ptr < bufSize; ptr ++) { - for (const uint32_t *const p_end = pixels + CAST_SIZE(bufSize); - pixels != p_end; - ++pixels) - { - uint8_t *const p = reinterpret_cast(pixels); - const unsigned int data = *pixels; + uint8_t *const p = reinterpret_cast(&pixels[ptr]); + const unsigned int data = pixels[ptr]; - STD_VECTOR::const_iterator it = mColors.begin(); - while (it != it_end) - { - const DyeColor &col = *it; - ++ it; - const DyeColor &col2 = *it; + STD_VECTOR::const_iterator it = mColors.begin(); + while (it != it_end) + { + const DyeColor &col = *it; + ++ it; + const DyeColor &col2 = *it; #if SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int coldata = (col.value[0] << 24U) - | (col.value[1] << 16U) - | (col.value[2] << 8U) - | col.value[3]; + const unsigned int coldata = (col.value[0] << 24U) + | (col.value[1] << 16U) + | (col.value[2] << 8U) + | col.value[3]; #else // SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int coldata = (col.value[0]) - | (col.value[1] << 8U) - | (col.value[2] << 16U) - | (col.value[3] << 24U); + const unsigned int coldata = (col.value[0]) + | (col.value[1] << 8U) + | (col.value[2] << 16U) + | (col.value[3] << 24U); #endif // SDL_BYTEORDER == SDL_BIG_ENDIAN - if (data == coldata) - { - p[0] = col2.value[0]; - p[1] = col2.value[1]; - p[2] = col2.value[2]; - p[3] = col2.value[3]; - break; - } - - ++ it; + if (data == coldata) + { + p[0] = col2.value[0]; + p[1] = col2.value[1]; + p[2] = col2.value[2]; + p[3] = col2.value[3]; + break; } + ++ it; } } } diff --git a/src/resources/dye/dyepalette_replacesoglcolor.cpp b/src/resources/dye/dyepalette_replacesoglcolor.cpp index 4483ed02b..a23c64f1f 100644 --- a/src/resources/dye/dyepalette_replacesoglcolor.cpp +++ b/src/resources/dye/dyepalette_replacesoglcolor.cpp @@ -109,77 +109,73 @@ void DyePalette::replaceSOGLColorSse2(uint32_t *restrict pixels, if ((sz % 2) != 0U) -- it_end; - if (bufSize >= 8) + const int mod = bufSize % 4; + const int bufEnd = bufSize - mod; + + for (int ptr = 0; ptr < bufEnd; ptr += 4) { - for (int ptr = 0; ptr < bufSize; ptr += 4) + __m128i mask = _mm_set1_epi32(0x00ffffff); +// __m128i base = _mm_load_si128(reinterpret_cast<__m128i*>( +// &pixels[ptr])); + __m128i base = _mm_loadu_si128(reinterpret_cast<__m128i*>( + &pixels[ptr])); + + STD_VECTOR::const_iterator it = mColors.begin(); + while (it != it_end) { - __m128i mask = _mm_set1_epi32(0x00ffffff); -// __m128i base = _mm_load_si128(reinterpret_cast<__m128i*>( -// &pixels[ptr])); - __m128i base = _mm_loadu_si128(reinterpret_cast<__m128i*>( - &pixels[ptr])); - - STD_VECTOR::const_iterator it = mColors.begin(); - while (it != it_end) - { - const DyeColor &col = *it; - ++ it; - const DyeColor &col2 = *it; - - __m128i base2 = _mm_and_si128(mask, base); - __m128i newMask = _mm_set1_epi32(col2.valueSOgl); - __m128i cmpMask = _mm_set1_epi32(col.valueSOgl); - __m128i cmpRes = _mm_cmpeq_epi32(base2, cmpMask); - cmpRes = _mm_and_si128(mask, cmpRes); - __m128i srcAnd = _mm_andnot_si128(cmpRes, base); - __m128i dstAnd = _mm_and_si128(cmpRes, newMask); - base = _mm_or_si128(srcAnd, dstAnd); - ++ it; - } -// _mm_store_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base); - _mm_storeu_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base); + const DyeColor &col = *it; + ++ it; + const DyeColor &col2 = *it; + + __m128i base2 = _mm_and_si128(mask, base); + __m128i newMask = _mm_set1_epi32(col2.valueSOgl); + __m128i cmpMask = _mm_set1_epi32(col.valueSOgl); + __m128i cmpRes = _mm_cmpeq_epi32(base2, cmpMask); + cmpRes = _mm_and_si128(mask, cmpRes); + __m128i srcAnd = _mm_andnot_si128(cmpRes, base); + __m128i dstAnd = _mm_and_si128(cmpRes, newMask); + base = _mm_or_si128(srcAnd, dstAnd); + ++ it; } +// _mm_store_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base); + _mm_storeu_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base); } - else + + for (int ptr = bufSize - mod; ptr < bufSize; ptr ++) { - for (const uint32_t *const p_end = pixels + CAST_SIZE(bufSize); - pixels != p_end; - ++pixels) - { - uint8_t *const p = reinterpret_cast(pixels); + uint8_t *const p = reinterpret_cast(&pixels[ptr]); #if SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int data = (*pixels) & 0xffffff00; + const unsigned int data = pixels[ptr] & 0xffffff00; #else // SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int data = (*pixels) & 0x00ffffff; + const unsigned int data = pixels[ptr] & 0x00ffffff; #endif // SDL_BYTEORDER == SDL_BIG_ENDIAN - STD_VECTOR::const_iterator it = mColors.begin(); - while (it != it_end) - { - const DyeColor &col = *it; - ++ it; - const DyeColor &col2 = *it; + STD_VECTOR::const_iterator it = mColors.begin(); + while (it != it_end) + { + const DyeColor &col = *it; + ++ it; + const DyeColor &col2 = *it; #if SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int coldata = (col.value[0] << 24) - | (col.value[1] << 16) | (col.value[2] << 8); + const unsigned int coldata = (col.value[0] << 24) + | (col.value[1] << 16) | (col.value[2] << 8); #else // SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int coldata = (col.value[0]) - | (col.value[1] << 8) | (col.value[2] << 16); + const unsigned int coldata = (col.value[0]) + | (col.value[1] << 8) | (col.value[2] << 16); #endif // SDL_BYTEORDER == SDL_BIG_ENDIAN - if (data == coldata) - { - p[0] = col2.value[0]; - p[1] = col2.value[1]; - p[2] = col2.value[2]; - break; - } - - ++ it; + if (data == coldata) + { + p[0] = col2.value[0]; + p[1] = col2.value[1]; + p[2] = col2.value[2]; + break; } + + ++ it; } } } @@ -195,79 +191,75 @@ void DyePalette::replaceSOGLColorAvx2(uint32_t *restrict pixels, if ((sz % 2) != 0U) -- it_end; - if (bufSize >= 8) + const int mod = bufSize % 8; + const int bufEnd = bufSize - mod; + + for (int ptr = 0; ptr < bufEnd; ptr += 8) { - for (int ptr = 0; ptr < bufSize; ptr += 8) - { - __m256i mask = _mm256_set1_epi32(0x00ffffff); + __m256i mask = _mm256_set1_epi32(0x00ffffff); // __m256i base = _mm256_load_si256(reinterpret_cast<__m256i*>( // &pixels[ptr])); - __m256i base = _mm256_loadu_si256(reinterpret_cast<__m256i*>( - &pixels[ptr])); + __m256i base = _mm256_loadu_si256(reinterpret_cast<__m256i*>( + &pixels[ptr])); - STD_VECTOR::const_iterator it = mColors.begin(); - while (it != it_end) - { - const DyeColor &col = *it; - ++ it; - const DyeColor &col2 = *it; - - __m256i base2 = _mm256_and_si256(mask, base); - __m256i newMask = _mm256_set1_epi32(col2.valueSOgl); - __m256i cmpMask = _mm256_set1_epi32(col.valueSOgl); - __m256i cmpRes = _mm256_cmpeq_epi32(base2, cmpMask); - cmpRes = _mm256_and_si256(mask, cmpRes); - __m256i srcAnd = _mm256_andnot_si256(cmpRes, base); - __m256i dstAnd = _mm256_and_si256(cmpRes, newMask); - base = _mm256_or_si256(srcAnd, dstAnd); - ++ it; - } + STD_VECTOR::const_iterator it = mColors.begin(); + while (it != it_end) + { + const DyeColor &col = *it; + ++ it; + const DyeColor &col2 = *it; + + __m256i base2 = _mm256_and_si256(mask, base); + __m256i newMask = _mm256_set1_epi32(col2.valueSOgl); + __m256i cmpMask = _mm256_set1_epi32(col.valueSOgl); + __m256i cmpRes = _mm256_cmpeq_epi32(base2, cmpMask); + cmpRes = _mm256_and_si256(mask, cmpRes); + __m256i srcAnd = _mm256_andnot_si256(cmpRes, base); + __m256i dstAnd = _mm256_and_si256(cmpRes, newMask); + base = _mm256_or_si256(srcAnd, dstAnd); + ++ it; + } // _mm256_store_si256(reinterpret_cast<__m256i*>(&pixels[ptr]), // base); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(&pixels[ptr]), - base); - } + _mm256_storeu_si256(reinterpret_cast<__m256i*>(&pixels[ptr]), + base); } - else + + for (int ptr = bufSize - mod; ptr < bufSize; ptr ++) { - for (const uint32_t *const p_end = pixels + CAST_SIZE(bufSize); - pixels != p_end; - ++pixels) - { - uint8_t *const p = reinterpret_cast(pixels); + uint8_t *const p = reinterpret_cast(&pixels[ptr]); #if SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int data = (*pixels) & 0xffffff00; + const unsigned int data = pixels[ptr] & 0xffffff00; #else // SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int data = (*pixels) & 0x00ffffff; + const unsigned int data = pixels[ptr] & 0x00ffffff; #endif // SDL_BYTEORDER == SDL_BIG_ENDIAN - STD_VECTOR::const_iterator it = mColors.begin(); - while (it != it_end) - { - const DyeColor &col = *it; - ++ it; - const DyeColor &col2 = *it; + STD_VECTOR::const_iterator it = mColors.begin(); + while (it != it_end) + { + const DyeColor &col = *it; + ++ it; + const DyeColor &col2 = *it; #if SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int coldata = (col.value[0] << 24) - | (col.value[1] << 16) | (col.value[2] << 8); + const unsigned int coldata = (col.value[0] << 24) + | (col.value[1] << 16) | (col.value[2] << 8); #else // SDL_BYTEORDER == SDL_BIG_ENDIAN - const unsigned int coldata = (col.value[0]) - | (col.value[1] << 8) | (col.value[2] << 16); + const unsigned int coldata = (col.value[0]) + | (col.value[1] << 8) | (col.value[2] << 16); #endif // SDL_BYTEORDER == SDL_BIG_ENDIAN - if (data == coldata) - { - p[0] = col2.value[0]; - p[1] = col2.value[1]; - p[2] = col2.value[2]; - break; - } - - ++ it; + if (data == coldata) + { + p[0] = col2.value[0]; + p[1] = col2.value[1]; + p[2] = col2.value[2]; + break; } + + ++ it; } } } -- cgit v1.2.3-60-g2f50