Fix possible out of buffer access in simd dye code.

2 files changed, 202 insertions, 219 deletions

diff --git a/src/resources/dye/dyepalette_replaceaoglcolor.cpp b/src/resources/dye/dyepalette_replaceaoglcolor.cpp
index e50136783..8631f3ebf 100644
--- a/src/resources/dye/dyepalette_replaceaoglcolor.cpp
+++ b/src/resources/dye/dyepalette_replaceaoglcolor.cpp
@@ -109,75 +109,71 @@ void DyePalette::replaceAOGLColorSse2(uint32_t *restrict pixels,
     if ((sz % 2) != 0U)
         -- it_end;
 
-    if (bufSize >= 8)
+    const int mod = bufSize % 4;
+    const int bufEnd = bufSize - mod;
+
+    for (int ptr = 0; ptr < bufEnd; ptr += 4)
     {
-        for (int ptr = 0; ptr < bufSize; ptr += 4)
+//        __m128i base = _mm_load_si128(reinterpret_cast<__m128i*>(
+//            &pixels[ptr]));
+        __m128i base = _mm_loadu_si128(reinterpret_cast<__m128i*>(
+            &pixels[ptr]));
+
+        STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
+        while (it != it_end)
         {
-//            __m128i base = _mm_load_si128(reinterpret_cast<__m128i*>(
-//                &pixels[ptr]));
-            __m128i base = _mm_loadu_si128(reinterpret_cast<__m128i*>(
-                &pixels[ptr]));
+            const DyeColor &col = *it;
+            ++ it;
+            const DyeColor &col2 = *it;
 
-            STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
-            while (it != it_end)
-            {
-                const DyeColor &col = *it;
-                ++ it;
-                const DyeColor &col2 = *it;
-
-                __m128i newMask = _mm_set1_epi32(col2.valueAOgl);
-                __m128i cmpMask = _mm_set1_epi32(col.valueAOgl);
-                __m128i cmpRes = _mm_cmpeq_epi32(base, cmpMask);
-                __m128i srcAnd = _mm_andnot_si128(cmpRes, base);
-                __m128i dstAnd = _mm_and_si128(cmpRes, newMask);
-                base = _mm_or_si128(srcAnd, dstAnd);
-
-                ++ it;
-            }
-//            _mm_store_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base);
-            _mm_storeu_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base);
+            __m128i newMask = _mm_set1_epi32(col2.valueAOgl);
+            __m128i cmpMask = _mm_set1_epi32(col.valueAOgl);
+            __m128i cmpRes = _mm_cmpeq_epi32(base, cmpMask);
+            __m128i srcAnd = _mm_andnot_si128(cmpRes, base);
+            __m128i dstAnd = _mm_and_si128(cmpRes, newMask);
+            base = _mm_or_si128(srcAnd, dstAnd);
+
+            ++ it;
         }
+//        _mm_store_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base);
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base);
     }
-    else
+
+    for (int ptr = bufSize - mod; ptr < bufSize; ptr ++)
     {
-        for (const uint32_t *const p_end = pixels + CAST_SIZE(bufSize);
-             pixels != p_end;
-             ++pixels)
-        {
-            uint8_t *const p = reinterpret_cast<uint8_t *>(pixels);
-            const unsigned int data = *pixels;
+        uint8_t *const p = reinterpret_cast<uint8_t *>(&pixels[ptr]);
+        const unsigned int data = pixels[ptr];
 
-            STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
-            while (it != it_end)
-            {
-                const DyeColor &col = *it;
-                ++ it;
-                const DyeColor &col2 = *it;
+        STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
+        while (it != it_end)
+        {
+            const DyeColor &col = *it;
+            ++ it;
+            const DyeColor &col2 = *it;
 
 #if SDL_BYTEORDER == SDL_BIG_ENDIAN
-                const unsigned int coldata = (col.value[0] << 24U)
-                    | (col.value[1] << 16U)
-                    | (col.value[2] << 8U)
-                    | col.value[3];
+            const unsigned int coldata = (col.value[0] << 24U)
+                | (col.value[1] << 16U)
+                | (col.value[2] << 8U)
+                | col.value[3];
 #else  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-                const unsigned int coldata = (col.value[0])
-                    | (col.value[1] << 8U)
-                    | (col.value[2] << 16U)
-                    | (col.value[3] << 24U);
+            const unsigned int coldata = (col.value[0])
+                | (col.value[1] << 8U)
+                | (col.value[2] << 16U)
+                | (col.value[3] << 24U);
 #endif  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-                if (data == coldata)
-                {
-                    p[0] = col2.value[0];
-                    p[1] = col2.value[1];
-                    p[2] = col2.value[2];
-                    p[3] = col2.value[3];
-                    break;
-                }
-
-                ++ it;
+            if (data == coldata)
+            {
+                p[0] = col2.value[0];
+                p[1] = col2.value[1];
+                p[2] = col2.value[2];
+                p[3] = col2.value[3];
+                break;
             }
+
+            ++ it;
         }
     }
 }
@@ -193,77 +189,72 @@ void DyePalette::replaceAOGLColorAvx2(uint32_t *restrict pixels,
     if ((sz % 2) != 0U)
         -- it_end;
 
-    if (bufSize >= 8)
+    const int mod = bufSize % 8;
+    const int bufEnd = bufSize - mod;
+
+    for (int ptr = 0; ptr < bufEnd; ptr += 8)
     {
-        for (int ptr = 0; ptr < bufSize; ptr += 8)
+//        __m256i base = _mm256_load_si256(reinterpret_cast<__m256i*>(
+//            &pixels[ptr]));
+        __m256i base = _mm256_loadu_si256(reinterpret_cast<__m256i*>(
+            &pixels[ptr]));
+
+        STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
+        while (it != it_end)
         {
-//            __m256i base = _mm256_load_si256(reinterpret_cast<__m256i*>(
-//                &pixels[ptr]));
-            __m256i base = _mm256_loadu_si256(reinterpret_cast<__m256i*>(
-                &pixels[ptr]));
+            const DyeColor &col = *it;
+            ++ it;
+            const DyeColor &col2 = *it;
 
-            STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
-            while (it != it_end)
-            {
-                const DyeColor &col = *it;
-                ++ it;
-                const DyeColor &col2 = *it;
-
-                __m256i newMask = _mm256_set1_epi32(col2.valueAOgl);
-                __m256i cmpMask = _mm256_set1_epi32(col.valueAOgl);
-                __m256i cmpRes = _mm256_cmpeq_epi32(base, cmpMask);
-                __m256i srcAnd = _mm256_andnot_si256(cmpRes, base);
-                __m256i dstAnd = _mm256_and_si256(cmpRes, newMask);
-                base = _mm256_or_si256(srcAnd, dstAnd);
-
-                ++ it;
-            }
-//            _mm256_store_si256(reinterpret_cast<__m256i*>(&pixels[ptr]),
-//                base);
-            _mm256_storeu_si256(reinterpret_cast<__m256i*>(&pixels[ptr]),
-                base);
+            __m256i newMask = _mm256_set1_epi32(col2.valueAOgl);
+            __m256i cmpMask = _mm256_set1_epi32(col.valueAOgl);
+            __m256i cmpRes = _mm256_cmpeq_epi32(base, cmpMask);
+            __m256i srcAnd = _mm256_andnot_si256(cmpRes, base);
+            __m256i dstAnd = _mm256_and_si256(cmpRes, newMask);
+            base = _mm256_or_si256(srcAnd, dstAnd);
+
+            ++ it;
         }
+//        _mm256_store_si256(reinterpret_cast<__m256i*>(&pixels[ptr]),
+//            base);
+        _mm256_storeu_si256(reinterpret_cast<__m256i*>(&pixels[ptr]),
+            base);
     }
-    else
+
+    for (int ptr = bufSize - mod; ptr < bufSize; ptr ++)
     {
-        for (const uint32_t *const p_end = pixels + CAST_SIZE(bufSize);
-             pixels != p_end;
-             ++pixels)
-        {
-            uint8_t *const p = reinterpret_cast<uint8_t *>(pixels);
-            const unsigned int data = *pixels;
+        uint8_t *const p = reinterpret_cast<uint8_t *>(&pixels[ptr]);
+        const unsigned int data = pixels[ptr];
 
-            STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
-            while (it != it_end)
-            {
-                const DyeColor &col = *it;
-                ++ it;
-                const DyeColor &col2 = *it;
+        STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
+        while (it != it_end)
+        {
+            const DyeColor &col = *it;
+            ++ it;
+            const DyeColor &col2 = *it;
 
 #if SDL_BYTEORDER == SDL_BIG_ENDIAN
-                const unsigned int coldata = (col.value[0] << 24U)
-                    | (col.value[1] << 16U)
-                    | (col.value[2] << 8U)
-                    | col.value[3];
+            const unsigned int coldata = (col.value[0] << 24U)
+                | (col.value[1] << 16U)
+                | (col.value[2] << 8U)
+                | col.value[3];
 #else  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-                const unsigned int coldata = (col.value[0])
-                    | (col.value[1] << 8U)
-                    | (col.value[2] << 16U)
-                    | (col.value[3] << 24U);
+            const unsigned int coldata = (col.value[0])
+                | (col.value[1] << 8U)
+                | (col.value[2] << 16U)
+                | (col.value[3] << 24U);
 #endif  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-                if (data == coldata)
-                {
-                    p[0] = col2.value[0];
-                    p[1] = col2.value[1];
-                    p[2] = col2.value[2];
-                    p[3] = col2.value[3];
-                    break;
-                }
-
-                ++ it;
+            if (data == coldata)
+            {
+                p[0] = col2.value[0];
+                p[1] = col2.value[1];
+                p[2] = col2.value[2];
+                p[3] = col2.value[3];
+                break;
             }
+            ++ it;
         }
     }
 }
diff --git a/src/resources/dye/dyepalette_replacesoglcolor.cpp b/src/resources/dye/dyepalette_replacesoglcolor.cpp
index 4483ed02b..a23c64f1f 100644
--- a/src/resources/dye/dyepalette_replacesoglcolor.cpp
+++ b/src/resources/dye/dyepalette_replacesoglcolor.cpp
@@ -109,77 +109,73 @@ void DyePalette::replaceSOGLColorSse2(uint32_t *restrict pixels,
     if ((sz % 2) != 0U)
         -- it_end;
 
-    if (bufSize >= 8)
+    const int mod = bufSize % 4;
+    const int bufEnd = bufSize - mod;
+
+    for (int ptr = 0; ptr < bufEnd; ptr += 4)
     {
-        for (int ptr = 0; ptr < bufSize; ptr += 4)
+        __m128i mask = _mm_set1_epi32(0x00ffffff);
+//        __m128i base = _mm_load_si128(reinterpret_cast<__m128i*>(
+//         &pixels[ptr]));
+        __m128i base = _mm_loadu_si128(reinterpret_cast<__m128i*>(
+            &pixels[ptr]));
+
+        STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
+        while (it != it_end)
         {
-            __m128i mask = _mm_set1_epi32(0x00ffffff);
-//            __m128i base = _mm_load_si128(reinterpret_cast<__m128i*>(
-//             &pixels[ptr]));
-            __m128i base = _mm_loadu_si128(reinterpret_cast<__m128i*>(
-                &pixels[ptr]));
-
-            STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
-            while (it != it_end)
-            {
-                const DyeColor &col = *it;
-                ++ it;
-                const DyeColor &col2 = *it;
-
-                __m128i base2 = _mm_and_si128(mask, base);
-                __m128i newMask = _mm_set1_epi32(col2.valueSOgl);
-                __m128i cmpMask = _mm_set1_epi32(col.valueSOgl);
-                __m128i cmpRes = _mm_cmpeq_epi32(base2, cmpMask);
-                cmpRes = _mm_and_si128(mask, cmpRes);
-                __m128i srcAnd = _mm_andnot_si128(cmpRes, base);
-                __m128i dstAnd = _mm_and_si128(cmpRes, newMask);
-                base = _mm_or_si128(srcAnd, dstAnd);
-                ++ it;
-            }
-//            _mm_store_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base);
-            _mm_storeu_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base);
+            const DyeColor &col = *it;
+            ++ it;
+            const DyeColor &col2 = *it;
+
+            __m128i base2 = _mm_and_si128(mask, base);
+            __m128i newMask = _mm_set1_epi32(col2.valueSOgl);
+            __m128i cmpMask = _mm_set1_epi32(col.valueSOgl);
+            __m128i cmpRes = _mm_cmpeq_epi32(base2, cmpMask);
+            cmpRes = _mm_and_si128(mask, cmpRes);
+            __m128i srcAnd = _mm_andnot_si128(cmpRes, base);
+            __m128i dstAnd = _mm_and_si128(cmpRes, newMask);
+            base = _mm_or_si128(srcAnd, dstAnd);
+            ++ it;
         }
+//        _mm_store_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base);
+        _mm_storeu_si128(reinterpret_cast<__m128i*>(&pixels[ptr]), base);
     }
-    else
+
+    for (int ptr = bufSize - mod; ptr < bufSize; ptr ++)
     {
-        for (const uint32_t *const p_end = pixels + CAST_SIZE(bufSize);
-             pixels != p_end;
-             ++pixels)
-        {
-            uint8_t *const p = reinterpret_cast<uint8_t *>(pixels);
+        uint8_t *const p = reinterpret_cast<uint8_t *>(&pixels[ptr]);
 #if SDL_BYTEORDER == SDL_BIG_ENDIAN
-            const unsigned int data = (*pixels) & 0xffffff00;
+        const unsigned int data = pixels[ptr] & 0xffffff00;
 #else  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-            const unsigned int data = (*pixels) & 0x00ffffff;
+        const unsigned int data = pixels[ptr] & 0x00ffffff;
 #endif  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-            STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
-            while (it != it_end)
-            {
-                const DyeColor &col = *it;
-                ++ it;
-                const DyeColor &col2 = *it;
+        STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
+        while (it != it_end)
+        {
+            const DyeColor &col = *it;
+            ++ it;
+            const DyeColor &col2 = *it;
 
 #if SDL_BYTEORDER == SDL_BIG_ENDIAN
-                const unsigned int coldata = (col.value[0] << 24)
-                    | (col.value[1] << 16) | (col.value[2] << 8);
+            const unsigned int coldata = (col.value[0] << 24)
+                | (col.value[1] << 16) | (col.value[2] << 8);
 #else  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-                const unsigned int coldata = (col.value[0])
-                    | (col.value[1] << 8) | (col.value[2] << 16);
+            const unsigned int coldata = (col.value[0])
+                | (col.value[1] << 8) | (col.value[2] << 16);
 #endif  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-                if (data == coldata)
-                {
-                    p[0] = col2.value[0];
-                    p[1] = col2.value[1];
-                    p[2] = col2.value[2];
-                    break;
-                }
-
-                ++ it;
+            if (data == coldata)
+            {
+                p[0] = col2.value[0];
+                p[1] = col2.value[1];
+                p[2] = col2.value[2];
+                break;
             }
+
+            ++ it;
         }
     }
 }
@@ -195,79 +191,75 @@ void DyePalette::replaceSOGLColorAvx2(uint32_t *restrict pixels,
     if ((sz % 2) != 0U)
         -- it_end;
 
-    if (bufSize >= 8)
+    const int mod = bufSize % 8;
+    const int bufEnd = bufSize - mod;
+
+    for (int ptr = 0; ptr < bufEnd; ptr += 8)
     {
-        for (int ptr = 0; ptr < bufSize; ptr += 8)
-        {
-            __m256i mask = _mm256_set1_epi32(0x00ffffff);
+        __m256i mask = _mm256_set1_epi32(0x00ffffff);
 //          __m256i base = _mm256_load_si256(reinterpret_cast<__m256i*>(
 //              &pixels[ptr]));
-            __m256i base = _mm256_loadu_si256(reinterpret_cast<__m256i*>(
-                &pixels[ptr]));
+        __m256i base = _mm256_loadu_si256(reinterpret_cast<__m256i*>(
+            &pixels[ptr]));
 
-            STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
-            while (it != it_end)
-            {
-                const DyeColor &col = *it;
-                ++ it;
-                const DyeColor &col2 = *it;
-
-                __m256i base2 = _mm256_and_si256(mask, base);
-                __m256i newMask = _mm256_set1_epi32(col2.valueSOgl);
-                __m256i cmpMask = _mm256_set1_epi32(col.valueSOgl);
-                __m256i cmpRes = _mm256_cmpeq_epi32(base2, cmpMask);
-                cmpRes = _mm256_and_si256(mask, cmpRes);
-                __m256i srcAnd = _mm256_andnot_si256(cmpRes, base);
-                __m256i dstAnd = _mm256_and_si256(cmpRes, newMask);
-                base = _mm256_or_si256(srcAnd, dstAnd);
-                ++ it;
-            }
+        STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
+        while (it != it_end)
+        {
+            const DyeColor &col = *it;
+            ++ it;
+            const DyeColor &col2 = *it;
+
+            __m256i base2 = _mm256_and_si256(mask, base);
+            __m256i newMask = _mm256_set1_epi32(col2.valueSOgl);
+            __m256i cmpMask = _mm256_set1_epi32(col.valueSOgl);
+            __m256i cmpRes = _mm256_cmpeq_epi32(base2, cmpMask);
+            cmpRes = _mm256_and_si256(mask, cmpRes);
+            __m256i srcAnd = _mm256_andnot_si256(cmpRes, base);
+            __m256i dstAnd = _mm256_and_si256(cmpRes, newMask);
+            base = _mm256_or_si256(srcAnd, dstAnd);
+            ++ it;
+        }
 //            _mm256_store_si256(reinterpret_cast<__m256i*>(&pixels[ptr]),
 //                base);
-            _mm256_storeu_si256(reinterpret_cast<__m256i*>(&pixels[ptr]),
-                base);
-        }
+        _mm256_storeu_si256(reinterpret_cast<__m256i*>(&pixels[ptr]),
+            base);
     }
-    else
+
+    for (int ptr = bufSize - mod; ptr < bufSize; ptr ++)
     {
-        for (const uint32_t *const p_end = pixels + CAST_SIZE(bufSize);
-             pixels != p_end;
-             ++pixels)
-        {
-            uint8_t *const p = reinterpret_cast<uint8_t *>(pixels);
+        uint8_t *const p = reinterpret_cast<uint8_t *>(&pixels[ptr]);
 #if SDL_BYTEORDER == SDL_BIG_ENDIAN
-            const unsigned int data = (*pixels) & 0xffffff00;
+        const unsigned int data = pixels[ptr] & 0xffffff00;
 #else  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-            const unsigned int data = (*pixels) & 0x00ffffff;
+        const unsigned int data = pixels[ptr] & 0x00ffffff;
 #endif  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-            STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
-            while (it != it_end)
-            {
-                const DyeColor &col = *it;
-                ++ it;
-                const DyeColor &col2 = *it;
+        STD_VECTOR<DyeColor>::const_iterator it = mColors.begin();
+        while (it != it_end)
+        {
+        const DyeColor &col = *it;
+            ++ it;
+            const DyeColor &col2 = *it;
 
 #if SDL_BYTEORDER == SDL_BIG_ENDIAN
-                const unsigned int coldata = (col.value[0] << 24)
-                    | (col.value[1] << 16) | (col.value[2] << 8);
+            const unsigned int coldata = (col.value[0] << 24)
+                | (col.value[1] << 16) | (col.value[2] << 8);
 #else  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-                const unsigned int coldata = (col.value[0])
-                    | (col.value[1] << 8) | (col.value[2] << 16);
+            const unsigned int coldata = (col.value[0])
+                | (col.value[1] << 8) | (col.value[2] << 16);
 #endif  // SDL_BYTEORDER == SDL_BIG_ENDIAN
 
-                if (data == coldata)
-                {
-                    p[0] = col2.value[0];
-                    p[1] = col2.value[1];
-                    p[2] = col2.value[2];
-                    break;
-                }
-
-                ++ it;
+            if (data == coldata)
+            {
+                p[0] = col2.value[0];
+                p[1] = col2.value[1];
+                p[2] = col2.value[2];
+                break;
             }
+
+            ++ it;
         }
     }
 }