diff mbox

[PULL,07/11] xbzrle.c: Avoid undefined behaviour with signed arithmetic

Message ID 1396940651-10513-8-git-send-email-mjt@msgid.tls.msk.ru
State Accepted
Commit 968fc24d843c9e9b24231ca1960b47ef2fc724ea
Headers show

Commit Message

Michael Tokarev April 8, 2014, 7:04 a.m. UTC 
From: Peter Maydell <peter.maydell@linaro.org>

Use unsigned types for doing bitwise arithmetic in the xzbrle
calculations, to avoid undefined behaviour:

 xbzrle.c:99:49: runtime error: left shift of 72340172838076673
 by 7 places cannot be represented in type 'long'

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Michael Tokarev <mjt@tls.msk.ru>
---
 xbzrle.c |    8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
diff mbox

Patch

diff --git a/xbzrle.c b/xbzrle.c
index fbcb35d..8e220bf 100644
--- a/xbzrle.c
+++ b/xbzrle.c
@@ -28,7 +28,7 @@  int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
 {
     uint32_t zrun_len = 0, nzrun_len = 0;
     int d = 0, i = 0;
-    long res, xor;
+    long res;
     uint8_t *nzrun_start = NULL;
 
     g_assert(!(((uintptr_t)old_buf | (uintptr_t)new_buf | slen) %
@@ -93,9 +93,11 @@  int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
         /* word at a time for speed, use of 32-bit long okay */
         if (!res) {
             /* truncation to 32-bit long okay */
-            long mask = (long)0x0101010101010101ULL;
+            unsigned long mask = (unsigned long)0x0101010101010101ULL;
             while (i < slen) {
-                xor = *(long *)(old_buf + i) ^ *(long *)(new_buf + i);
+                unsigned long xor;
+                xor = *(unsigned long *)(old_buf + i)
+                    ^ *(unsigned long *)(new_buf + i);
                 if ((xor - mask) & ~xor & (mask << 7)) {
                     /* found the end of an nzrun within the current long */
                     while (old_buf[i] != new_buf[i]) {