Marcus von Appen fixed bug 1535: SDL_UpdateWindowRects() SIGSEGV on X11 with out-of-bound rects
authorSam Lantinga <slouken@libsdl.org>
Thu, 05 Jul 2012 07:26:18 -0400
changeset 6351 8f5535d05e34
parent 6350 19545983ac76
child 6352 a9bcd26e7105
Marcus von Appen fixed bug 1535: SDL_UpdateWindowRects() SIGSEGV on X11 with out-of-bound rects Using SDL_UpdateWindowRects () with SDL_Rect instances, that do not clip to the bounds of the SDL_Window, causes a BadValue error on X11, if shared memory is enabled: X Error of failed request: BadValue (integer parameter out of range for operation) Major opcode of failed request: 142 (MIT-SHM) Minor opcode of failed request: 3 (X_ShmPutImage) Value in failed request: 0x5 Serial number of failed request: 67 Current serial number in output stream: 70 Attached is a test program to recreate the behaviour along with a simple patch for SDL_x11framebuffer.c, which reduces the rect area to the window bounds. I am not sure, if XPutImage() is more robust here, so I applied it to the non-shared memory version, too.
src/video/x11/SDL_x11framebuffer.c
--- a/src/video/x11/SDL_x11framebuffer.c	Tue Jul 03 23:52:02 2012 -0400
+++ b/src/video/x11/SDL_x11framebuffer.c	Thu Jul 05 07:26:18 2012 -0400
@@ -154,33 +154,68 @@
     SDL_WindowData *data = (SDL_WindowData *) window->driverdata;
     Display *display = data->videodata->display;
     int i;
-    SDL_Rect *rect;
-
+    int x, y, w ,h;
 #ifndef NO_SHARED_MEMORY
     if (data->use_mitshm) {
         for (i = 0; i < numrects; ++i) {
-            rect = &rects[i];
+            x = rects[i].x;
+            y = rects[i].y;
+            w = rects[i].w;
+            h = rects[i].h;
 
-            if (rect->w == 0 || rect->h == 0) { /* Clipped? */
+            if (w <= 0 || h <= 0 || (x + w) <= 0 || (y + h) <= 0) {
+                /* Clipped? */
                 continue;
             }
+            if (x < 0)
+            {
+                x += w;
+                w += rects[i].x;
+            }
+            if (y < 0)
+            {
+                y += h;
+                h += rects[i].y;
+            }
+            if (x + w > window->w)
+                w = window->w - x;
+            if (y + h > window->h)
+                h = window->h - y;
+
             XShmPutImage(display, data->xwindow, data->gc, data->ximage,
-                    rect->x, rect->y,
-                    rect->x, rect->y, rect->w, rect->h, False);
+                x, y, x, y, w, h, False);
         }
     }
     else
 #endif /* !NO_SHARED_MEMORY */
     {
         for (i = 0; i < numrects; ++i) {
-            rect = &rects[i];
+            x = rects[i].x;
+            y = rects[i].y;
+            w = rects[i].w;
+            h = rects[i].h;
 
-            if (rect->w == 0 || rect->h == 0) { /* Clipped? */
+            if (w <= 0 || h <= 0 || (x + w) <= 0 || (y + h) <= 0) {
+                /* Clipped? */
                 continue;
             }
+            if (x < 0)
+            {
+                x += w;
+                w += rects[i].x;
+            }
+            if (y < 0)
+            {
+                y += h;
+                h += rects[i].y;
+            }
+            if (x + w > window->w)
+                w = window->w - x;
+            if (y + h > window->h)
+                h = window->h - y;
+
             XPutImage(display, data->xwindow, data->gc, data->ximage,
-                  rect->x, rect->y,
-                  rect->x, rect->y, rect->w, rect->h);
+                x, y, x, y, w, h);
         }
     }