File sdl2-surface-pitch-overflow.patch of Package SDL2.23954

changeset:   13915:3f9b4e92c1d9
user:        Sam Lantinga <slouken@libsdl.org>
date:        Wed Jun 17 08:44:45 2020 -0700
summary:     Fixed overflow in surface pitch calculation

Backported by Mike Gorse <mgorse@suse.com>

diff -urp SDL2-2.0.8.orig/src/video/SDL_surface.c SDL2-2.0.8/src/video/SDL_surface.c
--- SDL2-2.0.8.orig/src/video/SDL_surface.c	2018-03-01 10:34:42.000000000 -0600
+++ SDL2-2.0.8/src/video/SDL_surface.c	2022-01-11 15:05:56.679255675 -0600
@@ -28,34 +28,26 @@
 #include "SDL_yuv_c.h"
 
 
-/* Check to make sure we can safely check multiplication of surface w and pitch and it won't overflow size_t */
-SDL_COMPILE_TIME_ASSERT(surface_size_assumptions,
-    sizeof(int) == sizeof(Sint32) && sizeof(size_t) >= sizeof(Sint32));
+/* Check to make sure we can safely check multiplication of surface w and pitch and it won't overflow Sint64 */
+SDL_COMPILE_TIME_ASSERT(surface_size_assumptions, sizeof(int) == sizeof(Sint32));
 
 /* Public routines */
 
 /*
  * Calculate the pad-aligned scanline width of a surface
  */
-int
+static Sint64
 SDL_CalculatePitch(Uint32 format, int width)
 {
-    int pitch;
+    Sint64 pitch;
 
-    /* Surface should be 4-byte aligned for speed */
-    pitch = width * SDL_BYTESPERPIXEL(format);
-    switch (SDL_BITSPERPIXEL(format)) {
-    case 1:
-        pitch = (pitch + 7) / 8;
-        break;
-    case 4:
-        pitch = (pitch + 1) / 2;
-        break;
-    default:
-        break;
-    }
-    pitch = (pitch + 3) & ~3;   /* 4-byte aligning */
-    return pitch;
+     if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) {
+        pitch = ((Sint64)width * SDL_BYTESPERPIXEL(format));
+     } else {
+        pitch = (((Sint64)width * SDL_BITSPERPIXEL(format)) + 7) / 8;
+     }
+     pitch = (pitch + 3) & ~3;   /* 4-byte aligning for speed */
+     return pitch;
 }
 
 /*
@@ -66,11 +58,19 @@ SDL_Surface *
 SDL_CreateRGBSurfaceWithFormat(Uint32 flags, int width, int height, int depth,
                                Uint32 format)
 {
+    Sint64 pitch;
     SDL_Surface *surface;
 
     /* The flags are no longer used, make the compiler happy */
     (void)flags;
 
+    pitch = SDL_CalculatePitch(format, width);
+    if (pitch < 0 || pitch > SDL_MAX_SINT32) {
+        /* Overflow... */
+        SDL_OutOfMemory();
+        return NULL;
+    }
+
     /* Allocate the surface */
     surface = (SDL_Surface *) SDL_calloc(1, sizeof(*surface));
     if (surface == NULL) {
@@ -85,7 +85,7 @@ SDL_CreateRGBSurfaceWithFormat(Uint32 fl
     }
     surface->w = width;
     surface->h = height;
-    surface->pitch = SDL_CalculatePitch(format, width);
+    surface->pitch = (int)pitch;
     SDL_SetClipRect(surface, NULL);
 
     if (SDL_ISPIXELFORMAT_INDEXED(surface->format->format)) {