File LibVNCServer-CVE-2014-6051.patch of Package LibVNCServer.1485

From 045a044e8ae79db9244593fbce154cdf6e843273 Mon Sep 17 00:00:00 2001
From: newsoft <newsoft@MacBook-Air-de-newsoft-2.local>
Date: Fri, 15 Aug 2014 16:31:13 +0200
Subject: [PATCH] Fix integer overflow in MallocFrameBuffer()

Promote integers to uint64_t to avoid integer overflow issue during
frame buffer allocation for very large screen sizes
---
 libvncclient/vncviewer.c | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

Index: LibVNCServer-0.9.9/libvncclient/vncviewer.c
===================================================================
--- LibVNCServer-0.9.9.orig/libvncclient/vncviewer.c
+++ LibVNCServer-0.9.9/libvncclient/vncviewer.c
@@ -82,9 +82,27 @@
 #endif
 }
 static rfbBool MallocFrameBuffer(rfbClient* client) {
+uint64_t allocSize;
+
   if(client->frameBuffer)
     free(client->frameBuffer);
-  client->frameBuffer=malloc(client->width*client->height*client->format.bitsPerPixel/8);
+
+  /* SECURITY: promote 'width' into uint64_t so that the multiplication does not overflow
+     'width' and 'height' are 16-bit integers per RFB protocol design
+     SIZE_MAX is the maximum value that can fit into size_t
+  */
+  allocSize = (uint64_t)client->width * client->height * client->format.bitsPerPixel/8;
+
+  if (allocSize >= SIZE_MAX) {
+    rfbClientErr("CRITICAL: cannot allocate frameBuffer, requested size is too large\n");
+    return FALSE;
+  }
+
+  client->frameBuffer=malloc( (size_t)allocSize );
+
+  if (client->frameBuffer == NULL)
+    rfbClientErr("CRITICAL: frameBuffer allocation failed, requested size too large or not enough memory?\n");
+
   return client->frameBuffer?TRUE:FALSE;
 }