File freerdp-CVE-2018-8788.patch of Package freerdp.6948

From d1112c279bd1a327e8e4d0b5f371458bf2579659 Mon Sep 17 00:00:00 2001
From: Armin Novak <armin.novak@thincast.com>
Date: Mon, 22 Oct 2018 16:52:21 +0200
Subject: [PATCH 5/6] Fixed CVE-2018-8788

Thanks to Eyal Itkin from Check Point Software Technologies.
---
 include/freerdp/codec/nsc.h   |  4 +-
 libfreerdp/codec/nsc.c        | 94 ++++++++++++++++++++++++++++++++++++-------
 libfreerdp/codec/nsc_encode.c | 62 +++++++++++++++++++---------
 libfreerdp/codec/nsc_encode.h |  2 +-
 libfreerdp/codec/nsc_sse2.c   |  4 +-
 5 files changed, 130 insertions(+), 36 deletions(-)

Index: b/libfreerdp/codec/nsc.c
===================================================================
--- a/libfreerdp/codec/nsc.c	2016-05-17 18:24:59.000000000 +0800
+++ b/libfreerdp/codec/nsc.c	2019-01-09 22:02:46.530302304 +0800
@@ -39,7 +39,7 @@
 #define NSC_INIT_SIMD(_nsc_context) do { } while (0)
 #endif
 
-static void nsc_decode(NSC_CONTEXT* context)
+static BOOL nsc_decode(NSC_CONTEXT* context)
 {
 	UINT16 x;
 	UINT16 y;
@@ -56,11 +56,18 @@ static void nsc_decode(NSC_CONTEXT* cont
 	INT16 g_val;
 	INT16 b_val;
 	BYTE* bmpdata;
+	size_t pos = 0;
+
+	if (!context)
+		return FALSE;
 
 	bmpdata = context->BitmapData;
 	rw = ROUND_UP_TO(context->width, 8);
 	shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
 
+	if (!bmpdata)
+		return FALSE;
+
 	WLog_Print(context->priv->log, WLOG_DEBUG, "NscDecode: width: %d height: %d ChromaSubsamplingLevel: %d",
 			context->width, context->height, context->ChromaSubsamplingLevel);
 
@@ -89,6 +96,11 @@ static void nsc_decode(NSC_CONTEXT* cont
 			r_val = y_val + co_val - cg_val;
 			g_val = y_val + cg_val;
 			b_val = y_val - co_val - cg_val;
+
+			if (pos + 4 > context->BitmapDataLength)
+				return FALSE;
+
+			pos += 4;
 			*bmpdata++ = MINMAX(b_val, 0, 0xFF);
 			*bmpdata++ = MINMAX(g_val, 0, 0xFF);
 			*bmpdata++ = MINMAX(r_val, 0, 0xFF);
@@ -99,9 +111,11 @@ static void nsc_decode(NSC_CONTEXT* cont
 			aplane++;
 		}
 	}
+
+	return TRUE;
 }
 
-static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 originalSize)
+static BOOL nsc_rle_decode(BYTE* in, BYTE* out, UINT32 outSize, UINT32 originalSize)
 {
 	UINT32 len;
 	UINT32 left;
@@ -115,6 +129,10 @@ static void nsc_rle_decode(BYTE* in, BYT
 
 		if (left == 5)
 		{
+			if (outSize < 1)
+				return FALSE;
+
+			outSize--;
 			*out++ = value;
 			left--;
 		}
@@ -134,27 +152,42 @@ static void nsc_rle_decode(BYTE* in, BYT
 				in += 4;
 			}
 
+			if (outSize < len)
+				return FALSE;
+
+			outSize -= len;
 			FillMemory(out, len, value);
 			out += len;
 			left -= len;
 		}
 		else
 		{
+			if (outSize < 1)
+				return FALSE;
+
+			outSize--;
 			*out++ = value;
 			left--;
 		}
 	}
 
-	*((UINT32*)out) = *((UINT32*)in);
+	if ((outSize < 4) || (left < 4))
+		return FALSE;
+
+	memcpy(out, in, 4);
+	return TRUE;
 }
 
-static void nsc_rle_decompress_data(NSC_CONTEXT* context)
+static BOOL nsc_rle_decompress_data(NSC_CONTEXT* context)
 {
 	UINT16 i;
 	BYTE* rle;
 	UINT32 planeSize;
 	UINT32 originalSize;
 
+	if (!context)
+		return FALSE;
+
 	rle = context->Planes;
 
 	for (i = 0; i < 4; i++)
@@ -163,14 +196,30 @@ static void nsc_rle_decompress_data(NSC_
 		planeSize = context->PlaneByteCount[i];
 
 		if (planeSize == 0)
+		{
+			if (context->priv->PlaneBuffersLength < originalSize)
+				return FALSE;
+
 			FillMemory(context->priv->PlaneBuffers[i], originalSize, 0xFF);
+		}
 		else if (planeSize < originalSize)
-			nsc_rle_decode(rle, context->priv->PlaneBuffers[i], originalSize);
+		{
+			if (!nsc_rle_decode(rle, context->priv->PlaneBuffers[i], context->priv->PlaneBuffersLength,
+			                    originalSize))
+				return FALSE;
+		}
 		else
+		{
+			if (context->priv->PlaneBuffersLength < originalSize)
+				return FALSE;
+
 			CopyMemory(context->priv->PlaneBuffers[i], rle, originalSize);
+		}
 
 		rle += planeSize;
 	}
+
+	return TRUE;
 }
 
 static BOOL nsc_stream_initialize(NSC_CONTEXT* context, wStream* s)
@@ -408,14 +457,26 @@ int nsc_process_message(NSC_CONTEXT* con
 		return -1;
 
 	/* RLE decode */
-	PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data);
-	nsc_rle_decompress_data(context);
-	PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data);
+	{
+		BOOL rc;
+		PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data);
+		rc = nsc_rle_decompress_data(context);
+		PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data);
+
+		if (!rc)
+			return -1;
+	}
 
 	/* Colorloss recover, Chroma supersample and AYCoCg to ARGB Conversion in one step */
-	PROFILER_ENTER(context->priv->prof_nsc_decode);
-	context->decode(context);
-	PROFILER_EXIT(context->priv->prof_nsc_decode);
+	{
+		BOOL rc;
+		PROFILER_ENTER(context->priv->prof_nsc_decode);
+		rc = context->decode(context);
+		PROFILER_EXIT(context->priv->prof_nsc_decode);
+
+		if (!rc)
+			return -1;
+	}
 
 	return 1;
 }
Index: b/libfreerdp/codec/nsc_encode.c
===================================================================
--- a/libfreerdp/codec/nsc_encode.c	2016-05-17 18:24:59.000000000 +0800
+++ b/libfreerdp/codec/nsc_encode.c	2019-01-09 21:57:47.684342345 +0800
@@ -69,7 +69,7 @@ static void nsc_context_initialize_encod
 	}
 }
 
-static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, BYTE* data, int scanline)
+static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, BYTE* data, int scanline)
 {
 	UINT16 x;
 	UINT16 y;
@@ -87,10 +87,20 @@ static void nsc_encode_argb_to_aycocg(NS
 	UINT32 tempWidth;
 	UINT32 tempHeight;
 
+	if (!context || data || (scanline == 0))
+		return FALSE;
+
 	tempWidth = ROUND_UP_TO(context->width, 8);
 	tempHeight = ROUND_UP_TO(context->height, 2);
 	rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width);
 	ccl = context->ColorLossLevel;
+
+	if (context->priv->PlaneBuffersLength < rw * scanline)
+		return FALSE;
+
+	if (rw < scanline * 2)
+		return FALSE;
+
 	yplane = context->priv->PlaneBuffers[0];
 	coplane = context->priv->PlaneBuffers[1];
 	cgplane = context->priv->PlaneBuffers[2];
@@ -211,32 +221,37 @@ static void nsc_encode_argb_to_aycocg(NS
 		CopyMemory(coplane + rw, coplane, rw);
 		CopyMemory(cgplane + rw, cgplane, rw);
 	}
+
+	return TRUE;
 }
 
-static void nsc_encode_subsampling(NSC_CONTEXT* context)
+static BOOL nsc_encode_subsampling(NSC_CONTEXT* context)
 {
 	UINT16 x;
 	UINT16 y;
-	BYTE* co_dst;
-	BYTE* cg_dst;
-	INT8* co_src0;
-	INT8* co_src1;
-	INT8* cg_src0;
-	INT8* cg_src1;
 	UINT32 tempWidth;
 	UINT32 tempHeight;
 
+	if (!context)
+		return FALSE;
+
 	tempWidth = ROUND_UP_TO(context->width, 8);
 	tempHeight = ROUND_UP_TO(context->height, 2);
 
+	if (tempHeight == 0)
+		return FALSE;
+
+	if (tempWidth > context->priv->PlaneBuffersLength / tempHeight)
+		return FALSE;
+
 	for (y = 0; y < tempHeight >> 1; y++)
 	{
-		co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
-		cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
-		co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
-		co_src1 = co_src0 + tempWidth;
-		cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
-		cg_src1 = cg_src0 + tempWidth;
+		BYTE* co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
+		BYTE* cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
+		const INT8* co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
+		const INT8* co_src1 = co_src0 + tempWidth;
+		const INT8* cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
+		const INT8* cg_src1 = cg_src0 + tempWidth;
 
 		for (x = 0; x < tempWidth >> 1; x++)
 		{
@@ -250,19 +265,28 @@ static void nsc_encode_subsampling(NSC_C
 			cg_src1 += 2;
 		}
 	}
+
+	return TRUE;
 }
 
-void nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
+BOOL nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
 {
-	nsc_encode_argb_to_aycocg(context, bmpdata, rowstride);
+	if (!context || !bmpdata || (rowstride == 0))
+		return FALSE;
+
+	if (!nsc_encode_argb_to_aycocg(context, bmpdata, rowstride))
+		return FALSE;
 
 	if (context->ChromaSubsamplingLevel)
 	{
-		nsc_encode_subsampling(context);
+		if (!nsc_encode_subsampling(context))
+			return FALSE;
 	}
+
+	return TRUE;
 }
 
-static UINT32 nsc_rle_encode(BYTE* in, BYTE* out, UINT32 originalSize)
+static UINT32 nsc_rle_encode(const BYTE* in, BYTE* out, UINT32 originalSize)
 {
 	UINT32 left;
 	UINT32 runlength = 1;
Index: b/libfreerdp/codec/nsc_encode.h
===================================================================
--- a/libfreerdp/codec/nsc_encode.h	2016-05-17 18:24:59.000000000 +0800
+++ b/libfreerdp/codec/nsc_encode.h	2019-01-09 21:57:47.684342345 +0800
@@ -20,6 +20,6 @@
 #ifndef __NSC_ENCODE_H
 #define __NSC_ENCODE_H
 
-void nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride);
+BOOL nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride);
 
 #endif
Index: b/libfreerdp/codec/nsc_sse2.c
===================================================================
--- a/libfreerdp/codec/nsc_sse2.c	2016-05-17 18:24:59.000000000 +0800
+++ b/libfreerdp/codec/nsc_sse2.c	2019-01-09 21:57:47.684342345 +0800
@@ -347,7 +347,7 @@ static void nsc_encode_subsampling_sse2(
 	}
 }
 
-static void nsc_encode_sse2(NSC_CONTEXT* context, BYTE* data, int scanline)
+static BOOL nsc_encode_sse2(NSC_CONTEXT* context, BYTE* data, int scanline)
 {
 	nsc_encode_argb_to_aycocg_sse2(context, data, scanline);
 
@@ -355,6 +355,8 @@ static void nsc_encode_sse2(NSC_CONTEXT*
 	{
 		nsc_encode_subsampling_sse2(context);
 	}
+
+	return TRUE;
 }
 
 void nsc_init_sse2(NSC_CONTEXT* context)
Index: b/include/freerdp/codec/nsc.h
===================================================================
--- a/include/freerdp/codec/nsc.h	2016-05-17 18:24:59.000000000 +0800
+++ b/include/freerdp/codec/nsc.h	2019-01-09 21:57:47.684342345 +0800
@@ -76,8 +76,8 @@ struct _NSC_CONTEXT
 	/* color palette allocated by the application */
 	const BYTE* palette;
 
-	void (*decode)(NSC_CONTEXT* context);
-	void (*encode)(NSC_CONTEXT* context, BYTE* BitmapData, int rowstride);
+	BOOL (*decode)(NSC_CONTEXT* context);
+	BOOL (*encode)(NSC_CONTEXT* context, BYTE* BitmapData, int rowstride);
 
 	NSC_CONTEXT_PRIV* priv;
 };
openSUSE Build Service is sponsored by