File freerdp-CVE-2018-8788.patch of Package freerdp.13065
From d1112c279bd1a327e8e4d0b5f371458bf2579659 Mon Sep 17 00:00:00 2001
From: Armin Novak <armin.novak@thincast.com>
Date: Mon, 22 Oct 2018 16:52:21 +0200
Subject: [PATCH 5/6] Fixed CVE-2018-8788
Thanks to Eyal Itkin from Check Point Software Technologies.
---
include/freerdp/codec/nsc.h | 4 +-
libfreerdp/codec/nsc.c | 94 ++++++++++++++++++++++++++++++++++++-------
libfreerdp/codec/nsc_encode.c | 62 +++++++++++++++++++---------
libfreerdp/codec/nsc_encode.h | 2 +-
libfreerdp/codec/nsc_sse2.c | 4 +-
5 files changed, 130 insertions(+), 36 deletions(-)
Index: b/libfreerdp/codec/nsc.c
===================================================================
--- a/libfreerdp/codec/nsc.c 2016-05-17 18:24:59.000000000 +0800
+++ b/libfreerdp/codec/nsc.c 2019-01-09 22:02:46.530302304 +0800
@@ -39,7 +39,7 @@
#define NSC_INIT_SIMD(_nsc_context) do { } while (0)
#endif
-static void nsc_decode(NSC_CONTEXT* context)
+static BOOL nsc_decode(NSC_CONTEXT* context)
{
UINT16 x;
UINT16 y;
@@ -56,11 +56,18 @@ static void nsc_decode(NSC_CONTEXT* cont
INT16 g_val;
INT16 b_val;
BYTE* bmpdata;
+ size_t pos = 0;
+
+ if (!context)
+ return FALSE;
bmpdata = context->BitmapData;
rw = ROUND_UP_TO(context->width, 8);
shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
+ if (!bmpdata)
+ return FALSE;
+
WLog_Print(context->priv->log, WLOG_DEBUG, "NscDecode: width: %d height: %d ChromaSubsamplingLevel: %d",
context->width, context->height, context->ChromaSubsamplingLevel);
@@ -89,6 +96,11 @@ static void nsc_decode(NSC_CONTEXT* cont
r_val = y_val + co_val - cg_val;
g_val = y_val + cg_val;
b_val = y_val - co_val - cg_val;
+
+ if (pos + 4 > context->BitmapDataLength)
+ return FALSE;
+
+ pos += 4;
*bmpdata++ = MINMAX(b_val, 0, 0xFF);
*bmpdata++ = MINMAX(g_val, 0, 0xFF);
*bmpdata++ = MINMAX(r_val, 0, 0xFF);
@@ -99,9 +111,11 @@ static void nsc_decode(NSC_CONTEXT* cont
aplane++;
}
}
+
+ return TRUE;
}
-static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 originalSize)
+static BOOL nsc_rle_decode(BYTE* in, BYTE* out, UINT32 outSize, UINT32 originalSize)
{
UINT32 len;
UINT32 left;
@@ -115,6 +129,10 @@ static void nsc_rle_decode(BYTE* in, BYT
if (left == 5)
{
+ if (outSize < 1)
+ return FALSE;
+
+ outSize--;
*out++ = value;
left--;
}
@@ -134,27 +152,42 @@ static void nsc_rle_decode(BYTE* in, BYT
in += 4;
}
+ if (outSize < len)
+ return FALSE;
+
+ outSize -= len;
FillMemory(out, len, value);
out += len;
left -= len;
}
else
{
+ if (outSize < 1)
+ return FALSE;
+
+ outSize--;
*out++ = value;
left--;
}
}
- *((UINT32*)out) = *((UINT32*)in);
+ if ((outSize < 4) || (left < 4))
+ return FALSE;
+
+ memcpy(out, in, 4);
+ return TRUE;
}
-static void nsc_rle_decompress_data(NSC_CONTEXT* context)
+static BOOL nsc_rle_decompress_data(NSC_CONTEXT* context)
{
UINT16 i;
BYTE* rle;
UINT32 planeSize;
UINT32 originalSize;
+ if (!context)
+ return FALSE;
+
rle = context->Planes;
for (i = 0; i < 4; i++)
@@ -163,14 +196,30 @@ static void nsc_rle_decompress_data(NSC_
planeSize = context->PlaneByteCount[i];
if (planeSize == 0)
+ {
+ if (context->priv->PlaneBuffersLength < originalSize)
+ return FALSE;
+
FillMemory(context->priv->PlaneBuffers[i], originalSize, 0xFF);
+ }
else if (planeSize < originalSize)
- nsc_rle_decode(rle, context->priv->PlaneBuffers[i], originalSize);
+ {
+ if (!nsc_rle_decode(rle, context->priv->PlaneBuffers[i], context->priv->PlaneBuffersLength,
+ originalSize))
+ return FALSE;
+ }
else
+ {
+ if (context->priv->PlaneBuffersLength < originalSize)
+ return FALSE;
+
CopyMemory(context->priv->PlaneBuffers[i], rle, originalSize);
+ }
rle += planeSize;
}
+
+ return TRUE;
}
static BOOL nsc_stream_initialize(NSC_CONTEXT* context, wStream* s)
@@ -408,14 +457,26 @@ int nsc_process_message(NSC_CONTEXT* con
return -1;
/* RLE decode */
- PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data);
- nsc_rle_decompress_data(context);
- PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data);
+ {
+ BOOL rc;
+ PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data);
+ rc = nsc_rle_decompress_data(context);
+ PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data);
+
+ if (!rc)
+ return -1;
+ }
/* Colorloss recover, Chroma supersample and AYCoCg to ARGB Conversion in one step */
- PROFILER_ENTER(context->priv->prof_nsc_decode);
- context->decode(context);
- PROFILER_EXIT(context->priv->prof_nsc_decode);
+ {
+ BOOL rc;
+ PROFILER_ENTER(context->priv->prof_nsc_decode);
+ rc = context->decode(context);
+ PROFILER_EXIT(context->priv->prof_nsc_decode);
+
+ if (!rc)
+ return -1;
+ }
return 1;
}
Index: b/libfreerdp/codec/nsc_encode.c
===================================================================
--- a/libfreerdp/codec/nsc_encode.c 2016-05-17 18:24:59.000000000 +0800
+++ b/libfreerdp/codec/nsc_encode.c 2019-01-09 21:57:47.684342345 +0800
@@ -69,7 +69,7 @@ static void nsc_context_initialize_encod
}
}
-static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, BYTE* data, int scanline)
+static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, BYTE* data, int scanline)
{
UINT16 x;
UINT16 y;
@@ -87,10 +87,20 @@ static void nsc_encode_argb_to_aycocg(NS
UINT32 tempWidth;
UINT32 tempHeight;
+ if (!context || data || (scanline == 0))
+ return FALSE;
+
tempWidth = ROUND_UP_TO(context->width, 8);
tempHeight = ROUND_UP_TO(context->height, 2);
rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width);
ccl = context->ColorLossLevel;
+
+ if (context->priv->PlaneBuffersLength < rw * scanline)
+ return FALSE;
+
+ if (rw < scanline * 2)
+ return FALSE;
+
yplane = context->priv->PlaneBuffers[0];
coplane = context->priv->PlaneBuffers[1];
cgplane = context->priv->PlaneBuffers[2];
@@ -211,32 +221,37 @@ static void nsc_encode_argb_to_aycocg(NS
CopyMemory(coplane + rw, coplane, rw);
CopyMemory(cgplane + rw, cgplane, rw);
}
+
+ return TRUE;
}
-static void nsc_encode_subsampling(NSC_CONTEXT* context)
+static BOOL nsc_encode_subsampling(NSC_CONTEXT* context)
{
UINT16 x;
UINT16 y;
- BYTE* co_dst;
- BYTE* cg_dst;
- INT8* co_src0;
- INT8* co_src1;
- INT8* cg_src0;
- INT8* cg_src1;
UINT32 tempWidth;
UINT32 tempHeight;
+ if (!context)
+ return FALSE;
+
tempWidth = ROUND_UP_TO(context->width, 8);
tempHeight = ROUND_UP_TO(context->height, 2);
+ if (tempHeight == 0)
+ return FALSE;
+
+ if (tempWidth > context->priv->PlaneBuffersLength / tempHeight)
+ return FALSE;
+
for (y = 0; y < tempHeight >> 1; y++)
{
- co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
- cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
- co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
- co_src1 = co_src0 + tempWidth;
- cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
- cg_src1 = cg_src0 + tempWidth;
+ BYTE* co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
+ BYTE* cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
+ const INT8* co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
+ const INT8* co_src1 = co_src0 + tempWidth;
+ const INT8* cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
+ const INT8* cg_src1 = cg_src0 + tempWidth;
for (x = 0; x < tempWidth >> 1; x++)
{
@@ -250,19 +265,28 @@ static void nsc_encode_subsampling(NSC_C
cg_src1 += 2;
}
}
+
+ return TRUE;
}
-void nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
+BOOL nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
{
- nsc_encode_argb_to_aycocg(context, bmpdata, rowstride);
+ if (!context || !bmpdata || (rowstride == 0))
+ return FALSE;
+
+ if (!nsc_encode_argb_to_aycocg(context, bmpdata, rowstride))
+ return FALSE;
if (context->ChromaSubsamplingLevel)
{
- nsc_encode_subsampling(context);
+ if (!nsc_encode_subsampling(context))
+ return FALSE;
}
+
+ return TRUE;
}
-static UINT32 nsc_rle_encode(BYTE* in, BYTE* out, UINT32 originalSize)
+static UINT32 nsc_rle_encode(const BYTE* in, BYTE* out, UINT32 originalSize)
{
UINT32 left;
UINT32 runlength = 1;
Index: b/libfreerdp/codec/nsc_encode.h
===================================================================
--- a/libfreerdp/codec/nsc_encode.h 2016-05-17 18:24:59.000000000 +0800
+++ b/libfreerdp/codec/nsc_encode.h 2019-01-09 21:57:47.684342345 +0800
@@ -20,6 +20,6 @@
#ifndef __NSC_ENCODE_H
#define __NSC_ENCODE_H
-void nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride);
+BOOL nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride);
#endif
Index: b/libfreerdp/codec/nsc_sse2.c
===================================================================
--- a/libfreerdp/codec/nsc_sse2.c 2016-05-17 18:24:59.000000000 +0800
+++ b/libfreerdp/codec/nsc_sse2.c 2019-01-09 21:57:47.684342345 +0800
@@ -347,7 +347,7 @@ static void nsc_encode_subsampling_sse2(
}
}
-static void nsc_encode_sse2(NSC_CONTEXT* context, BYTE* data, int scanline)
+static BOOL nsc_encode_sse2(NSC_CONTEXT* context, BYTE* data, int scanline)
{
nsc_encode_argb_to_aycocg_sse2(context, data, scanline);
@@ -355,6 +355,8 @@ static void nsc_encode_sse2(NSC_CONTEXT*
{
nsc_encode_subsampling_sse2(context);
}
+
+ return TRUE;
}
void nsc_init_sse2(NSC_CONTEXT* context)
Index: b/include/freerdp/codec/nsc.h
===================================================================
--- a/include/freerdp/codec/nsc.h 2016-05-17 18:24:59.000000000 +0800
+++ b/include/freerdp/codec/nsc.h 2019-01-09 21:57:47.684342345 +0800
@@ -76,8 +76,8 @@ struct _NSC_CONTEXT
/* color palette allocated by the application */
const BYTE* palette;
- void (*decode)(NSC_CONTEXT* context);
- void (*encode)(NSC_CONTEXT* context, BYTE* BitmapData, int rowstride);
+ BOOL (*decode)(NSC_CONTEXT* context);
+ BOOL (*encode)(NSC_CONTEXT* context, BYTE* BitmapData, int rowstride);
NSC_CONTEXT_PRIV* priv;
};
