File nss-3.23-DER_integer_decoding.patch of Package mozilla-nss.2738

# HG changeset patch
# Parent  ff4d7dcbc21e3ffe43a81b3f93263359032a0feb
Backport of fixes for MFSA-2016-62/CVE-2016-2834/bsc#983639

bmo#1221620

Upstream commit:
changeset:   11826:8d78a5ae260a
user:        Franziskus Kiefer <franziskuskiefer@gmail.com>
files:       lib/util/dersubr.c
description:
Bug 1221620 - DER integer decoding update, r=mt

diff --git a/lib/util/dersubr.c b/lib/util/dersubr.c
--- a/lib/util/dersubr.c
+++ b/lib/util/dersubr.c
@@ -174,43 +174,51 @@ DER_SetUInteger(PLArenaPool *arena, SECI
 /*
 ** Convert a der encoded *signed* integer into a machine integral value.
 ** If an underflow/overflow occurs, sets error code and returns min/max.
 */
 long
 DER_GetInteger(const SECItem *it)
 {
     long ival = 0;
-    unsigned len = it->len;
+    PRBool negative = PR_FALSE;
+    unsigned int len = it->len;
+    unsigned int originalLength = len;
     unsigned char *cp = it->data;
     unsigned long overflow = 0x1ffUL << (((sizeof(ival) - 1) * 8) - 1);
-    unsigned long ofloinit;
+    unsigned long mask = 1;
 
     PORT_Assert(len);
     if (!len) {
 	PORT_SetError(SEC_ERROR_INPUT_LEN);
 	return 0;
     }
 
-    if (*cp & 0x80)
-    	ival = -1L;
-    ofloinit = ival & overflow;
+    if (*cp & 0x80) {
+	negative = PR_TRUE;
+	overflow <<= 1;
+    }
 
     while (len) {
-	if ((ival & overflow) != ofloinit) {
+	if ((ival & overflow) != 0) {
 	    PORT_SetError(SEC_ERROR_BAD_DER);
-	    if (ival < 0) {
+	    if (negative) {
 		return LONG_MIN;
 	    }
 	    return LONG_MAX;
 	}
 	ival = ival << 8;
 	ival |= *cp++;
 	--len;
     }
+    if (negative && ival && (overflow & ival) == 0) {
+	mask <<=  ((originalLength  * 8) - 1);
+	ival &= ~mask;
+	ival -= mask;
+    }
     return ival;
 }
 
 /*
 ** Convert a der encoded *unsigned* integer into a machine integral value.
 ** If an overflow occurs, sets error code and returns max.
 */
 unsigned long