File nss-3.23-Unicode_conversion_length_estimation.patch of Package mozilla-nss.2738

# HG changeset patch
# Parent  8521caf3a60853632746e502f1613f004f6c5cc7
Backport of fixes for MFSA-2016-62/CVE-2016-2834/bsc#983639

Upstream commit:
changeset:   11855:5fde729fdbff
user:        Jed Davis <jld@mozilla.com>
date:        Fri Feb 05 11:15:12 2016 +0100
files:       lib/util/utf8.c
description:
Bug 1241034 - Fix UCS-2 to UTF-8 conversion length estimation. r=ttaubert

diff --git a/lib/util/utf8.c b/lib/util/utf8.c
--- a/lib/util/utf8.c
+++ b/lib/util/utf8.c
@@ -314,17 +314,17 @@ sec_port_ucs2_utf8_conversion_function
     unsigned int i, len = 0;
     PORT_Assert((inBufLen % 2) == 0);
     if ((inBufLen % 2) != 0) {
       *outBufLen = 0;
       return PR_FALSE;
     }
 
     for( i = 0; i < inBufLen; i += 2 ) {
-      if( (inBuf[i+H_0] == 0x00) && ((inBuf[i+H_0] & 0x80) == 0x00) ) len += 1;
+      if( (inBuf[i+H_0] == 0x00) && ((inBuf[i+H_1] & 0x80) == 0x00) ) len += 1;
       else if( inBuf[i+H_0] < 0x08 ) len += 2;
       else if( ((inBuf[i+0+H_0] & 0xDC) == 0xD8) ) {
         if( ((inBuf[i+2+H_0] & 0xDC) == 0xDC) && ((inBufLen - i) > 2) ) {
           i += 2;
           len += 4;
         } else {
           return PR_FALSE;
         }
@@ -1215,16 +1215,28 @@ test_ucs4_chars
     }
 
     if( (sizeof(back) != len) || (e->c != back) ) {
       dump_utf8("Wrong conversion of UTF-8", utf8, " to UCS-4:");
       fprintf(stdout, "expected 0x%08.8x, received 0x%08.8x\n", e->c, back);
       rv = PR_FALSE;
       continue;
     }
+
+    len = strlen(e->utf8) - 1;
+    result = sec_port_ucs4_utf8_conversion_function(PR_FALSE,
+      (unsigned char *)&e->c, sizeof(e->c), utf8 + sizeof(utf8) - len, len,
+      &len);
+
+    if( result || len != strlen(e->utf8) ) {
+      fprintf(stdout, "Length computation error converting UCS-4 0x%08.8x"
+        " to UTF-8\n", e->c);
+      rv = PR_FALSE;
+      continue;
+    }
   }
 
   return rv;
 }
 
 static PRBool
 test_ucs2_chars
 (
@@ -1272,16 +1284,28 @@ test_ucs2_chars
     }
 
     if( (sizeof(back) != len) || (e->c != back) ) {
       dump_utf8("Wrong conversion of UTF-8", utf8, "to UCS-2:");
       fprintf(stdout, "expected 0x%08.8x, received 0x%08.8x\n", e->c, back);
       rv = PR_FALSE;
       continue;
     }
+
+    len = strlen(e->utf8) - 1;
+    result = sec_port_ucs2_utf8_conversion_function(PR_FALSE,
+      (unsigned char *)&e->c, sizeof(e->c), utf8 + sizeof(utf8) - len, len,
+      &len);
+
+    if( result || len != strlen(e->utf8) ) {
+      fprintf(stdout, "Length computation error converting UCS-2 0x%04.4x"
+        " to UTF-8\n", e->c);
+      rv = PR_FALSE;
+      continue;
+    }
   }
 
   return rv;
 }
 
 static PRBool
 test_utf16_chars
 (