File nss-3.23-UTF-16_surrogate_handling_fix.patch of Package mozilla-nss.2738

# HG changeset patch
# Parent  5cbcebd9d8959dc6dec43299f5e45891cfab6b57
Backport of fixes for MFSA-2016-62/CVE-2016-2834/bsc#983639

Upstream commit:
changeset:   11884:329932eb1700
user:        Jed Davis <jld@mozilla.com>
date:        Thu Feb 11 08:37:35 2016 +0100
files:       lib/util/utf8.c
description:
Bug 1241037 - Fix UTF-16 surrogate handling to match only surrogates. r=ttaubert

diff --git a/lib/util/utf8.c b/lib/util/utf8.c
--- a/lib/util/utf8.c
+++ b/lib/util/utf8.c
@@ -316,18 +316,18 @@ sec_port_ucs2_utf8_conversion_function
     if ((inBufLen % 2) != 0) {
       *outBufLen = 0;
       return PR_FALSE;
     }
 
     for( i = 0; i < inBufLen; i += 2 ) {
       if( (inBuf[i+H_0] == 0x00) && ((inBuf[i+H_1] & 0x80) == 0x00) ) len += 1;
       else if( inBuf[i+H_0] < 0x08 ) len += 2;
-      else if( ((inBuf[i+0+H_0] & 0xDC) == 0xD8) ) {
-        if( ((inBufLen - i) > 2) && ((inBuf[i+2+H_0] & 0xDC) == 0xDC) ) {
+      else if( ((inBuf[i+0+H_0] & 0xFC) == 0xD8) ) {
+        if( ((inBufLen - i) > 2) && ((inBuf[i+2+H_0] & 0xFC) == 0xDC) ) {
           i += 2;
           len += 4;
         } else {
           return PR_FALSE;
         }
       }
       else len += 3;
     }
@@ -351,20 +351,20 @@ sec_port_ucs2_utf8_conversion_function
         /* 0080-07FF -> 110xxxxx 10xxxxxx */
         /* 00000abc defghijk -> 110abcde 10fghijk */
 
         outBuf[len+0] = 0xC0 | ((inBuf[i+H_0] & 0x07) << 2) 
                              | ((inBuf[i+H_1] & 0xC0) >> 6);
         outBuf[len+1] = 0x80 | ((inBuf[i+H_1] & 0x3F) >> 0);
 
         len += 2;
-      } else if( (inBuf[i+H_0] & 0xDC) == 0xD8 ) {
+      } else if( (inBuf[i+H_0] & 0xFC) == 0xD8 ) {
         int abcde, BCDE;
 
-        PORT_Assert(((inBufLen - i) > 2) && ((inBuf[i+2+H_0] & 0xDC) == 0xDC) );
+        PORT_Assert(((inBufLen - i) > 2) && ((inBuf[i+2+H_0] & 0xFC) == 0xDC) );
 
         /* D800-DBFF DC00-DFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
         /* 110110BC DEfghijk 110111lm nopqrstu ->
            { Let abcde = BCDE + 1 }
            11110abc 10defghi 10jklmno 10pqrstu */
 
         BCDE = ((inBuf[i+H_0] & 0x03) << 2) | ((inBuf[i+H_1] & 0xC0) >> 6);
         abcde = BCDE + 1;
@@ -847,16 +847,17 @@ struct ucs2 ucs2[] = {
   { 0x8080, "\xE8\x82\x80" },
   { 0x8100, "\xE8\x84\x80" },
   { 0x8200, "\xE8\x88\x80" },
   { 0x8400, "\xE8\x90\x80" },
   { 0x8800, "\xE8\xA0\x80" },
   { 0x9000, "\xE9\x80\x80" },
   { 0xA000, "\xEA\x80\x80" },
   { 0xC000, "\xEC\x80\x80" },
+  { 0xFB01, "\xEF\xAC\x81" },
   { 0xFFFF, "\xEF\xBF\xBF" }
 
 };
 
 /*
  * UTF-16 vectors
  */
 
@@ -1156,16 +1157,18 @@ char *utf8_bad[] = {
 /* illegal UTF-16 sequences, 0-terminated */
 uint16_t utf16_bad[][3] = {
   /* leading surrogate not followed by trailing surrogate */
   { 0xD800, 0, 0 },
   { 0xD800, 0x41, 0 },
   { 0xD800, 0xfe, 0 },
   { 0xD800, 0x3bb, 0 },
   { 0xD800, 0xD800, 0 },
+  { 0xD800, 0xFEFF, 0 },
+  { 0xD800, 0xFFFD, 0 },
 };
 
 static void
 dump_utf8
 (
   char *word,
   unsigned char *utf8,
   char *end