File nss-GCM-ctr.patch of Package mozilla-nss.6304

NIST SP-800-38D specifies, that GCM is not safe after 2^39 - 256 bits with the
same IV. Return error, should this limit be reached.

bmo#1053127
bnc#892530

diff --git a/nss/lib/freebl/gcm.c b/nss/lib/freebl/gcm.c
index 2212100..cf8e027 100644
--- a/nss/lib/freebl/gcm.c
+++ b/nss/lib/freebl/gcm.c
@@ -605,9 +605,15 @@ struct GCMContextStr {
     gcmHashContext ghash_context;
     CTRContext ctr_context;
     unsigned long tagBits;
+    unsigned long long gcm_iv_bytes;
     unsigned char tagKey[MAX_BLOCK_SIZE];
 };
 
+/* NIST SP-800-38D limits the use of GCM with a single IV to 2^39 - 256
+ * bits which translates to 2^32 - 2 128bit blocks or 2^36 - 32 bytes
+ */
+#define MAX_GCM_BYTES_PER_IV    ((1ULL << 36) - 32)
+
 GCMContext *
 GCM_CreateContext(void *context, freeblCipherFunc cipher,
                   const unsigned char *params, unsigned int blocksize)
@@ -677,6 +683,8 @@ GCM_CreateContext(void *context, freeblCipherFunc cipher,
         goto loser;
     }
 
+    gcm->gcm_iv_bytes = MAX_GCM_BYTES_PER_IV;
+
     /* finally mix in the AAD data */
     rv = gcmHash_Reset(ghash, gcmParams->pAAD, gcmParams->ulAADLen, blocksize);
     if (rv != SECSuccess) {
@@ -772,6 +780,13 @@ GCM_EncryptUpdate(GCMContext *gcm, unsigned char *outbuf,
     unsigned int tagBytes;
     unsigned int len;
 
+    /* bail out if this invocation requests processing more than what is
+     * considered to be a safe limit */
+    if (gcm->gcm_iv_bytes < (unsigned long long)inlen) {
+        PORT_SetError(SEC_ERROR_INPUT_LEN);
+        return SECFailure;
+    }
+
     tagBytes = (gcm->tagBits + (PR_BITS_PER_BYTE - 1)) / PR_BITS_PER_BYTE;
     if (UINT_MAX - inlen < tagBytes) {
         PORT_SetError(SEC_ERROR_INPUT_LEN);
@@ -800,6 +815,7 @@ GCM_EncryptUpdate(GCMContext *gcm, unsigned char *outbuf,
         *outlen = 0;
         return SECFailure;
     };
+    gcm->gcm_iv_bytes -= inlen;
     *outlen += len;
     return SECSuccess;
 }