File nss-GCM-ctr.patch of Package mozilla-nss.972

# HG changeset patch
# Parent c2351189803767939aaa10b67164187b47017191
# Parent  105cb8f05f697c89781cddfc29c3e11890d20751
NIST SP-800-38D specifies, that GCM is not safe after 2^39 - 256 bits with the
same IV. Return error, should this limit be reached.

bmo#1053127
bnc#892530

diff --git a/lib/freebl/gcm.c b/lib/freebl/gcm.c
--- a/lib/freebl/gcm.c
+++ b/lib/freebl/gcm.c
@@ -603,19 +603,25 @@ gcmHash_Reset(gcmHashContext *ghash, con
  *           Now implement the GCM using gcmHash and CTR                  *
  **************************************************************************/
 
 /* state to handle the full GCM operation (hash and counter) */
 struct GCMContextStr {
     gcmHashContext ghash_context;
     CTRContext ctr_context;
     unsigned long tagBits;
+    unsigned long long gcm_iv_bytes;
     unsigned char tagKey[MAX_BLOCK_SIZE];
 };
 
+/* NIST SP-800-38D limits the use of GCM with a single IV to 2^39 - 256
+ * bits which translates to 2^32 - 2 128bit blocks or 2^36 - 32 bytes
+ */
+#define MAX_GCM_BYTES_PER_IV    ((1ULL << 36) - 32)
+
 GCMContext *
 GCM_CreateContext(void *context, freeblCipherFunc cipher,
 		  const unsigned char *params, unsigned int blocksize)
 {
     GCMContext *gcm = NULL;
     gcmHashContext *ghash;
     unsigned char H[MAX_BLOCK_SIZE];
     unsigned int tmp;
@@ -675,16 +681,18 @@ GCM_CreateContext(void *context, freeblC
     /* calculate the final tag key. NOTE: gcm->tagKey is zero to start with.
      * if this assumption changes, we would need to explicitly clear it here */
     rv = CTR_Update(&gcm->ctr_context, gcm->tagKey, &tmp, blocksize,
 		    gcm->tagKey, blocksize, blocksize);
     if (rv != SECSuccess) {
 	goto loser;
     }
 
+    gcm->gcm_iv_bytes = MAX_GCM_BYTES_PER_IV;
+
     /* finally mix in the AAD data */
     rv = gcmHash_Reset(ghash, gcmParams->pAAD, gcmParams->ulAADLen, blocksize);
     if (rv != SECSuccess) {
 	goto loser;
     }
 
     return gcm;
 
@@ -771,16 +779,23 @@ GCM_EncryptUpdate(GCMContext *gcm, unsig
 		unsigned int *outlen, unsigned int maxout,
 		const unsigned char *inbuf, unsigned int inlen,
 		unsigned int blocksize)
 {
     SECStatus rv;
     unsigned int tagBytes;
     unsigned int len;
 
+    /* bail out if this invocation requests processing more than what is
+     * considered to be a safe limit */
+    if (gcm->gcm_iv_bytes < (unsigned long long)inlen) {
+	PORT_SetError(SEC_ERROR_INPUT_LEN);
+	return SECFailure;
+    }
+
     tagBytes = (gcm->tagBits + (PR_BITS_PER_BYTE-1)) / PR_BITS_PER_BYTE;
     if (UINT_MAX - inlen < tagBytes) {
 	PORT_SetError(SEC_ERROR_INPUT_LEN);
 	return SECFailure;
     }
     if (maxout < inlen + tagBytes) {
 	*outlen = inlen + tagBytes;
 	PORT_SetError(SEC_ERROR_OUTPUT_LEN);
@@ -799,16 +814,17 @@ GCM_EncryptUpdate(GCMContext *gcm, unsig
 	return SECFailure;
     }
     rv = gcm_GetTag(gcm, outbuf + *outlen, &len, maxout - *outlen, blocksize);
     if (rv != SECSuccess) {
 	PORT_Memset(outbuf, 0, *outlen); /* clear the output buffer */
 	*outlen = 0;
 	return SECFailure;
     };
+    gcm->gcm_iv_bytes -= inlen;
     *outlen += len;
     return SECSuccess;
 }
 
 /*
  * See The Galois/Counter Mode of Operation, McGrew and Viega.
  *  GCM is basically counter mode with a specific initialization and
  *  built in macing operation. NOTE: the only difference between Encrypt