File nss-fips-gcm-ctr.patch of Package mozilla-nss.14664

From 41dd171b242b0cb550d12760da110db7e2c21daf Mon Sep 17 00:00:00 2001
From: Hans Petter Jansson <hpj@cl.no>
Date: Wed, 20 Nov 2019 08:25:39 +0100
Subject: [PATCH] 22

---
 nss/lib/freebl/gcm.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/nss/lib/freebl/gcm.c b/nss/lib/freebl/gcm.c
index f1e16da..0f42525 100644
--- a/nss/lib/freebl/gcm.c
+++ b/nss/lib/freebl/gcm.c
@@ -500,9 +500,15 @@ struct GCMContextStr {
     gcmHashContext *ghash_context;
     CTRContext ctr_context;
     unsigned long tagBits;
+    unsigned long long gcm_iv_bytes;
     unsigned char tagKey[MAX_BLOCK_SIZE];
 };
 
+/* NIST SP-800-38D limits the use of GCM with a single IV to 2^39 - 256
+ * bits which translates to 2^32 - 2 128bit blocks or 2^36 - 32 bytes
+ */
+#define MAX_GCM_BYTES_PER_IV    ((1ULL << 36) - 32)
+
 GCMContext *
 GCM_CreateContext(void *context, freeblCipherFunc cipher,
                   const unsigned char *params)
@@ -576,6 +582,8 @@ GCM_CreateContext(void *context, freeblCipherFunc cipher,
         goto loser;
     }
 
+    gcm->gcm_iv_bytes = MAX_GCM_BYTES_PER_IV;
+
     /* finally mix in the AAD data */
     rv = gcmHash_Reset(ghash, gcmParams->pAAD, gcmParams->ulAADLen);
     if (rv != SECSuccess) {
@@ -672,6 +680,13 @@ GCM_EncryptUpdate(GCMContext *gcm, unsigned char *outbuf,
         return SECFailure;
     }
 
+    /* bail out if this invocation requests processing more than what is
+     * considered to be a safe limit */
+    if (gcm->gcm_iv_bytes < (unsigned long long)inlen) {
+        PORT_SetError(SEC_ERROR_INPUT_LEN);
+        return SECFailure;
+    }
+
     tagBytes = (gcm->tagBits + (PR_BITS_PER_BYTE - 1)) / PR_BITS_PER_BYTE;
     if (UINT_MAX - inlen < tagBytes) {
         PORT_SetError(SEC_ERROR_INPUT_LEN);
@@ -700,6 +715,7 @@ GCM_EncryptUpdate(GCMContext *gcm, unsigned char *outbuf,
         *outlen = 0;
         return SECFailure;
     };
+    gcm->gcm_iv_bytes -= inlen;
     *outlen += len;
     return SECSuccess;
 }
-- 
2.21.0