File nss-CC-ctr.patch of Package mozilla-nss.972

# HG changeset patch
# Parent a3d232b48740f32b2ef23f382625b84b1df45228
# Parent  51e9d34183bcbf06b497f6e7234d24b7da91d412
Handle counter rollovers while incrementing in CTR modes.

diff --git a/lib/freebl/ctr.c b/lib/freebl/ctr.c
--- a/lib/freebl/ctr.c
+++ b/lib/freebl/ctr.c
@@ -73,40 +73,53 @@ CTR_DestroyContext(CTRContext *ctr, PRBo
 }
 
 /*
  * Used by counter mode. Increment the counter block. Not all bits in the
  * counter block are part of the counter, counterBits tells how many bits
  * are part of the counter. The counter block is blocksize long. It's a
  * big endian value.
  *
- * XXX Does not handle counter rollover.
+ * Returns SECFailure when rollover of the counter happens, SECSuccess
+ * otherwise.
  */
-static void
+static SECStatus
 ctr_GetNextCtr(unsigned char *counter, unsigned int counterBits,
 	       unsigned int blocksize)
 {
     unsigned char *counterPtr = counter + blocksize - 1;
     unsigned char mask, count;
+    SECStatus rv = SECSuccess;
 
     PORT_Assert(counterBits <= blocksize*PR_BITS_PER_BYTE);
     while (counterBits >= PR_BITS_PER_BYTE) {
 	if (++(*(counterPtr--))) {
-	    return;
+	    goto check_rv;
 	}
 	counterBits -= PR_BITS_PER_BYTE;
     }
     if (counterBits == 0) {
-	return;
+	/* If we got here, it means that the last char processed overflowed
+	 * while containing exactly the last PR_BITS_PER_BYTE bits of the
+	 * counter. That in turn means that the counter as a whole overflowed
+	 * and we are thus in error mode */
+	rv = SECFailure;
+	goto check_rv;
     }
     /* increment the final partial byte */
     mask = (1 << counterBits)-1;
     count = ++(*counterPtr) & mask;
+    if (count == 0) {
+	/* the last counter bits overfloved */
+	rv = SECFailure;
+	goto check_rv;
+    }
     *counterPtr = ((*counterPtr) & ~mask) | count;
-    return;
+check_rv:
+    return rv;
 }
 
 static void
 ctr_xor(unsigned char *target, const unsigned char *x,
 	const unsigned char *y, unsigned int count)
 {
     unsigned int i;
     for (i=0; i < count; i++) {
@@ -141,32 +154,38 @@ CTR_Update(CTRContext *ctr, unsigned cha
 	    return SECSuccess;
 	}
 	PORT_Assert(ctr->bufPtr == blocksize);
     }
 
     while (inlen >= blocksize) {
 	rv = (*ctr->cipher)(ctr->context, ctr->buffer, &tmp, blocksize,
 			ctr->counter, blocksize, blocksize);
-	ctr_GetNextCtr(ctr->counter, ctr->counterBits, blocksize);
+	if (rv != SECSuccess) {
+	    return SECFailure;
+	}
+	rv = ctr_GetNextCtr(ctr->counter, ctr->counterBits, blocksize);
 	if (rv != SECSuccess) {
 	    return SECFailure;
 	}
 	ctr_xor(outbuf, inbuf, ctr->buffer, blocksize);
 	outbuf += blocksize;
 	inbuf += blocksize;
 	*outlen += blocksize;
 	inlen -= blocksize;
     }
     if (inlen == 0) {
 	return SECSuccess;
     }
     rv = (*ctr->cipher)(ctr->context, ctr->buffer, &tmp, blocksize,
 			ctr->counter, blocksize, blocksize);
-    ctr_GetNextCtr(ctr->counter, ctr->counterBits, blocksize);
+    if (rv != SECSuccess) {
+	return SECFailure;
+    }
+    rv = ctr_GetNextCtr(ctr->counter, ctr->counterBits, blocksize);
     if (rv != SECSuccess) {
 	return SECFailure;
     }
     ctr_xor(outbuf, inbuf, ctr->buffer, inlen);
     ctr->bufPtr = inlen;
     *outlen += inlen;
     return SECSuccess;
 }
@@ -212,17 +231,20 @@ CTR_Update_HW_AES(CTRContext *ctr, unsig
     inbuf += fullblocks;
     inlen -= fullblocks;
 
     if (inlen == 0) {
 	return SECSuccess;
     }
     rv = (*ctr->cipher)(ctr->context, ctr->buffer, &tmp, blocksize,
 			ctr->counter, blocksize, blocksize);
-    ctr_GetNextCtr(ctr->counter, ctr->counterBits, blocksize);
+    if (rv != SECSuccess) {
+	return SECFailure;
+    }
+    rv = ctr_GetNextCtr(ctr->counter, ctr->counterBits, blocksize);
     if (rv != SECSuccess) {
 	return SECFailure;
     }
     ctr_xor(outbuf, inbuf, ctr->buffer, inlen);
     ctr->bufPtr = inlen;
     *outlen += inlen;
     return SECSuccess;
 }