File rpm-digestalgo.diff of Package rpm

--- ./lib/fsm.c.orig	2011-06-15 16:23:38.000000000 +0000
+++ ./lib/fsm.c	2011-06-15 16:39:51.000000000 +0000
@@ -808,11 +808,13 @@ int fsmMapAttrs(FSM_t fsm)
 	     */
 	    if (ts != NULL && !(rpmtsFlags(ts) & RPMTRANS_FLAG_NOMD5)) {
 		fsm->fmd5sum = (fi->fmd5s ? fi->fmd5s[i] : NULL);
-		fsm->md5sum = (fi->md5s ? (fi->md5s + (16 * i)) : NULL);
+		fsm->md5sum = (fi->md5s ? (fi->md5s + (fi->digestlen * i)) : NULL);
 	    } else {
 		fsm->fmd5sum = NULL;
 		fsm->md5sum = NULL;
 	    }
+	    fsm->digestalgo = fi->digestalgo;
+	    fsm->digestlen = fi->digestlen;
 	}
 
     }
@@ -839,7 +841,7 @@ static int expandRegular(/*@special@*/ F
 	goto exit;
 
     if (st->st_size > 0 && (fsm->fmd5sum != NULL || fsm->md5sum != NULL))
-	fdInitDigest(fsm->wfd, PGPHASHALGO_MD5, 0);
+	fdInitDigest(fsm->wfd, fsm->digestalgo, 0);
 
     while (left) {
 
@@ -864,7 +866,7 @@ static int expandRegular(/*@special@*/ F
 	int asAscii = (fsm->md5sum == NULL ? 1 : 0);
 
 	(void) Fflush(fsm->wfd);
-	fdFiniDigest(fsm->wfd, PGPHASHALGO_MD5, &md5sum, NULL, asAscii);
+	fdFiniDigest(fsm->wfd, fsm->digestalgo, &md5sum, NULL, asAscii);
 
 	if (md5sum == NULL) {
 	    rc = CPIOERR_MD5SUM_MISMATCH;
@@ -872,7 +874,7 @@ static int expandRegular(/*@special@*/ F
 	}
 
 	if (fsm->md5sum != NULL) {
-	    if (memcmp(md5sum, fsm->md5sum, 16))
+	    if (memcmp(md5sum, fsm->md5sum, fsm->digestlen))
 		rc = CPIOERR_MD5SUM_MISMATCH;
 	} else {
 	    if (strcmp(md5sum, fsm->fmd5sum))
--- ./lib/fsm.h.orig	2011-06-15 16:37:18.000000000 +0000
+++ ./lib/fsm.h	2011-06-15 16:38:00.000000000 +0000
@@ -189,6 +189,8 @@ struct fsm_s {
     const char * fmd5sum;	/*!< Hex MD5 sum (NULL disables). */
 /*@shared@*/ /*@relnull@*/
     const char * md5sum;	/*!< Binary MD5 sum (NULL disables). */
+    int digestalgo;		/*!< digest algorithm */
+    int digestlen;		/*!< digest binary length */
 /*@dependent@*/ /*@observer@*/ /*@null@*/
     const char * fcontext;	/*!< File security context (NULL disables). */
     
--- ./lib/rpmfi.c.orig	2011-06-15 16:23:22.000000000 +0000
+++ ./lib/rpmfi.c	2011-06-15 17:16:18.000000000 +0000
@@ -211,12 +211,22 @@ const unsigned char * rpmfiMD5(rpmfi fi)
     if (fi != NULL && fi->i >= 0 && fi->i < fi->fc) {
 /*@-boundsread@*/
 	if (fi->md5s != NULL)
-	    MD5 = fi->md5s + (16 * fi->i);
+	    MD5 = fi->md5s + (fi->digestlen * fi->i);
 /*@=boundsread@*/
     }
     return MD5;
 }
 
+int rpmfiDigestAlgo(rpmfi fi)
+{
+   return fi ? fi->digestalgo : 0;
+}
+
+int rpmfiDigestLen(rpmfi fi)
+{
+   return fi ? fi->digestlen : 0;
+}
+
 const char * rpmfiFLink(rpmfi fi)
 {
     const char * flink = NULL;
@@ -536,7 +546,9 @@ int rpmfiCompare(const rpmfi afi, const
 	if (amd5 == bmd5) return 0;
 	if (amd5 == NULL) return 1;
 	if (bmd5 == NULL) return -1;
-	return memcmp(amd5, bmd5, 16);
+	/* can't meaningfully compare different hash types */
+	if (afi->digestalgo != bfi->digestalgo) return -1;
+	return memcmp(amd5, bmd5, afi->digestlen);
     }
 
     return 0;
@@ -596,20 +608,20 @@ fileAction rpmfiDecideFate(const rpmfi o
 	const unsigned char * omd5, * nmd5;
 	omd5 = rpmfiMD5(ofi);
 	if (diskWhat == REG) {
-	    if (domd5(fn, (unsigned char *)buffer, 0, NULL))
+	    if (domd5_algo(fn, (unsigned char *)buffer, 0, NULL, ofi->digestalgo))
 		return FA_CREATE;       /* assume file has been removed */
-	    if (omd5 && !memcmp(omd5, buffer, 16))
+	    if (omd5 && !memcmp(omd5, buffer, ofi->digestlen))
 		return FA_CREATE;       /* unmodified config file, replace. */
 	}
 
-	if (domd5(fn, buffer, 0, NULL))
+	if (domd5_algo(fn, buffer, 0, NULL, ofi->digestalgo))
 	    return FA_CREATE;	/* assume file has been removed */
 	omd5 = rpmfiMD5(ofi);
-	if (omd5 && !memcmp(omd5, buffer, 16))
+	if (omd5 && !memcmp(omd5, buffer, ofi->digestlen))
 	    return FA_CREATE;	/* unmodified config file, replace. */
 	nmd5 = rpmfiMD5(nfi);
 /*@-nullpass@*/
-	if (omd5 && nmd5 && !memcmp(omd5, nmd5, 16))
+	if (omd5 && nmd5 && ofi->digestalgo == nfi->digestalgo && !memcmp(omd5, nmd5, nfi->digestlen))
 	    return FA_SKIP;	/* identical file, don't bother. */
 /*@=nullpass@*/
     } else /* dbWhat == LINK */ {
@@ -663,10 +675,10 @@ int rpmfiConfigConflict(const rpmfi fi)
     memset(buffer, 0, sizeof(buffer));
     if (newWhat == REG) {
 	const unsigned char * nmd5;
-	if (domd5(fn, (unsigned char *)buffer, 0, NULL))
+	if (domd5_algo(fn, (unsigned char *)buffer, 0, NULL, fi->digestalgo))
 	    return 0;	/* assume file has been removed */
 	nmd5 = rpmfiMD5(fi);
-	if (nmd5 && !memcmp(nmd5, buffer, 16))
+	if (nmd5 && !memcmp(nmd5, buffer, fi->digestlen))
 	    return 0;	/* unmodified config file */
     } else /* newWhat == LINK */ {
 	const char * nFLink;
@@ -1268,7 +1280,7 @@ rpmfi rpmfiNew(const rpmts ts, Header h,
     rpmte p;
     rpmfi fi = NULL;
     const char * Type;
-    uint_32 * uip;
+    uint_32 * uip, * digestalgop;
     int dnlmax, bnlmax;
     unsigned char * t;
     int len;
@@ -1361,9 +1373,23 @@ if (fi->actions == NULL)
     fi->fmd5s = NULL;
     xx = hge(h, RPMTAG_FILEMD5S, NULL, (void **) &fi->fmd5s, NULL);
 
+    fi->digestalgo = PGPHASHALGO_MD5;
+    fi->digestlen = 16;
+    if (hge(h, RPMTAG_FILEDIGESTALGO, NULL, &digestalgop, NULL)) {
+	fi->digestalgo = *digestalgop;
+	if (fi->digestalgo == PGPHASHALGO_SHA1)
+	    fi->digestlen = 20;
+	else if (fi->digestalgo == PGPHASHALGO_SHA256)
+	    fi->digestlen = 32;
+	else if (fi->digestalgo == PGPHASHALGO_SHA512)
+	    fi->digestlen = 64;
+	else
+	    fi->digestalgo = PGPHASHALGO_MD5;
+    }
+
     fi->md5s = NULL;
     if (fi->fmd5s) {
-	t = xmalloc(fi->fc * 16);
+	t = xmalloc(fi->fc * fi->digestlen);
 	fi->md5s = t;
 	for (i = 0; i < fi->fc; i++) {
 	    const char * fmd5;
@@ -1371,11 +1397,11 @@ if (fi->actions == NULL)
 
 	    fmd5 = fi->fmd5s[i];
 	    if (!(fmd5 && *fmd5 != '\0')) {
-		memset(t, 0, 16);
-		t += 16;
+		memset(t, 0, fi->digestlen);
+		t += fi->digestlen;
 		continue;
 	    }
-	    for (j = 0; j < 16; j++, t++, fmd5 += 2)
+	    for (j = 0; j < fi->digestlen; j++, t++, fmd5 += 2)
 		*t = (nibble(fmd5[0]) << 4) | nibble(fmd5[1]);
 	}
 	fi->fmd5s = hfd(fi->fmd5s, -1);
--- ./lib/rpmfi.h.orig	2011-06-15 16:23:26.000000000 +0000
+++ ./lib/rpmfi.h	2011-06-15 17:13:52.000000000 +0000
@@ -125,6 +125,8 @@ struct rpmfi_s {
 /*@unused@*/
     int_32 * odil;		/*!< Original dirindex(s) (from header) */
 
+    int digestalgo;		/*!< File digest algorithm */
+    int digestlen;		/*!< File digest binary length */
 /*@only@*/ /*@relnull@*/
     unsigned char * md5s;	/*!< File md5 sums in binary. */
 
@@ -343,6 +345,9 @@ rpmfileState rpmfiFState(/*@null@*/ rpmf
 extern const unsigned char * rpmfiMD5(/*@null@*/ rpmfi fi)
 	/*@*/;
 
+extern int rpmfiDigestAlgo(rpmfi fi);
+extern int rpmfiDigestLen(rpmfi fi);
+
 /**
  * Return current file linkto (i.e. symlink(2) target) from file info set.
  * @param fi		file info set
--- ./lib/rpmlib.h.orig	2011-06-15 16:55:35.000000000 +0000
+++ ./lib/rpmlib.h	2011-06-15 16:57:13.000000000 +0000
@@ -460,6 +460,8 @@ typedef enum rpmTag_e {
     RPMTAG_SCRIPTSTATES		= 1174, /*!< i scriptlet exit codes */
     RPMTAG_SCRIPTMETRICS	= 1175, /*!< i scriptlet execution times */
     RPMTAG_BUILDCPUCLOCK	= 1176, /*!< i */
+    /* tags 1997-4999 reserved */
+    RPMTAG_FILEDIGESTALGO       = 5011, /* i file digest algorithm */
 
 /*@-enummemuse@*/
     RPMTAG_FIRSTFREE_TAG	/*!< internal */
--- ./lib/transaction.c.orig	2011-06-15 16:50:14.000000000 +0000
+++ ./lib/transaction.c	2011-06-15 17:11:17.000000000 +0000
@@ -650,9 +650,9 @@ assert(otherFi != NULL);
 	    }
 		
 	    /* Here is a pre-existing modified config file that needs saving. */
-	    {	char md5sum[50];
+	    {	char md5sum[64];
 		const unsigned char * MD5 = rpmfiMD5(fi);
-		if (!domd5(fn, md5sum, 0, NULL) && memcmp(MD5, md5sum, 16)) {
+		if (!domd5_algo(fn, md5sum, 0, NULL, fi->digestalgo) && memcmp(MD5, md5sum, fi->digestlen)) {
 		    fi->actions[i] = FA_BACKUP;
 		    /*@switchbreak@*/ break;
 		}
--- ./lib/verify.c.orig	2011-06-15 16:47:27.000000000 +0000
+++ ./lib/verify.c	2011-06-15 17:16:56.000000000 +0000
@@ -118,17 +118,17 @@ int rpmVerifyFile(const rpmts ts, const
 /*@=branchstate@*/
 
     if (flags & RPMVERIFY_MD5) {
-	unsigned char md5sum[16];
+	unsigned char md5sum[64];
 	size_t fsize;
 
 	/* XXX If --nomd5, then prelinked library sizes are not corrected. */
-	rc = domd5(fn, md5sum, 0, &fsize);
+	rc = domd5_algo(fn, md5sum, 0, &fsize, rpmfiDigestAlgo(fi));
 	sb.st_size = fsize;
 	if (rc)
 	    *res |= (RPMVERIFY_READFAIL|RPMVERIFY_MD5);
 	else {
 	    const unsigned char * MD5 = rpmfiMD5(fi);
-	    if (MD5 == NULL || memcmp(md5sum, MD5, sizeof(md5sum)))
+	    if (MD5 == NULL || memcmp(md5sum, MD5, rpmfiDigestLen(fi)))
 		*res |= RPMVERIFY_MD5;
 	}
     } 
--- ./rpmdb/legacy.c.orig	2011-06-15 16:41:13.000000000 +0000
+++ ./rpmdb/legacy.c	2011-06-15 16:43:54.000000000 +0000
@@ -145,7 +145,7 @@ exit:
     return fdno;
 }
 
-int domd5(const char * fn, unsigned char * digest, int asAscii, size_t *fsizep)
+int domd5_algo(const char * fn, unsigned char * digest, int asAscii, size_t *fsizep, int algo)
 {
     const char * path;
     urltype ut = urlPath(fn, &path);
@@ -193,7 +193,7 @@ int domd5(const char * fn, unsigned char
 #endif
 	}
 
-	ctx = rpmDigestInit(PGPHASHALGO_MD5, RPMDIGEST_NONE);
+	ctx = rpmDigestInit(algo, RPMDIGEST_NONE);
 	if (fsize)
 	    xx = rpmDigestUpdate(ctx, mapped, fsize);
 	xx = rpmDigestFinal(ctx, (void **)&md5sum, &md5len, asAscii);
@@ -219,11 +219,11 @@ int domd5(const char * fn, unsigned char
 	    break;
 	}
 	
-	fdInitDigest(fd, PGPHASHALGO_MD5, 0);
+	fdInitDigest(fd, algo, 0);
 	fsize = 0;
 	while ((rc = Fread(buf, sizeof(buf[0]), sizeof(buf), fd)) > 0)
 	    fsize += rc;
-	fdFiniDigest(fd, PGPHASHALGO_MD5, (void **)&md5sum, &md5len, asAscii);
+	fdFiniDigest(fd, algo, (void **)&md5sum, &md5len, asAscii);
 	if (Ferror(fd))
 	    rc = 1;
 
@@ -251,6 +251,11 @@ exit:
     return rc;
 }
 
+int domd5(const char * fn, unsigned char * digest, int asAscii, size_t *fsizep)
+{
+  return domd5_algo(fn, digest, asAscii, fsizep, PGPHASHALGO_MD5);
+}
+
 /*@-exportheadervar@*/
 /*@unchecked@*/
 int _noDirTokens = 0;
--- ./rpmdb/legacy.h.orig	2011-06-15 16:41:16.000000000 +0000
+++ ./rpmdb/legacy.h	2011-06-15 17:10:37.000000000 +0000
@@ -29,6 +29,8 @@ int domd5(const char * fn, /*@out@*/ uns
 		/*@null@*/ /*@out@*/ size_t *fsizep)
 	/*@globals h_errno, fileSystem, internalState @*/
 	/*@modifies digest, *fsizep, fileSystem, internalState @*/;
+int domd5_algo(const char * fn, /*@out@*/ unsigned char * digest, int asAscii,
+		/*@null@*/ /*@out@*/ size_t *fsizep, int algo);
 
 /**
  * Convert absolute path tag to (dirname,basename,dirindex) tags.