File tar4ibd_libtar-1.2.11.patch of Package xtrabackup

Overview Repositories Revisions Requests Users Attributes Meta

File tar4ibd_libtar-1.2.11.patch of Package xtrabackup (Revision 9b675f779b7138e14725cd1180d5472f)

Currently displaying revision 9b675f779b7138e14725cd1180d5472f , Show latest

diff -ru libtar-1.2.11_orig/lib/Makefile.in libtar-1.2.11/lib/Makefile.in
--- libtar-1.2.11_orig/lib/Makefile.in	2002-12-16 03:02:30.000000000 +0900
+++ libtar-1.2.11/lib/Makefile.in	2009-08-21 18:10:44.000000000 +0900
@@ -29,7 +29,7 @@
 		  -I${top_srcdir}/compat \
 		  -I../listhash \
 		  @CPPFLAGS@
-CFLAGS		= @CFLAGS@
+CFLAGS		= @CFLAGS@ -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
 LDFLAGS		= @LDFLAGS@
 LIBS		= @LIBS@
 LIBOBJS		= @LIBOBJS@
diff -ru libtar-1.2.11_orig/lib/append.c libtar-1.2.11/lib/append.c
--- libtar-1.2.11_orig/lib/append.c	2003-01-07 10:40:59.000000000 +0900
+++ libtar-1.2.11/lib/append.c	2009-07-09 17:49:26.000000000 +0900
@@ -27,6 +27,215 @@
 # include <unistd.h>
 #endif
 
+#ifdef POSIX_FADV_NORMAL
+#define USE_POSIX_FADVISE
+#endif
+
+/* all of these ripped from InnoDB code from MySQL 4.0.22 */
+#define UT_HASH_RANDOM_MASK     1463735687
+#define UT_HASH_RANDOM_MASK2    1653893711
+#define FIL_PAGE_LSN          16 
+#define FIL_PAGE_FILE_FLUSH_LSN 26
+#define FIL_PAGE_OFFSET     4
+#define FIL_PAGE_DATA       38
+#define FIL_PAGE_DATA_ALIGN_32  40
+#define FIL_PAGE_END_LSN_OLD_CHKSUM 8
+#define FIL_PAGE_SPACE_OR_CHKSUM 0
+//#define UNIV_PAGE_SIZE          (2 * 8192)
+#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL
+
+/* command line argument to do page checks (that's it) */
+/* another argument to specify page ranges... seek to right spot and go from there */
+
+typedef unsigned long int ulint;
+typedef unsigned char byte;
+
+/* innodb function in name; modified slightly to not have the ASM version (lots of #ifs that didn't apply) */
+ulint mach_read_from_4(byte *b)
+{
+  return( ((ulint)(b[0]) << 24)
+          + ((ulint)(b[1]) << 16)
+          + ((ulint)(b[2]) << 8)
+          + (ulint)(b[3])
+          );
+}
+
+ulint
+ut_fold_ulint_pair(
+/*===============*/
+            /* out: folded value */
+    ulint   n1, /* in: ulint */
+    ulint   n2) /* in: ulint */
+{
+    return(((((n1 ^ n2 ^ UT_HASH_RANDOM_MASK2) << 8) + n1)
+                        ^ UT_HASH_RANDOM_MASK) + n2);
+}
+
+ulint
+ut_fold_binary(
+/*===========*/
+            /* out: folded value */
+    byte*   str,    /* in: string of bytes */
+    ulint   len)    /* in: length */
+{
+    ulint   i;
+    ulint   fold= 0;
+
+    for (i= 0; i < len; i++)
+    {
+      fold= ut_fold_ulint_pair(fold, (ulint)(*str));
+
+      str++;
+    }
+
+    return(fold);
+}
+
+ulint
+ut_fold_binary_32(
+/*==============*/
+    byte*   str,
+    ulint   len)
+{
+    unsigned int*   str_end = (unsigned int*) (str + len);
+    unsigned int*   str_32 = (unsigned int*) str;
+    ulint   fold = 0;
+
+    if (sizeof(unsigned int) != 4) {
+        fprintf(stderr,
+                "size of int is not 32bit.\n");
+        exit(-1);
+    }
+
+    while (str_32 < str_end) {
+        fold = ut_fold_ulint_pair(fold, (ulint)(*str_32));
+
+        str_32++;
+    }
+
+    return(fold);
+}
+
+ulint
+buf_calc_page_new_checksum(
+/*=======================*/
+               /* out: checksum */
+    byte*    page, /* in: buffer page */
+    ulint    page_size)
+{
+    ulint checksum;
+
+    /* Since the fields FIL_PAGE_FILE_FLUSH_LSN and ..._ARCH_LOG_NO
+    are written outside the buffer pool to the first pages of data
+    files, we have to skip them in the page checksum calculation.
+    We must also skip the field FIL_PAGE_SPACE_OR_CHKSUM where the
+    checksum is stored, and also the last 8 bytes of page because
+    there we store the old formula checksum. */
+
+    checksum= ut_fold_binary(page + FIL_PAGE_OFFSET,
+                             FIL_PAGE_FILE_FLUSH_LSN - FIL_PAGE_OFFSET)
+            + ut_fold_binary(page + FIL_PAGE_DATA,
+                             page_size - FIL_PAGE_DATA
+                             - FIL_PAGE_END_LSN_OLD_CHKSUM);
+    checksum= checksum & 0xFFFFFFFF;
+
+    return(checksum);
+}
+
+ulint
+buf_calc_page_new_checksum_32(
+/*==========================*/
+    byte*    page,
+    ulint    page_size)
+{
+    ulint checksum;
+
+    checksum = ut_fold_binary(page + FIL_PAGE_OFFSET,
+                              FIL_PAGE_FILE_FLUSH_LSN - FIL_PAGE_OFFSET)
+             + ut_fold_binary(page + FIL_PAGE_DATA,
+                              FIL_PAGE_DATA_ALIGN_32 - FIL_PAGE_DATA)
+             + ut_fold_binary_32(page + FIL_PAGE_DATA_ALIGN_32,
+                                 page_size - FIL_PAGE_DATA_ALIGN_32
+                                 - FIL_PAGE_END_LSN_OLD_CHKSUM);
+    checksum= checksum & 0xFFFFFFFF;
+
+    return(checksum);
+}
+
+ulint
+buf_calc_page_old_checksum(
+/*=======================*/
+               /* out: checksum */
+    byte*    page) /* in: buffer page */
+{
+    ulint checksum;
+
+    checksum= ut_fold_binary(page, FIL_PAGE_FILE_FLUSH_LSN);
+
+    checksum= checksum & 0xFFFFFFFF;
+
+    return(checksum);
+}
+
+int
+buf_page_is_corrupted(
+/*==================*/
+				/* out: TRUE if corrupted */
+	byte*	read_buf,	/* in: a database page */
+	ulint	page_size)
+{
+	ulint	checksum;
+	ulint	old_checksum;
+	ulint	checksum_field;
+	ulint	old_checksum_field;
+
+	if (mach_read_from_4(read_buf + FIL_PAGE_LSN + 4)
+	     != mach_read_from_4(read_buf + page_size
+				- FIL_PAGE_END_LSN_OLD_CHKSUM + 4)) {
+
+		/* Stored log sequence numbers at the start and the end
+		of page do not match */
+
+		return(1);
+	}
+
+  /* If we use checksums validation, make additional check before returning
+  TRUE to ensure that the checksum is not equal to BUF_NO_CHECKSUM_MAGIC which
+  might be stored by InnoDB with checksums disabled.
+     Otherwise, skip checksum calculation and return FALSE */
+  
+    old_checksum = buf_calc_page_old_checksum(read_buf); 
+
+    old_checksum_field = mach_read_from_4(read_buf + page_size
+					- FIL_PAGE_END_LSN_OLD_CHKSUM);
+
+    /* There are 2 valid formulas for old_checksum_field:
+	  1. Very old versions of InnoDB only stored 8 byte lsn to the start
+	  and the end of the page.
+	  2. Newer InnoDB versions store the old formula checksum there. */
+	
+    if (old_checksum_field != mach_read_from_4(read_buf + FIL_PAGE_LSN)
+        && old_checksum_field != old_checksum
+        && old_checksum_field != BUF_NO_CHECKSUM_MAGIC) {
+
+      return(1);
+    }
+
+    checksum = buf_calc_page_new_checksum(read_buf, page_size);
+    checksum_field = mach_read_from_4(read_buf + FIL_PAGE_SPACE_OR_CHKSUM);
+
+    /* InnoDB versions < 4.0.14 and < 4.1.1 stored the space id
+	  (always equal to 0), to FIL_PAGE_SPACE_SPACE_OR_CHKSUM */
+
+    if (checksum_field != 0 && checksum_field != checksum
+        && checksum_field != buf_calc_page_new_checksum_32(read_buf, page_size)
+        && checksum_field != BUF_NO_CHECKSUM_MAGIC) {
+
+      return(1);
+    }
+  
+	return(0);
+}
 
 struct tar_dev
 {
@@ -210,10 +419,11 @@
 int
 tar_append_regfile(TAR *t, char *realname)
 {
-	char block[T_BLOCKSIZE];
+	unsigned char block[T_BLOCKSIZE * 32]; /* 16KB is MAX */
 	int filefd;
-	int i, j;
-	size_t size;
+	int j, k, n_retry;
+	size_t i, size;
+	int page_size = 4 * 1024; /* check 4K, 8K and 16K */
 
 	filefd = open(realname, O_RDONLY);
 	if (filefd == -1)
@@ -224,28 +434,61 @@
 		return -1;
 	}
 
+#ifdef USE_POSIX_FADVISE
+	posix_fadvise(filefd, 0, 0, POSIX_FADV_SEQUENTIAL);
+	posix_fadvise(filefd, 0, 0, POSIX_FADV_DONTNEED);
+#endif
+
+	n_retry = 0;
 	size = th_get_size(t);
-	for (i = size; i > T_BLOCKSIZE; i -= T_BLOCKSIZE)
+	for (i = size; i >= page_size; i -= page_size)
 	{
-		j = read(filefd, &block, T_BLOCKSIZE);
-		if (j != T_BLOCKSIZE)
+		j = read(filefd, &block[0], page_size);
+		if (j != page_size)
 		{
 			if (j != -1)
 				errno = EINVAL;
 			return -1;
 		}
-		if (tar_block_write(t, &block) == -1)
-			return -1;
+		if (buf_page_is_corrupted(block, (ulint)page_size)) {
+			if (n_retry > 9) {
+				fprintf(stderr,
+"The file '%s' may not be InnoDB datafile or may be corrupted.\n",
+					realname);
+				errno = EIO;
+				return -1;
+			}
+
+			/* retry */
+			if (lseek(filefd, - page_size, SEEK_CUR) == -1)
+				return -1;
+
+			if (i == size
+			    && page_size < 16 * 1024) {
+				page_size *= 2;
+			} else {
+				n_retry++;
+			}
+
+			i += page_size;
+
+			continue;
+		}
+		for (k = 0; k < page_size / T_BLOCKSIZE; k++)
+			if (tar_block_write(t, &block[T_BLOCKSIZE * k]) == -1)
+				return -1;
+		n_retry = 0;
 	}
 
 	if (i > 0)
 	{
-		j = read(filefd, &block, i);
+		j = read(filefd, &block[0], i);
 		if (j == -1)
 			return -1;
-		memset(&(block[i]), 0, T_BLOCKSIZE - i);
-		if (tar_block_write(t, &block) == -1)
-			return -1;
+		memset(&(block[i]), 0, T_BLOCKSIZE * 32 - i);
+		for (k = 0; k < ((j - 1) / T_BLOCKSIZE + 1); k++)
+			if (tar_block_write(t, &block[T_BLOCKSIZE * k]) == -1)
+				return -1;
 	}
 
 	close(filefd);
diff -ru libtar-1.2.11_orig/lib/extract.c libtar-1.2.11/lib/extract.c
--- libtar-1.2.11_orig/lib/extract.c	2003-03-03 08:58:07.000000000 +0900
+++ libtar-1.2.11/lib/extract.c	2009-07-09 17:49:26.000000000 +0900
@@ -158,11 +158,11 @@
 tar_extract_regfile(TAR *t, char *realname)
 {
 	mode_t mode;
-	size_t size;
+	size_t i, size;
 	uid_t uid;
 	gid_t gid;
 	int fdout;
-	int i, k;
+	int k;
 	char buf[T_BLOCKSIZE];
 	char *filename;
 
diff -ru libtar-1.2.11_orig/lib/libtar.h libtar-1.2.11/lib/libtar.h
--- libtar-1.2.11_orig/lib/libtar.h	2003-01-07 10:40:59.000000000 +0900
+++ libtar-1.2.11/lib/libtar.h	2010-02-03 17:03:22.000000000 +0900
@@ -180,7 +180,7 @@
 
 /* decode tar header info */
 #define th_get_crc(t) oct_to_int((t)->th_buf.chksum)
-#define th_get_size(t) oct_to_int((t)->th_buf.size)
+#define th_get_size(t) oct_to_longlong((t)->th_buf.size)
 #define th_get_mtime(t) oct_to_int((t)->th_buf.mtime)
 #define th_get_devmajor(t) oct_to_int((t)->th_buf.devmajor)
 #define th_get_devminor(t) oct_to_int((t)->th_buf.devminor)
@@ -204,9 +204,9 @@
 void th_set_group(TAR *t, gid_t gid);
 void th_set_mode(TAR *t, mode_t fmode);
 #define th_set_mtime(t, fmtime) \
-	int_to_oct_nonull((fmtime), (t)->th_buf.mtime, 12)
+	int_to_oct((fmtime), (t)->th_buf.mtime, 12)
 #define th_set_size(t, fsize) \
-	int_to_oct_nonull((fsize), (t)->th_buf.size, 12)
+	longlong_to_oct_nonull((fsize), (t)->th_buf.size, 12)
 
 /* encode everything at once (except the pathname and linkname) */
 void th_set_from_stat(TAR *t, struct stat *s);
@@ -268,13 +268,16 @@
 
 /* string-octal to integer conversion */
 int oct_to_int(char *oct);
+long oct_to_long(char *oct);
+long long oct_to_longlong(char *oct);
 
 /* integer to NULL-terminated string-octal conversion */
 #define int_to_oct(num, oct, octlen) \
-	snprintf((oct), (octlen), "%*lo ", (octlen) - 2, (unsigned long)(num))
+	snprintf((oct), (octlen), "%0*lo", (octlen) - 1, (unsigned long)(num))
 
 /* integer to string-octal conversion, no NULL */
 void int_to_oct_nonull(int num, char *oct, size_t octlen);
+void longlong_to_oct_nonull(long long num, char *oct, size_t octlen);
 
 
 /***** wrapper.c **********************************************************/
diff -ru libtar-1.2.11_orig/lib/util.c libtar-1.2.11/lib/util.c
--- libtar-1.2.11_orig/lib/util.c	2003-01-07 10:41:00.000000000 +0900
+++ libtar-1.2.11/lib/util.c	2009-07-09 17:56:31.000000000 +0900
@@ -138,6 +138,14 @@
 	return i;
 }
 
+long
+oct_to_long(char *oct)
+{
+	long i;
+	sscanf(oct, "%12lo", &i);
+	return i;
+}
+
 
 /* integer to string-octal conversion, no NULL */
 void
@@ -147,4 +155,46 @@
 	oct[octlen - 1] = ' ';
 }
 
+void
+longlong_to_oct_nonull(long long num, char *oct, size_t octlen)
+{
+	char buf[100];
+
+	if (num >= (long long)64 * 1024 * 1024 * 1024) {
+		oct[0] = 0x80;
+		oct[1] = 0;
+		oct[2] = 0;
+		oct[3] = 0;
+		oct[4] = (char)((num >> 56) & 0xFF);
+		oct[5] = (char)((num >> 48) & 0xFF);
+		oct[6] = (char)((num >> 40) & 0xFF);
+		oct[7] = (char)((num >> 32) & 0xFF);
+		oct[8] = (char)((num >> 24) & 0xFF);
+		oct[9] = (char)((num >> 16) & 0xFF);
+		oct[10] = (char)((num >> 8) & 0xFF);
+		oct[11] = (char)((num) & 0xFF);
+	} else {
+		snprintf(buf, octlen + 1, "%0*llo", octlen, num);
+		strncpy(oct, buf, octlen);
+	}
+}
+
+long long
+oct_to_longlong(char *oct)
+{
+	long long i;
 
+	if (oct[0] & 0x80) {
+		i = ((long long)((unsigned char)oct[4]) << 56)
+			| ((long long)((unsigned char)oct[5]) << 48)
+			| ((long long)((unsigned char)oct[6]) << 40)
+			| ((long long)((unsigned char)oct[7]) << 32)
+			| ((long long)((unsigned char)oct[8]) << 24)
+			| ((long long)((unsigned char)oct[9]) << 16)
+			| ((long long)((unsigned char)oct[10]) << 8)
+			| ((long long)((unsigned char)oct[11]));
+	} else {
+		sscanf(oct, "%12llo", &i);
+	}
+	return i;
+}
diff -ru libtar-1.2.11_orig/libtar/Makefile.in libtar-1.2.11/libtar/Makefile.in
--- libtar-1.2.11_orig/libtar/Makefile.in	2002-12-16 03:02:30.000000000 +0900
+++ libtar-1.2.11/libtar/Makefile.in	2009-08-21 18:14:52.000000000 +0900
@@ -29,7 +29,7 @@
 		  -I${top_srcdir}/lib \
 		  -I${top_srcdir}/compat \
 		  @CPPFLAGS@
-CFLAGS		= @CFLAGS@
+CFLAGS		= @CFLAGS@ -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
 LDFLAGS		= @LDFLAGS@
 LIBS		= @LIBS@
 LIBOBJS		= @LIBOBJS@
@@ -46,15 +46,15 @@
 		  ${top_srcdir}/lib/libtar.h \
 		  ../listhash/libtar_listhash.h
 LIBTAR_LIBS	= ../lib/libtar.a
-ALL		= libtar
+ALL		= tar4ibd
 
 
 all: ${ALL}
 
 .PHONY: clean distclean install
 
-libtar: ${LIBTAR_OBJS} ${LIBTAR_LIBS} ${LIBTAR_HDRS}
-	${CC} ${CFLAGS} ${LDFLAGS} -o libtar libtar.o ${LIBTAR_LIBS} ${LIBS}
+tar4ibd: ${LIBTAR_OBJS} ${LIBTAR_LIBS} ${LIBTAR_HDRS}
+	${CC} ${CFLAGS} ${LDFLAGS} -o tar4ibd libtar.o ${LIBTAR_LIBS} ${LIBS}
 
 ${LIBTAR_OBJS}: ${LIBTAR_HDRS}
 
@@ -69,5 +69,5 @@
 
 install: ${ALL}
 	${MKDIR} ${DESTDIR}${bindir}
-	${INSTALL_PROGRAM} libtar ${DESTDIR}${bindir}
+	${INSTALL_PROGRAM} tar4ibd ${DESTDIR}${bindir}
 
diff -ru libtar-1.2.11_orig/libtar/libtar.c libtar-1.2.11/libtar/libtar.c
--- libtar-1.2.11_orig/libtar/libtar.c	2003-01-07 10:41:00.000000000 +0900
+++ libtar-1.2.11/libtar/libtar.c	2009-07-09 17:49:26.000000000 +0900
@@ -109,7 +109,7 @@
 	char buf[MAXPATHLEN];
 	libtar_listptr_t lp;
 
-	if (tar_open(&t, tarfile,
+	if (tar_fdopen(&t, fileno(stdout), tarfile,
 #ifdef HAVE_LIBZ
 		     (use_zlib ? &gztype : NULL),
 #else
@@ -164,7 +164,7 @@
 	TAR *t;
 	int i;
 
-	if (tar_open(&t, tarfile,
+	if (tar_fdopen(&t, fileno(stdin), tarfile,
 #ifdef HAVE_LIBZ
 		     (use_zlib ? &gztype : NULL),
 #else
@@ -225,7 +225,7 @@
 #ifdef DEBUG
 	puts("opening tarfile...");
 #endif
-	if (tar_open(&t, tarfile,
+	if (tar_fdopen(&t, fileno(stdin), tarfile,
 #ifdef HAVE_LIBZ
 		     (use_zlib ? &gztype : NULL),
 #else
@@ -264,9 +264,9 @@
 void
 usage()
 {
-	printf("Usage: %s [-C rootdir] [-g] [-z] -x|-t filename.tar\n",
+	printf("Usage: %s [-C rootdir] [-g] [-z] -x|-t < filename.tar\n",
 	       progname);
-	printf("       %s [-C rootdir] [-g] [-z] -c filename.tar ...\n",
+	printf("       %s [-C rootdir] [-g] [-z] -c ... > filename.tar\n",
 	       progname);
 	exit(-1);
 }
@@ -327,7 +327,7 @@
 			usage();
 		}
 
-	if (!mode || ((argc - optind) < (mode == MODE_CREATE ? 2 : 1)))
+	if (!mode || ((argc - optind) < (mode == MODE_CREATE ? 1 : 0)))
 	{
 #ifdef DEBUG
 		printf("argc - optind == %d\tmode == %d\n", argc - optind,
@@ -343,15 +343,14 @@
 	switch (mode)
 	{
 	case MODE_EXTRACT:
-		return extract(argv[optind], rootdir);
+		return extract("<stdin>", rootdir);
 	case MODE_CREATE:
-		tarfile = argv[optind];
 		l = libtar_list_new(LIST_QUEUE, NULL);
-		for (c = optind + 1; c < argc; c++)
+		for (c = optind; c < argc; c++)
 			libtar_list_add(l, argv[c]);
-		return create(tarfile, rootdir, l);
+		return create("<stdout>", rootdir, l);
 	case MODE_LIST:
-		return list(argv[optind]);
+		return list("<stdin>");
 	default:
 		break;
 	}

Places

File tar4ibd_libtar-1.2.11.patch of Package xtrabackup (Revision 9b675f779b7138e14725cd1180d5472f)

Places