File blktrace-0.99.3..00a47cd169f2cc87b5f63fe93226b7... of Package blktrace

Overview Repositories Revisions Requests Users Attributes Meta

File blktrace-0.99.3..00a47cd169f2cc87b5f63fe93226b7231dee678c.diff of Package blktrace (Revision 3451e70dd1b9f12a28b5cbd7e8b58f00)

Currently displaying revision 3451e70dd1b9f12a28b5cbd7e8b58f00 , Show latest

diff --git a/Makefile b/Makefile
index e36f5a2..edc6ca9 100644
--- a/Makefile
+++ b/Makefile
@@ -5,20 +5,26 @@ PROGS	= blkparse blktrace verify_blkparse blkrawverify
 LIBS	= -lpthread
 SCRIPTS	= btrace
 
-ALL = $(PROGS) $(SCRIPTS) btt/btt
+ALL = $(PROGS) $(SCRIPTS) btt/btt btreplay/btrecord btreplay/btreplay
 
 all: $(ALL)
 
 btt/btt:
 	$(MAKE) -C btt
 
+btreplay/btrecord:
+	$(MAKE) -C btreplay
+
+btreplay/btreplay:
+	$(MAKE) -C btreplay
+
 %.o: %.c
 	$(CC) -o $*.o -c $(ALL_CFLAGS) $<
 
 blkparse: blkparse.o blkparse_fmt.o rbtree.o act_mask.o
 	$(CC) $(ALL_CFLAGS) -o $@ $(filter %.o,$^)
 
-blktrace: blktrace.o act_mask.o $(LIBS)
+blktrace: blktrace.o act_mask.o
 	$(CC) $(ALL_CFLAGS) -o $@ $(filter %.o,$^) $(LIBS)
 
 verify_blkparse: verify_blkparse.o
@@ -32,10 +38,12 @@ $(PROGS): | depend
 docs:
 	$(MAKE) -C doc all
 	$(MAKE) -C btt docs
+	$(MAKE) -C btreplay docs
 
 docsclean:
 	$(MAKE) -C doc clean
 	$(MAKE) -C btt clean
+	$(MAKE) -C btreplay clean
 
 depend:
 	@$(CC) -MM $(ALL_CFLAGS) *.c 1> .depend
@@ -63,6 +71,7 @@ rpm: dist
 clean: docsclean
 	-rm -f *.o $(PROGS) .depend btrace-1.0.tar.bz2
 	$(MAKE) -C btt clean
+	$(MAKE) -C btreplay clean
 
 install: all
 	$(INSTALL) -m 755 -d $(DESTDIR)$(bindir)
diff --git a/barrier.h b/barrier.h
index 19169d7..14d06b5 100644
--- a/barrier.h
+++ b/barrier.h
@@ -17,7 +17,7 @@
   #define store_barrier()         asm volatile("":::"memory")
 #elif defined(__sparc__)
   #define store_barrier()         asm volatile("":::"memory")
-#elif defined(__m68000__)
+#elif defined(__m68000__) || defined(__m68k__) || defined(mc68000) || defined(_M_M68K)
   #define store_barrier()         asm volatile("":::"memory")
 #elif defined(__mips__)  /* also mipsel */
   #define store_barrier()         do { } while(0)
diff --git a/blktrace.c b/blktrace.c
index 42613a7..4d458ac 100644
--- a/blktrace.c
+++ b/blktrace.c
@@ -1224,6 +1224,7 @@ static int start_devices(void)
 		fprintf(stderr, "Out of memory, threads (%d)\n", size * ndevs);
 		return 1;
 	}
+	memset(thread_information, 0, size * ndevs);
 
 	for_each_dip(dip, i) {
 		if (start_trace(dip)) {
diff --git a/btreplay/Makefile b/btreplay/Makefile
new file mode 100644
index 0000000..a8d2e3b
--- /dev/null
+++ b/btreplay/Makefile
@@ -0,0 +1,45 @@
+#
+# OCFLAGS:
+# 	COUNT_IOS	- Counts struct io's left at end
+# 	DEBUG		- Various and sundy debug asserts
+# 	NDEBUG		- Defined: no asserts, Undefined: asserts
+#
+
+CC	= gcc
+CFLAGS	= -Wall -W -O2 -g
+INCS	= -I. -I.. -I../btt
+OCFLAGS	= -UCOUNT_IOS -UDEBUG -DNDEBUG
+XCFLAGS	= -D_GNU_SOURCE -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
+override CFLAGS += $(INCS) $(XCFLAGS) $(OCFLAGS)
+
+PROGS	= btrecord btreplay
+LIBS	= -laio -lrt
+
+all: depend $(PROGS)
+
+$(PROGS): | depend
+
+docs:
+	$(MAKE) -C doc all
+
+docsclean:
+	$(MAKE) -C doc clean
+
+clean: docsclean
+	-rm -f *.o $(PROGS) .depend
+
+%.o: %.c
+	$(CC) $(CFLAGS) -c -o $*.o $<
+
+btrecord: btrecord.o
+	$(CC) $(CFLAGS) -o $@ $(filter %.o,$^) $(LIBS)
+
+btreplay: btreplay.o
+	$(CC) $(CFLAGS) -o $@ $(filter %.o,$^) $(LIBS)
+
+depend:
+	@$(CC) -MM $(CFLAGS) *.c 1> .depend
+
+ifneq ($(wildcard .depend),)
+include .depend
+endif
diff --git a/btreplay/btrecord.c b/btreplay/btrecord.c
new file mode 100644
index 0000000..09329f7
--- /dev/null
+++ b/btreplay/btrecord.c
@@ -0,0 +1,789 @@
+/*
+ * Blktrace record utility - Convert binary trace data into bunches of IOs
+ *
+ * Copyright (C) 2007 Alan D. Brunelle <Alan.Brunelle@hp.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+static char build_date[] = __DATE__ " at "__TIME__;
+
+#include <assert.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <stdarg.h>
+
+#if !defined(_GNU_SOURCE)
+#	define _GNU_SOURCE
+#endif
+#include <getopt.h>
+
+#include "list.h"
+#include "btrecord.h"
+#include "blktrace.h"
+
+/*
+ * Per input file information
+ *
+ * @head: 	Used to link up on input_files
+ * @devnm: 	Device name portion of this input file
+ * @file_name: 	Fully qualified name for this input file
+ * @cpu: 	CPU that this file was collected on
+ * @ifd: 	Input file descriptor (when opened)
+ * @tpkts: 	Total number of packets processed.
+ */
+struct ifile_info {
+	struct list_head head;
+	char *devnm, *file_name;
+	int cpu, ifd;
+	__u64 tpkts, genesis;
+};
+
+/*
+ * Per IO trace information
+ *
+ * @time: 	Time stamp when trace was emitted
+ * @sector: 	IO sector identifier
+ * @bytes: 	Number of bytes transferred
+ * @rw: 	Read (1) or write (0) 
+ */
+struct io_spec {
+	__u64 time;
+	__u64 sector;
+	__u32 bytes;
+	int rw;
+};
+
+/*
+ * Per output file information
+ *
+ * @ofp: 	Output file 
+ * @vfp:	Verbose output file
+ * @file_name: 	Fully qualified name for this file
+ * @vfn:	Fully qualified name for this file
+ * @cur: 	Current IO bunch being collected
+ * @iip: 	Input file this is associated with
+ * @start_time: Start time of th ecurrent bunch
+ * @last_time: 	Time of last packet put in
+ * @bunches: 	Number of bunches processed
+ * @pkts: 	Number of packets stored in bunches
+ */
+struct io_stream {
+	FILE *ofp, *vfp;
+	char *file_name, *vfn;
+	struct io_bunch *cur;
+	struct ifile_info *iip;
+	__u64 start_time, last_time, bunches, pkts;
+};
+
+int data_is_native;				// Indicates whether to swap
+static LIST_HEAD(input_files);			// List of all input files
+static char *idir = ".";			// Input directory base
+static char *odir = ".";			// Output directory base
+static char *obase = "replay";			// Output file base
+static __u64 max_bunch_tm = (10 * 1000 * 1000);	// 10 milliseconds
+static __u64 max_pkts_per_bunch = 8;		// Default # of pkts per bunch
+static int verbose = 0;				// Boolean: output stats
+static int find_traces = 0;			// Boolean: Find traces in dir
+
+static char usage_str[] =                                                  \
+        "\n"                                                               \
+	"\t[ -d <dir>  : --input-directory=<dir> ] Default: .\n"           \
+	"\t[ -D <dir>  : --output-directory=<dir>] Default: .\n"           \
+	"\t[ -F        : --find-traces           ] Default: Off\n"         \
+        "\t[ -h        : --help                  ] Default: Off\n"         \
+        "\t[ -m <nsec> : --max-bunch-time=<nsec> ] Default: 10 msec\n"     \
+	"\t[ -M <pkts> : --max-pkts=<pkts>       ] Default: 8\n"           \
+        "\t[ -o <base> : --output-base=<base>    ] Default: replay\n"      \
+        "\t[ -v        : --verbose               ] Default: Off\n"         \
+        "\t[ -V        : --version               ] Default: Off\n"         \
+	"\t<dev>...                                Default: None\n"	   \
+        "\n";
+
+#define S_OPTS	"d:D:Fhm:M:o:vV"
+static struct option l_opts[] = {
+	{
+		.name = "input-directory",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'd'
+	},
+	{
+		.name = "output-directory",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'D'
+	},
+	{
+		.name = "find-traces",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'F'
+	},
+	{
+		.name = "help",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'h'
+	},
+	{
+		.name = "max-bunch-time",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'm'
+	},
+	{
+		.name = "max_pkts",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'M'
+	},
+	{
+		.name = "output-base",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'o'
+	},
+	{
+		.name = "verbose",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'v'
+	},
+	{
+		.name = "version",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'V'
+	},
+	{
+		.name = NULL
+	}
+};
+
+#define ERR_ARGS			1
+#define ERR_SYSCALL			2
+static inline void fatal(const char *errstring, const int exitval,
+			 const char *fmt, ...)
+{
+	va_list ap;
+
+	if (errstring)
+		perror(errstring);
+
+	va_start(ap, fmt);
+	vfprintf(stderr, fmt, ap);
+	va_end(ap);
+
+	exit(exitval);
+	/*NOTREACHED*/
+}
+
+/**
+ * match - Return true if this trace is a proper QUEUE transaction
+ * @action: Action field from trace
+ */
+static inline int match(__u32 action)
+{
+	return ((action & 0xffff) == __BLK_TA_QUEUE) &&
+				       (action & BLK_TC_ACT(BLK_TC_QUEUE));
+}
+
+/**
+ * usage - Display usage string and version
+ */
+static void usage(void)
+{
+	fprintf(stderr, "Usage: btrecord -- version %s\n%s", 
+		my_btversion, usage_str);
+}
+
+/**
+ * write_file_hdr - Seek to and write btrecord file header
+ * @stream: Output file information
+ * @hdr: Header to write
+ */
+static void write_file_hdr(struct io_stream *stream, struct io_file_hdr *hdr)
+{
+	hdr->version = mk_btversion(btver_mjr, btver_mnr, btver_sub);
+
+	if (verbose) {
+		fprintf(stderr, "\t%s: %llx %llx %llx %llx\n", 
+			stream->file_name,
+			(long long unsigned)hdr->version,
+			(long long unsigned)hdr->genesis,
+			(long long unsigned)hdr->nbunches,
+			(long long unsigned)hdr->total_pkts);
+	}
+
+	fseek(stream->ofp, 0, SEEK_SET);
+	if (fwrite(hdr, sizeof(*hdr), 1, stream->ofp) != 1) {
+		fatal(stream->file_name, ERR_SYSCALL, "Hdr write failed\n");
+		/*NOTREACHED*/
+	}
+}
+
+/**
+ * io_bunch_create - Allocate & initialize an io_bunch
+ * @io_stream: IO stream being added to
+ * @pre_stall: Amount of time that this bunch should be delayed by
+ * @start_time: Records current start 
+ */
+static inline void io_bunch_create(struct io_stream *stream, __u64 start_time)
+{
+	struct io_bunch *cur = malloc(sizeof(*cur));
+
+	memset(cur, 0, sizeof(*cur));
+
+	cur->hdr.npkts = 0;
+	cur->hdr.time_stamp = stream->start_time = start_time;
+
+	stream->cur = cur;
+}
+
+/**
+ * io_bunch_add - Add an IO to the current bunch of IOs
+ * @stream: Per-output file stream information
+ * @spec: IO trace specification
+ *
+ * Returns update bunch information
+ */
+static void io_bunch_add(struct io_stream *stream, struct io_spec *spec)
+{
+	struct io_bunch *cur = stream->cur;
+	struct io_pkt iop = {
+		.sector = spec->sector,
+		.nbytes = spec->bytes,
+		.rw = spec->rw
+	};
+
+	assert(cur != NULL);
+	assert(cur->hdr.npkts < BT_MAX_PKTS);
+	assert(stream->last_time == 0 || stream->last_time <= spec->time);
+
+	cur->pkts[cur->hdr.npkts++] = iop;	// Struct copy
+	stream->last_time = spec->time;
+}
+
+/**
+ * rem_input_file - Release resources associated with an input file
+ * @iip: Per-input file information
+ */
+static void rem_input_file(struct ifile_info *iip)
+{
+	list_del(&iip->head);
+
+	close(iip->ifd);
+	free(iip->file_name);
+	free(iip->devnm);
+	free(iip);
+}
+
+/**
+ * __add_input_file - Allocate and initialize per-input file structure
+ * @cpu: CPU for this file
+ * @devnm: Device name for this file
+ * @file_name: Fully qualifed input file name
+ */
+static void __add_input_file(int cpu, char *devnm, char *file_name)
+{
+	struct ifile_info *iip = malloc(sizeof(*iip));
+
+	iip->cpu = cpu;
+	iip->tpkts = 0;
+	iip->genesis = 0;
+	iip->devnm = strdup(devnm);
+	iip->file_name = strdup(file_name);
+	iip->ifd = open(file_name, O_RDONLY);
+	if (iip->ifd < 0) {
+		fatal(file_name, ERR_ARGS, "Unable to open\n");
+		/*NOTREACHED*/
+	}
+
+	list_add_tail(&iip->head, &input_files);
+}
+
+/**
+ * add_input_file - Set up the input file name
+ * @devnm: Device name to use
+ */
+static void add_input_file(char *devnm)
+{
+	struct list_head *p;
+	int cpu, found = 0;
+
+	__list_for_each(p, &input_files) {
+		struct ifile_info *iip = list_entry(p, struct ifile_info, head);
+		if (strcmp(iip->devnm, devnm) == 0)
+			return;
+	}
+
+	for (cpu = 0; ; cpu++) {
+		char full_name[MAXPATHLEN];
+
+		sprintf(full_name, "%s/%s.blktrace.%d", idir, devnm, cpu);
+		if (access(full_name, R_OK) != 0)
+			break;
+
+		__add_input_file(cpu, devnm, full_name);
+		found++;
+	}
+
+	if (!found) {
+		fatal(NULL, ERR_ARGS, "No traces found for %s\n", devnm);
+		/*NOTREACHED*/
+	}
+}
+
+static void find_input_files(char *idir)
+{
+	struct dirent *ent;
+	DIR *dir = opendir(idir);
+
+	if (dir == NULL) {
+		fatal(idir, ERR_ARGS, "Unable to open %s\n", idir);
+		/*NOTREACHED*/
+	}
+
+	while ((ent = readdir(dir)) != NULL) {
+		char *p, *dsf = malloc(256);
+
+		if (strstr(ent->d_name, ".blktrace.") == NULL)
+			continue;
+
+		dsf = strdup(ent->d_name);
+		p = index(dsf, '.');
+		assert(p != NULL);
+		*p = '\0';
+		add_input_file(dsf);
+		free(dsf);
+	}
+
+	closedir(dir);
+}
+
+/**
+ * handle_args - Parse passed in argument list
+ * @argc: Number of arguments in argv
+ * @argv: Arguments passed in
+ *
+ * Does rudimentary parameter verification as well.
+ */
+void handle_args(int argc, char *argv[])
+{
+	int c;
+
+	while ((c = getopt_long(argc, argv, S_OPTS, l_opts, NULL)) != -1) {
+		switch (c) {
+		case 'd':
+			idir = optarg;
+			if (access(idir, R_OK | X_OK) != 0) {
+				fatal(idir, ERR_ARGS, 
+				      "Invalid input directory specified\n");
+				/*NOTREACHED*/
+			}
+			break;
+
+		case 'D':
+			odir = optarg;
+			if (access(odir, R_OK | X_OK) != 0) {
+				fatal(odir, ERR_ARGS, 
+				      "Invalid output directory specified\n");
+				/*NOTREACHED*/
+			}
+			break;
+
+		case 'F': 
+			find_traces = 1;
+			break;
+
+		case 'h': 
+			usage(); 
+			exit(0);
+			/*NOTREACHED*/
+
+		case 'm':
+			max_bunch_tm = (__u64)atoll(optarg);
+			if (max_bunch_tm < 1) {
+				fprintf(stderr, "Invalid bunch time %llu\n",
+					(unsigned long long)max_bunch_tm);
+				exit(ERR_ARGS);
+				/*NOTREACHED*/
+			}
+			break;
+
+		case 'M':
+			max_pkts_per_bunch = (__u64)atoll(optarg);
+			if (!((1 <= max_pkts_per_bunch) && 
+						(max_pkts_per_bunch < 513))) {
+				fprintf(stderr, "Invalid max pkts %llu\n",
+					(unsigned long long)max_pkts_per_bunch);
+				exit(ERR_ARGS);
+				/*NOTREACHED*/
+			}
+			break;
+
+		case 'o':
+			obase = optarg;
+			break;
+
+		case 'V':
+			fprintf(stderr, "btrecord -- version %s\n", 
+				my_btversion);
+			fprintf(stderr, "            Built on %s\n", build_date);
+			exit(0);
+			/*NOTREACHED*/
+
+		case 'v':
+			verbose++;
+			break;
+
+		default:
+			usage();
+			fatal(NULL, ERR_ARGS, "Invalid command line\n");
+			/*NOTREACHED*/
+		}
+	}
+
+	while (optind < argc)
+		add_input_file(argv[optind++]);
+
+	if (find_traces)
+		find_input_files(idir);
+
+	if (list_len(&input_files) == 0) {
+		fatal(NULL, ERR_ARGS, "Missing required input file name(s)\n");
+		/*NOTREACHED*/
+	}
+}
+
+/**
+ * next_io - Retrieve next Q trace from input stream
+ * @iip: Per-input file information
+ * @spec: IO specifier for trace
+ *
+ * Returns 0 on end of file, 1 if valid data returned.
+ */
+static int next_io(struct ifile_info *iip, struct io_spec *spec)
+{
+	ssize_t ret;
+	__u32 action;
+	__u16 pdu_len;
+	struct blk_io_trace t;
+
+again:
+	ret = read(iip->ifd, &t, sizeof(t));
+	if (ret < 0) {
+		fatal(iip->file_name, ERR_SYSCALL, "Read failed\n");
+		/*NOTREACHED*/
+	}
+	else if (ret == 0)
+		return 0;
+	else if (ret < (ssize_t)sizeof(t)) {
+		fprintf(stderr, "WARNING: Short read on %s (%d)\n", 
+			iip->file_name, (int)ret);
+		return 0;
+	}
+
+	if (data_is_native == -1)
+		check_data_endianness(t.magic);
+
+	assert(data_is_native >= 0);
+	if (data_is_native) {
+		spec->time = t.time;
+		spec->sector = t.sector;
+		spec->bytes = t.bytes;
+		action = t.action;
+		pdu_len = t.pdu_len;
+	}
+	else {
+		spec->time = be64_to_cpu(t.time);
+		spec->sector = be64_to_cpu(t.sector);
+		spec->bytes = be32_to_cpu(t.bytes);
+		action = be32_to_cpu(t.action);
+		pdu_len = be16_to_cpu(t.pdu_len);
+	}
+
+
+	if (pdu_len) {
+		char buf[pdu_len];
+
+		ret = read(iip->ifd, buf, pdu_len);
+		if (ret < 0) {
+			fatal(iip->file_name, ERR_SYSCALL, "Read PDU failed\n");
+			/*NOTREACHED*/
+		}
+		else if (ret < (ssize_t)pdu_len) {
+			fprintf(stderr, "WARNING: Short PDU read on %s (%d)\n", 
+				iip->file_name, (int)ret);
+			return 0;
+		}
+	}
+
+	iip->tpkts++;
+	if (!match(action))
+		goto again;
+
+	spec->rw = (action & BLK_TC_ACT(BLK_TC_READ)) ? 1 : 0;
+	if (verbose > 1)
+		fprintf(stderr, "%2d: %10llu+%10llu (%d) @ %10llx\n",
+			iip->cpu, (long long unsigned)spec->sector,
+			(long long unsigned)spec->bytes / 512LLU,
+			spec->rw, (long long unsigned)spec->time);
+
+	if (iip->genesis == 0) {
+		iip->genesis = spec->time;
+		if (verbose > 1)
+			fprintf(stderr, "\tSetting new genesis: %llx(%d)\n",
+				(long long unsigned)iip->genesis, iip->cpu);
+	}
+	else if (iip->genesis > spec->time)
+		fatal(NULL, ERR_SYSCALL, 
+			"Time inversion? %llu ... %llu\n",
+			(long long unsigned )iip->genesis, 
+			(long long unsigned )spec->time);
+
+	return 1;
+}
+
+/**
+ * bunch_output_hdr - Output bunch header
+ */
+static inline void bunch_output_hdr(struct io_stream *stream)
+{
+	struct io_bunch_hdr *hdrp = &stream->cur->hdr;
+
+	assert(0 < hdrp->npkts && hdrp->npkts <= BT_MAX_PKTS);
+	if (fwrite(hdrp, sizeof(struct io_bunch_hdr), 1, stream->ofp) != 1) {
+		fatal(stream->file_name, ERR_SYSCALL, "fwrite(hdr) failed\n");
+		/*NOTREACHED*/
+	}
+
+	if (verbose) {
+		__u64 off = hdrp->time_stamp - stream->iip->genesis;
+
+		assert(stream->vfp);
+		fprintf(stream->vfp, "------------------\n");
+		fprintf(stream->vfp, "%4llu.%09llu %3llu\n", 
+			(unsigned long long)off / (1000 * 1000 * 1000),
+			(unsigned long long)off % (1000 * 1000 * 1000), 
+			(unsigned long long)hdrp->npkts);
+		fprintf(stream->vfp, "------------------\n");
+	}
+}
+
+/**
+ * bunch_output_pkt - Output IO packets
+ */
+static inline void bunch_output_pkts(struct io_stream *stream)
+{
+	struct io_pkt *p = stream->cur->pkts;
+	size_t npkts = stream->cur->hdr.npkts;
+
+	assert(0 < npkts && npkts <= BT_MAX_PKTS);
+	if (fwrite(p, sizeof(struct io_pkt), npkts, stream->ofp) != npkts) {
+		fatal(stream->file_name, ERR_SYSCALL, "fwrite(pkts) failed\n");
+		/*NOTREACHED*/
+	}
+
+	if (verbose) {
+		size_t i;
+
+		assert(stream->vfp);
+		for (i = 0; i < npkts; i++, p++)
+			fprintf(stream->vfp, "\t%1d %10llu\t%10llu\n",
+				p->rw, 
+				(unsigned long long)p->sector,
+				(unsigned long long)p->nbytes / 512);
+	}
+}
+
+/**
+ * stream_flush - Flush current bunch of IOs out to the output stream
+ * @stream: Per-output file stream information
+ */
+static void stream_flush(struct io_stream *stream)
+{
+	struct io_bunch *cur = stream->cur;
+
+	if (cur) {
+		if (cur->hdr.npkts) {
+			assert(cur->hdr.npkts <= BT_MAX_PKTS);
+			bunch_output_hdr(stream);
+			bunch_output_pkts(stream);
+
+			stream->bunches++;
+			stream->pkts += cur->hdr.npkts;
+		}
+		free(cur);
+	}
+}
+
+/**
+ * bunch_done - Returns true if current bunch is either full, or next IO is late
+ * @stream: Output stream information
+ * @spec: IO trace specification
+ */
+static inline int bunch_done(struct io_stream *stream, struct io_spec *spec)
+{
+	if (stream->cur->hdr.npkts >= max_pkts_per_bunch)
+		return 1;
+
+	if ((spec->time - stream->start_time) > max_bunch_tm)
+		return 1;
+
+	return 0;
+}
+
+/**
+ * stream_add_io - Add an IO trace to the current stream
+ * @stream: Output stream information
+ * @spec: IO trace specification
+ */
+static void stream_add_io(struct io_stream *stream, struct io_spec *spec)
+{
+
+	if (stream->cur == NULL)
+		io_bunch_create(stream, spec->time);
+	else if (bunch_done(stream, spec)) {
+		stream_flush(stream);
+		io_bunch_create(stream, spec->time);
+	}
+
+	io_bunch_add(stream, spec);
+}
+
+/**
+ * stream_open - Open output stream for specified input stream
+ * @iip: Per-input file information
+ */
+static struct io_stream *stream_open(struct ifile_info *iip)
+{
+	char ofile_name[MAXPATHLEN];
+	struct io_stream *stream = malloc(sizeof(*stream));
+	struct io_file_hdr io_file_hdr = {
+		.genesis = 0,
+		.nbunches = 0,
+		.total_pkts = 0
+	};
+
+	memset(stream, 0, sizeof(*stream));
+
+	sprintf(ofile_name, "%s/%s.%s.%d", odir, iip->devnm, obase, iip->cpu);
+	stream->ofp = fopen(ofile_name, "w");
+	if (!stream->ofp) {
+		fatal(ofile_name, ERR_SYSCALL, "Open failed\n");
+		/*NOTREACHED*/
+	}
+
+	stream->iip = iip;
+	stream->cur = NULL;
+	stream->bunches = stream->pkts = 0;
+	stream->last_time = 0;
+	stream->file_name = strdup(ofile_name);
+
+	write_file_hdr(stream, &io_file_hdr);
+
+	if (verbose) {
+		char vfile_name[MAXPATHLEN];
+
+		sprintf(vfile_name, "%s/%s.%s.%d.rec", odir, iip->devnm,
+			obase, iip->cpu);
+		stream->vfp = fopen(vfile_name, "w");
+		if (!stream->vfp) {
+			fatal(vfile_name, ERR_SYSCALL, "Open failed\n");
+			/*NOTREACHED*/
+		}
+
+		stream->vfn = strdup(vfile_name);
+	}
+
+	data_is_native = -1;
+	return stream;
+}
+
+/**
+ * stream_close - Release resources associated with an output stream
+ * @stream: Stream to release
+ */
+static void stream_close(struct io_stream *stream)
+{
+	struct io_file_hdr io_file_hdr = {
+		.genesis = stream->iip->genesis,
+		.nbunches = stream->bunches,
+		.total_pkts = stream->pkts
+	};
+
+	stream_flush(stream);
+	write_file_hdr(stream, &io_file_hdr);
+	fclose(stream->ofp);
+
+	if (verbose && stream->bunches) {
+		fprintf(stderr, 
+			"%s:%d: %llu pkts (tot), %llu pkts (replay), "
+					"%llu bunches, %.1lf pkts/bunch\n",
+			stream->iip->devnm, stream->iip->cpu,
+			(unsigned long long)stream->iip->tpkts, 
+			(unsigned long long)stream->pkts, 
+			(unsigned long long)stream->bunches,
+			(double)(stream->pkts) / (double)(stream->bunches));
+
+		fclose(stream->vfp);
+		free(stream->vfn);
+	}
+
+	free(stream->file_name);
+	free(stream);
+}
+
+/**
+ * process - Process one input file to an output file
+ * @iip: Per-input file information
+ */
+static void process(struct ifile_info *iip)
+{
+	struct io_spec spec;
+	struct io_stream *stream;
+
+	stream = stream_open(iip);
+	while (next_io(iip, &spec))
+		stream_add_io(stream, &spec);
+	stream_close(stream);
+
+	rem_input_file(iip);
+}
+
+/**
+ * main - 
+ * @argc: Number of arguments
+ * @argv: Array of arguments
+ */
+int main(int argc, char *argv[])
+{
+	struct list_head *p, *q;
+
+	handle_args(argc, argv);
+	list_for_each_safe(p, q, &input_files)
+		process(list_entry(p, struct ifile_info, head));
+
+	return 0;
+}
diff --git a/btreplay/btrecord.h b/btreplay/btrecord.h
new file mode 100644
index 0000000..8026206
--- /dev/null
+++ b/btreplay/btrecord.h
@@ -0,0 +1,95 @@
+/*
+ * Blktrace record utility - Convert binary trace data into bunches of IOs
+ *
+ * Copyright (C) 2007 Alan D. Brunelle <Alan.Brunelle@hp.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#if !defined(__BTRECORD_H__)
+#define __BTRECORD_H__
+
+#include <asm/types.h>
+
+#define BT_MAX_PKTS	512
+
+/*
+ * Header for each bunch
+ *
+ * @nkts: 	Number of IO packets to process
+ * @time_stamp:	Time stamp for this bunch of IOs
+ */
+struct io_bunch_hdr {
+	__u64 npkts;
+	__u64 time_stamp;
+};
+
+/*
+ * IO specifer
+ *
+ * @sector:	Sector number of IO
+ * @nbytes:	Number of bytes to process
+ * @rw:		IO direction: 0 = write, 1 = read
+ */
+struct io_pkt {
+	__u64 sector;
+	__u64 nbytes;
+	__u32 rw;
+};
+
+/*
+ * Shorthand notion of a bunch of IOs
+ *
+ * @hdr: 	Header describing stall and how many IO packets follow
+ * @pkts: 	Individual IOs are described here
+ */
+struct io_bunch {
+	struct io_bunch_hdr hdr;
+	struct io_pkt pkts[BT_MAX_PKTS];
+};
+
+/*
+ * Header for each recorded file
+ *
+ * @version:	Version information
+ * @genesis:	Time stamp for earliest bunch
+ * @nbunches:	Number of bunches put into the file
+ * @total_pkts:	Number of packets to be processed
+ */
+struct io_file_hdr {
+	__u64 version;
+	__u64 genesis;
+	__u64 nbunches;
+	__u64 total_pkts;
+};
+
+static inline __u64 mk_btversion(int mjr, int mnr, int sub)
+{
+	return ((mjr & 0xff) << 16) | ((mnr & 0xff) << 8) | (sub & 0xff);
+}
+
+static inline void get_btversion(__u64 version, int *mjr, int *mnr, int *sub)
+{
+	*mjr = (int)((version >> 16) & 0xff);
+	*mnr = (int)((version >>  8) & 0xff);
+	*sub = (int)((version >>  0) & 0xff);
+}
+
+static char my_btversion[] = "0.9.3";
+static int btver_mjr = 0;
+static int btver_mnr = 9;
+static int btver_sub = 3;
+
+#endif
diff --git a/btreplay/btreplay.c b/btreplay/btreplay.c
new file mode 100644
index 0000000..8e6b7dc
--- /dev/null
+++ b/btreplay/btreplay.c
@@ -0,0 +1,1599 @@
+/*
+ * Blktrace replay utility - Play traces back
+ *
+ * Copyright (C) 2007 Alan D. Brunelle <Alan.Brunelle@hp.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+static char build_date[] = __DATE__ " at "__TIME__;
+
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libaio.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <stdarg.h>
+
+#if !defined(_GNU_SOURCE)
+#	define _GNU_SOURCE
+#endif
+#include <getopt.h>
+
+#include "list.h"
+#include "btrecord.h"
+
+/* 
+ * ========================================================================
+ * ==== STRUCTURE DEFINITIONS =============================================
+ * ========================================================================
+ */
+
+/**
+ * Each device map has one of these:
+ * 
+ * @head:	Linked on to map_devs
+ * @from_dev:	Device name as seen on recorded system
+ * @to_dev:	Device name to be used on replay system
+ */
+struct map_dev {
+	struct list_head head;
+	char *from_dev, *to_dev;
+};
+
+/**
+ * Each device name specified has one of these (until threads are created)
+ *
+ * @head: 	Linked onto input_devs
+ * @devnm: 	Device name -- 'sd*'
+ */
+struct dev_info {
+	struct list_head head;
+	char *devnm;
+};
+
+/*
+ * Per input file information
+ *
+ * @head: 	Used to link up on input_files
+ * @free_iocbs: List of free iocb's available for use
+ * @used_iocbs: List of iocb's currently outstanding
+ * @mutex: 	Mutex used with condition variable to protect volatile values
+ * @cond: 	Condition variable used when waiting on a volatile value change
+ * @naios_out: 	Current number of AIOs outstanding on this context
+ * @naios_free: Number of AIOs on the free list (short cut for list_len)
+ * @send_wait: 	Boolean: When true, the sub thread is waiting on free IOCBs
+ * @reap_wait: 	Boolean: When true, the rec thread is waiting on used IOCBs
+ * @send_done: 	Boolean: When true, the sub thread has completed work
+ * @reap_done: 	Boolean: When true, the rec thread has completed work
+ * @sub_thread: Thread used to submit IOs.
+ * @rec_thread: Thread used to reclaim IOs.
+ * @ctx: 	IO context
+ * @devnm: 	Copy of the device name being managed by this thread
+ * @file_name: 	Full name of the input file
+ * @cpu: 	CPU this thread is pinned to
+ * @ifd: 	Input file descriptor
+ * @ofd: 	Output file descriptor
+ * @iterations: Remaining iterations to process
+ * @vfp:	For verbose dumping of actions performed
+ */
+struct thr_info {
+	struct list_head head, free_iocbs, used_iocbs;
+	pthread_mutex_t mutex;
+	pthread_cond_t cond;
+	volatile long naios_out, naios_free;
+	volatile int send_wait, reap_wait, send_done, reap_done;
+	pthread_t sub_thread, rec_thread;
+	io_context_t ctx;
+	char *devnm, *file_name;
+	int cpu, ifd, ofd, iterations;
+	FILE *vfp;
+};
+
+/*
+ * Every Asynchronous IO used has one of these (naios per file/device).
+ *
+ * @iocb:	IOCB sent down via io_submit
+ * @head:	Linked onto file_list.free_iocbs or file_list.used_iocbs
+ * @tip:	Pointer to per-thread information this IO is associated with
+ * @nbytes:	Number of bytes in buffer associated with iocb
+ */
+struct iocb_pkt {
+	struct iocb iocb;
+	struct list_head head;
+	struct thr_info *tip;
+	int nbytes;
+};
+
+/* 
+ * ========================================================================
+ * ==== GLOBAL VARIABLES ==================================================
+ * ========================================================================
+ */
+
+static volatile int signal_done = 0;	// Boolean: Signal'ed, need to quit
+
+static char *ibase = "replay";		// Input base name
+static char *idir = ".";		// Input directory base
+static int cpus_to_use = -1;		// Number of CPUs to use
+static int def_iterations = 1;		// Default number of iterations
+static int naios = 512;			// Number of AIOs per thread
+static int ncpus = 0;			// Number of CPUs in the system
+static int verbose = 0;			// Boolean: Output some extra info
+static int write_enabled = 0;		// Boolean: Enable writing
+static __u64 genesis = ~0;		// Earliest time seen
+static __u64 rgenesis;			// Our start time
+static size_t pgsize;			// System Page size
+static int nb_sec = 512;		// Number of bytes per sector
+static LIST_HEAD(input_devs);		// List of devices to handle
+static LIST_HEAD(input_files);		// List of input files to handle
+static LIST_HEAD(map_devs);		// List of device maps
+static int nfiles = 0;			// Number of files to handle
+static int no_stalls = 0;		// Boolean: Disable pre-stalls
+static int find_records = 0;		// Boolean: Find record files auto
+
+/*
+ * Variables managed under control of condition variables.
+ *
+ * n_reclaims_done: 	Counts number of reclaim threads that have completed.
+ * n_replays_done:	Counts number of replay threads that have completed.
+ * n_replays_ready:	Counts number of replay threads ready to start.
+ * n_iters_done:	Counts number of replay threads done one iteration.
+ * iter_start:		Starts an iteration for the replay threads.
+ */
+static volatile int n_reclaims_done = 0;
+static pthread_mutex_t reclaim_done_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t reclaim_done_cond = PTHREAD_COND_INITIALIZER;
+
+static volatile int n_replays_done = 0;
+static pthread_mutex_t replay_done_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t replay_done_cond = PTHREAD_COND_INITIALIZER;
+
+static volatile int n_replays_ready = 0;
+static pthread_mutex_t replay_ready_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t replay_ready_cond = PTHREAD_COND_INITIALIZER;
+
+static volatile int n_iters_done = 0;
+static pthread_mutex_t iter_done_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t iter_done_cond = PTHREAD_COND_INITIALIZER;
+
+static volatile int iter_start = 0;
+static pthread_mutex_t iter_start_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t iter_start_cond = PTHREAD_COND_INITIALIZER;
+
+/* 
+ * ========================================================================
+ * ==== FORWARD REFERENECES ===============================================
+ * ========================================================================
+ */
+
+static void *replay_sub(void *arg);
+static void *replay_rec(void *arg);
+static char usage_str[];
+
+/* 
+ * ========================================================================
+ * ==== INLINE ROUTINES ===================================================
+ * ========================================================================
+ */
+
+/*
+ * The 'fatal' macro will output a perror message (if errstring is !NULL)
+ * and display a string (with variable arguments) and then exit with the 
+ * specified exit value.
+ */
+#define ERR_ARGS			1
+#define ERR_SYSCALL			2
+static inline void fatal(const char *errstring, const int exitval,
+			 const char *fmt, ...)
+{
+	va_list ap;
+
+	if (errstring)
+		perror(errstring);
+
+	va_start(ap, fmt);
+	vfprintf(stderr, fmt, ap);
+	va_end(ap);
+
+	exit(exitval);
+	/*NOTREACHED*/
+}
+
+static inline long long unsigned du64_to_sec(__u64 du64)
+{
+	return (long long unsigned)du64 / (1000 * 1000 * 1000);
+}
+
+static inline long long unsigned du64_to_nsec(__u64 du64)
+{
+	return llabs((long long)du64) % (1000 * 1000 * 1000);
+}
+
+/**
+ * min - Return minimum of two integers
+ */
+static inline int min(int a, int b)
+{ 
+	return a < b ? a : b;
+}
+
+/**
+ * minl - Return minimum of two longs
+ */
+static inline long minl(long a, long b)
+{ 
+	return a < b ? a : b;
+}
+
+/**
+ * usage - Display usage string and version
+ */
+static inline void usage(void)
+{
+	fprintf(stderr, "Usage: btreplay -- version %s\n%s", 
+		my_btversion, usage_str);
+}
+
+/**
+ * is_send_done - Returns true if sender should quit early
+ * @tip: Per-thread information
+ */
+static inline int is_send_done(struct thr_info *tip)
+{
+	return signal_done || tip->send_done;
+}
+
+/**
+ * is_reap_done - Returns true if reaper should quit early
+ * @tip: Per-thread information
+ */
+static inline int is_reap_done(struct thr_info *tip)
+{
+	return tip->send_done && tip->naios_out == 0;
+}
+
+/**
+ * ts2ns - Convert timespec values to a nanosecond value
+ */
+#define NS_TICKS		((__u64)1000 * (__u64)1000 * (__u64)1000)
+static inline __u64 ts2ns(struct timespec *ts)
+{
+	return ((__u64)(ts->tv_sec) * NS_TICKS) + (__u64)(ts->tv_nsec);
+}
+
+/**
+ * ts2ns - Convert timeval values to a nanosecond value
+ */
+static inline __u64 tv2ns(struct timeval *tp)
+{
+	return ((__u64)(tp->tv_sec)) + ((__u64)(tp->tv_usec) * (__u64)1000);
+}
+
+/**
+ * touch_memory - Force physical memory to be allocating it
+ * 
+ * For malloc()ed memory we need to /touch/ it to make it really
+ * exist. Otherwise, for write's (to storage) things may not work
+ * as planned - we see Linux just use a single area to /read/ from
+ * (as there isn't any memory that has been associated with the 
+ * allocated virtual addresses yet).
+ */
+static inline void touch_memory(char *buf, size_t bsize)
+{
+#if defined(PREP_BUFS)
+	memset(buf, 0, bsize);
+#else
+	size_t i;
+
+	for (i = 0; i < bsize; i += pgsize)
+		buf[i] = 0;
+#endif
+}
+
+/**
+ * buf_alloc - Returns a page-aligned buffer of the specified size
+ * @nbytes: Number of bytes to allocate
+ */
+static inline void *buf_alloc(size_t nbytes)
+{
+	void *buf;
+
+	if (posix_memalign(&buf, pgsize, nbytes)) {
+		fatal("posix_memalign", ERR_SYSCALL, "Allocation failed\n");
+		/*NOTREACHED*/
+	}
+
+	return buf;
+}
+
+/**
+ * gettime - Returns current time 
+ */
+static inline __u64 gettime(void)
+{
+	static int use_clock_gettime = -1;		// Which clock to use
+
+	if (use_clock_gettime < 0) {
+		use_clock_gettime = clock_getres(CLOCK_MONOTONIC, NULL) == 0;
+		if (use_clock_gettime) {
+			struct timespec ts = {
+				.tv_sec = 0,
+				.tv_nsec = 0
+			};
+			clock_settime(CLOCK_MONOTONIC, &ts);
+		}
+	}
+
+	if (use_clock_gettime) {
+		struct timespec ts;
+		clock_gettime(CLOCK_MONOTONIC, &ts);
+		return ts2ns(&ts);
+	}
+	else {
+		struct timeval tp;
+		gettimeofday(&tp, NULL);
+		return tv2ns(&tp);
+	}
+}
+
+/**
+ * setup_signal - Set up a signal handler for the specified signum
+ */
+static inline void setup_signal(int signum, sighandler_t handler)
+{
+	if (signal(signum, handler) == SIG_ERR) {
+		fatal("signal", ERR_SYSCALL, "Failed to set signal %d\n",
+			signum);
+		/*NOTREACHED*/
+	}
+}
+
+/* 
+ * ========================================================================
+ * ==== CONDITION VARIABLE ROUTINES =======================================
+ * ========================================================================
+ */
+
+/**
+ * __set_cv - Increments a variable under condition variable control.
+ * @pmp: 	Pointer to the associated mutex
+ * @pcp: 	Pointer to the associated condition variable
+ * @vp: 	Pointer to the variable being incremented
+ * @mxv: 	Max value for variable (Used only when ASSERTS are on)
+ */
+static inline void __set_cv(pthread_mutex_t *pmp, pthread_cond_t *pcp,
+			    volatile int *vp, 
+			    __attribute__((__unused__))int mxv)
+{
+	pthread_mutex_lock(pmp);
+	assert(*vp < mxv);
+	*vp += 1;
+	pthread_cond_signal(pcp);
+	pthread_mutex_unlock(pmp);
+}
+
+/**
+ * __wait_cv - Waits for a variable under cond var control to hit a value
+ * @pmp: 	Pointer to the associated mutex
+ * @pcp: 	Pointer to the associated condition variable
+ * @vp: 	Pointer to the variable being incremented
+ * @mxv: 	Value to wait for
+ */
+static inline void __wait_cv(pthread_mutex_t *pmp, pthread_cond_t *pcp,
+			     volatile int *vp, int mxv)
+{
+	pthread_mutex_lock(pmp);
+	while (*vp < mxv)
+		pthread_cond_wait(pcp, pmp);
+	*vp = 0;
+	pthread_mutex_unlock(pmp);
+}
+
+static inline void set_reclaim_done(void)
+{
+	__set_cv(&reclaim_done_mutex, &reclaim_done_cond, &n_reclaims_done,
+		 nfiles);
+}
+
+static inline void wait_reclaims_done(void)
+{
+	__wait_cv(&reclaim_done_mutex, &reclaim_done_cond, &n_reclaims_done,
+		  nfiles);
+}
+
+static inline void set_replay_ready(void)
+{
+	__set_cv(&replay_ready_mutex, &replay_ready_cond, &n_replays_ready,
+		 nfiles);
+}
+
+static inline void wait_replays_ready(void)
+{
+	__wait_cv(&replay_ready_mutex, &replay_ready_cond, &n_replays_ready,
+		  nfiles);
+}
+
+static inline void set_replay_done(void)
+{
+	__set_cv(&replay_done_mutex, &replay_done_cond, &n_replays_done,
+		nfiles);
+}
+
+static inline void wait_replays_done(void)
+{
+	__wait_cv(&replay_done_mutex, &replay_done_cond, &n_replays_done,
+		  nfiles);
+}
+
+static inline void set_iter_done(void)
+{
+	__set_cv(&iter_done_mutex, &iter_done_cond, &n_iters_done,
+		nfiles);
+}
+
+static inline void wait_iters_done(void)
+{
+	__wait_cv(&iter_done_mutex, &iter_done_cond, &n_iters_done,
+		  nfiles);
+}
+
+/**
+ * wait_iter_start - Wait for an iteration to start 
+ * 
+ * This is /slightly/ different: we are waiting for a value to become
+ * non-zero, and then we decrement it and go on. 
+ */
+static inline void wait_iter_start(void)
+{
+	pthread_mutex_lock(&iter_start_mutex);
+	while (iter_start == 0)
+		pthread_cond_wait(&iter_start_cond, &iter_start_mutex);
+	assert(1 <= iter_start && iter_start <= nfiles);
+	iter_start--;
+	pthread_mutex_unlock(&iter_start_mutex);
+}
+
+/**
+ * start_iter - Start an iteration at the replay thread level
+ */
+static inline void start_iter(void)
+{
+	pthread_mutex_lock(&iter_start_mutex);
+	assert(iter_start == 0);
+	iter_start = nfiles;
+	pthread_cond_broadcast(&iter_start_cond);
+	pthread_mutex_unlock(&iter_start_mutex);
+}
+
+/* 
+ * ========================================================================
+ * ==== CPU RELATED ROUTINES ==============================================
+ * ========================================================================
+ */
+
+/**
+ * get_ncpus - Sets up the global 'ncpus' value
+ */
+static void get_ncpus(void)
+{
+	cpu_set_t cpus;
+
+	if (sched_getaffinity(getpid(), sizeof(cpus), &cpus)) {
+		fatal("sched_getaffinity", ERR_SYSCALL, "Can't get CPU info\n");
+		/*NOTREACHED*/
+	}
+
+	/*
+	 * XXX This assumes (perhaps wrongly) that there are no /holes/ 
+	 * XXX in the mask.
+	 */
+	for (ncpus = 0; ncpus < CPU_SETSIZE && CPU_ISSET(ncpus, &cpus); ncpus++)
+		;
+	if (ncpus == 0) {
+		fatal(NULL, ERR_SYSCALL, "Insufficient number of CPUs\n");
+		/*NOTREACHED*/
+	}
+}
+
+/**
+ * pin_to_cpu - Pin this thread to a specific CPU
+ * @tip: Thread information
+ */
+static void pin_to_cpu(struct thr_info *tip)
+{
+	cpu_set_t cpus;
+
+	assert(0 <= tip->cpu && tip->cpu < ncpus);
+
+	CPU_ZERO(&cpus);
+	CPU_SET(tip->cpu, &cpus);
+	if (sched_setaffinity(getpid(), sizeof(cpus), &cpus)) {
+		fatal("sched_setaffinity", ERR_SYSCALL, "Failed to pin CPU\n");
+		/*NOTREACHED*/
+	}
+
+	if (verbose > 1) {
+		int i;
+		cpu_set_t now;
+
+		(void)sched_getaffinity(getpid(), sizeof(now), &now);
+		fprintf(tip->vfp, "Pinned to CPU %02d ", tip->cpu);
+		for (i = 0; i < ncpus; i++)
+			fprintf(tip->vfp, "%1d", CPU_ISSET(i, &now));
+		fprintf(tip->vfp, "\n");
+	}
+}
+
+/* 
+ * ========================================================================
+ * ==== INPUT DEVICE HANDLERS =============================================
+ * ========================================================================
+ */
+
+/**
+ * add_input_dev - Add a device ('sd*') to the list of devices to handle
+ */
+static void add_input_dev(char *devnm)
+{
+	struct list_head *p;
+	struct dev_info *dip;
+
+	__list_for_each(p, &input_devs) {
+		dip = list_entry(p, struct dev_info, head);
+		if (strcmp(dip->devnm, devnm) == 0)
+			return;
+	}
+
+	dip = malloc(sizeof(*dip));
+	dip->devnm = strdup(devnm);
+	list_add_tail(&dip->head, &input_devs);
+}
+
+/**
+ * rem_input_dev - Remove resources associated with this device
+ */
+static void rem_input_dev(struct dev_info *dip)
+{
+	list_del(&dip->head);
+	free(dip->devnm);
+	free(dip);
+}
+
+static void find_input_devs(char *idir)
+{
+	struct dirent *ent;
+	DIR *dir = opendir(idir);
+
+	if (dir == NULL) {
+		fatal(idir, ERR_ARGS, "Unable to open %s\n", idir);
+		/*NOTREACHED*/
+	}
+
+	while ((ent = readdir(dir)) != NULL) {
+		char *p, *dsf = malloc(256);
+
+		if (strstr(ent->d_name, ".replay.") == NULL)
+			continue;
+
+		dsf = strdup(ent->d_name);
+		p = index(dsf, '.');
+		assert(p != NULL);
+		*p = '\0';
+		add_input_dev(dsf);
+		free(dsf);
+	}
+
+	closedir(dir);
+}
+
+/* 
+ * ========================================================================
+ * ==== MAP DEVICE INTERFACES =============================================
+ * ========================================================================
+ */
+
+/**
+ * read_map_devs - Read in a set of device mapping from the provided file.
+ * @file_name:	File containing device maps
+ *
+ * We support the notion of multiple such files being specifed on the cmd line
+ */
+static void read_map_devs(char *file_name)
+{
+	FILE *fp;
+	char *from_dev, *to_dev;
+
+	fp = fopen(file_name, "r");
+	if (!fp) {
+		fatal(file_name, ERR_SYSCALL, "Could not open map devs file\n");
+		/*NOTREACHED*/
+	}
+
+	while (fscanf(fp, "%as %as", &from_dev, &to_dev) == 2) {
+		struct map_dev *mdp = malloc(sizeof(*mdp));
+
+		mdp->from_dev = from_dev;
+		mdp->to_dev = to_dev;
+		list_add_tail(&mdp->head, &map_devs);
+	}
+
+	fclose(fp);
+}
+
+/**
+ * release_map_devs - Release resources associated with device mappings.
+ */
+static void release_map_devs(void)
+{
+	struct list_head *p, *q;
+
+	list_for_each_safe(p, q, &map_devs) {
+		struct map_dev *mdp = list_entry(p, struct map_dev, head);
+
+		list_del(&mdp->head);
+
+		free(mdp->from_dev);
+		free(mdp->to_dev);
+		free(mdp);
+	}
+}
+
+/**
+ * map_dev - Return the mapped device for that specified
+ * @from_dev:	Device name as seen on recorded system
+ *
+ * Note: If there is no such mapping, we return the same name.
+ */
+static char *map_dev(char *from_dev)
+{
+	struct list_head *p;
+
+	__list_for_each(p, &map_devs) {
+		struct map_dev *mdp = list_entry(p, struct map_dev, head);
+
+		if (strcmp(from_dev, mdp->from_dev) == 0)
+			return mdp->to_dev;
+	}
+
+	return from_dev;
+}
+
+/* 
+ * ========================================================================
+ * ==== IOCB MANAGEMENT ROUTINES ==========================================
+ * ========================================================================
+ */
+
+/**
+ * iocb_init - Initialize the fields of an IOCB
+ * @tip: Per-thread information
+ * iocbp: IOCB pointer to update
+ */
+static void iocb_init(struct thr_info *tip, struct iocb_pkt *iocbp)
+{
+	iocbp->tip = tip;
+	iocbp->nbytes = 0;
+	iocbp->iocb.u.c.buf = NULL;
+}
+
+/**
+ * iocb_setup - Set up an iocb with this AIOs information
+ * @iocbp: IOCB pointer to update
+ * @rw: Direction (0 == write, 1 == read)
+ * @n: Number of bytes to transfer
+ * @off: Offset (in bytes)
+ */
+static void iocb_setup(struct iocb_pkt *iocbp, int rw, int n, long long off)
+{
+	char *buf;
+	struct iocb *iop = &iocbp->iocb;
+
+	assert(rw == 0 || rw == 1);
+	assert(0 < n && (n % nb_sec) == 0);
+	assert(0 <= off);
+
+	if (iocbp->nbytes) {
+		if (iocbp->nbytes >= n) {
+			buf = iop->u.c.buf;
+			goto prep;
+		}
+
+		assert(iop->u.c.buf);
+		free(iop->u.c.buf);
+	}
+
+	buf = buf_alloc(n);
+	iocbp->nbytes = n;
+
+prep:
+	if (rw)
+		io_prep_pread(iop, iocbp->tip->ofd, buf, n, off);
+	else {
+		assert(write_enabled);
+		io_prep_pwrite(iop, iocbp->tip->ofd, buf, n, off);
+		touch_memory(buf, n);
+	}
+
+	iop->data = iocbp;
+}
+
+/* 
+ * ========================================================================
+ * ==== PER-THREAD SET UP & TEAR DOWN =====================================
+ * ========================================================================
+ */
+
+/**
+ * tip_init - Per thread initialization function
+ */
+static void tip_init(struct thr_info *tip)
+{
+	int i;
+
+	INIT_LIST_HEAD(&tip->free_iocbs);
+	INIT_LIST_HEAD(&tip->used_iocbs);
+
+	pthread_mutex_init(&tip->mutex, NULL);
+	pthread_cond_init(&tip->cond, NULL);
+
+	if (io_setup(naios, &tip->ctx)) {
+		fatal("io_setup", ERR_SYSCALL, "io_setup failed\n");
+		/*NOTREACHED*/
+	}
+
+	tip->ofd = -1;
+	tip->naios_out = 0;
+	tip->send_done = tip->reap_done = 0;
+	tip->send_wait = tip->reap_wait = 0;
+
+	memset(&tip->sub_thread, 0, sizeof(tip->sub_thread));
+	memset(&tip->rec_thread, 0, sizeof(tip->rec_thread));
+
+	for (i = 0; i < naios; i++) {
+		struct iocb_pkt *iocbp = buf_alloc(sizeof(*iocbp));
+
+		iocb_init(tip, iocbp);
+		list_add_tail(&iocbp->head, &tip->free_iocbs);
+	}
+	tip->naios_free = naios;
+
+	if (verbose > 1) {
+		char fn[MAXPATHLEN];
+
+		sprintf(fn, "%s/%s.%s.%d.rep", idir, tip->devnm, ibase, 
+			tip->cpu);
+		tip->vfp = fopen(fn, "w");
+		if (!tip->vfp) {
+			fatal(fn, ERR_SYSCALL, "Failed to open report\n");
+			/*NOTREACHED*/
+		}
+
+		setlinebuf(tip->vfp);
+	}
+
+	if (pthread_create(&tip->sub_thread, NULL, replay_sub, tip)) {
+		fatal("pthread_create", ERR_SYSCALL, 
+			"thread create failed\n");
+		/*NOTREACHED*/
+	}
+
+	if (pthread_create(&tip->rec_thread, NULL, replay_rec, tip)) {
+		fatal("pthread_create", ERR_SYSCALL, 
+			"thread create failed\n");
+		/*NOTREACHED*/
+	}
+}
+
+/**
+ * tip_release - Release resources associated with this thread
+ */
+static void tip_release(struct thr_info *tip)
+{
+	struct list_head *p, *q;
+
+	assert(tip->send_done);
+	assert(tip->reap_done);
+	assert(list_len(&tip->used_iocbs) == 0);
+	assert(tip->naios_free == naios);
+
+	if (pthread_join(tip->sub_thread, NULL)) {
+		fatal("pthread_join", ERR_SYSCALL, "pthread sub join failed\n");
+		/*NOTREACHED*/
+	}
+	if (pthread_join(tip->rec_thread, NULL)) {
+		fatal("pthread_join", ERR_SYSCALL, "pthread rec join failed\n");
+		/*NOTREACHED*/
+	}
+
+	io_destroy(tip->ctx);
+
+	list_splice(&tip->used_iocbs, &tip->free_iocbs);
+	list_for_each_safe(p, q, &tip->free_iocbs) {
+		struct iocb_pkt *iocbp = list_entry(p, struct iocb_pkt, head);
+
+		list_del(&iocbp->head);
+		if (iocbp->nbytes) 
+			free(iocbp->iocb.u.c.buf);
+		free(iocbp);
+	}
+
+	pthread_cond_destroy(&tip->cond);
+	pthread_mutex_destroy(&tip->mutex);
+}
+
+/**
+ * add_input_file - Allocate and initialize per-input file structure
+ * @cpu: CPU for this file
+ * @devnm: Device name for this file
+ * @file_name: Fully qualifed input file name
+ */
+static void add_input_file(int cpu, char *devnm, char *file_name)
+{
+	struct stat buf;
+	struct io_file_hdr hdr;
+	struct thr_info *tip = buf_alloc(sizeof(*tip));
+	__u64 my_version = mk_btversion(btver_mjr, btver_mnr, btver_sub);
+
+	assert(0 <= cpu && cpu < ncpus);
+
+	memset(&hdr, 0, sizeof(hdr));
+	memset(tip, 0, sizeof(*tip));
+	tip->cpu = cpu % cpus_to_use;
+	tip->iterations = def_iterations;
+
+	tip->ifd = open(file_name, O_RDONLY);
+	if (tip->ifd < 0) {
+		fatal(file_name, ERR_ARGS, "Unable to open\n");
+		/*NOTREACHED*/
+	}
+	if (fstat(tip->ifd, &buf) < 0) {
+		fatal(file_name, ERR_SYSCALL, "fstat failed\n");
+		/*NOTREACHED*/
+	}
+	if (buf.st_size < (off_t)sizeof(hdr)) {
+		if (verbose)
+			fprintf(stderr, "\t%s empty\n", file_name);
+		goto empty_file;
+	}
+
+	if (read(tip->ifd, &hdr, sizeof(hdr)) != sizeof(hdr)) {
+		fatal(file_name, ERR_ARGS, "Header read failed\n");
+		/*NOTREACHED*/
+	}
+
+	if (hdr.version != my_version) {
+		fprintf(stderr, "%llx %llx %llx %llx\n", 
+			(long long unsigned)hdr.version,
+			(long long unsigned)hdr.genesis,
+			(long long unsigned)hdr.nbunches,
+			(long long unsigned)hdr.total_pkts);
+		fatal(NULL, ERR_ARGS, 
+			"BT version mismatch: %lx versus my %lx\n",
+			(long)hdr.version, (long)my_version);
+			
+	}
+
+	if (hdr.nbunches == 0) {
+empty_file:
+		close(tip->ifd);
+		free(tip);
+		return;
+	}
+
+	if (hdr.genesis < genesis) {
+		if (verbose > 1)
+			fprintf(stderr, "Setting genesis to %llu.%llu\n",
+				du64_to_sec(hdr.genesis),
+				du64_to_nsec(hdr.genesis));
+		genesis = hdr.genesis;
+	}
+
+	tip->devnm = strdup(devnm);
+	tip->file_name = strdup(file_name);
+
+	list_add_tail(&tip->head, &input_files);
+
+	if (verbose)
+		fprintf(stderr, "Added %s %llu\n", file_name, 
+			(long long)hdr.genesis);
+}
+
+/**
+ * rem_input_file - Release resources associated with an input file
+ * @tip: Per-input file information
+ */
+static void rem_input_file(struct thr_info *tip)
+{
+	list_del(&tip->head);
+
+	tip_release(tip);
+
+	close(tip->ofd);
+	close(tip->ifd);
+	free(tip->file_name);
+	free(tip->devnm);
+	free(tip);
+}
+
+/**
+ * rem_input_files - Remove all input files
+ */
+static void rem_input_files(void)
+{
+	struct list_head *p, *q;
+
+	list_for_each_safe(p, q, &input_files) {
+		rem_input_file(list_entry(p, struct thr_info, head));
+	}
+}
+
+/**
+ * __find_input_files - Find input files associated with this device (per cpu)
+ */
+static void __find_input_files(struct dev_info *dip)
+{
+	int cpu = 0;
+
+	for (;;) {
+		char full_name[MAXPATHLEN];
+
+		sprintf(full_name, "%s/%s.%s.%d", idir, dip->devnm, ibase, cpu);
+		if (access(full_name, R_OK) != 0)
+			break;
+
+		add_input_file(cpu, dip->devnm, full_name);
+		cpu++;
+	}
+
+	if (!cpu) {
+		fatal(NULL, ERR_ARGS, "No traces found for %s\n", dip->devnm);
+		/*NOTREACHED*/
+	}
+
+	rem_input_dev(dip);
+}
+
+
+/**
+ * find_input_files - Find input files for all devices
+ */
+static void find_input_files(void)
+{
+	struct list_head *p, *q;
+
+	list_for_each_safe(p, q, &input_devs) {
+		__find_input_files(list_entry(p, struct dev_info, head));
+	}
+}
+
+/* 
+ * ========================================================================
+ * ==== RECLAIM ROUTINES ==================================================
+ * ========================================================================
+ */
+
+/**
+ * reap_wait_aios - Wait for and return number of outstanding AIOs
+ *
+ * Will return 0 if we are done
+ */
+static int reap_wait_aios(struct thr_info *tip)
+{
+	int naios = 0;
+
+	if (!is_reap_done(tip)) {
+		pthread_mutex_lock(&tip->mutex);
+		while (tip->naios_out == 0) {
+			tip->reap_wait = 1;
+			if (pthread_cond_wait(&tip->cond, &tip->mutex)) {
+				fatal("pthread_cond_wait", ERR_SYSCALL, 
+					"nfree_current cond wait failed\n");
+				/*NOTREACHED*/
+			}
+		}
+		naios = tip->naios_out;
+		pthread_mutex_unlock(&tip->mutex);
+	}
+	assert(is_reap_done(tip) || naios > 0);
+
+	return is_reap_done(tip) ? 0 : naios;
+}
+
+/**
+ * reclaim_ios - Reclaim AIOs completed, recycle IOCBs
+ * @tip: Per-thread information
+ * @naios_out: Number of AIOs we have outstanding (min)
+ */
+static void reclaim_ios(struct thr_info *tip, long naios_out)
+{
+	long i, ndone;
+	struct io_event *evp, events[naios_out];
+
+again:
+	assert(naios > 0);
+	for (;;) {
+		ndone = io_getevents(tip->ctx, 1, naios_out, events, NULL);
+		if (ndone > 0)
+			break;
+
+		if (errno && errno != EINTR) {
+			fatal("io_getevents", ERR_SYSCALL, 
+				"io_getevents failed\n");
+			/*NOTREACHED*/
+		}
+	}
+	assert(0 < ndone && ndone <= naios_out);
+
+	pthread_mutex_lock(&tip->mutex);
+	for (i = 0, evp = events; i < ndone; i++, evp++) {
+		struct iocb_pkt *iocbp = evp->data;
+
+                if (evp->res != iocbp->iocb.u.c.nbytes) {
+                        fatal(NULL, ERR_SYSCALL,
+                              "Event failure %ld/%ld\t(%ld + %ld)\n",
+                              (long)evp->res, (long)evp->res2,
+                              (long)iocbp->iocb.u.c.offset / nb_sec, 
+			      (long)iocbp->iocb.u.c.nbytes / nb_sec);
+                        /*NOTREACHED*/
+                }
+
+		list_move_tail(&iocbp->head, &tip->free_iocbs);
+	}
+
+	tip->naios_free += ndone;
+	tip->naios_out -= ndone;
+	naios_out = minl(naios_out, tip->naios_out);
+
+	if (tip->send_wait) {
+		tip->send_wait = 0;
+		pthread_cond_signal(&tip->cond);
+	}
+	pthread_mutex_unlock(&tip->mutex);
+
+	/*
+	 * Short cut: If we /know/ there are some more AIOs, go handle them
+	 */
+	if (naios_out)
+		goto again;
+}
+
+/**
+ * replay_rec - Worker thread to reclaim AIOs
+ * @arg: Pointer to thread information
+ */
+static void *replay_rec(void *arg)
+{
+	long naios_out;
+	struct thr_info *tip = arg;
+
+	while ((naios_out = reap_wait_aios(tip)) > 0) 
+		reclaim_ios(tip, naios_out);
+
+	assert(tip->send_done);
+	tip->reap_done = 1;
+	set_reclaim_done();
+
+	return NULL;
+}
+
+/* 
+ * ========================================================================
+ * ==== REPLAY ROUTINES ===================================================
+ * ========================================================================
+ */
+
+/**
+ * next_bunch - Retrieve next bunch of AIOs to process
+ * @tip: Per-thread information
+ * @bunch: Bunch information
+ *
+ * Returns TRUE if we recovered a bunch of IOs, else hit EOF
+ */
+static int next_bunch(struct thr_info *tip, struct io_bunch *bunch)
+{
+	size_t count, result;
+	
+	result = read(tip->ifd, &bunch->hdr, sizeof(bunch->hdr));
+	if (result != sizeof(bunch->hdr)) {
+		if (result == 0)
+			return 0;
+
+		fatal(tip->file_name, ERR_SYSCALL, "Short hdr(%ld)\n", 
+			(long)result);
+		/*NOTREACHED*/
+	}
+	assert(bunch->hdr.npkts <= BT_MAX_PKTS);
+
+	count = bunch->hdr.npkts * sizeof(struct io_pkt);
+	result = read(tip->ifd, &bunch->pkts, count);
+	if (result != count) {
+		fatal(tip->file_name, ERR_SYSCALL, "Short pkts(%ld/%ld)\n", 
+			(long)result, (long)count);
+		/*NOTREACHED*/
+	}
+
+	return 1;
+}
+
+/**
+ * nfree_current - Returns current number of AIOs that are free
+ *
+ * Will wait for available ones...
+ *
+ * Returns 0 if we have some condition that causes us to exit
+ */
+static int nfree_current(struct thr_info *tip)
+{
+	int nfree = 0;
+
+	pthread_mutex_lock(&tip->mutex);
+	while (!is_send_done(tip) && ((nfree = tip->naios_free) == 0)) {
+		tip->send_wait = 1;
+		if (pthread_cond_wait(&tip->cond, &tip->mutex)) {
+			fatal("pthread_cond_wait", ERR_SYSCALL, 
+				"nfree_current cond wait failed\n");
+			/*NOTREACHED*/
+		}
+	}
+	pthread_mutex_unlock(&tip->mutex);
+
+	return nfree;
+}
+
+/**
+ * stall - Stall for the number of nanoseconds requested
+ *
+ * We may be late, in which case we just return.
+ */
+static void stall(struct thr_info *tip, long long oclock)
+{
+	struct timespec req;
+	long long dreal, tclock = gettime() - rgenesis;
+
+	if (verbose > 1)
+		fprintf(tip->vfp, "   stall(%lld.%09lld, %lld.%09lld)\n",
+			du64_to_sec(oclock), du64_to_nsec(oclock),
+			du64_to_sec(tclock), du64_to_nsec(tclock));
+
+	while (!is_send_done(tip) && tclock < oclock) {
+		dreal = oclock - tclock;
+		req.tv_sec = dreal / (1000 * 1000 * 1000);
+		req.tv_nsec = dreal % (1000 * 1000 * 1000);
+
+		if (verbose > 1) {
+			fprintf(tip->vfp, "++ stall(%lld.%09lld) ++\n",
+				(long long)req.tv_sec,
+				(long long)req.tv_nsec);
+		}
+
+		if (nanosleep(&req, NULL) < 0 && signal_done)
+			break;
+
+		tclock = gettime() - rgenesis;
+	}
+}
+
+/**
+ * iocbs_map - Map a set of AIOs onto a set of IOCBs
+ * @tip: Per-thread information
+ * @list: List of AIOs created
+ * @pkts: AIOs to map
+ * @ntodo: Number of AIOs to map
+ */
+static void iocbs_map(struct thr_info *tip, struct iocb **list, 
+					     struct io_pkt *pkts, int ntodo)
+{
+	int i;
+	struct io_pkt *pkt;
+
+	assert(0 < ntodo && ntodo <= naios);
+
+	pthread_mutex_lock(&tip->mutex);
+	assert(ntodo <= list_len(&tip->free_iocbs));
+	for (i = 0, pkt = pkts; i < ntodo; i++, pkt++) {
+		__u32 rw = pkt->rw;
+		struct iocb_pkt *iocbp;
+
+		if (!pkt->rw && !write_enabled)
+			rw = 1;
+
+		if (verbose > 1)
+			fprintf(tip->vfp, "\t%10llu + %10llu %c%c\n",
+				(unsigned long long)pkt->sector, 
+				(unsigned long long)pkt->nbytes / nb_sec,
+				rw ? 'R' : 'W', 
+				(rw == 1 && pkt->rw == 0) ? '!' : ' ');
+		
+		iocbp = list_entry(tip->free_iocbs.next, struct iocb_pkt, head);
+		iocb_setup(iocbp, rw, pkt->nbytes, pkt->sector * nb_sec);
+
+		list_move_tail(&iocbp->head, &tip->used_iocbs);
+		list[i] = &iocbp->iocb;
+	}
+
+	tip->naios_free -= ntodo;
+	assert(tip->naios_free >= 0);
+	pthread_mutex_unlock(&tip->mutex);
+}
+
+/**
+ * process_bunch - Process a bunch of requests
+ * @tip: Per-thread information
+ * @bunch: Bunch to process
+ */
+static void process_bunch(struct thr_info *tip, struct io_bunch *bunch)
+{
+	__u64 i = 0;
+	struct iocb *list[bunch->hdr.npkts];
+
+	assert(0 < bunch->hdr.npkts && bunch->hdr.npkts <= BT_MAX_PKTS);
+	while (!is_send_done(tip) && (i < bunch->hdr.npkts)) {
+		long ndone;
+		int ntodo = min(nfree_current(tip), bunch->hdr.npkts - i);
+
+		assert(0 < ntodo && ntodo <= naios);
+		iocbs_map(tip, list, &bunch->pkts[i], ntodo);
+		if (!no_stalls)
+			stall(tip, bunch->hdr.time_stamp - genesis);
+
+		if (ntodo) {
+			if (verbose > 1)
+				fprintf(tip->vfp, "submit(%d)\n", ntodo);
+			ndone = io_submit(tip->ctx, ntodo, list);
+			if (ndone != (long)ntodo) {
+				fatal("io_submit", ERR_SYSCALL,
+					"%d: io_submit(%d:%ld) failed (%s)\n", 
+					tip->cpu, ntodo, ndone, 
+					strerror(labs(ndone)));
+				/*NOTREACHED*/
+			}
+
+			pthread_mutex_lock(&tip->mutex);
+			tip->naios_out += ndone;
+			assert(tip->naios_out <= naios);
+			if (tip->reap_wait) {
+				tip->reap_wait = 0;
+				pthread_cond_signal(&tip->cond);
+			}
+			pthread_mutex_unlock(&tip->mutex);
+
+			i += ndone;
+			assert(i <= bunch->hdr.npkts);
+		}
+	}
+}
+
+/**
+ * reset_input_file - Reset the input file for the next iteration
+ * @tip: Thread information
+ *
+ * We also do a dummy read of the file header to get us to the first bunch.
+ */
+static void reset_input_file(struct thr_info *tip)
+{
+	struct io_file_hdr hdr;
+
+	lseek(tip->ifd, 0, 0);
+
+	if (read(tip->ifd, &hdr, sizeof(hdr)) != sizeof(hdr)) {
+		fatal(tip->file_name, ERR_ARGS, "Header reread failed\n");
+		/*NOTREACHED*/
+	}
+}
+
+/**
+ * replay_sub - Worker thread to submit AIOs that are being replayed
+ */
+static void *replay_sub(void *arg)
+{
+	char path[MAXPATHLEN];
+	struct io_bunch bunch;
+	struct thr_info *tip = arg;
+
+	pin_to_cpu(tip);
+
+	sprintf(path, "/dev/%s", map_dev(tip->devnm));
+	tip->ofd = open(path, O_RDWR | O_DIRECT | O_NOATIME);
+	if (tip->ofd < 0) {
+		fatal(path, ERR_SYSCALL, "Failed device open\n");
+		/*NOTREACHED*/
+	}
+
+	set_replay_ready();
+	while (!is_send_done(tip) && tip->iterations--) {
+		wait_iter_start();
+		if (verbose > 1)
+			fprintf(tip->vfp, "\n=== %d ===\n", tip->iterations);
+		while (!is_send_done(tip) && next_bunch(tip, &bunch))
+			process_bunch(tip, &bunch);
+		set_iter_done();
+		reset_input_file(tip);
+	}
+	tip->send_done = 1;
+	set_replay_done();
+
+	return NULL;
+}
+
+/* 
+ * ========================================================================
+ * ==== COMMAND LINE ARGUMENT HANDLING ====================================
+ * ========================================================================
+ */
+
+static char usage_str[] = 						\
+        "\n"                                                           \
+        "\t[ -c <cpus> : --cpus=<cpus>           ] Default: 1\n"        \
+        "\t[ -d <dir>  : --input-directory=<dir> ] Default: .\n"        \
+	"\t[ -F        : --find-records          ] Default: Off\n"         \
+        "\t[ -h        : --help                  ] Default: Off\n"      \
+        "\t[ -i <base> : --input-base=<base>     ] Default: replay\n"   \
+        "\t[ -I <iters>: --iterations=<iters>    ] Default: 1\n"        \
+        "\t[ -M <file> : --map-devs=<file>       ] Default: None\n"     \
+        "\t[ -N        : --no-stalls             ] Default: Off\n"      \
+        "\t[ -v        : --verbose               ] Default: Off\n"      \
+        "\t[ -V        : --version               ] Default: Off\n"      \
+        "\t[ -W        : --write-enable          ] Default: Off\n"      \
+        "\t<dev...>                                Default: None\n"     \
+        "\n";
+
+#define S_OPTS	"c:d:Fhi:I:M:Nt:vVW"
+static struct option l_opts[] = {
+	{
+		.name = "cpus",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'c'
+	},
+	{
+		.name = "input-directory",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'd'
+	},
+	{
+		.name = "find-records",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'F'
+	},
+	{
+		.name = "help",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'h'
+	},
+	{
+		.name = "input-base",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'i'
+	},
+	{
+		.name = "iterations",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'I'
+	},
+	{
+		.name = "map-devs",
+		.has_arg = required_argument,
+		.flag = NULL,
+		.val = 'M'
+	},
+	{
+		.name = "no-stalls",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'N'
+	},
+	{
+		.name = "verbose",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'v'
+	},
+	{
+		.name = "version",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'V'
+	},
+	{
+		.name = "write-enable",
+		.has_arg = no_argument,
+		.flag = NULL,
+		.val = 'W'
+	},
+	{
+		.name = NULL
+	}
+};
+
+/**
+ * handle_args: Parse passed in argument list
+ * @argc: Number of arguments in argv
+ * @argv: Arguments passed in
+ *
+ * Does rudimentary parameter verification as well.
+ */
+static void handle_args(int argc, char *argv[])
+{
+	int c;
+
+	while ((c = getopt_long(argc, argv, S_OPTS, l_opts, NULL)) != -1) {
+		switch (c) {
+		case 'c': 
+			cpus_to_use = atoi(optarg);
+			if (cpus_to_use <= 0 || cpus_to_use > ncpus) {
+				fatal(NULL, ERR_ARGS, 
+				      "Invalid number of cpus %d (0<x<%d)\n",
+				      cpus_to_use, ncpus);
+				/*NOTREACHED*/
+			}
+			break;
+
+		case 'd':
+			idir = optarg;
+			if (access(idir, R_OK | X_OK) != 0) {
+				fatal(idir, ERR_ARGS, 
+				      "Invalid input directory specified\n");
+				/*NOTREACHED*/
+			}
+			break;
+
+		case 'F': 
+			find_records = 1;
+			break;
+
+		case 'h': 
+			usage(); 
+			exit(0);
+			/*NOTREACHED*/
+
+		case 'i': 
+			ibase = optarg;
+			break;
+
+		case 'I':
+			def_iterations = atoi(optarg);
+			if (def_iterations <= 0) {
+				fprintf(stderr, 
+					"Invalid number of iterations %d\n",
+					def_iterations);
+				exit(ERR_ARGS);
+				/*NOTREACHED*/
+			}
+			break;
+
+		case 'M':
+			read_map_devs(optarg);
+			break;
+
+		case 'N':
+			no_stalls = 1;
+			break;
+
+		case 'V':
+			fprintf(stderr, "btreplay -- version %s\n", 
+				my_btversion);
+			fprintf(stderr, "            Built on %s\n", 
+				build_date);
+			exit(0);
+			/*NOTREACHED*/
+
+		case 'v':
+			verbose++;
+			break;
+
+		case 'W':
+			write_enabled = 1;
+			break;
+
+		default:
+			usage();
+			fatal(NULL, ERR_ARGS, 
+			      "Invalid command line argument %c\n", c);
+			/*NOTREACHED*/
+		}
+	}
+
+	while (optind < argc)
+		add_input_dev(argv[optind++]);
+
+	if (find_records)
+		find_input_devs(idir);
+
+	if (list_len(&input_devs) == 0) {
+		fatal(NULL, ERR_ARGS, "Missing required input dev name(s)\n");
+		/*NOTREACHED*/
+	}
+
+	if (cpus_to_use < 0)
+		cpus_to_use = ncpus;
+}
+
+/* 
+ * ========================================================================
+ * ==== MAIN ROUTINE ======================================================
+ * ========================================================================
+ */
+
+/**
+ * set_signal_done - Signal handler, catches signals & sets signal_done
+ */
+static void set_signal_done(__attribute__((__unused__))int signum)
+{
+	signal_done = 1;
+}
+
+/**
+ * main - 
+ * @argc: Number of arguments
+ * @argv: Array of arguments
+ */
+int main(int argc, char *argv[])
+{
+	int i;
+	struct list_head *p;
+
+	pgsize = getpagesize();
+	assert(pgsize > 0);
+
+	setup_signal(SIGINT, set_signal_done);
+	setup_signal(SIGTERM, set_signal_done);
+
+	get_ncpus();
+	handle_args(argc, argv);
+	find_input_files();
+
+	nfiles = list_len(&input_files);
+	__list_for_each(p, &input_files) {
+		tip_init(list_entry(p, struct thr_info, head));
+	}
+
+	wait_replays_ready();
+	for (i = 0; i < def_iterations; i++) {
+		rgenesis = gettime();
+		start_iter();
+		if (verbose)
+			fprintf(stderr, "I");
+		wait_iters_done();
+	}
+
+	wait_replays_done();
+	wait_reclaims_done();
+
+	if (verbose)
+		fprintf(stderr, "\n");
+
+	rem_input_files();
+	release_map_devs();
+
+	return 0;
+}
diff --git a/btreplay/doc/Makefile b/btreplay/doc/Makefile
new file mode 100644
index 0000000..e3b383e
--- /dev/null
+++ b/btreplay/doc/Makefile
@@ -0,0 +1,18 @@
+DOCTMP  = btreplay.log btreplay.aux btreplay.dvi btreplay.toc
+
+all: btreplay.dvi btreplay.pdf
+
+btreplay.tex:
+	@touch btreplay.tex
+
+btreplay.dvi: btreplay.tex abstract.tex
+	@latex btreplay.tex
+	@latex btreplay.tex
+
+btreplay.pdf: btreplay.dvi
+	@dvipdfm -p letter btreplay
+
+clean:
+	-rm -f $(DOCTMP)
+	-rm -f *.bak *.ps *.pdf
+	@rm -rf btreplay
diff --git a/btreplay/doc/abstract.tex b/btreplay/doc/abstract.tex
new file mode 100644
index 0000000..314d820
--- /dev/null
+++ b/btreplay/doc/abstract.tex
@@ -0,0 +1,34 @@
+%
+% Copyright (C) 2007 Alan D. Brunelle <Alan.Brunelle@hp.com>
+%
+%  This program is free software; you can redistribute it and/or modify
+%  it under the terms of the GNU General Public License as published by
+%  the Free Software Foundation; either version 2 of the License, or
+%  (at your option) any later version.
+%
+%  This program is distributed in the hope that it will be useful,
+%  but WITHOUT ANY WARRANTY; without even the implied warranty of
+%  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+%  GNU General Public License for more details.
+%
+%  You should have received a copy of the GNU General Public License
+%  along with this program; if not, write to the Free Software
+%  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+%
+%  vi :set textwidth=75
+%
+The \texttt{btrecord} and \texttt{btreplay} tools provide the ability to
+record and replay IOs captured by the \texttt{blktrace} utility. Attempts
+are made to maintain ordering, CPU mappings and time-separation of IOs. The
+general workflow is expected to be:
+
+\begin{enumerate}
+  \item Initiate \texttt{blktrace} to capture traces
+  \item Generate traces\ldots
+  \item Stop \texttt{blktrace}
+  \item Run \texttt{btrecord} to convert traces into IO records 
+  \item Utilize \texttt{btreplay} to replay IOs 
+\end{enumerate}
+
+This document will discuss the operating characteristics of
+\texttt{btreplay} and provide detailed command line option descriptions.
diff --git a/btreplay/doc/btreplay.tex b/btreplay/doc/btreplay.tex
new file mode 100644
index 0000000..beec720
--- /dev/null
+++ b/btreplay/doc/btreplay.tex
@@ -0,0 +1,521 @@
+%
+% Copyright (C) 2007 Alan D. Brunelle <Alan.Brunelle@hp.com>
+%
+%  This program is free software; you can redistribute it and/or modify
+%  it under the terms of the GNU General Public License as published by
+%  the Free Software Foundation; either version 2 of the License, or
+%  (at your option) any later version.
+%
+%  This program is distributed in the hope that it will be useful,
+%  but WITHOUT ANY WARRANTY; without even the implied warranty of
+%  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+%  GNU General Public License for more details.
+%
+%  You should have received a copy of the GNU General Public License
+%  along with this program; if not, write to the Free Software
+%  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+%
+%  vi :set textwidth=75
+%
+\documentclass{article}
+\usepackage{multirow,graphicx,placeins}
+
+\begin{document}
+%---------------------
+\title{\texttt{btrecord} and \texttt{btreplay} User Guide}
+\author{Alan D. Brunelle (Alan.Brunelle@hp.com)}
+\date{\today}
+\maketitle
+\begin{abstract}
+\input{abstract.tex}
+\end{abstract}
+\thispagestyle{empty}\newpage
+%---------------------
+\tableofcontents\thispagestyle{empty}\newpage
+%---------------------
+\section{Introduction}
+\input{abstract.tex}
+
+\bigskip 
+This document presents the command line overview for
+\texttt{btrecord} and \texttt{btreplay}, and shows some commonly used
+example usages of it in everyday work here at OSLO's Scalability and
+Performance Group.
+
+\subsection*{Build Note}
+
+To build these tools, one needs to
+place the source directory next to a valid
+\texttt{blktrace}\footnote{\texttt{git://git.kernel.dk/blktrace.git}}
+directory, as it includes \texttt{../blktrace} in the \texttt{Makefile}.
+
+
+%---------------------
+\newpage\section{\texttt{btrecord} and \texttt{btreplay} Operating Model}
+
+The \texttt{blktrace} utility provides the ability to collect detailed
+traces from the kernel for each IO processed by the block IO layer. The
+traces provide a complete timeline for each IO processed, including
+detailed information concerning when an IO was first received by the block
+IO layer -- indicating the device, CPU number, time stamp, IO direction,
+sector number and IO size (number of sectors). Using this information,
+one is able to \emph{replay} the IO again on the same machine or another
+set up entirely.
+
+\subsection{Basic Workflow}
+The basic operating work-flow to replay IOs would be something like:
+
+\begin{enumerate}
+  \item Run \texttt{blktrace} to collect traces. Here you specify the
+  device or devices that you wish to trace and later replay IOs upon. Note:
+  the only traces you are interested in are \emph{QUEUE} requests --
+  thus, to save system resources (including storage for traces), one could
+  specify the \texttt{-a queue} command line option to \texttt{blktrace}.
+
+  \item While \texttt{blktrace} is running, you run the workload that you
+  are interested in. 
+
+  \item When the work load has completed, you stop the \texttt{blktrace}
+  utility (thus saving all traces over the complete workload). 
+
+  \item You extract the pertinent IO information from the traces saved by
+  \texttt{blktrace} using the \texttt{btrecord} utility. This will parse
+  each trace file created by \texttt{blktrace}, and crafty IO descriptions
+  to be used in the next phase of the workload processing.
+
+  \item Once \texttt{btrecord} has successfully created a series of data
+  files to be processed, you can run the \texttt{btreplay} utility which
+  attempts to generate the same IOs seen during the sample workload phase.
+\end{enumerate}
+
+\subsection{IO Stream Replay Characteristics}
+  The major characteristics of the IO stream that are kept intact include:
+
+  \begin{description}
+    \item[Device] The IOs are replayed on the same device as was seen
+    during the sample workload.
+
+    \item[IO direction] The same IO direction (read/write) is maintained.
+
+    \item[IO offset] The same device offset is maintained.
+
+    \item[IO size] The same number of sectors are transferred.
+
+    \item[Time differential] The time stamps stored during the
+    \texttt{blktrace} run are used to determine the amount of time between
+    IOs during the sample workload. \texttt{btreplay} \emph{attempts} to
+    maintain the same time differential between IOs, but no guarantees as
+    to complete accuracy are provided by the utility.
+
+    \item[Device IO Stream Ordering] All IOs on a device are submitted in
+    the precise order they were seen during the sample workload run. 
+  \end{description}
+
+  As noted above, the time between IOs may not be accurately maintained
+  during replays. In addition the actual ordering of IOs \emph{between}
+  devices is not necessarily maintained. (Each device with an IO stream
+  maintains its own concept of time, and thus there may be slippage of the
+  time kept between managing threads.)
+
+  \begin{quotation}
+    We have prototyped a different approach, wherein a single managing
+    thread handles all IOs across all devices. This approach, while
+    guaranteeing correct ordering of IOs across all devices, resulted in
+    much worse timing on a per IO basis. 
+  \end{quotation}
+
+\subsection{\texttt{btrecord/btreplay} Method of Operation}
+
+As noted above, \texttt{btrecord} extracts \texttt{QUEUE} operations from
+\texttt{blktrace} output. These \texttt{QUEUE} operations indicate the
+entrance of IOs into the block IO layer. In order to replay these IOs with
+some accuracy in regards to ordering and timeliness, we decided to take
+multiple sequential (in time) IOs and put them in a single \emph{bunch} of
+IOs that will be processed as a single \emph{asynchronous IO} call to the
+kernel\footnote{Attempts to do them individually resulted in too large of a
+turnaround time penalty (user-space to kernel and back). Note that in a
+number of workloads, the IOs are coming in from the page cache handling
+code, and thus are submitted to the block IO layer with \emph{very small}
+time intervals between issues.}. To manage the size of the \emph{bunches},
+the \texttt{btrecord} utility provides you with two controlling knobs:
+
+\begin{description}
+  \item[\texttt{--max-bunch-time}] This is the amount of time to encompass
+  in one bunch -- only IOs within the time specified are eligible
+  for \emph{bunching.} The default time is 10 milliseconds (10,000,000
+  nanoseconds). Refer to section~\ref{sec:c-o-m} on page~\pageref{sec:c-o-m}
+  for more information.
+
+  \item[\texttt{--max-pkts}] A \emph{bunch} size can be anywhere from
+  1 to 512 packets in size and by default we max a bunch to contain no
+  more than 8 individual IOs. With this option, one can increase or
+  decrease the maximum \emph{bunch} size.  Refer to section~\ref{sec:c-o-M}
+  on page~\pageref{sec:c-o-M} for more information.
+\end{description}
+
+Each input data file (one per device per CPU) results in a new record
+data file (again, one per device per CPU) which contains information
+about \emph{bunches} of IOs to be replayed. \texttt{btreplay} operates on
+these record data files by spawning a new pair of threads per file. One
+thread managed the submitting of AIOs per bunch in the record data file,
+while the other thread manages reclaiming AIOs completed\footnote{We
+have found that having the same thread do both results in a further
+reduction in replay timing accuracty.}.
+
+Each submitting thread simply reads the input file of \emph{bunches}
+recorded by \texttt{btrecord}, and attempts to faithfully reproduce the
+ordering and timing of IOs seen during the sample workload. The reclaiming
+thread simply wait for AIO completions, freeing up resources for the
+submitting thread to utilize to submit new AIOs.
+
+The number of CPUs being used on the replay system can be different from
+the number on the recorded system. To help with mappings here the
+\texttt{--cpus} option allows one to state how many CPUs on the replay
+system to utilize. If the number of CPUs on the replay system is less than
+on the recording system, we wrap CPU IDs. This \emph{may} result in an
+overload of CPU processing capabilities on the replay system. (Refer to
+section~\ref{sec:p-o-c} on page~\pageref{sec:p-o-c} for more details about the
+\texttt{--cpus} option.)
+
+\newpage\subsection{Known Deficiencies and Proposed Possible Fixes}
+
+The overall known deficiencies with this current set of utilities is
+outlined here, in some cases ideas on additions and/or improvements are
+included as well.
+
+\begin{enumerate}
+  \item Lack of IO ordering across devices. 
+
+  \begin{quote}
+    \emph{We could institute the notion of global time across threads,
+    and thus ensure IO ordering across devices, with some reduction in
+    timing accuracy.}
+  \end{quote}
+
+  \item Lack of IO timing accuracy -- additional time between IO bunches.
+
+  \begin{quote}
+    \emph{This is the primary problem with any IO replay mechanism -- how
+    to guarantee per-IO timing accuracy with respect to other replayed IOs?
+    One idea to reduce errors in this area would be to push the IO replay
+    into the kernel, where you \emph{may} receive more responsive timings.}
+  \end{quote}
+
+  \item Bunching of IOs results in reduced time amongst IOs within a bunch.
+
+  \begin{quote}
+    \emph{The user has \emph{some} control over this (via the
+    \texttt{--max-pkts} option). One \emph{could} simply specify
+    \texttt{-max-pkts=1} and then each IO would be treated individualy. Of
+    course, this would probably then run into the problem of excessive
+    inter-IO times.}
+  \end{quote}
+
+  \item 1-to-1 mapping of devices -- for now the devices on the replay
+  machine must be the same as on the recording machine. 
+
+  \begin{quote}
+    \emph{It should be relatively trivial to add in the notion of
+    mapping -- simply include a file that is read which maps devices
+    on one machine to devices (with offsets and sizes) on the replay
+    machine\footnote{The notion of an offset and device size to replay on
+    could be used to both allow for a single device to masquerade as more
+    than one device, and could be utilized in case the replay device is
+    smaller than the recorded device.}.}
+    
+    \medskip\emph{One could also add in the notion of CPU mappings as well --
+    device $D_{rec}$ managed by CPU $C_{rec}$ on the recorded system
+    shall be replayed on device $D_{rep}$ and CPU $C_{rep}$ on the
+    replay machine.}
+
+    \bigskip
+    \begin{quote}
+      With version 0.9.1 we now support the \texttt{-M} option to do this
+      -- see section~\ref{sec:p-o-M} on page~\pageref{sec:p-o-M} for more
+      information on device mapping.
+    \end{quote}
+  \end{quote}
+
+\end{enumerate}
+
+%---------------------
+\newpage\section{\label{sec:command-line}Command Line Options}
+\subsection{\texttt{btrecord} Command Line Options}
+\begin{figure}[h!]
+\begin{verbatim}
+Usage: btrecord -- version 0.9.3
+
+	[ -d <dir>  : --input-directory=<dir> ] Default: .
+	[ -D <dir>  : --output-directory=<dir>] Default: .
+	[ -F        : --find-traces           ] Default: Off
+	[ -h        : --help                  ] Default: Off
+	[ -m <nsec> : --max-bunch-time=<nsec> ] Default: 10 msec
+	[ -M <pkts> : --max-pkts=<pkts>       ] Default: 8
+	[ -o <base> : --output-base=<base>    ] Default: replay
+	[ -v        : --verbose               ] Default: Off
+	[ -V        : --version               ] Default: Off
+	<dev>...                                Default: None
+\end{verbatim}
+\caption{\label{fig:btrecord--help}\texttt{btrecord --help} Output}
+\end{figure}
+\FloatBarrier
+
+\subsubsection{\label{sec:c-o-d}\texttt{-d} or
+\texttt{--input-directory}\\Set Input Directory}
+
+The \texttt{-d} option requires a single parameter providing the directory
+name for where input files are to be found. The default directory is the
+current directory (\texttt{.}).
+
+\subsubsection{\label{sec:c-o-D}\texttt{-D} or
+\texttt{--output-directory}\\Set Output Directory}
+
+The \texttt{-D} option requires a single parameter providing the directory
+name for where output files are to be placed. The default directory is the
+current directory (\texttt{.}).
+
+\subsubsection{\texttt{-F} or \texttt{--find-traces}\\Find Trace Files
+Automatically}
+
+The \texttt{-F} option instructs \texttt{btrecord} to go find all the
+trace files in the directory specified (either via the \texttt{-d}
+option, or in the default directory '.').
+
+\subsubsection{\texttt{-h} or \texttt{--help}\\Display Help Message}
+\subsubsection{\texttt{-V} or \texttt{--version}\\Display
+\texttt{btrecord}Version}
+
+The \texttt{-h} option displays the command line options and
+defaults, as presented in figure~\ref{fig:btrecord--help} on
+page~\pageref{fig:btrecord--help}.
+
+The \texttt{-V} option displays the \texttt{btreplay} version, as shown here:
+
+\begin{verbatim}
+$ btrecord --version
+btrecord -- version 0.9.0
+\end{verbatim}
+
+Both commands exit immediately after processing the option.
+
+\subsubsection{\label{sec:c-o-m}\texttt{-m} or
+\texttt{--max-bunch-time}\\Set Maximum Time Per Bunch}
+
+The \texttt{-m} option requires a single parameter which specifies an
+amount of time (in nanoseconds) to include in any one bunch of IOs that
+are to be processed. The smaller the value, the smaller the number of
+IOs processed at one time -- perhaps yielding in more realistic replay.
+However, after a certain point the amount of overhead per bunch may result
+in additonal real replay time, thus yielding less accurate replay times.
+
+The default value is 10,000,000 nanoseconds (10 milliseconds).
+
+\subsubsection{\label{sec:c-o-M}\texttt{-M} or
+\texttt{--max-pkts}\\Set Maximum Packets Per Bunch}
+
+The \texttt{-M} option requires a single parameter which specifies the
+maximum number of IOs to store in a single bunch. As with the \texttt{-m}
+option (section~\ref{sec:c-o-m}), smaller values \emph{may} or \emph{may not}
+yield more accurate replay times.
+
+The default value is 8, with a maximum value of up to 512 being supported.
+
+\subsubsection{\label{sec:c-o-o}\texttt{-o} or
+\texttt{--output-base}\\Set Base Name for Output Files}
+
+Each output file has 3 fields:
+
+\begin{enumerate}
+  \item Device identifier (taken directly from the device name of the
+  \texttt{blktrace} output file).
+
+  \item \texttt{btrecord} base name -- by default ``replay''.
+
+  \item And the CPU number (again, taken directly from the
+  \texttt{blktrace} output file name).
+\end{enumerate}
+
+This option requires a single parameter that will override the default name
+(replay), and replace it with the specified value.
+
+\subsubsection{\label{sec:c-o-v}\texttt{-v} or
+\texttt{--verbose}\\Select Verbose Output}
+
+This option will output some simple statistics at the end of a successful
+run. Figure~\ref{fig:verb-out} (page~\pageref{fig:verb-out}) shows
+an example of some output, while figure~\ref{fig:verb-defs}
+(page~\pageref{fig:verb-defs}) shows what the fields mean.
+
+\begin{figure}[h!]
+\begin{verbatim}
+sdab:0: 580661 pkts (tot), 126030 pkts (replay), 89809 bunches, 1.4 pkts/bunch
+sdab:1: 2559775 pkts (tot), 430172 pkts (replay), 293029 bunches, 1.5 pkts/bunch
+sdab:2: 653559 pkts (tot), 136522 pkts (replay), 102288 bunches, 1.3 pkts/bunch
+sdab:3: 474773 pkts (tot), 117849 pkts (replay), 69572 bunches, 1.7 pkts/bunch
+\end{verbatim}
+\caption{\label{fig:verb-out}Verbose Output Example}
+\end{figure}
+\FloatBarrier
+
+\begin{figure}[h!]
+\begin{description}
+  \item[Field 1] The first field contains the device name and CPU
+  identrifer. Thus: \texttt{sdab:0:} means the device \texttt{sdab} and
+  traces on CPU 0. 
+
+  \item[Field 2] The second field contains the total number of packets
+  processed for each device file. 
+
+  \item[Field 3] The next field shows the number of packets eligible for
+  replay. 
+
+  \item[Field 4] The fourth field contains the total number of IO bunches. 
+
+  \item[Field 5] The last field shows the average number of IOs per bunch
+  recorded.
+\end{description}
+\caption{\label{fig:verb-defs}Verbose Field Definitions}
+\end{figure}
+\FloatBarrier
+
+%---------------------
+\newpage\subsection{\texttt{btreplay} Command Line Options}
+\begin{figure}[h!]
+\begin{verbatim}
+Usage: btreplay -- version 0.9.3
+
+	[ -c <cpus> : --cpus=<cpus>           ] Default: 1
+	[ -d <dir>  : --input-directory=<dir> ] Default: .
+	[ -F        : --find-records           ] Default: Off
+	[ -h        : --help                  ] Default: Off
+	[ -i <base> : --input-base=<base>     ] Default: replay
+	[ -I <iters>: --iterations=<iters>    ] Default: 1
+	[ -M <file> : --map-devs=<file>       ] Default: None
+	[ -N        : --no-stalls             ] Default: Off
+	[ -v        : --verbose               ] Default: Off
+	[ -V        : --version               ] Default: Off
+	[ -W        : --write-enable          ] Default: Off
+	<dev...>                                Default: None
+\end{verbatim}
+\caption{\label{fig:btreplay--help}\texttt{btreplay --help} Output}
+\end{figure}
+\FloatBarrier
+
+\subsubsection{\label{sec:p-o-c}\texttt{-c} or
+\texttt{--cpus}\\Set Number of CPUs to Use}
+
+\subsubsection{\label{sec:p-o-d}\texttt{-d} or
+\texttt{--input-directory}\\Set Input Directory}
+
+The \texttt{-d} option requires a single parameter providing the directory
+name for where input files are to be found. The default directory is the
+current directory (\texttt{.}).
+
+\subsubsection{\texttt{-F} or \texttt{--find-records}\\Find RecordFiles
+Automatically}
+
+The \texttt{-F} option instructs \texttt{btreplay} to go find all the
+record files in the directory specified (either via the \texttt{-d}
+option, or in the default directory '.').
+
+\subsubsection{\texttt{-h} or \texttt{--help}\\Display Help Message}
+\subsubsection{\texttt{-V} or \texttt{--version}\\Display
+\texttt{btreplay}Version}
+
+The \texttt{-h} option displays the command line options and
+defaults, as presented in figure~\ref{fig:btreplay--help} on
+page~\pageref{fig:btreplay--help}.
+
+The \texttt{-V} option displays the \texttt{btreplay} version, as show here:
+
+\begin{verbatim}
+$ btreplay --version
+btreplay -- version 0.9.0
+\end{verbatim}
+
+Both commands exit immediately after processing the option.
+
+\subsubsection{\label{sec:p-o-i}\texttt{-i} or
+\texttt{--input-base}\\Set Base Name for Input Files}
+
+Each input file has 3 fields:
+
+\begin{enumerate}
+  \item Device identifier (taken directly from the device name of the
+  \texttt{blktrace} output file).
+
+  \item \texttt{btrecord} base name -- by default ``replay''.
+
+  \item And the CPU number (again, taken directly from the
+  \texttt{blktrace} output file name).
+\end{enumerate}
+
+This option requires a single parameter that will override the default name
+(replay), and replace it with the specified value.
+
+\subsubsection{\label{sec:p-o-I}\texttt{-I} or
+\texttt{--iterations}\\Set Number of Iterations to Run}
+
+This option requires a single parameter which specifies the number of times
+to run through the input files. The default value is 1.
+
+\subsubsection{\label{sec:p-o-M}\texttt{-M} or \texttt{map-devs}\\
+Specify Device Mappings}
+
+This option requires a single paramter which specifies the name of a
+file contain device mappings. The file must be very simply managed, with
+just two pieces of data per line:
+
+\begin{enumerate}
+  \item The device name on the recorded system (with the \texttt{'/dev/'}
+  removed). Example: \texttt{/dev/sda} would just be \texttt{sda}.
+
+  \item The device name on the replay system to use (again, without the
+  \texttt{'/dev/'} path prepended).
+\end{enumerate}
+
+An example file for when one would map devices \texttt{/dev/sda} and
+\texttt{/dev/sdb} on the recorded system to \texttt{dev/sdg} and
+\texttt{sdh} on the replay system would be:
+
+\begin{verbatim}
+sda sdg
+sdb sdh
+\end{verbatim}
+
+The only entries in the file that are allowed are these two element lines
+-- we do not (yet?) support the notion of blank lines, or comment lines, or
+the like.
+
+The utility \emph{does} allow for multiple \texttt{-M} options to be
+supplied on the command line.
+
+\subsubsection{\label{sec:o-N}\texttt{-N} or \texttt{--no-stalls}\\Disable
+Pre-bunch Stalls}
+
+When specified on the command line, all pre-bunch stall indicators will be
+ignored. IOs will be replayed without inter-bunch delays.
+
+\subsubsection{\label{sec:p-o-v}\texttt{-v} or
+\texttt{--verbose}\\Select Verbose Output}
+
+When specified on the command line, this option instructs \texttt{btreplay}
+to store information concerning each \emph{stall} and IO operation
+performed by \texttt{btreplay}. The name of each file so created will be
+the input file name used with an extension of \texttt{.rep} appended onto
+it. Thus, an input file of the name \texttt{sdab.replay.3} would generate a
+verbose output file with the name \texttt{sdab.replay.3.rep} in the
+directory specified for input files.
+
+In addition, \texttt{btreplay} will also output to \texttt{stderr} the
+names of the input files being processed.
+
+\subsubsection{\label{sec:p-o-W}\texttt{-W} or
+\texttt{--write-enable}\\Enable Writing During Replay}
+
+As a precautionary measure, by default \texttt{btreplay} will \emph{not}
+process \emph{write} requests. In order to enable \texttt{btreplay} to
+actually \emph{write} to devices one must explicitly specify the
+\texttt{-W} option.
+
+\end{document}
diff --git a/btt/Makefile b/btt/Makefile
index 5281a94..094f77f 100644
--- a/btt/Makefile
+++ b/btt/Makefile
@@ -17,7 +17,7 @@ LIBS	= $(PLIBS) $(ELIBS)
 OBJS	= args.o bt_timeline.o devmap.o devs.o dip_rb.o iostat.o latency.o \
 	  misc.o output.o proc.o seek.o trace.o trace_complete.o trace_im.o \
 	  trace_issue.o trace_queue.o trace_remap.o trace_requeue.o \
-	  ../rbtree.o mmap.o trace_plug.o bno_dump.o unplug_hist.o
+	  ../rbtree.o mmap.o trace_plug.o bno_dump.o unplug_hist.o q2d.o
 
 all: depend $(PROGS)
 
diff --git a/btt/bt_timeline.c b/btt/bt_timeline.c
index f38f6a2..b1f480a 100644
--- a/btt/bt_timeline.c
+++ b/btt/bt_timeline.c
@@ -25,7 +25,7 @@
 #include <time.h>
 #include "globals.h"
 
-char bt_timeline_version[] = "0.99.1";
+char bt_timeline_version[] = "2.01";
 
 char *devices, *exes, *input_name, *output_name, *seek_name, *bno_dump_name;
 char *d2c_name, *q2c_name, *per_io_name, *unplug_hist_name;
@@ -40,13 +40,10 @@ LIST_HEAD(all_devs);
 LIST_HEAD(all_procs);
 LIST_HEAD(free_ios);
 LIST_HEAD(free_bilinks);
-LIST_HEAD(rmhd);
-LIST_HEAD(retries);
 __u64 q_histo[N_HIST_BKTS], d_histo[N_HIST_BKTS];
 
 double range_delta = 0.1;
 __u64 last_q = (__u64)-1;
-__u64 next_retry_check = 0;
 
 struct region_info all_regions = {
 	.qranges = LIST_HEAD_INIT(all_regions.qranges),
@@ -90,7 +87,6 @@ int main(int argc, char *argv[])
 	dip_exit();
 	pip_exit();
 	io_free_all();
-	bilink_free_all();
 	region_exit(&all_regions);
 
 	free(input_name);
@@ -120,7 +116,6 @@ int process(void)
 	}
 
 	io_release(iop);
-	do_retries(0);
 	gettimeofday(&tve, NULL);
 
 	if (verbose) {
diff --git a/btt/devs.c b/btt/devs.c
index 397a945..61dd427 100644
--- a/btt/devs.c
+++ b/btt/devs.c
@@ -58,10 +58,7 @@ void __dump_rb_node(struct rb_node *n)
 	struct io *iop = rb_entry(n, struct io, rb_node);
 
 	dbg_ping();
-	if (iop->type == IOP_A)
-		__dump_iop2(stdout, iop, bilink_first_down(iop, NULL));
-	else
-		__dump_iop(stdout, iop, 0);
+	__dump_iop(stdout, iop, 0);
 	if (n->rb_left)
 		__dump_rb_node(n->rb_left);
 	if (n->rb_right)
@@ -92,10 +89,8 @@ void dump_rb_trees(void)
 			dip = list_entry(p, struct d_info, hash_head);
 			printf("Trees for %3d,%-3d\n", MAJOR(dip->device),
 			       MINOR(dip->device));
-			for (type = IOP_Q; type < N_IOP_TYPES; type++) {
-				if (type != IOP_L)
-					__dump_rb_tree(dip, type);
-			}
+			for (type = IOP_Q; type < N_IOP_TYPES; type++)
+				__dump_rb_tree(dip, type);
 		}
 	}
 }
@@ -133,17 +128,31 @@ void dip_exit(void)
 		__destroy_heads(dip->heads);
 		region_exit(&dip->regions);
 		seeki_exit(dip->seek_handle);
+		seeki_exit(dip->q2q_handle);
 		bno_dump_exit(dip->bno_dump_handle);
 		unplug_hist_exit(dip->unplug_hist_handle);
+		if (output_all_data)
+			q2d_release(dip->q2d_priv);
 		free(dip);
 	}
 }
 
+static inline char *mkhandle(char *str, __u32 device, char *post)
+{
+	int mjr = device >> MINORBITS;
+	int mnr = device & ((1 << MINORBITS) - 1);
+
+	sprintf(str, "%03d,%03d%s", mjr, mnr, post);
+	return str;
+}
+
 struct d_info *dip_add(__u32 device, struct io *iop)
 {
 	struct d_info *dip = __dip_find(device);
 
 	if (dip == NULL) {
+		char str[256];
+
 		dip = malloc(sizeof(struct d_info));
 		memset(dip, 0, sizeof(*dip));
 		dip->heads = dip_rb_mkhds();
@@ -151,14 +160,17 @@ struct d_info *dip_add(__u32 device, struct io *iop)
 		dip->device = device;
 		dip->last_q = (__u64)-1;
 		dip->map = dev_map_find(device);
-		dip->seek_handle = seeki_init(device);
 		dip->bno_dump_handle = bno_dump_init(device);
 		dip->unplug_hist_handle = unplug_hist_init(device);
+		dip->seek_handle = seeki_init(mkhandle(str, device, "_d2d"));
+		dip->q2q_handle = seeki_init(mkhandle(str, device, "_q2q"));
 		latency_init(dip);
 		list_add_tail(&dip->hash_head, &dev_heads[DEV_HASH(device)]);
 		list_add_tail(&dip->all_head, &all_devs);
 		dip->start_time = BIT_TIME(iop->t.time);
 		dip->pre_culling = 1;
+		if (output_all_data)
+			dip->q2d_priv = q2d_init();
 		n_devs++;
 	}
 
@@ -252,15 +264,21 @@ void dip_plug(__u32 dev, double cur_time)
 	dip->last_plug = cur_time;
 }
 
-void dip_unplug(__u32 dev, double cur_time, int is_timer)
+void dip_unplug(__u32 dev, double cur_time)
 {
 	struct d_info *dip = __dip_find(dev);
 
-	if (!dip || !dip->is_plugged) return;
+	if (dip && dip->is_plugged) {
+		dip->nplugs++;
+		dip->plugged_time += (cur_time - dip->last_plug);
+		dip->is_plugged = 0;
+	}
+}
 
-	dip->nplugs++;
-	if (is_timer) dip->n_timer_unplugs++;
+void dip_unplug_tm(__u32 dev)
+{
+	struct d_info *dip = __dip_find(dev);
 
-	dip->plugged_time += (cur_time - dip->last_plug);
-	dip->is_plugged = 0;
+	if (dip && dip->is_plugged)
+		dip->n_timer_unplugs++;
 }
diff --git a/btt/doc/btt.tex b/btt/doc/btt.tex
index e8762e8..0fe44ef 100644
--- a/btt/doc/btt.tex
+++ b/btt/doc/btt.tex
@@ -22,14 +22,16 @@
 
 \title{\texttt{btt} User Guide}
 \author{Alan D. Brunelle (Alan.Brunelle@hp.com)}
-\date{16 April 2007}
+\date{13 November 2007}
 
 \begin{document}
 \maketitle
 %--------------
 \section{\label{sec:intro}Introduction}
 
-\texttt{btt} is a post-processing tool for the block layer IO tracing tool called blktrace. As noted in its Users Guide, blktrace 
+\texttt{btt} is a post-processing tool for the block layer IO tracing
+tool called blktrace. As noted in its Users Guide, blktrace
+
   \begin{quotation}
     is a block layer IO tracing mechanism which provides detailed
     information about request queue operations up to user space.
@@ -56,7 +58,7 @@ easier.
 
 \bigskip
   This document refers to the output formats generated by \texttt{btt}
-  version 0.99.1.  However, the descriptions are general enough to cover
+  version 2.00.  However, the descriptions are general enough to cover
   output formats prior to that.
 
 \newpage\tableofcontents
@@ -109,11 +111,24 @@ easier.
     \begin{enumerate}
       \item Time needed to insert or merge an incoming IO onto the request
       queue. This is the average time from when the IO enters the block
-      IO layer (queue trace) until it is inserted (insert trace) or merged
-      (back merge or front merge trace).
+      IO layer (queue trace) until it is inserted (insert trace).
 
       This is denoted as \emph{Q2I} time.
 
+      This is also broken down into two component times:
+
+        \begin{description}
+	  \item[Q2G] Time needed to \emph{get} a request (get request
+	  trace).
+
+	  \item[G2I] Time needed to put that request onto the request 
+	  queue (insert trace).
+        \end{description}
+
+      For \emph{merged} requests -- an incoming request that is merged
+      with a previously submitted request -- we calculate \emph{Q2M}, the
+      amount of time between the queue trace and the merge trace.
+
       \item Time spent on the request queue. The average time from when
       the IO is inserted or merged onto the request queue, until it is
       issued (issue trace) to the lower level driver.
@@ -163,16 +178,31 @@ D2C    0.000193721   0.030406554   1.634221408     2262311
 Q2C    0.000207665   0.125405263   1.830917198     2262311
 \end{verbatim}
 
+  When tracking \emph{device mapper} devices, we also break down the
+  \emph{Q2A} and \emph{Q2C} times for those IOs.
+
   \item[Device Overhead]
 
   Using the data from the previous chart, we can then provide some idea
   as to where IO spend most of the time on average. The following output
-  shows the percentage of time spent in each of the 3 phases of an IO:
+  shows the percentage of time spent in each of the phases of an
+IO\footnote{It should be noted that incoming requests either go through:
+
+\begin{enumerate}
+  \item Q2G + Q2I 
+
+  or
+
+  \item Q2M
+\end{enumerate}
+  before proceeding to I2D and D2C.}
 
 \begin{verbatim}
-       DEV |    Q2I    I2D    D2C
----------- | ------ ------ ------
- ( 68, 64) |   0.0%  75.7%  24.2%
+       DEV |       Q2G       G2I       Q2M       I2D       D2C
+---------- | --------- --------- --------- --------- ---------
+ (  8, 80) |   0.0013%   0.0004%   0.0006%  88.5005%  11.4988%
+---------- | --------- --------- --------- --------- ---------
+   Overall |   0.0003%   0.0001%   0.0041%  21.4998%  78.4958%
 \end{verbatim}
 
   \item[Device Merge Information]
@@ -199,13 +229,20 @@ Q2C    0.000207665   0.125405263   1.830917198     2262311
   \item[Device Seek Information]
 
   Another useful measure is the variability in the sector distances
-  between consecutive IOs. The next session provides some rudimentary
+  between consecutively \emph{recieved -- queued} and \emph{submitted
+  -- issued} IOs. The next two sections provides some rudimentary
   statistics to gauge the general nature of the sector differences
   between IOs. Values provided include the number of seeks (number of IOs
   submitted to lower level drivers), the \emph{mean} distance between
   IOs, the \emph{median} value for all seeks, and the \emph{mode} -
   the value(s) and the counts are provided for the latter.
 
+  The first of the two sections displays values for Q2Q seek distances --
+  providing a set of indicators showing how close incoming IO requests
+  are to each other. The second section shows D2D seek distances --
+  providing a set of indicators showing how close the IO requests are
+  that are handled by underlying drivers.
+
 \begin{verbatim}
       DEV | NSEEKS    MEAN MEDIAN | MODE
 --------- | ------ ------- ------ | -------
@@ -219,7 +256,6 @@ Q2C    0.000207665   0.125405263   1.830917198     2262311
   the block IO layer in adjacent sectors. (Obviously, the higher this
   percentage, the better the underlying subsystems can handle them.)
 
-
   \item[Request Queue Plug Information]
 
   During normal operation, requests queues are \emph{plugged} and during
@@ -245,8 +281,35 @@ Q2C    0.000207665   0.125405263   1.830917198     2262311
 
   The total number of unplugs is equal to the number of plugs less the
   ones due to timer unplugs.
+
+  \item[Active Requests At Q Information]
+
+  An important consideration when analyzing block IO schedulers is to
+  know how many requests the scheduler has to work with. The metric
+  provided in this section details how many requests (on average) were
+  being held by the IO scheduler when an incoming IO request was being
+  handled. To determine this, \texttt{btt} keeps track of how many Q
+  requests came in, and subtacts requests that have been issued (D).
+
+  Here is a sample output of this sections:
+
+\begin{verbatim}
+==================== Active Requests At Q Information ====================
+
+       DEV |  Avg Reqs @ Q
+---------- | -------------
+ ( 65, 80) |          12.0
+ ( 65,240) |          16.9
+...
+ ( 66,112) |          44.2
+---------- | -------------
+   Overall | Avgs Reqs @ Q
+   Average |          17.4
+\end{verbatim}
+
 \end{description}
 
+\newpage
 \subsection{\label{sec:detailed-data}Detailed Data}
 
   In addition to the default sections output, if one supplies the
@@ -298,14 +361,32 @@ pdflush       0.000000790 0.000006752 0.247231307  179791
 ( 65,160) 0.000139915 0.001175554 0.073512063 133973
 ( 65,176) 0.000141254 0.001104870 0.073231310 145764
 ( 65,192) 0.000141453 0.001234460 0.167622507 140618
-( 65,208) 0.000143901 0.001126610 0.144651899 178548
-( 65,224) 0.000145020 0.001226478 0.124902029 206241
-( 65,240) 0.000144315 0.001199571 0.102415459 129154
 ...
 \end{verbatim}
 
   \item[Per Device Averages] The average columns from the above charts,
   are also presented in their own chart.
+
+  \item[Q2D Histogram] A display of histogram buckets for the Q to D times
+  -- basically, from where an IO enters the block IO layer for a given
+  device, and when it is dispatched. The buckets are arranged via the
+  time in seconds, as in:
+
+\begin{verbatim}
+==================== Q2D Histogram ====================
+
+       DEV | <.005 <.010 <.025 <.050 <.075 <.100 <.250 <.500 < 1.0 >=1.0
+ --------- | ===== ===== ===== ===== ===== ===== ===== ===== ===== =====
+ ( 66, 80) |  61.2   7.9  12.1   7.9   3.0   1.4   1.5   0.2   0.0   4.6
+ ( 65,192) |  42.3   5.0   8.7  30.0   8.9   3.0   1.8   0.1   0.0   0.1
+ ( 65,128) |  34.3   5.3   8.9  32.0   9.7   3.7   5.3   0.6   0.0   0.1
+...
+ ( 65, 64) |  59.9   4.2   6.0  24.6   4.2   0.8   0.1   0.0   0.0   0.1
+ ( 66, 64) |  62.6   8.1  12.7   7.9   2.4   0.6   0.1   0.0   0.0   5.4
+========== | ===== ===== ===== ===== ===== ===== ===== ===== ===== =====
+       AVG |  52.9   6.2  10.0  20.1   5.3   1.7   1.4   0.2   0.0   2.1
+\end{verbatim}
+
 \end{description}
 
 \newpage\section{\label{sec:data-files}Data Files Output}
@@ -343,7 +424,7 @@ pdflush       0.000000790 0.000006752 0.247231307  179791
     on a per-IO basis can be generated. These are described in
     sections~\ref{sec:lat-q2c} and~\ref{sec:lat-d2c}.
 
-    \item[seek details] A data file containing all IO-to-IO
+    \item[seek details] A set of data files containing all IO-to-IO
     sector differences can be output, with details found in
     section~\ref{sec:seek}.
 
@@ -513,36 +594,45 @@ Device:       rrqm/s   wrqm/s     r/s     w/s    rsec/s    wsec/s
 
   The \emph{--per-io-dump} or \emph{-p} option triggers this behavior,
   and will produce a file containing streams of IOs (separated by blank
-  spaces), here is an example:
+  spaces). As an example, here is a snippet of 4 IOs that were merged
+  together, you will note there are 3 merged IOs, and 1 inserted in the
+  stream. The issue and completion traces are replicated per IO.
 
 \begin{verbatim}
-   20.002179731   8,32  Q         34+8
-   20.002181199   8,32  I         34+8
-   20.098329567   8,32  D         34+16
-   20.002182760   8,32  Q         42+8
-   20.002183613   8,32  M         42+8
-   20.098329567   8,32  D         34+16
-   20.692643206   8,32  C         34+16
+ 66,0  :     0.763283556 Q       6208+8   
+             0.763300157 I       6208+8   
+             0.763296365 G       6208+8   
+             0.763338848 D       6208+32  
+             0.763705760 C       6208+32  
+
+ 66,0  :     0.763314550 Q       6224+8   
+             0.763315341 M       6224+8   
+             0.763338848 D       6208+32  
+             0.763705760 C       6208+32  
+
+ 66,0  :     0.763321010 Q       6232+8   
+             0.763321775 M       6232+8   
+             0.763338848 D       6208+32  
+             0.763705760 C       6208+32  
+
+ 65,240:     0.763244173 Q       6216+8   
+             0.763244974 M       6216+8   
+             0.763374288 D       6208+32  
+             0.763826610 C       6208+32  
 \end{verbatim}
 
   The columns provide the following information:
 
   \begin{enumerate}
-    \item Time of the trace (seconds from the start of the run)
-
     \item Device major/minor.
 
+    \item Time of the trace (seconds from the start of the run)
+
     \item Trace type 
 
     \item start block + number of blocks
   \end{enumerate}
   
-  For this pair of initial IOs (Q traces at 20.002179731 and
-  20.002182760), we can follow the insert and merge traces (20.002181199
-  and 20.002183613 respectively), and the issue (duplicate entries for
-  20.098329567), and the completion at 20.692643206. Every Q has its
-  corresponding issue trace bounding any insert or merge operations.
-
 \newpage\section{\label{sec:lat}\label{sec:lat-q2c}\label{sec:lat-d2c}Latency Data Files}
 
   The latency data files which can be optionally produced by \texttt{btt}
@@ -555,7 +645,7 @@ Device:       rrqm/s   wrqm/s     r/s     w/s    rsec/s    wsec/s
 
 \newpage\section{\label{sec:seek}Seek Data File}
 
-  \texttt{btt} can also produce a data file containing all IO-to-IO sector
+  \texttt{btt} can also produce two data files containing all IO-to-IO sector
   deltas, providing seek information which can then be plotted. The
   produced data file contains 3 sets of data:
 
@@ -567,6 +657,25 @@ Device:       rrqm/s   wrqm/s     r/s     w/s    rsec/s    wsec/s
      \item Write data -- just seek deltas for writes
   \end{enumerate}
 
+  The format of the output file names is to have the name generated by 
+  the following fields separated by underscores (\texttt{\_}):
+  
+  \begin{itemize}
+    \item The prefix provided as the argument to the \texttt{-s} option.
+    \item The major and minor numbers of the device separated by a comma.
+    \item The string \texttt{q2q} or \texttt{d2d}, indicating the Q2Q or 
+          D2D seeks, respectively.
+    \item One of the following characters:
+    	\begin{description}
+	  \item[r] For read (device to system) IOs
+	  \item[w] For write (system to device) IOs
+	  \item[c] Combined -- both read and write IOs
+	\end{description}
+  \end{itemize}
+
+  An example name would be after specifying \texttt{-s seek} would be:
+  \texttt{seek\_065,048\_q2q\_w.dat}.
+
   The format of the data is to have the runtime values (seconds since
   the start of the run) in column 1 (X values); and the difference in
   sectors from the previous IO in column 2 (Y values). Here is a snippet
@@ -614,7 +723,7 @@ Device:       rrqm/s   wrqm/s     r/s     w/s    rsec/s    wsec/s
 \newpage\section{\label{sec:cmd-line}Command Line}
 
 \begin{verbatim}
-Usage: \texttt{btt} 0.99.1 
+Usage: btt 2.01 
 [ -a               | --seek-absolute ]
 [ -A               | --all-data ]
 [ -B <output name> | --dump-blocknos=<output name> ]
@@ -823,248 +932,5 @@ Output}
   
   Here is a complete output file from a btt run, illustrating a lot of the
   capabilities of btt.
-
-\begin{verbatim}
-==================== All Devices ====================
-
-            ALL           MIN           AVG           MAX           N
---------------- ------------- ------------- ------------- -----------
-Q2Q               0.000000001   0.000015439   0.067431982     4485897
-Q2I               0.000000277   0.000005085  12.844603081     4485736
-I2D               0.000000869   0.000721745  12.845117057     4485736
-D2C               0.000151807   0.005254051   0.097569048     4485736
-Q2C               0.000156268   0.005980882  12.864868116     4485736
-
-\end{verbatim}
-\newpage\begin{verbatim}
-
-==================== Device Overhead ====================
-
-       DEV |    Q2I    I2D    D2C
----------- | ------ ------ ------
- (  8,160) |   0.0%   1.6%  98.4%
- (  8,176) |   0.1%  20.0%  79.9%
- (  8,208) |   1.2%  69.2%  29.5%
- ( 65, 32) |   0.0%   1.1%  98.8%
- ( 65, 64) |   0.0%   3.6%  96.4%
- ( 65,176) |   0.0%   2.5%  97.4%
- ( 65, 96) |   0.0%  15.3%  84.6%
- (  8,224) |   0.0%   1.7%  98.3%
- ( 65,112) |   0.0%   2.4%  97.6%
- (  8,240) |   0.0%   2.3%  97.7%
- ( 65,192) |   0.0%  19.4%  80.6%
- ( 65,240) |   0.0%   1.3%  98.7%
- ( 65, 48) |   0.0%   2.3%  97.7%
- (  8, 32) |   0.0%   7.0%  93.0%
- ( 66, 80) |   0.1%   9.9%  90.0%
- ( 66, 32) |   0.0%   0.7%  99.3%
- ( 65,224) |   0.0%  18.1%  81.9%
- ( 65,144) |   0.1%  39.5%  60.5%
- (  8,144) |   0.0%   0.7%  99.3%
- ( 66,144) |   0.0%   1.5%  98.5%
- ( 66,128) |   0.0%   3.0%  97.0%
- ( 66,176) |   0.0%  12.5%  87.4%
- ( 66,224) |   0.0%  10.2%  89.8%
- ( 66,192) |   0.0%   2.1%  97.9%
- ( 66,160) |   0.0%   9.4%  90.5%
- ( 66,240) |   0.0%   9.8%  90.2%
- ( 66,112) |   0.1%  24.2%  75.7%
- (  8, 64) |   0.0%   9.6%  90.4%
- ( 65,160) |   0.2%  26.2%  73.7%
- ( 65,208) |   0.0%   2.0%  98.0%
- ( 66, 16) |   0.0%   4.5%  95.5%
- ( 65,  0) |   0.0%   2.2%  97.8%
- ( 65, 16) |   0.0%   1.8%  98.1%
- ( 66,208) |   0.0%   2.7%  97.3%
- (  8,128) |   0.2%  23.7%  76.1%
- ( 65, 80) |   0.0%  20.0%  80.0%
- (  8,112) |   0.0%   1.2%  98.7%
- ( 65,128) |   0.0%   1.9%  98.1%
- ( 66, 64) |   0.0%  12.1%  87.9%
- ( 66,  0) |   0.0%   7.2%  92.8%
- ( 66, 48) |   0.0%   2.6%  97.4%
- (  8,192) |   0.0%   2.3%  97.7%
- ( 67, 16) |   0.0%   0.8%  99.2%
- ( 66, 96) |   0.0%  12.3%  87.7%
- (  8, 96) |   0.0%  10.3%  89.7%
- (  8, 80) |   0.0%   1.7%  98.3%
- (  8, 48) |   0.0%   0.7%  99.2%
- ( 67,  0) |   0.0%   2.6%  97.4%
-
-\end{verbatim}
-\newpage\begin{verbatim}
-
-==================== Device Merge Information ====================
-
-       DEV |       #Q       #D   Ratio |   BLKmin   BLKavg   BLKmax    Total
----------- | -------- -------- ------- | -------- -------- -------- --------
- (  8,160) |    75145    47890     1.6 |        8       12     1024   601160
- (  8,176) |    91374    55492     1.6 |        8       13     1024   730992
- (  8,208) |   101039    63944     1.6 |        8       12     1024   809256
- ( 65, 32) |    67919    44494     1.5 |        8       12     1024   543352
- ( 65, 64) |   114614    77396     1.5 |        8       11     1024   916968
- ( 65,176) |    93808    62746     1.5 |        8       11     1024   750464
- ( 65, 96) |    95537    51705     1.8 |        8       14     1024   764296
- (  8,224) |    95435    49753     1.9 |        8       15     1024   765560
- ( 65,112) |   100020    63530     1.6 |        8       12     1024   800160
- (  8,240) |    72282    44700     1.6 |        8       12     1024   578256
- ( 65,192) |    95175    59010     1.6 |        8       12     1024   761400
- ( 65,240) |    86334    53548     1.6 |        8       12     1024   690984
- ( 65, 48) |    69623    44652     1.6 |        8       12     1024   556984
- (  8, 32) |    97816    63116     1.5 |        8       12     1024   782528
- ( 66, 80) |   110873    71526     1.6 |        8       12     1024   886984
- ( 66, 32) |    79242    53134     1.5 |        8       11     1024   633936
- ( 65,224) |   122788    66180     1.9 |        8       14     1024   982304
- ( 65,144) |   116636    70205     1.7 |        8       13     1024   933416
- (  8,144) |    72014    49047     1.5 |        8       11     1024   576112
- ( 66,144) |   109244    70613     1.5 |        8       12     1024   873952
- ( 66,128) |   104688    65381     1.6 |        8       12     1024   837504
- ( 66,176) |    79627    47894     1.7 |        8       13     1024   637016
- ( 66,224) |    88754    58159     1.5 |        8       12     1024   710032
- ( 66,192) |    88919    55417     1.6 |        8       12     1024   711464
- ( 66,160) |   102908    71156     1.4 |        8       11     1024   823264
- ( 66,240) |    94190    66472     1.4 |        8       11     1024   753520
- ( 66,112) |   138799    82027     1.7 |        8       13     1024  1110392
- (  8, 64) |   105011    63892     1.6 |        8       13     1024   840112
- ( 65,160) |    93383    55579     1.7 |        8       13     1024   747064
- ( 65,208) |   109771    67987     1.6 |        8       12     1024   878168
- ( 66, 16) |    96703    56613     1.7 |        8       13     1024   773624
- ( 65,  0) |    83752    51532     1.6 |        8       13     1024   670032
- ( 65, 16) |    64538    35982     1.8 |        8       14     1024   516320
- ( 66,208) |    90636    54306     1.7 |        8       13     1024   725088
- (  8,128) |    96202    59653     1.6 |        8       13     1024   776192
- ( 65, 80) |   107945    65672     1.6 |        8       13     1024   863704
- (  8,112) |    78235    52847     1.5 |        8       11     1024   625880
- ( 65,128) |   104631    63106     1.7 |        8       13     1024   837048
- ( 66, 64) |    86365    50956     1.7 |        8       13     1024   690920
- ( 66,  0) |    90413    46722     1.9 |        8       15     1024   723304
- ( 66, 48) |   106631    65219     1.6 |        8       13     1024   853048
- (  8,192) |    80591    47154     1.7 |        8       13     1024   644728
- ( 67, 16) |    72314    48487     1.5 |        8       11     1024   578512
- ( 66, 96) |    90990    54454     1.7 |        8       13     1024   727920
- (  8, 96) |   110805    73522     1.5 |        8       12     1024   886440
- (  8, 80) |    85032    56643     1.5 |        8       12     1024   680400
- (  8, 48) |    87374    57544     1.5 |        8       12     1024   698992
- ( 67,  0) |    79611    53515     1.5 |        8       11     1024   636888
----------- | -------- -------- ------- | -------- -------- -------- --------
-       DEV |       #Q       #D   Ratio |   BLKmin   BLKavg   BLKmax    Total
-     TOTAL |  4485736  2790572     1.6 |        8       12     1024 35896640
-
-\end{verbatim}
-\newpage\begin{verbatim}
-
-==================== Device Seek Information ====================
-
-       DEV |          NSEEKS            MEAN          MEDIAN | MODE           
----------- | --------------- --------------- --------------- | ---------------
- (  8,160) |           47890        203945.5               0 | 0(2496)
- (  8,176) |           55492        252948.8               0 | 0(2360)
- (  8,208) |           63944        167845.3               0 | 0(4327)
- ( 65, 32) |           44494        224708.3               0 | 0(2683)
- ( 65, 64) |           77396        197838.0               0 | 0(2532)
- ( 65,176) |           62746        168400.5               0 | 0(2675)
- ( 65, 96) |           51705        162500.2               0 | 0(2778)
- (  8,224) |           49753        206665.7               0 | 0(2753)
- ( 65,112) |           63530        146289.8               0 | 0(2598)
- (  8,240) |           44700        167258.3               0 | 0(2735)
- ( 65,192) |           59010        217004.7               0 | 0(2724)
- ( 65,240) |           53548        278194.8               0 | 0(2415)
- ( 65, 48) |           44652        180710.3               0 | 0(2660)
- (  8, 32) |           63116        234049.5               0 | 0(2473)
- ( 66, 80) |           71526        184981.9               0 | 0(2455)
- ( 66, 32) |           53134        198369.5               0 | 0(2415)
- ( 65,224) |           66180        157948.5               0 | 0(2859)
- ( 65,144) |           70205        195689.7               0 | 0(2734)
- (  8,144) |           49047        169282.7               0 | 0(2574)
- ( 66,144) |           70613        149265.7               0 | 0(2717)
- ( 66,128) |           65381        165931.1               0 | 0(2687)
- ( 66,176) |           47894        199744.8               0 | 0(3096)
- ( 66,224) |           58159        161603.9               0 | 0(2738)
- ( 66,192) |           55417        153055.3               0 | 0(2828)
- ( 66,160) |           71156        126479.4               0 | 0(2739)
- ( 66,240) |           66472        142014.0               0 | 0(2933)
- ( 66,112) |           82027        115471.3               0 | 0(2961)
- (  8, 64) |           63892        136632.0               0 | 0(2655)
- ( 65,160) |           55579        154668.3               0 | 0(3377)
- ( 65,208) |           67987        152829.1               0 | 0(2523)
- ( 66, 16) |           56613        150652.8               0 | 0(2886)
- ( 65,  0) |           51532        186889.7               0 | 0(2765)
- ( 65, 16) |           35982        327187.6               0 | 0(2756)
- ( 66,208) |           54306        243784.1               0 | 0(3076)
- (  8,128) |           59653        170797.2               0 | 0(2800)
- ( 65, 80) |           65672        224501.4               0 | 0(2452)
- (  8,112) |           52847        213100.3               0 | 0(2417)
- ( 65,128) |           63106        207821.8               0 | 0(2686)
- ( 66, 64) |           50956        198045.0               0 | 0(2896)
- ( 66,  0) |           46722        205476.3               0 | 0(3159)
- ( 66, 48) |           65219        142716.6               0 | 0(2869)
- (  8,192) |           47154        248778.6               0 | 0(2584)
- ( 67, 16) |           48487        193335.5               0 | 0(2879)
- ( 66, 96) |           54454        185100.7               0 | 0(2158)
- (  8, 96) |           73522        216187.1               0 | 0(2476)
- (  8, 80) |           56643        172561.8               0 | 0(2612)
- (  8, 48) |           57544        219104.1               0 | 0(2472)
- ( 67,  0) |           53515        161613.8               0 | 0(2990)
----------- | --------------- --------------- --------------- | ---------------
-   Overall |          NSEEKS            MEAN          MEDIAN | MODE           
-   Average |         2790572        185170.0               0 | 0(131433)
-
-\end{verbatim}
-\newpage\begin{verbatim}
-==================== Plug Information ====================
-
-       DEV |    # Plugs # Timer Us  | % Time Q Plugged
----------- | ---------- ----------  | ----------------
- (  8,160) |      45092(         0) |   0.003021834%
- (  8,176) |      49430(         0) |   0.003724997%
- (  8,208) |      58950(         0) |   0.004506453%
- ( 65, 32) |      41617(         0) |   0.002710008%
- ( 65, 64) |      74294(         0) |   0.004971739%
- ( 65,176) |      59764(         0) |   0.003827258%
- ( 65, 96) |      47495(         0) |   0.004012641%
- (  8,224) |      46806(         0) |   0.003724031%
- ( 65,112) |      60539(         0) |   0.004272802%
- (  8,240) |      41797(         0) |   0.002832794%
- ( 65,192) |      54754(         0) |   0.004049758%
- ( 65,240) |      50757(         0) |   0.003466684%
- ( 65, 48) |      41749(         0) |   0.002783118%
- (  8, 32) |      59486(         0) |   0.004102902%
- ( 66, 80) |      67519(         0) |   0.004698104%
- ( 66, 32) |      50441(         0) |   0.003229586%
- ( 65,224) |      60219(         0) |   0.005114778%
- ( 65,144) |      64699(         0) |   0.005185315%
- (  8,144) |      46237(         0) |   0.002824578%
- ( 66,144) |      67605(         0) |   0.004459997%
- ( 66,128) |      62418(         0) |   0.004598612%
- ( 66,176) |      43007(         0) |   0.003277143%
- ( 66,224) |      54724(         0) |   0.003682546%
- ( 66,192) |      52395(         0) |   0.003611178%
- ( 66,160) |      67775(         0) |   0.004347445%
- ( 66,240) |      62892(         0) |   0.003906526%
- ( 66,112) |      72351(         0) |   0.005906198%
- (  8, 64) |      59642(         0) |   0.004275726%
- ( 65,160) |      50303(         0) |   0.003841735%
- ( 65,208) |      64750(         0) |   0.004655374%
- ( 66, 16) |      53443(         0) |   0.003936557%
- ( 65,  0) |      48450(         0) |   0.003301599%
- ( 65, 16) |      33047(         0) |   0.002447028%
- ( 66,208) |      51060(         0) |   0.003674090%
- (  8,128) |      52664(         0) |   0.004009472%
- ( 65, 80) |      61974(         0) |   0.004623080%
- (  8,112) |      50106(         0) |   0.003284028%
- ( 65,128) |      60047(         0) |   0.004267589%
- ( 66, 64) |      47590(         0) |   0.003646022%
- ( 66,  0) |      43413(         0) |   0.003600655%
- ( 66, 48) |      61984(         0) |   0.004440717%
- (  8,192) |      44294(         0) |   0.003120507%
- ( 67, 16) |      45482(         0) |   0.002852274%
- ( 66, 96) |      50099(         0) |   0.003970858%
- (  8, 96) |      70266(         0) |   0.004743310%
- (  8, 80) |      53676(         0) |   0.003530690%
- (  8, 48) |      54672(         0) |   0.003527266%
- ( 67,  0) |      50418(         0) |   0.003217817%
----------- | ---------- ----------  | ----------------
-       DEV |    # Plugs # Timer Us  | % Time Q Plugged
-   OVERALL |      54420(         0) |   0.003871155%
-\end{verbatim}
+\input{sample-btt-output.tex}
 \end{document}
diff --git a/btt/doc/sample-btt-output.tex b/btt/doc/sample-btt-output.tex
new file mode 100644
index 0000000..5cb42f5
--- /dev/null
+++ b/btt/doc/sample-btt-output.tex
@@ -0,0 +1,138 @@
+\begin{verbatim}
+==================== All Devices ====================
+
+            ALL           MIN           AVG           MAX           N
+--------------- ------------- ------------- ------------- -----------
+Q2Q               0.000000001   0.000171953   0.248458647     3163303
+Q2A               0.000285034   0.000285034   0.000285034           1
+Q2I               0.000000400   0.000010041   0.004243885     3163305
+I2D               0.000000001   0.000048286   0.077015932     3163304
+D2C               0.000000001   0.001732394   0.360111706     3163299
+Q2C               0.000131596   0.001790722   0.360143154     3163299
+\end{verbatim}\newpage\begin{verbatim}
+==================== Device Overhead ====================
+
+       DEV |    Q2I    I2D    D2C
+---------- | ------ ------ ------
+ ( 65, 16) |   0.5%   8.1%  91.4%
+ ( 65,176) |   0.5%   1.5%  98.0%
+ ( 65,160) |   0.5%   1.5%  98.0%
+ ( 65,192) |   0.5%   1.2%  98.3%
+ ( 65,208) |   0.5%   6.1%  93.4%
+ ( 65,240) |   0.5%   1.1%  98.3%
+ ( 65,224) |   0.5%   5.8%  93.7%
+ ( 66,  0) |   0.6%   1.1%  98.3%
+ ( 66, 16) |   0.6%   1.1%  98.3%
+ ( 66, 32) |   0.6%   1.1%  98.3%
+ ( 66, 48) |   0.6%   1.0%  98.4%
+ ( 65, 32) |   0.6%   9.7%  89.7%
+ ( 66, 64) |   0.6%   3.1%  96.3%
+ ( 65, 48) |   0.6%   1.5%  97.9%
+ ( 65, 64) |   0.6%   1.5%  98.0%
+ ( 65, 80) |   0.6%   1.5%  98.0%
+ ( 65, 96) |   0.6%   1.4%  98.0%
+ ( 65,112) |   0.6%   1.4%  98.0%
+ ( 65,128) |   0.6%   1.4%  98.0%
+ ( 65,144) |   0.6%   1.3%  98.1%
+\end{verbatim}\newpage\begin{verbatim}
+==================== Device Merge Information ====================
+
+       DEV |       #Q       #D   Ratio |   BLKmin   BLKavg   BLKmax    Total
+---------- | -------- -------- ------- | -------- -------- -------- --------
+ ( 65, 16) |   165163   163803     1.0 |        1        9      104  1605272
+ ( 65,176) |   159070   158236     1.0 |        8        9       96  1524016
+ ( 65,160) |   159580   158705     1.0 |        1        9      120  1525712
+ ( 65,192) |   158697   157879     1.0 |        8        9       96  1520184
+ ( 65,208) |   161893   161061     1.0 |        3        9       96  1544184
+ ( 65,240) |   157983   157175     1.0 |        8        9      128  1512600
+ ( 65,224) |   162596   161750     1.0 |        6        9      120  1575528
+ ( 66,  0) |   158474   157646     1.0 |        8        9       96  1518704
+ ( 66, 16) |   158453   157656     1.0 |        8        9      112  1516704
+ ( 66, 32) |   156270   155467     1.0 |        8        9      112  1499744
+ ( 66, 48) |   156296   155472     1.0 |        8        9       96  1501064
+ ( 65, 32) |   162500   161664     1.0 |        1       10      128  1625368
+ ( 66, 64) |   157908   156648     1.0 |        1        9       96  1504648
+ ( 65, 48) |   155031   154203     1.0 |        8        9      128  1490840
+ ( 65, 64) |   155879   155042     1.0 |        8        9      128  1495192
+ ( 65, 80) |   155590   154756     1.0 |        6        9      128  1494080
+ ( 65, 96) |   155967   155139     1.0 |        6        9       96  1500504
+ ( 65,112) |   155780   154968     1.0 |        8        9      128  1495784
+ ( 65,128) |   154916   154105     1.0 |        8        9      128  1487712
+ ( 65,144) |   155253   154440     1.0 |        8        9      128  1491592
+---------- | -------- -------- ------- | -------- -------- -------- --------
+       DEV |       #Q       #D   Ratio |   BLKmin   BLKavg   BLKmax    Total
+     TOTAL |  3163299  3145815     1.0 |        1        9      128 30429432
+\end{verbatim}\newpage\begin{verbatim}
+==================== Device Q2Q Seek Information ====================
+
+       DEV |          NSEEKS            MEAN          MEDIAN | MODE           
+---------- | --------------- --------------- --------------- | ---------------
+ ( 65, 16) |          165164      34513601.2               0 | 0(10240)
+ ( 65,176) |          159070      35875810.3               0 | 0(7279)
+ ( 65,160) |          159580      35686855.3               0 | 0(7396)
+ ( 65,192) |          158697      35873254.0               0 | 0(7097)
+ ( 65,208) |          161895      35251205.6               0 | 0(8950)
+ ( 65,240) |          157983      35690510.7               0 | 0(7058)
+ ( 65,224) |          162597      35215774.0               0 | 0(8931)
+ ( 66,  0) |          158474      35679927.3               0 | 0(7089)
+ ( 66, 16) |          158453      35953391.0               0 | 0(7115)
+ ( 66, 32) |          156270      35835785.0               0 | 0(7107)
+ ( 66, 48) |          156296      35693042.3               0 | 0(7135)
+ ( 65, 32) |          162501      34412411.2               0 | 0(7749)
+ ( 66, 64) |          157908      35159261.5               0 | 0(9040)
+ ( 65, 48) |          155031      35670294.2               0 | 0(7010)
+ ( 65, 64) |          155879      35634563.5               0 | 0(7094)
+ ( 65, 80) |          155590      35768901.5               0 | 0(7126)
+ ( 65, 96) |          155967      35774790.5               0 | 0(7160)
+ ( 65,112) |          155780      35715302.8               0 | 0(7096)
+ ( 65,128) |          154916      35863718.6               0 | 0(7102)
+ ( 65,144) |          155253      35831897.7               0 | 0(7158)
+---------- | --------------- --------------- --------------- | ---------------
+   Overall |          NSEEKS            MEAN          MEDIAN | MODE           
+   Average |         3163304      35549147.0               0 | 0(151932)
+\end{verbatim}\newpage\begin{verbatim}
+==================== Device D2D Seek Information ====================
+
+       DEV |          NSEEKS            MEAN          MEDIAN | MODE           
+---------- | --------------- --------------- --------------- | ---------------
+ ( 65, 16) |          163803      34673643.1               0 | 0(9165)
+ ( 65,176) |          158236      36061875.3               0 | 0(6488)
+ ( 65,160) |          158705      35885601.9               0 | 0(6575)
+ ( 65,192) |          157879      36058734.5               0 | 0(6294)
+ ( 65,208) |          161061      35327230.2               0 | 0(8538)
+ ( 65,240) |          157175      35867982.1               0 | 0(6282)
+ ( 65,224) |          161750      35312998.9               0 | 0(8512)
+ ( 66,  0) |          157646      35867654.4               0 | 0(6287)
+ ( 66, 16) |          157656      36130706.7               0 | 0(6355)
+ ( 66, 32) |          155467      36012662.4               0 | 0(6334)
+ ( 66, 48) |          155472      35867912.7               0 | 0(6346)
+ ( 65, 32) |          161664      34441442.0               0 | 0(7107)
+ ( 66, 64) |          156648      35403718.1               0 | 0(7957)
+ ( 65, 48) |          154203      35841703.8               0 | 0(6214)
+ ( 65, 64) |          155042      35821650.7               0 | 0(6294)
+ ( 65, 80) |          154756      35957188.1               0 | 0(6318)
+ ( 65, 96) |          155139      35961290.3               0 | 0(6345)
+ ( 65,112) |          154968      35897688.9               0 | 0(6299)
+ ( 65,128) |          154105      36039309.9               0 | 0(6306)
+ ( 65,144) |          154440      36008517.2               0 | 0(6381)
+---------- | --------------- --------------- --------------- | ---------------
+   Overall |          NSEEKS            MEAN          MEDIAN | MODE           
+   Average |         3145815      35715829.0               0 | 0(136397)
+\end{verbatim}\newpage\begin{verbatim}
+==================== Plug Information ====================
+
+       DEV |    # Plugs # Timer Us  | % Time Q Plugged
+---------- | ---------- ----------  | ----------------
+ ( 65, 16) |          2(         1) |   0.183841368%
+ ( 65, 32) |          2(         1) |   0.207033397%
+ ( 65, 48) |          2(         1) |   0.207120028%
+ ( 65, 64) |          2(         1) |   0.207174813%
+ ( 65, 80) |          2(         1) |   0.207215340%
+ ( 65, 96) |          2(         1) |   0.207275962%
+ ( 65,112) |          2(         1) |   0.207305990%
+ ( 65,128) |          2(         1) |   0.207349645%
+ ( 65,144) |          2(         1) |   0.207413334%
+---------- | ---------- ----------  | ----------------
+   Overall |    # Plugs # Timer Us  | % Time Q Plugged
+   Average |          2(         1) |   0.204636653%
+\end{verbatim}
diff --git a/btt/globals.h b/btt/globals.h
index 5a6982f..163986b 100644
--- a/btt/globals.h
+++ b/btt/globals.h
@@ -72,14 +72,14 @@ enum iop_type {
 	IOP_Q = 0,
 	IOP_X = 1,
 	IOP_A = 2,
-	IOP_I = 3,
+	IOP_G = 3,
 	IOP_M = 4,
 	IOP_D = 5,
 	IOP_C = 6,
 	IOP_R = 7,
-	IOP_L = 8, 	// IOP_A -> IOP_A + IOP_L
+	IOP_I = 8
 };
-#define N_IOP_TYPES	(IOP_L + 1)
+#define N_IOP_TYPES	(IOP_I + 1)
 
 struct file_info {
 	struct file_info *next;
@@ -106,12 +106,19 @@ struct avg_info {
 };
 
 struct avgs_info {
+        struct avg_info q2q_dm;
+        struct avg_info q2a_dm;
+        struct avg_info q2c_dm;
+
         struct avg_info q2q;
-	struct avg_info q2c;
-	struct avg_info q2a;		/* Q to (A or X) */
-	struct avg_info q2i;		/* Q to (I or M) */
-	struct avg_info i2d;		/* (I or M) to D */
+	struct avg_info q2a;
+	struct avg_info q2g;
+	struct avg_info g2i;
+	struct avg_info q2m;
+	struct avg_info i2d;
+	struct avg_info m2d;
 	struct avg_info d2c;
+	struct avg_info q2c;
 
 	struct avg_info blks;		/* Blocks transferred */
 };
@@ -158,11 +165,13 @@ struct d_info {
 	void *heads;
 	struct region_info regions;
 	struct devmap *map;
-	void *seek_handle, *bno_dump_handle, *unplug_hist_handle;
+	void *q2q_handle, *seek_handle, *bno_dump_handle, *unplug_hist_handle;
+	void *q2d_priv;
 	FILE *d2c_ofp, *q2c_ofp;
 	struct avgs_info avgs;
 	struct stats stats, all_stats;
-	__u64 last_q, n_ds;
+	__u64 last_q, n_qs, n_ds;
+	__u64 n_act_q, t_act_q;	/* # currently active when Q comes in */
 	__u32 device;
 
 	int pre_culling;
@@ -172,26 +181,23 @@ struct d_info {
 
 struct io {
 	struct rb_node rb_node;
-	struct list_head f_head, c_pending, retry, rm_head;
-	struct list_head down_list, up_list;
+	struct list_head f_head;
 	struct d_info *dip;
 	struct p_info *pip;
 	void *pdu;
+	__u64 bytes_left, g_time, i_time, m_time, d_time, c_time, d_sec, c_sec;
+	__u32 d_nsec, c_nsec;
+
 	struct blk_io_trace t;
-	__u64 bytes_left;
-	int linked, on_retry_list, down_len, up_len, on_rm_list;
+
+	int linked;
 	enum iop_type type;
+
 #if defined(COUNT_IOS)
 	struct list_head cio_head;
 #endif
 };
 
-struct bilink {
-	struct list_head down_head, up_head;
-	struct io *diop, *uiop;
-};
-#define bilink_free_head	down_head
-
 /* bt_timeline.c */
 
 extern char bt_timeline_version[], *devices, *exes, *input_name, *output_name;
@@ -202,11 +208,11 @@ extern FILE *ranges_ofp, *avgs_ofp, *iostat_ofp, *per_io_ofp;
 extern int verbose, done, time_bounded, output_all_data, seek_absolute;
 extern unsigned int n_devs;
 extern unsigned long n_traces;
-extern struct list_head all_devs, all_procs, retries, rmhd;
+extern struct list_head all_devs, all_procs;
 extern struct avgs_info all_avgs;
-extern __u64 last_q, next_retry_check;
+extern __u64 last_q;
 extern struct region_info all_regions;
-extern struct list_head free_ios, free_bilinks;
+extern struct list_head free_ios;
 extern __u64 iostat_interval, iostat_last_stamp;
 extern time_t genesis, last_vtrace;
 extern double t_astart, t_aend;
@@ -241,7 +247,8 @@ void dip_foreach(struct io *iop, enum iop_type type,
 struct io *dip_find_sec(struct d_info *dip, enum iop_type type, __u64 sec);
 void dip_foreach_out(void (*func)(struct d_info *, void *), void *arg);
 void dip_plug(__u32 dev, double cur_time);
-void dip_unplug(__u32 dev, double cur_time, int is_timer);
+void dip_unplug(__u32 dev, double cur_time);
+void dip_unplug_tm(__u32 dev);
 void dip_exit(void);
 
 /* dip_rb.c */
@@ -253,7 +260,7 @@ void rb_foreach(struct rb_node *n, struct io *iop,
 
 /* iostat.c */
 void iostat_init(void);
-void iostat_insert(struct io *iop);
+void iostat_getrq(struct io *iop);
 void iostat_merge(struct io *iop);
 void iostat_issue(struct io *iop);
 void iostat_unissue(struct io *iop);
@@ -298,8 +305,18 @@ void bno_dump_exit(void *param);
 void bno_dump_add(void *handle, struct io *iop);
 void bno_dump_clean(void);
 
+/* q2d.c */
+void q2d_histo_add(void *priv, __u64 q2d);
+void *q2d_init(void);
+void q2d_release(void *priv);
+void q2d_display_header(FILE *fp);
+void q2d_display_dashes(FILE *fp);
+void q2d_display(FILE *fp, void *priv);
+int q2d_ok(void *priv);
+void q2d_acc(void *a1, void *a2);
+
 /* seek.c */
-void *seeki_init(__u32 device);
+void *seeki_init(char *str);
 void seeki_exit(void *param);
 void seek_clean(void);
 void seeki_add(void *handle, struct io *iop);
@@ -309,15 +326,10 @@ long long seeki_median(void *handle);
 int seeki_mode(void *handle, struct mode *mp);
 
 /* trace.c */
-void __dump_iop(FILE *ofp, struct io *iop, int extra_nl);
-void __dump_iop2(FILE *ofp, struct io *a_iop, struct io *l_iop);
-void release_iops(void);
 void add_trace(struct io *iop);
-void do_retries(__u64 now);
 
 /* trace_complete.c */
 void trace_complete(struct io *c_iop);
-void retry_complete(struct io *c_iop, __u64 now);
 
 /* trace_im.c */
 void run_im(struct io *im_iop, struct io *d_iop, struct io *c_iop);
@@ -325,6 +337,7 @@ void run_unim(struct io *im_iop, struct io *d_iop, struct io *c_iop);
 int ready_im(struct io *im_iop, struct io *c_iop);
 void trace_insert(struct io *i_iop);
 void trace_merge(struct io *m_iop);
+void trace_getrq(struct io *g_iop);
 
 /* trace_issue.c */
 void run_issue(struct io *d_iop, struct io *u_iop, struct io *c_iop);
diff --git a/btt/inlines.h b/btt/inlines.h
index 25e5b21..875b75b 100644
--- a/btt/inlines.h
+++ b/btt/inlines.h
@@ -19,6 +19,12 @@
  *
  */
 
+static inline int remapper_dev(__u32 dev)
+{
+	int mjr = MAJOR(dev);
+	return mjr == 9 || mjr == 253 || mjr == 254;
+}
+
 static inline void region_init(struct region_info *reg)
 {
 	INIT_LIST_HEAD(&reg->qranges);
@@ -118,7 +124,10 @@ static inline void update_lq(__u64 *last_q, struct avg_info *avg, __u64 time)
 
 static inline void dip_update_q(struct d_info *dip, struct io *iop)
 {
-	update_lq(&dip->last_q, &dip->avgs.q2q, iop->t.time);
+	if (remapper_dev(dip->device))
+		update_lq(&dip->last_q, &dip->avgs.q2q_dm, iop->t.time);
+	else
+		update_lq(&dip->last_q, &dip->avgs.q2q, iop->t.time);
 	update_qregion(&dip->regions, iop->t.time);
 }
 
@@ -143,13 +152,9 @@ static inline struct io *io_alloc(void)
 	}
 
 	memset(iop, 0, sizeof(struct io));
-	INIT_LIST_HEAD(&iop->down_list);
-	INIT_LIST_HEAD(&iop->up_list);
 
 #	if defined(DEBUG)
 		iop->f_head.next = LIST_POISON1;
-		iop->c_pending.next = LIST_POISON1;
-		iop->retry.next = LIST_POISON1;
 #	endif
 
 #	if defined(COUNT_IOS)
@@ -199,10 +204,6 @@ static inline int io_setup(struct io *iop, enum iop_type type)
 static inline void io_release(struct io *iop)
 {
 	ASSERT(iop->f_head.next == LIST_POISON1);
-	ASSERT(iop->c_pending.next == LIST_POISON1);
-	ASSERT(iop->retry.next == LIST_POISON1);
-	ASSERT(list_empty(&iop->up_list));
-	ASSERT(list_empty(&iop->down_list));
 
 	if (iop->linked)
 		dip_rem(iop);
@@ -236,7 +237,10 @@ static inline void update_q2c(struct io *iop, __u64 c_time)
 		if (per_io_ofp) 
 			fprintf(per_io_ofp, "q2c %13.9f\n", BIT_TIME(c_time));
 #	endif
-	UPDATE_AVGS(q2c, iop, iop->pip, c_time);
+	if (remapper_dev(iop->dip->device))
+		UPDATE_AVGS(q2c_dm, iop, iop->pip, c_time);
+	else
+		UPDATE_AVGS(q2c, iop, iop->pip, c_time);
 }
 
 static inline void update_q2a(struct io *iop, __u64 a_time)
@@ -245,22 +249,55 @@ static inline void update_q2a(struct io *iop, __u64 a_time)
 		if (per_io_ofp) 
 			fprintf(per_io_ofp, "q2a %13.9f\n", BIT_TIME(a_time));
 #	endif
-	UPDATE_AVGS(q2a, iop, iop->pip, a_time);
+	if (remapper_dev(iop->dip->device))
+		UPDATE_AVGS(q2a_dm, iop, iop->pip, a_time);
+	else
+		UPDATE_AVGS(q2a, iop, iop->pip, a_time);
+}
+
+static inline void update_q2g(struct io *iop, __u64 g_time)
+{
+#	if defined(DEBUG)
+		if (per_io_ofp) 
+			fprintf(per_io_ofp, "q2g %13.9f\n", BIT_TIME(g_time));
+#	endif
+
+	UPDATE_AVGS(q2g, iop, iop->pip, g_time);
+}
+
+static inline void unupdate_q2g(struct io *iop, __u64 g_time)
+{
+	UNUPDATE_AVGS(q2g, iop, iop->pip, g_time);
+}
+
+static inline void update_g2i(struct io *iop, __u64 i_time)
+{
+#	if defined(DEBUG)
+		if (per_io_ofp) 
+			fprintf(per_io_ofp, "g2i %13.9f\n", BIT_TIME(i_time));
+#	endif
+
+	UPDATE_AVGS(g2i, iop, iop->pip, i_time);
+}
+
+static inline void unupdate_g2i(struct io *iop, __u64 i_time)
+{
+	UNUPDATE_AVGS(g2i, iop, iop->pip, i_time);
 }
 
-static inline void update_q2i(struct io *iop, __u64 i_time)
+static inline void update_q2m(struct io *iop, __u64 m_time)
 {
 #	if defined(DEBUG)
 		if (per_io_ofp) 
-			fprintf(per_io_ofp, "q2i %13.9f\n", BIT_TIME(i_time));
+			fprintf(per_io_ofp, "q2m %13.9f\n", BIT_TIME(m_time));
 #	endif
 
-	UPDATE_AVGS(q2i, iop, iop->pip, i_time);
+	UPDATE_AVGS(q2m, iop, iop->pip, m_time);
 }
 
-static inline void unupdate_q2i(struct io *iop, __u64 i_time)
+static inline void unupdate_q2m(struct io *iop, __u64 m_time)
 {
-	UNUPDATE_AVGS(q2i, iop, iop->pip, i_time);
+	UNUPDATE_AVGS(q2m, iop, iop->pip, m_time);
 }
 
 static inline void update_i2d(struct io *iop, __u64 d_time)
@@ -278,15 +315,29 @@ static inline void unupdate_i2d(struct io *iop, __u64 d_time)
 	UNUPDATE_AVGS(i2d, iop, iop->pip, d_time);
 }
 
-static inline void update_d2c(struct io *iop, int n, __u64 c_time)
+static inline void update_m2d(struct io *iop, __u64 d_time)
+{
+#	if defined(DEBUG)
+		if (per_io_ofp) 
+			fprintf(per_io_ofp, "m2d %13.9f\n", BIT_TIME(d_time));
+#	endif
+
+	UPDATE_AVGS(m2d, iop, iop->pip, d_time);
+}
+
+static inline void unupdate_m2d(struct io *iop, __u64 d_time)
+{
+	UNUPDATE_AVGS(m2d, iop, iop->pip, d_time);
+}
+
+static inline void update_d2c(struct io *iop, __u64 c_time)
 {
 #	if defined(DEBUG)
 		if (per_io_ofp) 
-			fprintf(per_io_ofp, "d2c %13.9f\n", 
-							n*BIT_TIME(c_time));
+			fprintf(per_io_ofp, "d2c %13.9f\n", BIT_TIME(c_time));
 #	endif
 
-	UPDATE_AVGS_N(d2c, iop, iop->pip, c_time, n);
+	UPDATE_AVGS(d2c, iop, iop->pip, c_time);
 }
 
 static inline void update_blks(struct io *iop)
@@ -333,55 +384,9 @@ static inline struct io *dip_rb_find_sec(struct d_info *dip,
 	return rb_find_sec(__get_root(dip, type), sec);
 }
 
-static inline void bump_retry(__u64 now)
-{
-	if (!list_empty(&retries))
-		next_retry_check = now + (100 * 1000); // 100 usec
-	else 
-		next_retry_check = 0;
-}
-
-static inline void add_retry(struct io *iop)
-{
-	bump_retry(iop->t.time);
-	if (!iop->on_retry_list) {
-		list_add_tail(&iop->retry, &retries);
-		iop->on_retry_list = 1;
-	}
-}
-
-static inline void del_retry(struct io *iop)
-{
-	if (iop->on_retry_list) {
-		LIST_DEL(&iop->retry);
-		iop->on_retry_list = 0;
-	}
-	bump_retry(iop->t.time);
-}
-
-static inline __u64 tdelta(struct io *iop1, struct io *iop2)
-{
-	__u64 t1 = iop1->t.time;
-	__u64 t2 = iop2->t.time;
-	return (t1 < t2) ? (t2 - t1) : 1;
-}
-
-static inline int remapper_dev(__u32 dev)
-{
-	int mjr = MAJOR(dev);
-	return mjr == 9 || mjr == 253 || mjr == 254;
-}
-
-static inline void dump_iop(struct io *iop, int extra_nl)
+static inline __u64 tdelta(__u64 from, __u64 to)
 {
-	if (per_io_ofp) 
-		__dump_iop(per_io_ofp, iop, extra_nl);
-}
-
-static inline void dump_iop2(struct io *a_iop, struct io *l_iop)
-{
-	if (per_io_ofp) 
-		__dump_iop2(per_io_ofp, a_iop, l_iop);
+	return (from < to) ? (to - from) : 1;
 }
 
 static inline int type2c(enum iop_type type)
@@ -397,130 +402,54 @@ static inline int type2c(enum iop_type type)
 	case IOP_D: c = 'D'; break;
 	case IOP_C: c = 'C'; break;
 	case IOP_R: c = 'R'; break;
-	case IOP_L: c = 'L'; break;
+	case IOP_G: c = 'G'; break;
 	default   : c = '?'; break;
 	}
 
 	return c;
 }
 
-static inline void bilink_free(struct bilink *blp)
-{
-	list_add_tail(&blp->bilink_free_head, &free_bilinks);
-}
-
-static inline void bilink_free_all(void)
-{
-	struct bilink *blp;
-	struct list_head *p, *q;
-
-	list_for_each_safe(p, q, &free_bilinks) {
-		blp = list_entry(p, struct bilink, bilink_free_head);
-		free(blp);
-	}
-}
-
-static inline struct bilink *bilink_alloc(struct io *diop, struct io *uiop)
-{
-	struct bilink *blp;
-
-	if (!list_empty(&free_bilinks)) {
-		blp = list_entry(free_bilinks.prev, struct bilink, 
-							bilink_free_head);
-		LIST_DEL(&blp->bilink_free_head);
-	}
-	else
-		blp = malloc(sizeof(*blp));
-
-	blp->diop = diop;
-	blp->uiop = uiop;
-
-	return blp;
-}
-
-static inline void bilink(struct io *diop, struct io *uiop)
+static inline int histo_idx(__u64 nbytes)
 {
-	struct bilink *blp = bilink_alloc(diop, uiop);
-
-	list_add_tail(&blp->down_head, &diop->up_list);
-	list_add_tail(&blp->up_head, &uiop->down_list);
-
-	diop->up_len++;
-	uiop->down_len++;
+	int idx = (nbytes >> 9) - 1;
+	return min(idx, N_HIST_BKTS-1);
 }
 
-static inline void biunlink(struct bilink *blp)
+static inline void update_q_histo(__u64 nbytes)
 {
-	LIST_DEL(&blp->down_head);
-	LIST_DEL(&blp->up_head);
-	blp->diop->up_len--;
-	blp->uiop->down_len--;
-	bilink_free(blp);
+	q_histo[histo_idx(nbytes)]++;
 }
 
-static inline struct io *bilink_first_down(struct io *iop, 
- 							struct bilink **blp_p)
+static inline void update_d_histo(__u64 nbytes)
 {
-	struct bilink *blp;
-
-	if (list_empty(&iop->down_list))
-		return NULL;
-	blp = list_entry(iop->down_list.next, struct bilink, up_head);
-
-	if (blp_p != NULL) 
-		*blp_p = blp;
-	return blp->diop;
+	d_histo[histo_idx(nbytes)]++;
 }
 
-static inline struct io *bilink_first_up(struct io *iop, struct bilink **blp_p)
+static inline struct io *io_first_list(struct list_head *head)
 {
-	struct bilink *blp;
-
-	if (list_empty(&iop->up_list))
+	if (list_empty(head))
 		return NULL;
-	blp = list_entry(iop->up_list.next, struct bilink, down_head);
-
-	if (blp_p != NULL) 
-		*blp_p = blp;
-	return blp->uiop;
-}
-
-typedef void (*bilink_func)(struct io *diop, struct io *uiop, 
-							struct io *c_iop);
-static inline void bilink_for_each_down(bilink_func func, struct io *uiop,
-				        struct io *c_iop, int ul)
-{
-	struct bilink *blp;
-	struct list_head *p, *q;
-
-	list_for_each_safe(p, q, &uiop->down_list) {
-		blp = list_entry(p, struct bilink, up_head);
-		func(blp->diop, uiop, c_iop);
-		if (ul)
-			biunlink(blp);
-	}
-}
-
-static inline int histo_idx(__u64 nbytes)
-{
-	int idx = (nbytes >> 9) - 1;
-	return min(idx, N_HIST_BKTS-1);
-}
 
-static inline void update_q_histo(__u64 nbytes)
-{
-	q_histo[histo_idx(nbytes)]++;
+	return list_entry(head->next, struct io, f_head);
 }
 
-static inline void update_d_histo(__u64 nbytes)
+static inline void __dump_iop(FILE *ofp, struct io *iop, int extra_nl)
 {
-	d_histo[histo_idx(nbytes)]++;
+	fprintf(ofp, "%5d.%09lu %3d,%-3d %c %10llu+%-4u\n",
+		(int)SECONDS(iop->t.time),
+		(unsigned long)NANO_SECONDS(iop->t.time),
+		MAJOR(iop->t.device), MINOR(iop->t.device), type2c(iop->type),
+		(unsigned long long)iop->t.sector, t_sec(&iop->t));
+	if (extra_nl) fprintf(ofp, "\n");
 }
 
-static inline void add_rmhd(struct io *iop)
+static inline void __dump_iop2(FILE *ofp, struct io *a_iop, struct io *l_iop)
 {
-	if (!iop->on_rm_list) {
-		list_add_tail(&iop->rm_head, &rmhd);
-		iop->on_rm_list = 1;
-	}
+	fprintf(ofp, "%5d.%09lu %3d,%-3d %c %10llu+%-4u <- (%3d,%-3d) %10llu\n",
+		(int)SECONDS(a_iop->t.time),
+		(unsigned long)NANO_SECONDS(a_iop->t.time),
+		MAJOR(a_iop->t.device), MINOR(a_iop->t.device), 
+		type2c(a_iop->type), (unsigned long long)a_iop->t.sector, 
+		t_sec(&a_iop->t), MAJOR(l_iop->t.device), 
+		MINOR(l_iop->t.device), (unsigned long long)l_iop->t.sector);
 }
diff --git a/btt/iostat.c b/btt/iostat.c
index 360d972..f46d572 100644
--- a/btt/iostat.c
+++ b/btt/iostat.c
@@ -60,11 +60,6 @@ void dump_hdr(void)
 			    "avgrq-sz avgqu-sz   await   svctm  %%util   Stamp\n");
 }
 
-void im2d2c_func(struct io *c_iop, struct io *im_iop)
-{
-	ADD_STAT(c_iop->dip, wait, tdelta(im_iop, c_iop));
-}
-
 void iostat_init(void)
 {
 	last_start = (__u64)-1;
@@ -247,7 +242,7 @@ void iostat_check_time(__u64 stamp)
 	}
 }
 
-void iostat_insert(struct io *iop)
+void iostat_getrq(struct io *iop)
 {
 	update_tot_qusz(iop->dip, TO_SEC(iop->t.time));
 	INC_STAT(iop->dip, cur_qusz);
@@ -281,13 +276,15 @@ void iostat_unissue(struct io *iop)
 	DEC_STAT(dip, cur_dev);
 }
 
-void iostat_complete(struct io *d_iop, struct io *c_iop)
+void iostat_complete(struct io *q_iop, struct io *c_iop)
 {
 	double now = TO_SEC(c_iop->t.time);
-	struct d_info *dip = d_iop->dip;
+	struct d_info *dip = q_iop->dip;
 
-	dip_foreach(d_iop, IOP_I, im2d2c_func, 0);
-	dip_foreach(d_iop, IOP_M, im2d2c_func, 0);
+	if (q_iop->i_time != (__u64)-1)
+		ADD_STAT(c_iop->dip, wait, tdelta(q_iop->i_time,c_iop->t.time));
+	else if (q_iop->m_time != (__u64)-1)
+		ADD_STAT(c_iop->dip, wait, tdelta(q_iop->m_time,c_iop->t.time));
 
 	update_tot_qusz(dip, now);
 	DEC_STAT(dip, cur_qusz);
@@ -295,5 +292,5 @@ void iostat_complete(struct io *d_iop, struct io *c_iop)
 	update_idle_time(dip, now, 0);
 	DEC_STAT(dip, cur_dev);
 
-	ADD_STAT(dip, svctm, tdelta(d_iop, c_iop));
+	ADD_STAT(dip, svctm, tdelta(q_iop->t.time, c_iop->t.time));
 }
diff --git a/btt/list.h b/btt/list.h
index f28f0fb..363c888 100644
--- a/btt/list.h
+++ b/btt/list.h
@@ -1,6 +1,28 @@
 #ifndef _LINUX_LIST_H
 #define _LINUX_LIST_H
 
+#include <stdio.h>
+
+#ifndef offsetof
+/**
+ * Get offset of a member
+ */
+#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#endif
+
+#ifndef container_of
+/**
+ * Casts a member of a structure out to the containing structure
+ * @param ptr        the pointer to the member.
+ * @param type       the type of the container struct this is embedded in.
+ * @param member     the name of the member within the struct.
+ *
+ */
+#define container_of(ptr, type, member) ({                      \
+        const typeof( ((type *)0)->member ) *__mptr = (ptr);    \
+	        (type *)( (char *)__mptr - offsetof(type,member) );})
+#endif
+
 /*
  * These are non-NULL pointers that will result in page faults
  * under normal circumstances, used to verify that nobody uses
@@ -166,4 +188,29 @@ static inline void list_move_tail(struct list_head *list,
         list_add_tail(list, head);
 }
 
+static inline void __list_splice(struct list_head *list,
+                                 struct list_head *head)
+{
+        struct list_head *first = list->next;
+        struct list_head *last = list->prev;
+        struct list_head *at = head->next;
+
+        first->prev = head;
+        head->next = first;
+
+        last->next = at;
+        at->prev = last;
+}
+
+/**
+ *  * list_splice - join two lists
+ *   * @list: the new list to add.
+ *    * @head: the place to add it in the first list.
+ *     */
+static inline void list_splice(struct list_head *list, struct list_head *head)
+{
+        if (!list_empty(list))
+                __list_splice(list, head);
+}
+
 #endif
diff --git a/btt/output.c b/btt/output.c
index 18d1d53..fd22a9e 100644
--- a/btt/output.c
+++ b/btt/output.c
@@ -22,18 +22,30 @@
 #include "globals.h"
 
 typedef struct avg_info *ai_dip_t;
+ai_dip_t dip_q2q_dm_avg(struct d_info *dip) { return &dip->avgs.q2q_dm; }
+ai_dip_t dip_q2a_dm_avg(struct d_info *dip) { return &dip->avgs.q2a_dm; }
+ai_dip_t dip_q2c_dm_avg(struct d_info *dip) { return &dip->avgs.q2c_dm; }
+
 ai_dip_t dip_q2q_avg(struct d_info *dip) { return &dip->avgs.q2q; }
 ai_dip_t dip_q2c_avg(struct d_info *dip) { return &dip->avgs.q2c; }
 ai_dip_t dip_q2a_avg(struct d_info *dip) { return &dip->avgs.q2a; }
-ai_dip_t dip_q2i_avg(struct d_info *dip) { return &dip->avgs.q2i; }
+ai_dip_t dip_q2g_avg(struct d_info *dip) { return &dip->avgs.q2g; }
+ai_dip_t dip_g2i_avg(struct d_info *dip) { return &dip->avgs.g2i; }
+ai_dip_t dip_q2m_avg(struct d_info *dip) { return &dip->avgs.q2m; }
 ai_dip_t dip_i2d_avg(struct d_info *dip) { return &dip->avgs.i2d; }
 ai_dip_t dip_d2c_avg(struct d_info *dip) { return &dip->avgs.d2c; }
 
 typedef struct avg_info *ai_pip_t;
+ai_pip_t pip_q2q_dm_avg(struct p_info *pip) { return &pip->avgs.q2q_dm; }
+ai_pip_t pip_q2a_dm_avg(struct p_info *pip) { return &pip->avgs.q2a_dm; }
+ai_pip_t pip_q2c_dm_avg(struct p_info *pip) { return &pip->avgs.q2c_dm; }
+
 ai_pip_t pip_q2q_avg(struct p_info *pip) { return &pip->avgs.q2q; }
 ai_pip_t pip_q2c_avg(struct p_info *pip) { return &pip->avgs.q2c; }
 ai_pip_t pip_q2a_avg(struct p_info *pip) { return &pip->avgs.q2a; }
-ai_pip_t pip_q2i_avg(struct p_info *pip) { return &pip->avgs.q2i; }
+ai_pip_t pip_q2g_avg(struct p_info *pip) { return &pip->avgs.q2g; }
+ai_pip_t pip_g2i_avg(struct p_info *pip) { return &pip->avgs.g2i; }
+ai_pip_t pip_q2m_avg(struct p_info *pip) { return &pip->avgs.q2m; }
 ai_pip_t pip_i2d_avg(struct p_info *pip) { return &pip->avgs.i2d; }
 ai_pip_t pip_d2c_avg(struct p_info *pip) { return &pip->avgs.d2c; }
 
@@ -60,12 +72,6 @@ void __output_avg(FILE *ofp, char *hdr, struct avg_info *ap)
 	}
 }
 
-void output_hdr2(FILE *ofp, char*hdr)
-{
-	fprintf(ofp, "%15s %13s %13s %13s %13s %13s %13s\n", hdr, "Q2Q", "Q2A", "Q2I", "I2D", "D2C", "Q2C");
-	fprintf(ofp, "--------------- ------------- ------------- ------------- ------------- ------------- -------------\n");
-}
-
 static inline char *avg2string(struct avg_info *ap, char *string)
 {
 	if (ap->n > 0)
@@ -75,30 +81,6 @@ static inline char *avg2string(struct avg_info *ap, char *string)
 	return string;
 }
 
-void __output_avg2(FILE *ofp, char *hdr, struct avgs_info *ap)
-{
-	char c1[16], c2[16], c3[16], c4[16], c5[16], c6[16];
-
-	if (ap->q2q.n > 0 || ap->q2a.n > 0 || ap->q2i.n > 0 ||
-			ap->i2d.n > 0 || ap->d2c.n > 0 || ap->q2c.n > 0) {
-		fprintf(ofp, "%-15s %13s %13s %13s %13s %13s %13s\n", hdr,
-			avg2string(&ap->q2q,c1), avg2string(&ap->q2a,c2),
-			avg2string(&ap->q2i,c3), avg2string(&ap->i2d,c4),
-			avg2string(&ap->d2c,c5), avg2string(&ap->q2c,c6));
-	}
-}
-
-void __pip_output_avg2(struct p_info *pip, void *arg)
-{
-	__output_avg2((FILE *)arg, pip->name, &pip->avgs);
-}
-
-void __dip_output_avg2(struct d_info *dip, void *arg)
-{
-	char dev_info[15];
-	__output_avg2((FILE *)arg, make_dev_hdr(dev_info, 15, dip), &dip->avgs);
-}
-
 char *make_dev_hdr(char *pad, size_t len, struct d_info *dip)
 {
 	if (dip->map == NULL)
@@ -133,6 +115,49 @@ void output_dip_avg(FILE *ofp, char *hdr, ai_dip_t (*func)(struct d_info *))
 	fprintf(ofp, "\n");
 }
 
+struct __q2d {
+	FILE *ofp;
+	void *q2d_all;
+	int n;
+};
+void __output_q2d_histo(struct d_info *dip, void *arg)
+{
+	struct __q2d *q2dp = arg;
+
+	if (q2d_ok(dip->q2d_priv)) {
+		char scratch[15];
+		FILE *ofp = q2dp->ofp;
+
+		fprintf(q2dp->ofp, "%10s | ", make_dev_hdr(scratch, 15, dip));
+		q2d_display(ofp, dip->q2d_priv);
+		q2d_acc(q2dp->q2d_all, dip->q2d_priv);
+		q2dp->n++;
+	}
+}
+
+void output_q2d_histo(FILE *ofp)
+{
+	struct __q2d __q2d = {
+		.ofp = ofp,
+		.q2d_all = q2d_init(),
+		.n = 0
+	};
+
+	fprintf(ofp, "%10s | ", "DEV");
+	q2d_display_header(ofp);
+	fprintf(ofp, "--------- | ");
+	q2d_display_dashes(ofp);
+	dip_foreach_out(__output_q2d_histo, &__q2d);
+
+	if (__q2d.n) {
+		fprintf(ofp, "========== | ");
+		q2d_display_dashes(ofp);
+		fprintf(ofp, "%10s | ", "AVG");
+		q2d_display(ofp, __q2d.q2d_all);
+		fprintf(ofp, "\n");
+	}
+}
+
 int n_merges = 0;
 struct {
 	unsigned long long nq, nd, blkmin, blkmax, total;
@@ -141,15 +166,25 @@ void __output_dip_merge_ratio(struct d_info *dip, void *arg)
 {
 	double blks_avg;
 	char scratch[15];
-	double ratio, q2c_n = dip->avgs.q2c.n, d2c_n = dip->n_ds;
+	double ratio, q2c_n, d2c_n;
 
+	if (dip->n_qs == 0 || dip->n_ds == 0)
+		return;
+	else if (dip->n_qs < dip->n_ds)
+		dip->n_qs = dip->n_ds;
+
+	q2c_n = dip->n_qs;
+	d2c_n = dip->n_ds;
 	if (q2c_n > 0.0 && d2c_n > 0.0) {
-		ratio = q2c_n / d2c_n;
+		if (q2c_n < d2c_n)
+			ratio = 1.0;
+		else
+			ratio = q2c_n / d2c_n;
 		blks_avg = (double)dip->avgs.blks.total / d2c_n;
 		fprintf((FILE *)arg, 
 			"%10s | %8llu %8llu %7.1lf | %8llu %8llu %8llu %8llu\n",
 			make_dev_hdr(scratch, 15, dip),
-			(unsigned long long)dip->avgs.q2c.n,
+			(unsigned long long)dip->n_qs,
 			(unsigned long long)dip->n_ds,
 			ratio,
 			(unsigned long long)dip->avgs.blks.min,
@@ -162,7 +197,7 @@ void __output_dip_merge_ratio(struct d_info *dip, void *arg)
 			merge_data.blkmax = dip->avgs.blks.max;
 		}
 
-		merge_data.nq += dip->avgs.q2c.n;
+		merge_data.nq += dip->n_qs;
 		merge_data.nd += dip->n_ds;
 		merge_data.total += dip->avgs.blks.total;
 		if (dip->avgs.blks.min < merge_data.blkmin)
@@ -191,73 +226,113 @@ void output_dip_merge_ratio(FILE *ofp)
 	fprintf(ofp, "\n");
 }
 
+struct __ohead_data {
+	__u64 total;
+	int n;
+};
+
+struct ohead_data {
+	FILE *ofp;
+	struct __ohead_data q2g, g2i, q2m, i2d, d2c;
+};
+
 #define AVG(a,b) (100.0 * ((double)(a) / (double)(b)))
 #define CALC_AVG(ap) (ap)->avg = ((ap)->n == 0 ? 0.0 :			  \
                                                  (BIT_TIME((ap)->total) / \
 							(double)(ap)->n))
-char *q2i_v_q2C(struct d_info *dip, char *s)
-{
-	double q2c;
-
-	if (dip->avgs.q2i.n == 0) return " ";
-
-	q2c = dip->avgs.q2i.avg + dip->avgs.i2d.avg + dip->avgs.d2c.avg;
-	sprintf(s, "%5.1lf%%", AVG(dip->avgs.q2i.avg, q2c));
 
+#define x_v_q2C(fld, dip, s, odp)					\
+	double q2c;							\
+	if (dip->avgs. fld .n == 0) return " ";				\
+	q2c = dip->avgs.g2i.avg + dip->avgs.g2i.avg +			\
+				dip->avgs.i2d.avg + dip->avgs.d2c.avg;	\
+	sprintf(s, "%8.4lf%%", AVG(dip->avgs. fld .avg, q2c));		\
+	odp-> fld .n += dip->avgs. fld .n;				\
+	odp-> fld .total += dip->avgs. fld .total;			\
 	return s;
-}
 
-char *i2d_v_q2C(struct d_info *dip, char *s)
+char *q2g_v_q2C(struct d_info *dip, char *s, struct ohead_data *odp)
 {
-	double q2c;
-
-	if (dip->avgs.d2c.n == 0) return " ";
-
-	q2c = dip->avgs.q2i.avg + dip->avgs.i2d.avg + dip->avgs.d2c.avg;
-	sprintf(s, "%5.1lf%%", AVG(dip->avgs.i2d.avg, q2c));
-
-	return s;
+	x_v_q2C(q2g, dip, s, odp);
 }
 
-char *d2c_v_q2C(struct d_info *dip, char *s)
+char *g2i_v_q2C(struct d_info *dip, char *s, struct ohead_data *odp)
 {
-	double q2c;
+	x_v_q2C(g2i, dip, s, odp);
+}
 
-	if (dip->avgs.d2c.n == 0) return " ";
+char *q2m_v_q2C(struct d_info *dip, char *s, struct ohead_data *odp)
+{
+	x_v_q2C(q2m, dip, s, odp);
+}
 
-	q2c = dip->avgs.q2i.avg + dip->avgs.i2d.avg + dip->avgs.d2c.avg;
-	sprintf(s, "%5.1lf%%", AVG(dip->avgs.d2c.avg, q2c));
+char *i2d_v_q2C(struct d_info *dip, char *s, struct ohead_data *odp)
+{
+	x_v_q2C(i2d, dip, s, odp);
+}
 
-	return s;
+char *d2c_v_q2C(struct d_info *dip, char *s, struct ohead_data *odp)
+{
+	x_v_q2C(d2c, dip, s, odp);
 }
 
 void __output_dip_prep_ohead(struct d_info *dip, void *arg)
 {
 	char dev_info[15];
-	char s1[16], s2[16], s3[16];
-
-	if ((dip->avgs.q2i.n > 0 && dip->avgs.i2d.n > 0 &&
-						dip->avgs.d2c.n > 0)) {
-		CALC_AVG(&dip->avgs.q2i);
+	char s1[16], s2[16], s3[16], s4[16], s5[16];
+	struct ohead_data *odp = arg;
+
+	if (dip->avgs.q2g.n > 0 && dip->avgs.g2i.n > 0 && 
+				   dip->avgs.i2d.n > 0 && dip->avgs.d2c.n > 0) {
+		CALC_AVG(&dip->avgs.q2g);
+		CALC_AVG(&dip->avgs.g2i);
+		CALC_AVG(&dip->avgs.q2m);
 		CALC_AVG(&dip->avgs.i2d);
 		CALC_AVG(&dip->avgs.d2c);
 
-		fprintf((FILE *)arg, "%10s | %6s %6s %6s\n",
+		fprintf(odp->ofp, "%10s | %9s %9s %9s %9s %9s\n",
 			make_dev_hdr(dev_info, 15, dip),
-			q2i_v_q2C(dip, s1), i2d_v_q2C(dip, s2),
-			d2c_v_q2C(dip, s3));
+			q2g_v_q2C(dip, s1, odp), 
+			g2i_v_q2C(dip, s2, odp), 
+			q2m_v_q2C(dip, s3, odp),
+			i2d_v_q2C(dip, s4, odp),
+			d2c_v_q2C(dip, s5, odp));
 	}
 }
 
+#define OD_AVG(od, fld, q2c)						\
+	(od. fld .n == 0) ? (double)0.0 :				\
+		(100.0 * ((double)((od). fld . total) / q2c))
+
 void output_dip_prep_ohead(FILE *ofp)
 {
-	fprintf(ofp, "%10s | %6s %6s %6s\n", "DEV", "Q2I", "I2D", "D2C");
-	fprintf(ofp, "---------- | ------ ------ ------\n");
-	dip_foreach_out(__output_dip_prep_ohead, ofp);
+	double q2c;
+	struct ohead_data od;
+
+	memset(&od, 0, sizeof(od));
+	od.ofp = ofp;
+
+	fprintf(ofp, "%10s | %9s %9s %9s %9s %9s\n", 
+				"DEV", "Q2G", "G2I", "Q2M", "I2D", "D2C");
+	fprintf(ofp, "---------- | --------- --------- --------- --------- ---------\n");
+	dip_foreach_out(__output_dip_prep_ohead, &od);
+
+	if (od.q2g.n == 0 && od.g2i.n == 0 && od.q2m.n == 0 && 
+						od.i2d.n == 0 && od.d2c.n == 0)
+		goto out;
+
+	q2c = od.q2g.total + od.g2i.total + od.q2m.total + 	
+						od.i2d.total + od.d2c.total;
+	fprintf(ofp, "---------- | --------- --------- --------- --------- ---------\n");
+	fprintf(ofp, "%10s | %8.4lf%% %8.4lf%% %8.4lf%% %8.4lf%% %8.4lf%%\n", "Overall", 
+			OD_AVG(od, q2g, q2c), OD_AVG(od, g2i, q2c), 
+			OD_AVG(od, q2m, q2c), OD_AVG(od, i2d, q2c), 
+			OD_AVG(od, d2c, q2c));
+
+out:
 	fprintf(ofp, "\n");
 }
 
-int n_seeks;
 struct seek_mode_info {
 	struct seek_mode_info *next;
 	long long mode;
@@ -267,12 +342,9 @@ struct o_seek_info {
 	long long nseeks, median;
 	double mean;
 	struct seek_mode_info *head;
-} seek_info = {
-	.nseeks = 0L,
-	.median = 0L,
-	.mean = 0.0,
-	.head = NULL
-};
+} seek_info;
+int n_seeks;
+
 void output_seek_mode_info(FILE *ofp, struct o_seek_info *sip)
 {
 	struct seek_mode_info *p, *this, *new_list = NULL;
@@ -280,46 +352,30 @@ void output_seek_mode_info(FILE *ofp, struct o_seek_info *sip)
 	ASSERT(sip->head != NULL);
 	while ((this = sip->head) != NULL) {
 		sip->head = this->next;
-		if (new_list == NULL) {
-			this->next = NULL;
+		this->next = NULL;
+
+		if (new_list == NULL || this->nseeks > new_list->nseeks)
 			new_list = this;
-			continue;
-		}
-		if (this->nseeks < new_list->nseeks) {
-			free(this);
-			continue;
-		}
-		if (this->nseeks > new_list->nseeks) {
-			while ((p = new_list) != NULL) {
-				new_list = p->next;
-				free(p);
-			}
-			this->next = NULL;
+		else if (this->nseeks == new_list->nseeks) {
+			assert(this->nseeks == new_list->nseeks);
+			for (p = new_list; p != NULL; p = p->next)
+				if (p->mode == this->mode)
+					break;
+
+			if (p)
+				this->nseeks += p->nseeks;
+			else
+				this->next = new_list;
 			new_list = this;
-			continue;
 		}
-		for (p = new_list; p; p++) 
-			if (p->mode == this->mode) {
-				this->nseeks += p->nseeks;
-				free(p);
-				break;
-			}
-		if (p)
-			p->next = new_list;
 	}
 
 	fprintf(ofp, "%10s | %15lld %15.1lf %15lld | %lld(%d)",
 	        "Average", sip->nseeks, sip->mean / sip->nseeks, 
 		sip->median / sip->nseeks, new_list->mode, new_list->nseeks);
 
-	p = new_list;
-	new_list = p->next;
-	free(p);
-	while((p = new_list) != NULL) {
-		fprintf(ofp, " %lld", p->mode);
-		new_list = p->next;
-		free(p);
-	}
+	for (p = new_list->next; p != NULL; p = p->next)
+		fprintf(ofp, " %lld(%d)", p->mode, p->nseeks);
 }
 
 void add_seek_mode_info(struct o_seek_info *sip, struct mode *mp)
@@ -329,9 +385,9 @@ void add_seek_mode_info(struct o_seek_info *sip, struct mode *mp)
 	struct seek_mode_info *smip;
 
 	n_seeks++;
-	for (i = 0; i < mp->nmds; i++) {
+	for (i = 0; i < mp->nmds; i++, lp++) {
 		for (smip = sip->head; smip; smip = smip->next) {
-			if (smip->mode == lp[i]) {
+			if (smip->mode == *lp) {
 				smip->nseeks += mp->most_seeks;
 				break;
 			}
@@ -341,7 +397,7 @@ void add_seek_mode_info(struct o_seek_info *sip, struct mode *mp)
 
 			new->next = sip->head; 
 			sip->head = new;
-			new->mode = lp[i];
+			new->mode = *lp;
 			new->nseeks = mp->most_seeks;
 
 			add_buf(new);
@@ -349,7 +405,7 @@ void add_seek_mode_info(struct o_seek_info *sip, struct mode *mp)
 	}
 }
 
-void __output_dip_seek_info(struct d_info *dip, void *arg)
+static void do_output_dip_seek_info(struct d_info *dip, FILE *ofp, int is_q2q)
 {
 	double mean;
 	int i, nmodes;
@@ -357,13 +413,13 @@ void __output_dip_seek_info(struct d_info *dip, void *arg)
 	char dev_info[15];
 	long long median;
 	struct mode m;
-	FILE *ofp = arg;
+	void *handle = is_q2q ? dip->q2q_handle : dip->seek_handle;
 
-	nseeks = seeki_nseeks(dip->seek_handle);
+	nseeks = seeki_nseeks(handle);
 	if (nseeks > 0) {
-		mean = seeki_mean(dip->seek_handle);
-		median = seeki_median(dip->seek_handle);
-		nmodes = seeki_mode(dip->seek_handle, &m);
+		mean = seeki_mean(handle);
+		median = seeki_median(handle);
+		nmodes = seeki_mode(handle, &m);
 
 		fprintf(ofp, "%10s | %15lld %15.1lf %15lld | %lld(%d)",
 			make_dev_hdr(dev_info, 15, dip), nseeks, mean, median, 
@@ -380,8 +436,21 @@ void __output_dip_seek_info(struct d_info *dip, void *arg)
 	}
 }
 
+void __output_dip_seek_info(struct d_info *dip, void *arg)
+{
+	do_output_dip_seek_info(dip, (FILE *)arg, 0);
+}
+
+void __output_dip_q2q_seek_info(struct d_info *dip, void *arg)
+{
+	do_output_dip_seek_info(dip, (FILE *)arg, 1);
+}
+
 void output_dip_seek_info(FILE *ofp)
 {
+	n_seeks = 1;
+	memset(&seek_info, 0, sizeof(seek_info));
+
 	fprintf(ofp, "%10s | %15s %15s %15s | %-15s\n", "DEV", "NSEEKS", 
 			"MEAN", "MEDIAN", "MODE");
 	fprintf(ofp, "---------- | --------------- --------------- --------------- | ---------------\n");
@@ -396,6 +465,25 @@ void output_dip_seek_info(FILE *ofp)
 	fprintf(ofp, "\n");
 }
 
+void output_dip_q2q_seek_info(FILE *ofp)
+{
+	n_seeks = 1;
+	memset(&seek_info, 0, sizeof(seek_info));
+
+	fprintf(ofp, "%10s | %15s %15s %15s | %-15s\n", "DEV", "NSEEKS", 
+			"MEAN", "MEDIAN", "MODE");
+	fprintf(ofp, "---------- | --------------- --------------- --------------- | ---------------\n");
+	dip_foreach_out(__output_dip_q2q_seek_info, ofp);
+	if (n_seeks > 1) {
+		fprintf(ofp, "---------- | --------------- --------------- --------------- | ---------------\n");
+		fprintf(ofp, "%10s | %15s %15s %15s | %-15s\n", 
+		        "Overall", "NSEEKS", "MEAN", "MEDIAN", "MODE");
+		output_seek_mode_info(ofp, &seek_info);
+		fprintf(ofp, "\n");
+	}
+	fprintf(ofp, "\n");
+}
+
 struct __opa {
 	FILE *ofp;
 	ai_pip_t (*func)(struct p_info *);
@@ -424,20 +512,6 @@ void output_pip_avg(FILE *ofp, char *hdr, ai_pip_t (*func)(struct p_info *))
 	fprintf(ofp, "\n");
 }
 
-void output_dip_avgs(FILE *ofp)
-{
-	output_hdr2(ofp,"Dev");
-	dip_foreach_out(__dip_output_avg2, ofp);
-	fprintf(ofp, "\n");
-}
-
-void output_pip_avgs(FILE *ofp)
-{
-	output_hdr2(ofp,"Exe");
-	pip_foreach_out(__pip_output_avg2, ofp);
-	fprintf(ofp, "\n");
-}
-
 int n_plugs;
 struct plug_info {
 	long n_plugs, n_timer_unplugs;
@@ -451,7 +525,7 @@ void __dip_output_plug(struct d_info *dip, void *arg)
 	double delta, pct;
 
 	if (dip->nplugs > 0) {
-		if (dip->is_plugged) dip_unplug(dip->device, dip->end_time, 0);
+		if (dip->is_plugged) dip_unplug(dip->device, dip->end_time);
 		delta = dip->end_time - dip->start_time;
 		pct = 100.0 * ((dip->plugged_time / delta) / delta);
 
@@ -488,6 +562,45 @@ void output_plug_info(FILE *ofp)
 	fprintf(ofp, "\n");
 }
 
+int n_actQs;
+struct actQ_info {
+	__u64 t_qs;
+	__u64 t_act_qs;
+} actQ_info;
+
+void __dip_output_actQ(struct d_info *dip, void *arg)
+{
+	if (dip->n_qs > 0 && !remapper_dev(dip->device)) {
+		char dev_info[15];
+		double a_actQs = (double)dip->t_act_q / (double)dip->n_qs;
+
+		fprintf((FILE *)arg, "%10s | %13.1lf\n", 
+			make_dev_hdr(dev_info, 15, dip), a_actQs);
+			
+		n_actQs++;
+		actQ_info.t_qs += dip->n_qs;
+		actQ_info.t_act_qs += dip->t_act_q;
+	}
+}
+
+void __dip_output_actQ_all(FILE *ofp, struct actQ_info *p)
+{
+	fprintf(ofp, "---------- | -------------\n");
+	fprintf(ofp, "%10s | %13s\n", "Overall", "Avgs Reqs @ Q");
+	fprintf(ofp, "%10s | %13.1lf\n", "Average", 
+		(double)p->t_act_qs / (double)p->t_qs);
+}
+
+void output_actQ_info(FILE *ofp)
+{
+	fprintf(ofp, "%10s | %13s\n", "DEV", "Avg Reqs @ Q");
+	fprintf(ofp, "---------- | -------------\n");
+	dip_foreach_out(__dip_output_actQ, ofp);
+	if (n_actQs > 1)
+		__dip_output_actQ_all(ofp, &actQ_info);
+	fprintf(ofp, "\n");
+}
+
 void output_histos(void)
 {
 	int i;
@@ -530,18 +643,32 @@ int output_avgs(FILE *ofp)
 	if (output_all_data) {
 		if (exes == NULL || *exes != '\0') {
 			output_section_hdr(ofp, "Per Process");
+			output_pip_avg(ofp, "Q2Qdm", pip_q2q_dm_avg);
+			output_pip_avg(ofp, "Q2Adm", pip_q2a_dm_avg);
+			output_pip_avg(ofp, "Q2Cdm", pip_q2c_dm_avg);
+			fprintf(ofp, "\n");
+
 			output_pip_avg(ofp, "Q2Q", pip_q2q_avg);
 			output_pip_avg(ofp, "Q2A", pip_q2a_avg);
-			output_pip_avg(ofp, "Q2I", pip_q2i_avg);
+			output_pip_avg(ofp, "Q2G", pip_q2g_avg);
+			output_pip_avg(ofp, "G2I", pip_g2i_avg);
+			output_pip_avg(ofp, "Q2M", pip_q2m_avg);
 			output_pip_avg(ofp, "I2D", pip_i2d_avg);
 			output_pip_avg(ofp, "D2C", pip_d2c_avg);
 			output_pip_avg(ofp, "Q2C", pip_q2c_avg);
 		}
 
 		output_section_hdr(ofp, "Per Device");
+		output_dip_avg(ofp, "Q2Qdm", dip_q2q_dm_avg);
+		output_dip_avg(ofp, "Q2Adm", dip_q2a_dm_avg);
+		output_dip_avg(ofp, "Q2Cdm", dip_q2c_dm_avg);
+		fprintf(ofp, "\n");
+
 		output_dip_avg(ofp, "Q2Q", dip_q2q_avg);
 		output_dip_avg(ofp, "Q2A", dip_q2a_avg);
-		output_dip_avg(ofp, "Q2I", dip_q2i_avg);
+		output_dip_avg(ofp, "Q2G", dip_q2g_avg);
+		output_dip_avg(ofp, "G2I", dip_g2i_avg);
+		output_dip_avg(ofp, "Q2M", dip_q2m_avg);
 		output_dip_avg(ofp, "I2D", dip_i2d_avg);
 		output_dip_avg(ofp, "D2C", dip_d2c_avg);
 		output_dip_avg(ofp, "Q2C", dip_q2c_avg);
@@ -549,10 +676,18 @@ int output_avgs(FILE *ofp)
 
 	output_section_hdr(ofp, "All Devices");
 	output_hdr(ofp, "ALL");
+	__output_avg(ofp, "Q2Qdm", &all_avgs.q2q_dm);
+	__output_avg(ofp, "Q2Adm", &all_avgs.q2a_dm);
+	__output_avg(ofp, "Q2Cdm", &all_avgs.q2c_dm);
+	fprintf(ofp, "\n");
+
 	__output_avg(ofp, "Q2Q", &all_avgs.q2q);
 	__output_avg(ofp, "Q2A", &all_avgs.q2a);
-	__output_avg(ofp, "Q2I", &all_avgs.q2i);
+	__output_avg(ofp, "Q2G", &all_avgs.q2g);
+	__output_avg(ofp, "G2I", &all_avgs.g2i);
+	__output_avg(ofp, "Q2M", &all_avgs.q2m);
 	__output_avg(ofp, "I2D", &all_avgs.i2d);
+	__output_avg(ofp, "M2D", &all_avgs.m2d);
 	__output_avg(ofp, "D2C", &all_avgs.d2c);
 	__output_avg(ofp, "Q2C", &all_avgs.q2c);
 	fprintf(ofp, "\n");
@@ -560,27 +695,28 @@ int output_avgs(FILE *ofp)
 	output_section_hdr(ofp, "Device Overhead");
 	output_dip_prep_ohead(ofp);
 
-	if (output_all_data) {
-		if (exes == NULL || *exes != '\0') {
-			output_section_hdr(ofp, "Per Process (avgs)");
-			output_pip_avgs(ofp);
-		}
-
-		output_section_hdr(ofp, "Per Device (avgs)");
-		output_dip_avgs(ofp);
-	}
-
 	output_section_hdr(ofp, "Device Merge Information");
 	output_dip_merge_ratio(ofp);
 
-	output_section_hdr(ofp, "Device Seek Information");
+	output_section_hdr(ofp, "Device Q2Q Seek Information");
+	output_dip_q2q_seek_info(ofp);
+
+	output_section_hdr(ofp, "Device D2D Seek Information");
 	output_dip_seek_info(ofp);
 
 	output_section_hdr(ofp, "Plug Information");
 	output_plug_info(ofp);
 
+	output_section_hdr(ofp, "Active Requests At Q Information");
+	output_actQ_info(ofp);
+
 	output_histos();
 
+	if (output_all_data) {
+		output_section_hdr(ofp, "Q2D Histogram");
+		output_q2d_histo(ofp);
+	}
+
 	return 0;
 }
 
diff --git a/btt/proc.c b/btt/proc.c
index e24948f..5818db4 100644
--- a/btt/proc.c
+++ b/btt/proc.c
@@ -89,20 +89,7 @@ struct p_info *__find_process_name(char *name)
 	return NULL;
 }
 
-struct p_info *find_process(__u32 pid, char *name)
-{
-	struct p_info *pip;
-
-	if (pid != ((__u32)-1) && ((pip = __find_process_pid(pid)) != NULL))
-		return pip;
-
-	if (name)
-		return __find_process_name(name);
-
-	return NULL;
-}
-
-static void insert_pid(struct p_info *that)
+static void insert_pid(struct p_info *that, __u32 pid)
 {
 	struct pn_info *this;
 	struct rb_node *parent = NULL;
@@ -112,9 +99,9 @@ static void insert_pid(struct p_info *that)
 		parent = *p;
 		this = rb_entry(parent, struct pn_info, rb_node);
 
-		if (that->pid < this->u.pid)
+		if (pid < this->u.pid)
 			p = &(*p)->rb_left;
-		else if (that->pid > this->u.pid)
+		else if (pid > this->u.pid)
 			p = &(*p)->rb_right;
 		else {
 			ASSERT(strcmp(that->name, this->pip->name) == 0);
@@ -123,7 +110,7 @@ static void insert_pid(struct p_info *that)
 	}
 
 	this = malloc(sizeof(struct pn_info));
-	this->u.pid = that->pid;
+	this->u.pid = pid;
 	this->pip = that;
 
 	rb_link_node(&this->rb_node, parent, p);
@@ -160,29 +147,79 @@ static void insert_name(struct p_info *that)
 
 static void insert(struct p_info *pip)
 {
-	insert_pid(pip);
+	insert_pid(pip, pip->pid);
 	insert_name(pip);
 }
 
+static inline struct p_info *pip_alloc(void)
+{
+	return memset(malloc(sizeof(struct p_info)), 0, sizeof(struct p_info));
+}
+
+struct p_info *find_process(__u32 pid, char *name)
+{
+	struct p_info *pip;
+
+	if (pid != ((__u32)-1)) {
+		if ((pip = __find_process_pid(pid)) != NULL)
+			return pip;
+		else if (name) {
+			pip = __find_process_name(name);
+
+			if (pip && pid != pip->pid) {
+				/*
+				 * This is a process with the same name
+				 * as another, but a different PID.
+				 *
+				 * We'll store a reference in the PID
+				 * tree...
+				 */
+				insert_pid(pip, pid);
+			}
+			return pip;
+		}
+
+		/*
+		 * We're here because we have a pid, and no name, but
+		 * we didn't find a process ... 
+		 *
+		 * We'll craft one using the pid...
+		 */
+
+		name = alloca(256);
+		sprintf(name, "pid%09u", pid);
+		add_process(pid, name);
+		return __find_process_pid(pid);
+	}
+
+	assert(name != NULL);
+	return __find_process_name(name);
+}
+
 void add_process(__u32 pid, char *name)
 {
 	struct p_info *pip = find_process(pid, name);
 
 	if (pip == NULL) {
-		pip = memset(malloc(sizeof(*pip)), 0, sizeof(*pip));
+		pip = pip_alloc();
 		pip->pid = pid;
 		region_init(&pip->regions);
 		pip->last_q = (__u64)-1;
 		pip->name = strdup(name);
 
-		insert(pip);
+	 	insert(pip);
 	}
 }
 
 void pip_update_q(struct io *iop)
 {
 	if (iop->pip) {
-		update_lq(&iop->pip->last_q, &iop->pip->avgs.q2q, iop->t.time);
+		if (remapper_dev(iop->dip->device))
+			update_lq(&iop->pip->last_q, &iop->pip->avgs.q2q_dm, 
+								iop->t.time);
+		else
+			update_lq(&iop->pip->last_q, &iop->pip->avgs.q2q, 
+								iop->t.time);
 		update_qregion(&iop->pip->regions, iop->t.time);
 	}
 }
diff --git a/btt/q2d.c b/btt/q2d.c
new file mode 100644
index 0000000..ae61e20
--- /dev/null
+++ b/btt/q2d.c
@@ -0,0 +1,121 @@
+/*
+ * blktrace output analysis: generate a timeline & gather statistics
+ *
+ * (C) Copyright 2007 Hewlett-Packard Development Company, L.P.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "globals.h"
+
+#define Q2D_MAX_HISTO 9
+struct q2d_info {
+	unsigned long nhistos;
+	unsigned long histos[Q2D_MAX_HISTO + 1];
+};
+
+void q2d_histo_add(void *priv, __u64 q2d_in)
+{
+	int index;
+	struct q2d_info *q2dp = priv;
+	double q2d = BIT_TIME(q2d_in);
+	long msec = (long)(q2d / 0.001);
+
+	switch (msec) {
+	default:          index = 9; break;
+	case 500 ... 999: index = 8; break;
+	case 250 ... 499: index = 7; break;
+	case 100 ... 249: index = 6; break;
+	case  75 ...  99: index = 5; break;
+	case  50 ...  74: index = 4; break;
+	case  25 ...  49: index = 3; break;
+	case  10 ...  24: index = 2; break;
+	case   5 ...   9: index = 1; break;
+	case   0 ...   4: index = 0; break;
+	}
+
+	q2dp->histos[index]++;
+	q2dp->nhistos++;
+}
+
+void *q2d_init(void)
+{
+	struct q2d_info *q2dp = malloc(sizeof(*q2dp));
+
+	return memset(q2dp, 0, sizeof(*q2dp));
+}
+
+void q2d_release(void *priv)
+{
+	free(priv);
+}
+
+void q2d_display_header(FILE *fp)
+{
+	fprintf(fp, "%5s ", "<.005");
+	fprintf(fp, "%5s ", "<.010");
+	fprintf(fp, "%5s ", "<.025");
+	fprintf(fp, "%5s ", "<.050");
+	fprintf(fp, "%5s ", "<.075");
+	fprintf(fp, "%5s ", "<.100");
+	fprintf(fp, "%5s ", "<.250");
+	fprintf(fp, "%5s ", "<.500");
+	fprintf(fp, "%5s ", "< 1.0");
+	fprintf(fp, "%5s ", ">=1.0\n");
+}
+
+void q2d_display_dashes(FILE *fp)
+{
+	int i;
+
+	for (i = 0; i <= Q2D_MAX_HISTO; i++)
+		fprintf(fp, "===== ");
+	fprintf(fp, "\n");
+}
+
+void q2d_display(FILE *fp, void *priv)
+{
+	int i;
+	struct q2d_info *q2dp = priv;
+	double nh = (double)q2dp->nhistos;
+
+	for (i = 0; i <= Q2D_MAX_HISTO; i++) {
+		double p = 100.0 * (double)q2dp->histos[i] / nh;
+		fprintf(fp, "%5.1lf ", p);
+	}
+	fprintf(fp, "\n");
+}
+
+int q2d_ok(void *priv)
+{
+	struct q2d_info *q2dp = priv;
+
+	return q2dp && q2dp->nhistos > 0;
+}
+
+void q2d_acc(void *a1, void *a2)
+{
+	int i;
+	struct q2d_info *ap = a1;
+	struct q2d_info *tp = a2;
+
+	for (i = 0; i <= Q2D_MAX_HISTO; i++)
+		ap->histos[i] += tp->histos[i];
+	ap->nhistos += tp->nhistos;
+}
diff --git a/btt/seek.c b/btt/seek.c
index 8738698..cbbd1a3 100644
--- a/btt/seek.c
+++ b/btt/seek.c
@@ -36,19 +36,15 @@ struct seeki {
 	long long last_start, last_end;
 };
 
-static FILE *seek_open(__u32 device, char rw)
+static FILE *seek_open(char *str, char rw)
 {
 	FILE *fp;
 	char *oname;
-	int mjr, mnr;
 
 	if (seek_name == NULL) return NULL;
 
-	mjr = device >> MINORBITS;
-	mnr = device & ((1 << MINORBITS) - 1);
-
-	oname = malloc(strlen(seek_name)+32);
-	sprintf(oname, "%s_%03d,%03d_%c.dat", seek_name, mjr, mnr, rw);
+	oname = malloc(strlen(seek_name) + strlen(str) + 32);
+	sprintf(oname, "%s_%s_%c.dat", seek_name, str, rw);
 	if ((fp = fopen(oname, "w")) == NULL)
 		perror(oname);
 	else
@@ -124,13 +120,13 @@ long long seek_dist(struct seeki *sip, struct io *iop)
 	return dist;
 }
 
-void *seeki_init(__u32 device)
+void *seeki_init(char *str)
 {
 	struct seeki *sip = malloc(sizeof(struct seeki));
 
-	sip->rfp = seek_open(device, 'r');
-	sip->wfp = seek_open(device, 'w');
-	sip->cfp = seek_open(device, 'c');
+	sip->rfp = seek_open(str, 'r');
+	sip->wfp = seek_open(str, 'w');
+	sip->cfp = seek_open(str, 'c');
 	sip->tot_seeks = 0;
 	sip->total_sectors = 0.0;
 	sip->last_start = sip->last_end = 0;
diff --git a/btt/trace.c b/btt/trace.c
index 7f8e689..1348e89 100644
--- a/btt/trace.c
+++ b/btt/trace.c
@@ -20,65 +20,9 @@
  */
 #include "globals.h"
 
-int dump_level;
-
-void __dump_iop(FILE *ofp, struct io *iop, int extra_nl)
-{
-	fprintf(ofp, "%5d.%09lu %3d,%-3d %c %10llu+%-4u\n",
-		(int)SECONDS(iop->t.time),
-		(unsigned long)NANO_SECONDS(iop->t.time),
-		MAJOR(iop->t.device), MINOR(iop->t.device), type2c(iop->type),
-		(unsigned long long)iop->t.sector, t_sec(&iop->t));
-	if (extra_nl) fprintf(ofp, "\n");
-}
-
-void __dump_iop2(FILE *ofp, struct io *a_iop, struct io *l_iop)
-{
-	fprintf(ofp, "%5d.%09lu %3d,%-3d %c %10llu+%-4u <- (%3d,%-3d) %10llu\n",
-		(int)SECONDS(a_iop->t.time),
-		(unsigned long)NANO_SECONDS(a_iop->t.time),
-		MAJOR(a_iop->t.device), MINOR(a_iop->t.device), 
-		type2c(a_iop->type), (unsigned long long)a_iop->t.sector, 
-		t_sec(&a_iop->t), MAJOR(l_iop->t.device), 
-		MINOR(l_iop->t.device), (unsigned long long)l_iop->t.sector);
-}
-
-void release_iops(void)
-{
-	struct io *x_iop;
-	struct list_head *p, *q;
-
-	list_for_each_safe(p, q, &rmhd) {
-		x_iop = list_entry(p, struct io, rm_head);
-		LIST_DEL(&x_iop->rm_head);
-		io_release(x_iop);
-	}
-}
-
-void do_retries(__u64 now)
-{
-	struct io *iop;
-	struct list_head *p, *q;
-
-	list_for_each_safe(p, q, &retries) {
-		iop = list_entry(p, struct io, retry);
-		ASSERT(iop->type == IOP_C);
-
-		// iop could be gone after call...
-		retry_complete(iop, now);
-	}
-}
-
-static inline int retry_check_time(__u64 t)
-{
-	return next_retry_check && (t > next_retry_check);
-}
-
 static void __add_trace(struct io *iop)
 {
 	time_t now = time(NULL);
-	__u64 tstamp = iop->t.time;
-	int run_retry = retry_check_time(iop->t.time);
 
 	n_traces++;
 	iostat_check_time(iop->t.time);
@@ -86,8 +30,8 @@ static void __add_trace(struct io *iop)
 	if (verbose && ((now - last_vtrace) > 0)) {
 
 #	if defined(DEBUG)
-		printf("%10lu t\tretries=|%10d|\ttree size=|%10d|\r", 
-			n_traces, list_len(&retries), rb_tree_size);
+		printf("%10lu t\ttree size=|%10d|\r", 
+			n_traces, rb_tree_size);
 #	else
 		printf("%10lu t\r", n_traces);
 #	endif
@@ -100,7 +44,8 @@ static void __add_trace(struct io *iop)
 	switch (iop->t.action & 0xffff) {
 	case __BLK_TA_QUEUE:		trace_queue(iop); break;
 	case __BLK_TA_REMAP:		trace_remap(iop); break;
-	case __BLK_TA_GETRQ:		trace_insert(iop); break;
+	case __BLK_TA_INSERT:		trace_insert(iop); break;
+	case __BLK_TA_GETRQ:		trace_getrq(iop); break;
 	case __BLK_TA_BACKMERGE:	trace_merge(iop); break;
 	case __BLK_TA_FRONTMERGE:	trace_merge(iop); break;
 	case __BLK_TA_REQUEUE:		trace_requeue(iop); break;
@@ -113,24 +58,21 @@ static void __add_trace(struct io *iop)
 		io_release(iop); 
 		return;
 	}
-
-	if (run_retry && !list_empty(&retries)) {
-		do_retries(tstamp);
-		bump_retry(tstamp);
-	}
 }
 
 void add_trace(struct io *iop)
 {
 	if (iop->t.action & BLK_TC_ACT(BLK_TC_NOTIFY)) {
-		if (iop->t.pid == 0) 
-			add_process(0, "kernel");
-		else {
-			char *slash = strchr(iop->pdu, '/');
-			if (slash)
-				*slash = '\0';
-
-			add_process(iop->t.pid, iop->pdu);
+		if (iop->t.action == BLK_TN_PROCESS) {
+			if (iop->t.pid == 0) 
+				add_process(0, "kernel");
+			else {
+				char *slash = strchr(iop->pdu, '/');
+				if (slash)
+					*slash = '\0';
+
+				add_process(iop->t.pid, iop->pdu);
+			}
 		}
 		io_release(iop);
 	}
diff --git a/btt/trace_complete.c b/btt/trace_complete.c
index b48bf79..b4dcadd 100644
--- a/btt/trace_complete.c
+++ b/btt/trace_complete.c
@@ -20,170 +20,80 @@
  */
 #include "globals.h"
 
-LIST_HEAD(pending_cs);
-
-static inline void __run_complete(struct io *c_iop)
+static inline void __out(FILE *ofp, __u64 tm, enum iop_type type, 
+					__u64 sec, __u32 nsec, int indent)
 {
-	if (remapper_dev(c_iop->t.device)) {
-		struct bilink *blp = blp;
-		struct io *iop = bilink_first_down(c_iop, &blp);
-
-		if (iop->type == IOP_Q) {
-			run_queue(iop, c_iop, c_iop);
-			biunlink(blp);
-		}
-		else
-			bilink_for_each_down(run_remap, c_iop, c_iop, 1);
+	if (tm != (__u64)-1) {
+		if (indent) 
+			fprintf(ofp, "         ");
+		fprintf(ofp, "%5d.%09lu %c %10llu+%-4u\n", 
+			(int)SECONDS(tm), (unsigned long)NANO_SECONDS(tm),
+			type2c(type), (unsigned long long)sec, nsec);
 	}
-	else
-		bilink_for_each_down(run_issue, c_iop, c_iop, 1);
-
-	dump_iop(c_iop, 1);
-
-	LIST_DEL(&c_iop->c_pending);
-	del_retry(c_iop);
-	add_rmhd(c_iop);
+}
 
-	release_iops();
+static void display_io_track(FILE *ofp, struct io *iop)
+{
+	fprintf(ofp, "%3d,%-3d: ", MAJOR(iop->t.device), MINOR(iop->t.device));
+	__out(ofp, iop->t.time, IOP_Q, iop->t.sector, t_sec(&iop->t), 0);
+
+	if (iop->g_time != (__u64)-1)
+		__out(ofp, iop->g_time, IOP_G, iop->t.sector, t_sec(&iop->t),1);
+	if (iop->i_time != (__u64)-1)
+		__out(ofp, iop->i_time, IOP_I, iop->t.sector, t_sec(&iop->t),1);
+	if (iop->m_time != (__u64)-1)
+		__out(ofp, iop->i_time, IOP_M, iop->t.sector, t_sec(&iop->t),1);
+
+	__out(ofp, iop->d_time, IOP_D, iop->d_sec, iop->d_nsec, 1);
+	__out(ofp, iop->c_time, IOP_C, iop->c_sec, iop->c_nsec, 1);
+	fprintf(ofp, "\n");
 }
 
-static int ready_complete_remapper(struct io *c_iop)
+static void handle_complete(struct io *c_iop)
 {
 	LIST_HEAD(head);
 	struct list_head *p, *q;
 
-	dip_foreach_list(c_iop, IOP_L, &head);
-	if (list_empty(&head)) {
-		struct io *q_iop;
-
-		dip_foreach_list(c_iop, IOP_Q, &head);
-		list_for_each_safe(p, q, &head) {
-			q_iop = list_entry(p, struct io, f_head);
-			LIST_DEL(&q_iop->f_head);
+	update_blks(c_iop);
+	update_cregion(&all_regions, c_iop->t.time);
+	update_cregion(&c_iop->dip->regions, c_iop->t.time);
+	if (c_iop->pip)
+		update_cregion(&c_iop->pip->regions, c_iop->t.time);
 
-			ASSERT(list_empty(&q_iop->up_list));
-			ASSERT(list_empty(&q_iop->down_list));
-			ASSERT(q_iop->t.bytes == c_iop->t.bytes);
-			if (ready_queue(q_iop, c_iop)) {
-				dip_rem(q_iop);
-				bilink(q_iop, c_iop);
-				c_iop->bytes_left -= q_iop->t.bytes;
-			}
-		}
-	}
-	else {
-		struct io *l_iop, *a_iop;
-
-		list_for_each_safe(p, q, &head) {
-			l_iop = list_entry(p, struct io, f_head);
-			LIST_DEL(&l_iop->f_head);
-
-			ASSERT(!list_empty(&l_iop->up_list));
-			a_iop = bilink_first_up(l_iop, NULL);
-			if (ready_remap(a_iop, c_iop)) {
-				dip_rem(l_iop);
-				bilink(a_iop, c_iop);
-				c_iop->bytes_left -= a_iop->t.bytes;
-			}
-		}
-	}
+	dip_foreach_list(c_iop, IOP_Q, &head);
+	list_for_each_safe(p, q, &head) {
+		struct io *q_iop = list_entry(p, struct io, f_head);
+		__u64 q2c = tdelta(q_iop->t.time, c_iop->t.time);
 
-	return c_iop->bytes_left == 0;
-}
+		c_iop->bytes_left -= q_iop->t.bytes;
 
-int ready_complete(struct io *c_iop)
-{
-	__u64 d2c;
-	struct io *d_iop;
+		update_q2c(q_iop, q2c);
+		latency_q2c(q_iop->dip, q_iop->t.time, q2c);
 
-	if (c_iop->bytes_left == 0)
-		return 1;
+		if (q_iop->d_time != (__u64)-1) {
+			__u64 d2c = tdelta(q_iop->d_time, c_iop->t.time);
 
-	if (remapper_dev(c_iop->t.device))
-		return ready_complete_remapper(c_iop);
-
-	if (!list_empty(&c_iop->down_list))
-		return 1;
+			update_d2c(q_iop, d2c);
+			latency_d2c(q_iop->dip, c_iop->t.time, d2c);
+			iostat_complete(q_iop, c_iop);
+		}
 
-	d_iop = dip_find_sec(c_iop->dip, IOP_D, BIT_START(c_iop));
-	if (!d_iop)
-		return -1;
+		if (per_io_ofp) {
+			q_iop->c_time = c_iop->t.time;
+			q_iop->c_sec = c_iop->t.sector;
+			q_iop->c_nsec = t_sec(&c_iop->t);
+			display_io_track(per_io_ofp, q_iop);
+		}
 
-	if (c_iop->t.bytes != d_iop->t.bytes) {
-		fprintf(stderr, 
-			"\nFATAL: Probable time anomaly detected\n");
-		fprintf(stderr, 
-			"D @ %15.9lf missing C, later C @ %15.9lf\n", 
-			BIT_TIME(d_iop->t.time), 
-			BIT_TIME(c_iop->t.time));
-		exit(1);
+		LIST_DEL(&q_iop->f_head);
+		io_release(q_iop);
 	}
-
-	if (!ready_issue(d_iop, c_iop))
-		return 0;
-
-	c_iop->bytes_left = 0;
-
-	d2c = tdelta(d_iop, c_iop);
-	update_d2c(d_iop, d_iop->down_len, d2c);
-	latency_d2c(d_iop->dip, c_iop->t.time, d2c);
-	iostat_complete(d_iop, c_iop);
-
-	bilink(d_iop, c_iop);
-	dip_rem(d_iop);
-	return 1;
 }
 
 void trace_complete(struct io *c_iop)
 {
-	if (io_setup(c_iop, IOP_C)) {
-		update_blks(c_iop);
-		update_cregion(&all_regions, c_iop->t.time);
-		update_cregion(&c_iop->dip->regions, c_iop->t.time);
-		if (c_iop->pip)
-			update_cregion(&c_iop->pip->regions, c_iop->t.time);
+	if (io_setup(c_iop, IOP_C))
+		handle_complete(c_iop);
 
-		list_add_tail(&c_iop->c_pending, &pending_cs);
-		switch (ready_complete(c_iop)) {
-		case  1: 
-			__run_complete(c_iop); 
-			break;
-		case  0: 
-			add_retry(c_iop); 
-			break;
-		case -1: 
-			LIST_DEL(&c_iop->c_pending);
-			del_retry(c_iop);
-			io_release(c_iop);
-			break;
-		}
-	}
-	else 
-		io_release(c_iop);
-}
-
-void retry_complete(struct io *c_iop, __u64 now)
-{
-	double tc = BIT_TIME(c_iop->t.time);
-
-	switch (ready_complete(c_iop)) {
-	case  1: 
-#		if defined(DEBUG)
-			fprintf(stderr, "Retried %15.9lf success!\n", tc);
-#		endif
-
-		__run_complete(c_iop); 
-		break;
-	case  0:
-		if (now == 0 || ((BIT_TIME(now) - tc) < 1.0))
-			break;
-		if (!list_empty(&c_iop->down_list))
-			break;
-		/*FALLTHROUGH*/
-	case -1: 
-		LIST_DEL(&c_iop->c_pending);
-		del_retry(c_iop);
-		io_release(c_iop);
-		break;
-	}
+	io_release(c_iop);
 }
diff --git a/btt/trace_im.c b/btt/trace_im.c
index c16f1a8..b1d1560 100644
--- a/btt/trace_im.c
+++ b/btt/trace_im.c
@@ -20,88 +20,62 @@
  */
 #include "globals.h"
 
-static void __run_im(struct io *q_iop, struct io *im_iop, struct io *c_iop)
+static void handle_g(struct io *g_iop)
 {
-	run_queue(q_iop, im_iop, c_iop);
-	dump_iop(im_iop, 0);
-}
-
-static void __run_unim(struct io *q_iop, struct io *im_iop, 
-		       __attribute__((__unused__))struct io *c_iop)
-{
-	if (q_iop->bytes_left == 0) {
-		q_iop->linked = dip_rb_ins(q_iop->dip, q_iop);
-		ASSERT(q_iop->linked);
+	struct io *q_iop = dip_find_sec(g_iop->dip, IOP_Q, g_iop->t.sector);
 
-#		if defined(DEBUG)
-			rb_tree_size++;
-#		endif
+	iostat_getrq(g_iop);
+	if (q_iop) {
+		q_iop->g_time = g_iop->t.time;
+		update_q2g(q_iop, tdelta(q_iop->t.time, g_iop->t.time));
 	}
-
-	q_iop->bytes_left += im_iop->t.bytes;
-	unupdate_q2i(q_iop, tdelta(q_iop, im_iop));
 }
 
-void run_im(struct io *im_iop, __attribute__((__unused__))struct io *d_iop, 
-	    struct io *c_iop)
+static void handle_i(struct io *i_iop)
 {
-	bilink_for_each_down(__run_im, im_iop, c_iop, 1);
-	add_rmhd(im_iop);
-}
+	struct io *q_iop = dip_find_sec(i_iop->dip, IOP_Q, i_iop->t.sector);
 
-void run_unim(struct io *im_iop, __attribute__((__unused__))struct io *d_iop, 
-	      struct io *c_iop)
-{
-	bilink_for_each_down(__run_unim, im_iop, c_iop, 1);
-	add_rmhd(im_iop);
+	if (q_iop) {
+		q_iop->i_time = i_iop->t.time;
+		if (q_iop->g_time != (__u64)-1)
+			update_g2i(q_iop, tdelta(q_iop->g_time, i_iop->t.time));
+	}
 }
 
-int ready_im(struct io *im_iop, struct io *c_iop)
+static void handle_m(struct io *m_iop)
 {
-	if (im_iop->bytes_left > 0) {
-		__u64 xfer;
-		LIST_HEAD(head);
-		struct io *q_iop;
-		struct list_head *p, *q;
+	struct io *q_iop = dip_find_sec(m_iop->dip, IOP_Q, m_iop->t.sector);
 
-		dip_foreach_list(im_iop, IOP_Q, &head);
-		list_for_each_safe(p, q, &head) {
-			q_iop = list_entry(p, struct io, f_head);
-			LIST_DEL(&q_iop->f_head);
-
-			if (ready_queue(q_iop, c_iop)) {
-				update_q2i(q_iop, tdelta(q_iop, im_iop));
-
-				bilink(q_iop, im_iop);
-				dip_rem(q_iop);
+	iostat_merge(m_iop);
+	if (q_iop) {
+		q_iop->m_time = m_iop->t.time;
+		update_q2m(q_iop, tdelta(q_iop->t.time, m_iop->t.time));
+	}
 
-				xfer = min(im_iop->bytes_left, 
-							    q_iop->bytes_left);
-				im_iop->bytes_left -= xfer;
-				q_iop->bytes_left -= xfer;
+	if (m_iop->dip->n_act_q != 0)
+		m_iop->dip->n_act_q--;
+}
 
-				if (q_iop->bytes_left == 0)
-					dip_rem(q_iop);
-			}
-		}
-	}
+void trace_getrq(struct io *g_iop)
+{
+	if (io_setup(g_iop, IOP_G))
+		handle_g(g_iop);
 
-	return im_iop->bytes_left == 0;
+	io_release(g_iop);
 }
 
 void trace_insert(struct io *i_iop)
 {
 	if (io_setup(i_iop, IOP_I))
-		iostat_insert(i_iop);
-	else
-		io_release(i_iop);
+		handle_i(i_iop);
 
+	io_release(i_iop);
 }
 
 void trace_merge(struct io *m_iop)
 {
 	if (io_setup(m_iop, IOP_M))
-		iostat_merge(m_iop);
-	else
-		io_release(m_iop);
+		handle_m(m_iop);
+
+	io_release(m_iop);
 }
diff --git a/btt/trace_issue.c b/btt/trace_issue.c
index d2b0832..f081353 100644
--- a/btt/trace_issue.c
+++ b/btt/trace_issue.c
@@ -20,71 +20,49 @@
  */
 #include "globals.h"
 
-static void __run_issue(struct io *im_iop, struct io *d_iop, struct io *c_iop)
+static void handle_issue(struct io *d_iop)
 {
-	update_i2d(im_iop, tdelta(im_iop, d_iop));
-	run_im(im_iop, d_iop, c_iop);
-	dump_iop(d_iop, 0);
-}
+	LIST_HEAD(head);
+	struct list_head *p, *q;
 
-static void __run_unissue(struct io *im_iop, struct io *d_iop, 
-			  struct io *c_iop)
-{
-	unupdate_i2d(im_iop, tdelta(im_iop, d_iop));
-	run_unim(im_iop, d_iop, c_iop);
-}
+	if (d_iop->dip->n_act_q != 0)
+		d_iop->dip->n_act_q--;
 
-void run_issue(struct io *d_iop, __attribute__((__unused__))struct io *u_iop, 
-							struct io *c_iop)
-{
-	bilink_for_each_down(__run_issue, d_iop, c_iop, 1);
-	add_rmhd(d_iop);
-}
+	seeki_add(d_iop->dip->seek_handle, d_iop);
+	bno_dump_add(d_iop->dip->bno_dump_handle, d_iop);
+	iostat_issue(d_iop);
+	d_iop->dip->n_ds++;
+	if (!remapper_dev(d_iop->t.device))
+		update_d_histo(d_iop->t.bytes);
 
-void run_unissue(struct io *d_iop, 
-		 __attribute__((__unused__))struct io *u_iop, 
-		 struct io *c_iop)
-{
-	bilink_for_each_down(__run_unissue, d_iop, c_iop, 1);
-	add_rmhd(d_iop);
-}
+	dip_foreach_list(d_iop, IOP_Q, &head);
+	list_for_each_safe(p, q, &head) {
+		struct io *q_iop = list_entry(p, struct io, f_head);
+		
+		if (q_iop->i_time != (__u64)-1)
+			update_i2d(q_iop, tdelta(q_iop->i_time, d_iop->t.time));
+		else if (q_iop->m_time != (__u64)-1)
+			update_m2d(q_iop, tdelta(q_iop->m_time, d_iop->t.time));
 
-int ready_issue(struct io *d_iop, struct io *c_iop)
-{
-	if (d_iop->bytes_left > 0) {
-		LIST_HEAD(head);
-		struct io *im_iop;
-		struct list_head *p, *q;
+		d_iop->bytes_left -= q_iop->t.bytes;
+		LIST_DEL(&q_iop->f_head);
 
-		dip_foreach_list(d_iop, IOP_I, &head);
-		dip_foreach_list(d_iop, IOP_M, &head);
-		list_for_each_safe(p, q, &head) {
-			im_iop = list_entry(p, struct io, f_head);
-			LIST_DEL(&im_iop->f_head);
+		q_iop->d_time = d_iop->t.time;
+		q_iop->d_sec = d_iop->t.sector;
+		q_iop->d_nsec = t_sec(&d_iop->t);
 
-			ASSERT(d_iop->bytes_left >= im_iop->t.bytes);
-			if (ready_im(im_iop, c_iop)) {
-				bilink(im_iop, d_iop);
-				dip_rem(im_iop);
-				d_iop->bytes_left -= im_iop->t.bytes;
-			}
-		}
+		if (output_all_data)
+			q2d_histo_add(q_iop->dip->q2d_priv, 
+						d_iop->t.time - q_iop->t.time);
 	}
 
-	return d_iop->bytes_left == 0;
+	assert(d_iop->bytes_left == 0);
 }
 
 void trace_issue(struct io *d_iop)
 {
-	if (io_setup(d_iop, IOP_D)) {
-		seeki_add(d_iop->dip->seek_handle, d_iop);
-		bno_dump_add(d_iop->dip->bno_dump_handle, d_iop);
-		iostat_issue(d_iop);
-		d_iop->dip->n_ds++;
-		if (!remapper_dev(d_iop->t.device))
-			update_d_histo(d_iop->t.bytes);
-	}
-	else
-		io_release(d_iop);
+	if (io_setup(d_iop, IOP_D))
+		handle_issue(d_iop);
+	io_release(d_iop);
 
 }
diff --git a/btt/trace_plug.c b/btt/trace_plug.c
index 06d9fc9..03076d7 100644
--- a/btt/trace_plug.c
+++ b/btt/trace_plug.c
@@ -20,21 +20,17 @@
  */
 #include "globals.h"
 
-static inline void trace_unplug(struct io *u_iop, int is_timer)
+void trace_unplug_io(struct io *u_iop)
 {
 	unplug_hist_add(u_iop);
-	dip_unplug(u_iop->t.device, BIT_TIME(u_iop->t.time), is_timer);
+	dip_unplug(u_iop->t.device, BIT_TIME(u_iop->t.time));
 	io_release(u_iop);
 }
 
-void trace_unplug_io(struct io *u_iop)
-{
-	trace_unplug(u_iop, 0);
-}
-
-void trace_unplug_timer(struct io *u_iop)
+void trace_unplug_timer(struct io *ut_iop)
 {
-	trace_unplug(u_iop, 1);
+	dip_unplug_tm(ut_iop->t.device);
+	io_release(ut_iop);
 }
 
 void trace_plug(struct io *p_iop)
diff --git a/btt/trace_queue.c b/btt/trace_queue.c
index ad5e30c..60294b6 100644
--- a/btt/trace_queue.c
+++ b/btt/trace_queue.c
@@ -20,60 +20,31 @@
  */
 #include "globals.h"
 
-static inline void __update_q2c(struct io *q_iop, struct io *c_iop)
+static void handle_queue(struct io *q_iop)
 {
-	__u64 q2c = tdelta(q_iop, c_iop);
-
-	update_q2c(q_iop, q2c);
-	latency_q2c(q_iop->dip, q_iop->t.time, q2c);
-}
-
-void run_queue(struct io *q_iop, __attribute__((__unused__))struct io *u_iop,
-	       struct io *c_iop)
-{
-	struct bilink *blp;
-	struct io *a_iop = bilink_first_down(q_iop, &blp);
-
-	if (a_iop) {
-		run_remap(a_iop, q_iop, c_iop);
-		biunlink(blp);
+	seeki_add(q_iop->dip->q2q_handle, q_iop);
+	update_qregion(&all_regions, q_iop->t.time);
+	dip_update_q(q_iop->dip, q_iop);
+	pip_update_q(q_iop);
+	if (remapper_dev(q_iop->t.device))
+		update_lq(&last_q, &all_avgs.q2q_dm, q_iop->t.time);
+	else {
+		update_q_histo(q_iop->t.bytes);
+		update_lq(&last_q, &all_avgs.q2q, q_iop->t.time);
 	}
 
-	__update_q2c(q_iop, c_iop);
-	dump_iop(q_iop, 0);
-	add_rmhd(q_iop);
-}
-
-int ready_queue(struct io *q_iop, struct io *c_iop)
-{
-	struct io *a_iop;
-
-	if (!list_empty(&q_iop->down_list))
-		return 1;
-
-	a_iop = dip_find_sec(q_iop->dip, IOP_A, BIT_START(q_iop));
-	if (!a_iop)
-		return 1;
+	q_iop->i_time = q_iop->g_time = q_iop->i_time = q_iop->m_time = 
+						q_iop->d_time = (__u64)-1;
+	q_iop->dip->n_qs++;
 
-	if (!ready_remap(a_iop, c_iop))
-		return 0;
-
-	ASSERT(q_iop->t.bytes == a_iop->t.bytes);
-	bilink(a_iop, q_iop);
-	dip_rem(a_iop);
-	return 1;
+	q_iop->dip->t_act_q += q_iop->dip->n_act_q;
+	q_iop->dip->n_act_q++;
 }
 
 void trace_queue(struct io *q_iop)
 {
-	if (io_setup(q_iop, IOP_Q)) {
-		update_lq(&last_q, &all_avgs.q2q, q_iop->t.time);
-		update_qregion(&all_regions, q_iop->t.time);
-		dip_update_q(q_iop->dip, q_iop);
-		pip_update_q(q_iop);
-		if (!remapper_dev(q_iop->t.device))
-			update_q_histo(q_iop->t.bytes);
-	}
+	if (io_setup(q_iop, IOP_Q))
+		handle_queue(q_iop);
 	else
 		io_release(q_iop);
 }
diff --git a/btt/trace_remap.c b/btt/trace_remap.c
index c5203e8..22d340b 100644
--- a/btt/trace_remap.c
+++ b/btt/trace_remap.c
@@ -20,95 +20,37 @@
  */
 #include "globals.h"
 
-void run_remap(struct io *a_iop, __attribute__((__unused__)) struct io *u_iop,
-				 struct io *c_iop)
+static inline void cvt_pdu_remap(struct blk_io_trace_remap *rp)
 {
-	struct bilink *blp = blp, *blp2;
-	struct io *q_iop, *l_iop = bilink_first_down(a_iop, &blp);
-
-	ASSERT(l_iop);
-	q_iop = bilink_first_down(l_iop, &blp2);
-	if (q_iop) {
-		run_queue(q_iop, a_iop, c_iop);
-		biunlink(blp2);
-	}
-
-	dump_iop2(a_iop, l_iop);
-	biunlink(blp);
-
-	add_rmhd(l_iop);
-	add_rmhd(a_iop);
-}
-
-int ready_dev_remap(struct io *l_iop, struct io *c_iop)
-{
-	struct io *q_iop;
-
-	if (!list_empty(&l_iop->down_list)) 
-		return 1;
-
-	q_iop = dip_find_sec(l_iop->dip, IOP_Q, l_iop->t.sector);
-	if (!q_iop || !ready_queue(q_iop, c_iop))
-		return 0;
-
-	ASSERT(l_iop->t.bytes <= q_iop->t.bytes);
-	update_q2a(q_iop, tdelta(q_iop, l_iop));
-	bilink(q_iop, l_iop);
-
-	q_iop->bytes_left -= l_iop->t.bytes;
-	if (q_iop->bytes_left == 0)
-		dip_rem(q_iop);
-	return 1;
-}
-
-int ready_self_remap(struct io *l_iop)
-{
-	struct io *a_iop = dip_find_sec(l_iop->dip, IOP_A, l_iop->t.sector);
-
-	if (a_iop) {
-		update_q2a(a_iop, tdelta(a_iop, l_iop));
-		dip_rem(a_iop);
-	}
-
-	return 1;
-}
-
-int ready_remap(struct io *a_iop, struct io *c_iop)
-{
-	struct io *l_iop = bilink_first_down(a_iop, NULL);
-
-	ASSERT(l_iop);
-	if (remapper_dev(l_iop->t.device))
-		return ready_dev_remap(l_iop, c_iop);
-	else
-		return ready_self_remap(l_iop);
+	rp->device = be32_to_cpu(rp->device);
+	rp->device_from = be32_to_cpu(rp->device_from);
+	rp->sector = be64_to_cpu(rp->sector);
 }
 
+/*
+ * q_iop == volume device
+ * a_iop == underlying device
+ */
 void trace_remap(struct io *a_iop)
 {
-	struct io *l_iop;
+	struct io *q_iop;
+	struct d_info *q_dip;
 	struct blk_io_trace_remap *rp = a_iop->pdu;
 
-	a_iop->t.device = be32_to_cpu(rp->device_from);
-	if (!io_setup(a_iop, IOP_A)) {
-		io_release(a_iop);
-		return;
-	}
+	cvt_pdu_remap(rp);
+
+	a_iop->t.device = rp->device_from;
+	if (!io_setup(a_iop, IOP_A))
+		goto out;
 
-	l_iop = io_alloc();
-	memcpy(&l_iop->t, &a_iop->t, sizeof(a_iop->t));
-	if (l_iop->t.pdu_len) {
-		l_iop->pdu = malloc(l_iop->t.pdu_len);
-		memcpy(l_iop->pdu, a_iop->pdu, l_iop->t.pdu_len);
-	}
+	q_dip = __dip_find(rp->device);
+	if (!q_dip)
+		goto out;
 
-	l_iop->t.device = be32_to_cpu(rp->device);
-	l_iop->t.sector = be64_to_cpu(rp->sector);
-	if (!io_setup(l_iop, IOP_L)) {
-		io_release(l_iop);
-		io_release(a_iop);
-		return;
-	}
+	q_iop = dip_find_sec(q_dip, IOP_Q, rp->sector);
+	if (q_iop)
+		update_q2a(q_iop, tdelta(q_iop->t.time, a_iop->t.time));
 
-	bilink(l_iop, a_iop);
+out:
+	io_release(a_iop);
 }
diff --git a/btt/trace_requeue.c b/btt/trace_requeue.c
index 4baf66e..f1324d9 100644
--- a/btt/trace_requeue.c
+++ b/btt/trace_requeue.c
@@ -20,26 +20,16 @@
  */
 #include "globals.h"
 
-void trace_requeue(struct io *r_iop)
+static void handle_requeue(struct io *r_iop)
 {
-	struct io *d_iop;
-
-	if ((io_setup(r_iop, IOP_R) == 0) ||
-	    (d_iop = dip_find_sec(r_iop->dip, IOP_D, 
-	    					BIT_START(r_iop))) == NULL) {
-		io_release(r_iop);
-		return;
-	}
-	dip_rem(d_iop);
-
-#	if defined(DEBUG)
-		ASSERT(ready_issue(d_iop, r_iop) != 0);
-#	else
-		(void)ready_issue(d_iop, r_iop);
-#	endif
-
-	run_unissue(d_iop, r_iop, r_iop);
-	add_rmhd(r_iop);
+	r_iop->dip->n_qs++;
+	r_iop->dip->t_act_q += r_iop->dip->n_act_q;
+	r_iop->dip->n_act_q++;
+}
 
-	release_iops();
+void trace_requeue(struct io *r_iop)
+{
+	if (io_setup(r_iop, IOP_R))
+		handle_requeue(r_iop);
+	io_release(r_iop);
 }
diff --git a/btt/unplug_hist.c b/btt/unplug_hist.c
index 4971f31..aaa4c39 100644
--- a/btt/unplug_hist.c
+++ b/btt/unplug_hist.c
@@ -23,7 +23,7 @@
 #define BKT_WIDTH	5
 #define MAX_BKT		19
 #define EXCESS_BKT	20
-#define NBKTS		EXCESS_BKT
+#define NBKTS		(EXCESS_BKT + 1)
 
 struct hist_bkt {
 	__u32 dev;
@@ -56,7 +56,7 @@ void unplug_hist_add(struct io *u_iop)
 		struct hist_bkt *hbp = dip->unplug_hist_handle;
 
 		idx = (n_unplugs / BKT_WIDTH);
-		if (idx > MAX_BKT)
+		if (idx > EXCESS_BKT)
 			idx = EXCESS_BKT;
 
 		assert((0 <= idx) && (idx <= EXCESS_BKT));
diff --git a/doc/btrecord.8 b/doc/btrecord.8
new file mode 100644
index 0000000..d1ec686
--- /dev/null
+++ b/doc/btrecord.8
@@ -0,0 +1,210 @@
+.TH BTREPLAY 1 "December  8, 2007" "blktrace git\-20071207142532" ""
+
+
+.SH NAME
+btrecord \- recreate IO loads recorded by blktrace
+
+
+.SH SYNOPSIS
+.B Usage: 
+
+btrecord [ \fIoptions\fR ] <\fIdev\fR...>
+
+
+.SH DESCRIPTION
+
+.P
+The \fIbtrecord\fR and \fIbtreplay\fR tools provide the ability to
+record and replay IOs captured by the \fIblktrace\fR utility. Attempts
+are made to maintain ordering, CPU mappings and time-separation of IOs. 
+
+
+.P
+The \fIblktrace\fR utility provides the ability to collect detailed
+traces from the kernel for each IO processed by the block IO layer. The
+traces provide a complete timeline for each IO processed, including
+detailed information concerning when an IO was first received by the block
+IO layer \(em indicating the device, CPU number, time stamp, IO direction,
+sector number and IO size (number of sectors). Using this information,
+one is able to \fBreplay\fR the IO again on the same machine or another
+set up entirely.
+
+.P
+The basic operating work-flow to replay IOs would be something like:
+
+.IP \- 2
+  Run \fIblktrace\fR to collect traces. Here you specify the
+  device or devices that you wish to trace and later replay IOs upon. Note:
+  the only traces you are interested in are \fBQUEUE\fR requests \(em
+  thus, to save system resources (including storage for traces), one could
+  specify the \fI-a queue\fR command line option to \fIblktrace\fR.
+
+.IP \- 2
+  While \fIblktrace\fR is running, you run the workload that you
+  are interested in. 
+
+.IP \- 2
+  When the work load has completed, you stop the \fIblktrace\fR
+  utility (thus saving all traces over the complete workload). 
+
+.IP \- 2
+  You extract the pertinent IO information from the traces saved by
+  \fIblktrace\fR using the \fIbtrecord\fR utility. This will parse
+  each trace file created by \fIblktrace\fR, and crafty IO descriptions
+  to be used in the next phase of the workload processing.
+
+.IP \- 2
+  Once \fIbtrecord\fR has successfully created a series of data
+  files to be processed, you can run the \fIbtreplay\fR utility which
+  attempts to generate the same IOs seen during the sample workload phase.
+
+
+.SH OPTIONS
+
+\-d <\fIdir\fR>
+.br
+\-\-input\-directory=<\fIdir\fR>
+.RS
+Set input directory.
+This option requires a single parameter providing the directory
+name for where input files are to be found. The default directory is the
+current directory (\fI.\fR).
+.RE
+
+\-D <\fIdir\fR>
+.br
+\-\-output\-directory=<\fIdir\fR>
+.RS
+Set output directory.
+This option requires a single parameter providing the directory
+name for where output files are to be found. The default directory is the
+current directory (\fI.\fR).
+.RE
+
+\-F
+.br
+\-\-find\-traces
+.RS
+Find trace files automatically
+This option instructs \fIbtreplay\fR to go find all the trace files in the
+directory specified (either via the \fI-d\fR option, or in the default
+directory (\fI.\fR).
+.RE
+
+\-h
+.br
+\-\-help
+.RS
+Show help and exit.
+.RE
+
+\-V
+.br
+\-\-version
+.RS
+Show version number and exit.
+.RE
+
+\-m <\fInanoceconds\fR>
+.br
+\-\-input\-base=<\fInanoceconds\fR>
+.RS
+The \fI\-m\fR option requires a single parameter which specifies an
+amount of time (in nanoseconds) to include in any one bunch of IOs that
+are to be processed. The smaller the value, the smaller the number of
+IOs processed at one time \(em perhaps yielding in more realistic replay.
+However, after a certain point the amount of overhead per bunch may result
+in additonal real replay time, thus yielding less accurate replay times.
+.P
+The default value is 10,000,000 nanoseconds (10 milliseconds).
+.RE
+
+\-M <\fInum\fR>
+.br
+\-\-max\-packets=<\fInum\fR>
+.RS
+Set maximum number of packets per bunch.
+The \fI\-M\fR option requires a single parameter which specifies the
+maximum number of IOs to store in a single bunch. As with the \fI\-m\fR
+option, smaller values may or may not yield more accurate replay times.
+
+The default value is 8, with a maximum value of up to 512 being supported.
+.RE
+
+\-o <\fIbasename\fR>
+.br
+\-\-output\-base=<\fIbasename\fR>
+.RS
+Set base name for output files.
+Each output file has 3 fields:
+.IP 1. 3
+ Device identifier (taken directly from the device name of the
+ \fIblktrace\fR output file).
+.IP 2. 3
+ \fIbtrecord\fR base name \(em by default ``replay''.
+.IP 3. 3
+ The CPU number (again, taken directly from the
+ \fIblktrace\fR output file name).
+.P
+This option requires a single parameter that will override the default name
+(replay), and replace it with the specified value.
+.RE
+
+\-v
+.br
+\-\-verbose
+.RS
+Enable verbose output.
+This option will output some simple statistics at the end of a successful
+run.  Example output is:
+.nf
+.P
+sdab:0: 580661 pkts (tot), 126030 pkts (replay), 89809 bunches, 1.4 pkts/bunch
+sdab:1: 2559775 pkts (tot), 430172 pkts (replay), 293029 bunches, 1.5 pkts/bunch
+sdab:2: 653559 pkts (tot), 136522 pkts (replay), 102288 bunches, 1.3 pkts/bunch
+sdab:3: 474773 pkts (tot), 117849 pkts (replay), 69572 bunches, 1.7 pkts/bunch
+.fi
+.P
+The meaning of the columns is:
+.IP 1. 3
+  The first field contains the device name and CPU identrifer. Thus:
+  \fIsdab:0:\fR means the device \fIsdab\fR and traces on CPU 0. 
+.IP 2.
+  The second field contains the total number of packets processed for each
+  device file. 
+.IP 3.
+  The next field shows the number of packets eligible for replay. 
+.IP 4.
+  The fourth field contains the total number of IO bunches. 
+.IP 5.
+  The last field shows the average number of IOs per bunch recorded.
+.RE
+
+
+.SH AUTHORS
+\fIbtrecord\fR was written by Alan D. Brunelle.  This
+man page was created from the \fIbtreplay\fR documentation by Bas Zoetekouw.
+
+
+.SH "REPORTING BUGS"
+Report bugs to <linux\-btrace@vger.kernel.org>
+
+.SH COPYRIGHT
+Copyright \(co 2007 Alan D. Brunelle, Alan D. Brunelle and Nathan Scott.
+.br
+This is free software.  You may redistribute copies of it under the terms of
+the GNU General Public License <http://www.gnu.org/licenses/gpl.html>.
+There is NO WARRANTY, to the extent permitted by law.
+.br
+This manual page was created for Debian by Bas Zoetekouw.  It was derived from
+the documentation provided by the authors and it may be used, distributed and
+modified under the terms of the GNU General Public License, version 2.
+.br
+On Debian systems, the text of the GNU General Public License can be found in
+/usr/share/common\-licenses/GPL\-2.
+
+.SH "SEE ALSO"
+The full documentation for btreplay can be found in /usr/share/doc/blktrace on Debian systems.
+.br
+blktrace (8), blkparse (1), btreplay (8)
+
diff --git a/doc/btreplay.8 b/doc/btreplay.8
new file mode 100644
index 0000000..8971e99
--- /dev/null
+++ b/doc/btreplay.8
@@ -0,0 +1,235 @@
+.TH BTREPLAY 1 "December  8, 2007" "blktrace git\-20071207142532" ""
+
+
+.SH NAME
+btreplay \- recreate IO loads recorded by blktrace
+
+
+.SH SYNOPSIS
+.B Usage: 
+
+btreplay [ \fIoptions\fR ] <\fIdev\fR...>
+
+
+.SH DESCRIPTION
+
+.P
+The \fIbtrecord\fR and \fIbtreplay\fR tools provide the ability to
+record and replay IOs captured by the \fIblktrace\fR utility. Attempts
+are made to maintain ordering, CPU mappings and time-separation of IOs. 
+
+
+.P
+The \fIblktrace\fR utility provides the ability to collect detailed
+traces from the kernel for each IO processed by the block IO layer. The
+traces provide a complete timeline for each IO processed, including
+detailed information concerning when an IO was first received by the block
+IO layer \(em indicating the device, CPU number, time stamp, IO direction,
+sector number and IO size (number of sectors). Using this information,
+one is able to \fBreplay\fR the IO again on the same machine or another
+set up entirely.
+
+.P
+The basic operating work-flow to replay IOs would be something like:
+
+.IP \- 2
+  Run \fIblktrace\fR to collect traces. Here you specify the
+  device or devices that you wish to trace and later replay IOs upon. Note:
+  the only traces you are interested in are \fBQUEUE\fR requests \(em
+  thus, to save system resources (including storage for traces), one could
+  specify the \fI-a queue\fR command line option to \fIblktrace\fR.
+
+.IP \- 2
+  While \fIblktrace\fR is running, you run the workload that you
+  are interested in. 
+
+.IP \- 2
+  When the work load has completed, you stop the \fIblktrace\fR
+  utility (thus saving all traces over the complete workload). 
+
+.IP \- 2
+  You extract the pertinent IO information from the traces saved by
+  \fIblktrace\fR using the \fIbtrecord\fR utility. This will parse
+  each trace file created by \fIblktrace\fR, and crafty IO descriptions
+  to be used in the next phase of the workload processing.
+
+.IP \- 2
+  Once \fIbtrecord\fR has successfully created a series of data
+  files to be processed, you can run the \fIbtreplay\fR utility which
+  attempts to generate the same IOs seen during the sample workload phase.
+
+
+.SH OPTIONS
+
+\-c <\fInum\fR>
+.br
+\-\-cpus=<\fInum\fR>
+.RS
+Set number of CPUs to use.
+.RE
+
+\-d <\fIdir\fR>
+.br
+\-\-input\-directory=<\fIdir\fR>
+.RS
+Set input directory.
+This option requires a single parameter providing the directory
+name for where input files are to be found. The default directory is the
+current directory (\fI.\fR).
+.RE
+
+\-F
+.br
+\-\-find\-records
+.RS
+Find record files automatically
+This option instructs \fIbtreplay\fR to go find all the record files in the
+directory specified (either via the \fI-d\fR option, or in the default
+directory (\fI.\fR).
+.RE
+
+\-h
+.br
+\-\-help
+.RS
+Show help and exit.
+.RE
+
+\-i <\fIbasename\fR>
+.br
+\-\-input\-base=<\fIbasename\fR>
+.RS
+Set base name for input files.
+Each input file has 3 fields:
+.IP 1. 3
+ Device identifier (taken directly from the device name of the
+ \fIblktrace\fR output file).
+.IP 2. 3
+ \fIbtrecord\fR base name \(em by default ``replay''.
+.IP 3. 3
+ The CPU number (again, taken directly from the
+ \fIblktrace\fR output file name).
+.P
+This option requires a single parameter that will override the default name
+(replay), and replace it with the specified value.
+.RE
+
+\-I <\fInum\fR>
+.br
+\-\-iterations=<\fInum\fR>
+.RS
+Set number of iterations to run.
+This option requires a single parameter which specifies the number of times
+to run through the input files. The default value is 1
+.RE
+
+\-M <\fIfilename\fR>
+.br
+\-\-map\-devs=<\fIfilename\fR>
+.RS
+Specify device mappings.  
+This option requires a single paramter which specifies the name of a
+file contain device mappings. The file must be very simply managed, with
+just two pieces of data per line:
+
+.IP \- 2
+  The device name on the recorded system (with the '\fI/dev/\fR'
+  removed). Example: \fI/dev/sda\fR would just be \fIsda\fR.
+
+.IP \- 2
+  The device name on the replay system to use (again, without the
+  '\fI/dev/\fR' path prepended).
+
+.P
+An example file for when one would map devices \fI/dev/sda\fR and
+\fI/dev/sdb\fR on the recorded system to \fIdev/sdg\fR and
+\fIsdh\fR on the replay system would be:
+
+.nf
+.IP
+sda sdg
+sdb sdh
+.fi
+
+.P
+The only entries in the file that are allowed are these two element lines \(em
+we do not (yet?) support the notion of blank lines, or comment lines, or the
+like.
+
+.P
+The utility allows for multiple \fI-M\fR options to be
+supplied on the command line.
+.RE
+
+\-N
+.br
+\-\-no\-stalls
+.RS
+Disable pre-bunch stalls.
+When specified on the command line, all pre-bunch stall indicators will be
+ignored. IOs will be replayed without inter-bunch delays.
+.RE
+
+\-v
+.br
+\-\-verbose
+.RS
+Enable verbose output.
+When specified on the command line, this option instructs \fIbtreplay\fR
+to store information concerning each \fBstall\fR and IO operation
+performed by \fIbtreplay\fR. The name of each file so created will be
+the input file name used with an extension of \fI.rep\fR appended onto
+it. Thus, an input file of the name \fIsdab.replay.3\fR would generate a
+verbose output file with the name \fIsdab.replay.3.rep\fR in the
+directory specified for input files.
+.P
+In addition, \fIbtreplay\fR will also output to \fIstderr\fR the
+names of the input files being processed.
+.RE
+
+\-V
+.br
+\-\-version
+.RS
+Show version number and exit.
+.RE
+
+\-W
+.br
+\-\-write-enable
+.RS
+Enable writing during replay.
+As a precautionary measure, by default \texttt{btreplay} will not
+process \fBwrite requests. In order to enable \fIbtreplay\fR to
+actually \fBwrite\fR to devices one must explicitly specify the
+\fI\-W\fR option.
+.RE
+
+
+.SH AUTHORS
+\fIbtreplay\fR was written by Alan D. Brunelle.  This
+man page was created from the \fIbtreplay\fR documentation by Bas Zoetekouw.
+
+
+.SH "REPORTING BUGS"
+Report bugs to <linux\-btrace@vger.kernel.org>
+
+.SH COPYRIGHT
+Copyright \(co 2007 Alan D. Brunelle, Alan D. Brunelle and Nathan Scott.
+.br
+This is free software.  You may redistribute copies of it under the terms of
+the GNU General Public License <http://www.gnu.org/licenses/gpl.html>.
+There is NO WARRANTY, to the extent permitted by law.
+.br
+This manual page was created for Debian by Bas Zoetekouw.  It was derived from
+the documentation provided by the authors and it may be used, distributed and
+modified under the terms of the GNU General Public License, version 2.
+.br
+On Debian systems, the text of the GNU General Public License can be found in
+/usr/share/common\-licenses/GPL\-2.
+
+.SH "SEE ALSO"
+The full documentation for btreplay can be found in /usr/share/doc/blktrace on Debian systems.
+.br
+blktrace (8), blkparse (1), btrecord (8)
+
diff --git a/doc/btt.1 b/doc/btt.1
index febf784..a6b85cf 100644
--- a/doc/btt.1
+++ b/doc/btt.1
@@ -1,4 +1,4 @@
-.TH BTT 1 "March 06, 2007" "blktrace git\-20070306202522" ""
+.TH BTT 1 "September 29, 2007" "blktrace git\-20070910192508" ""
 
 
 .SH NAME
@@ -6,65 +6,130 @@ btt \- analyse block i/o traces produces by blktrace
 
 
 .SH SYNOPSIS
-.B btt [options] \-i <\fIinput file\fR>
+.B btt 
+[ \-a               | \-\-seek\-absolute ]
+.RS 4
+[ \-A               | \-\-all\-data ]
+.br
+[ \-B <\fIoutput name\fR> | \-\-dump\-blocknos=<\fIoutput name\fR> ]
+.br
+[ \-d <\fIseconds\fR>     | \-\-range\-delta=<\fIseconds\fR> ]
+.br
+[ \-D <\fIdev;...\fR>     | \-\-devices=<\fIdev;...\fR> ]
+.br
+[ \-e <\fIexe,...\fR>     | \-\-exes=<\fIexe,...\fR>  ]
+.br
+[ \-h               | \-\-help ]
+.br
+[ \-i <\fIinput name\fR>  | \-\-input\-file=<\fIinput name\fR> ]
+.br
+[ \-I <\fIoutput name\fR> | \-\-iostat=<\fIoutput name\fR> ]
+.br
+[ \-l <\fIoutput name\fR> | \-\-d2c\-latencies=<\fIoutput name\fR> ]
+.br
+[ \-M <\fIdev map\fR>     | \-\-dev\-maps=<\fIdev map\fR>
+.br
+[ \-o <\fIoutput name\fR> | \-\-output\-file=<\fIoutput name\fR> ]
+.br
+[ \-p <\fIoutput name\fR> | \-\-per\-io\-dump=<\fIoutput name\fR> ]
+.br
+[ \-q <\fIoutput name\fR> | \-\-q2c\-latencies=<\fIoutput name\fR> ]
+.br
+[ \-s <\fIoutput name\fR> | \-\-seeks=<\fIoutput name\fR> ]
+.br
+[ \-S <\fIinterval\fR>    | \-\-iostat\-interval=<\fIinterval\fR> ]
+.br
+[ \-t <\fIsec\fR>         | \-\-time\-start=<\fIsec\fR> ]
 .br
-.B btt \-h | \-\-help
+[ \-T <\fIsec\fR>         | \-\-time\-end=<\fIsec\fR> ]
 .br
-.B btt \-V | \-\-version
+[ \-u <\fIoutput name\fR> | \-\-unplug\-hist=<\fIoutput name\fR> ]
 .br
+[ \-v               | \-\-verbose ]
+.br
+[ \-V               | \-\-version ]
 
 
 .SH DESCRIPTION
 
+btt is a post\-processing tool for the block layer IO tracing tool called
+blktrace(8).  As noted in its documentation, blktrace 
+is a block layer IO tracing mechanism which provides detailed
+information about request queue operations up to user space.
+
 btt will take in binary dump data from blkparse, and analyse the events,
 producing a series of output from the analysis. It will also build .dat
-files containing "range data" -- showing things like Q activity (periods
+files containing "range data" \-\- showing things like Q activity (periods
 of time while Q events are being produced), C activity (likewise for
 command completions), and etc.
 
 
 .SH OPTIONS
 
-.B \-h
+.B \-a
 .br
-.B \-\-help
+.B \-\-seek\-absolute
 .RS 4
-Shows a short summary of possible command line option
+When specified on the command line, this directs btt to calculate
+seek distances based solely upon the ending block address of one IO,
+and the start of the next.  By default \fBbtt\fR uses the concept
+of the closeness to either the beginning or end of the previous IO. See
+the Users Manual for more details about seek distances.
 .RE
 
-.B \-V
+.B \-A
 .br
-.B \-\-version
+.B \-\-all\-data
 .RS 4
-Shows the version of btt.
+Normally \fBbtt\fR will not print out verbose information concerning
+per-process and per-device data.  If you desire that level of detail you can
+specify this option.
 .RE
 
-.B \-i
+.B \-B <\fIoutput name\fR>
 .br
+.B \-\-dump\-blocknos=<\fIoutput name\fR>
 .RS 4
-Specifies the input file to analyse.  This should be a trace file as produces
-by \fIblktrace\fR (8).
+This option will output absolute block numbers to three files prefixed
+by the specified output name:
+.HP
+.I prefix_device_r.dat
+.br
+All read block numbers are output, first column is time (seconds), second is
+the block number, and the third column is the ending block number.
+.HP
+.I prefix_device_w.dat
+.br
+All write block numbers are output, first column is time (seconds), second is
+the block number, and the third column is the ending block number.
+.HP
+.I prefix_device_c.dat
+.br
+All block numbers (read and write) are output, first column is time (seconds),
+second is the block number, and the third column is the ending block number.
 .RE
 
 .B \-d <\fIseconds\fR>
 .br
 .B \-\-range\-delta=<\fIseconds\fR>
 .RS 4
-The \-d option allows you to specify the granularity which determines
-"activity" with regard to the .dat files \-\- this specific the time
-(in seconds) that must elapse without a particular event occurring to
-signify inactivity. The larger the number, the fewer ranges output \-\-
-the default is 0.1 seconds.
+\fBbtt\fR outputs a file containing Q and C activity, the notion of active
+traces simply means that there are Q or C traces occurring within a certain
+period of each other. The default values is 0.1 seconds; with this option
+allowing one to change that granularity. The smaller the value, the more data
+points provided.
 .RE
 
 .B \-D <\fIdev;...\fR>
 .br
 .B \-\-devices=<\fIdev;...\fR>
 .RS 4
-The \-D option supplies the devices which should be looked at when
-analysing the input. This is a ":" separated list of devices, devices are
-specified by a mjr,mnr tuple (e.g.: \-D "8,0:8,8" specifies two devices
-with major 8 and minor 0 and 8 respectively).
+Normally, \fBbtt\fR will produce data for all devices detected in the
+traces parsed. With this option, one can reduce the analysis to one or more
+devices provided in the string passed to this option. The device identifiers
+are the major and minor number of each device, and each device identifier is
+separated by a colon (:). A valid specifier for devices 8,0 and 8,8 would then
+be: \fI8,0:8,8\fR.
 .RE
 
 .B \-e <\fIexe,...\fR>
@@ -75,6 +140,21 @@ The \-e option supplies the list of executables that will have I/Os
 analysed.
 .RE
 
+.B \-h
+.br
+.B \-\-help
+.RS 4
+Shows a short summary of possible command line option
+.RE
+
+.B \-i <\fIinput name\fR>
+.br
+.B \-\-input\-file <\fIinput file\fR>
+.RS 4
+Specifies the input file to analyse.  This should be a trace file produced
+by \fIblktrace\fR (8).
+.RE
+
 .B \-I <\fIoutput name\fR>
 .br
 .B \-\-iostat=<\fIoutput name\fR>
@@ -157,6 +237,41 @@ analysis will occur for all traces after the time specified. Similarly,
 if only \-T is specified, analysis stops after \-T's seconds.)
 .RE
 
+.B \-u <\fIoutput name\fR>
+.br
+.B \-\-unplug\-hist=<\fIoutput name\fR>
+.RS 4
+This option instructs \fBbtt\fR to generate a data file containing histogram
+information for unplug traces on a per device basis. It shows how many
+times an unplug was hit with a specified number of IOs released. There are 21
+output values into the file, as follows:
+
+.RS 4
+a value of 0 represents 0..4 counts
+.br
+a value of 1 represents 5..9 counts
+.br
+a value of 2 represents 10..14 counts
+.br
+etc, until
+.br
+a value of 20 represents 100+ counts
+.br
+.RE
+
+The file name(s) generated use the text string passed as an argument for
+the prefix, followed by the device identifier in \fImajor,minor\fR
+form, with a \fI.dat\fR extension.  For example, with \fI\-u
+up_hist\fR specified on the command line: \fIup_hist_008,032.dat\fR.
+.RE
+
+.B \-V
+.br
+.B \-\-version
+.RS 4
+Shows the version of btt.
+.RE
+
 .B \-v
 .br
 .B \-\-verbose
@@ -188,5 +303,7 @@ On Debian systems, the text of the GNU General Public License can be found in
 /usr/share/common\-licenses/GPL\-2.
 
 .SH "SEE ALSO"
+The btt Users Guide, which can be found in /usr/share/doc/blktrace/btt.pdf
+.br
 blktrace (8), blkparse (1), verify_blkparse (1), blkrawverify (1), btt (1)

Places

File blktrace-0.99.3..00a47cd169f2cc87b5f63fe93226b7231dee678c.diff of Package blktrace (Revision 3451e70dd1b9f12a28b5cbd7e8b58f00)

Places