###################################################################################
# ION SR-71mach5+ 
# (8) Intel 750 400GB SSDs in (2) RAID 5 array of (4) NVMe SSDs each
# (16) Intel DC S3610 1600GB SSDs in (2) RAID 5 array of (8) SATA SSDs each
# Each Array has (5) Partitions; All Partitions on each array are xfs
# format and mount options listed below
###################################################################################

2015 December 21, 13:37
CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
300 s ramp + 7200 s run, per test

###################################################################################
# 
# SR-71mach5 Configuration
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################
Product Name = Intel(R) Integrated RAID Module RMS3CC080
FW Package Build = 24.10.0-0020
Driver Name = megaraid_sas
Driver Version = 06.809.18.00
Product Name = Intel(R) RAID Controller RS3DC080
FW Package Build = 24.10.0-0020
Driver Name = megaraid_sas
Driver Version = 06.809.18.00
Drive /c0/e252/s0 :
252:0    10 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601DZ1P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c0/e252/s1 :
252:1    11 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601GG1P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c0/e252/s2 :
252:2     8 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601G51P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c0/e252/s3 :
252:3     9 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601GK1P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c0/e252/s4 :
252:4    14 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601AW1P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c0/e252/s5 :
252:5    13 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601C71P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c0/e252/s6 :
252:6    12 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601C81P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c0/e252/s7 :
252:7    15 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601GW1P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c1/e252/s0 :
252:0     8 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600MS1P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c1/e252/s1 :
252:1     9 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600T21P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c1/e252/s2 :
252:2    10 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600Q01P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c1/e252/s3 :
252:3    11 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600N51P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c1/e252/s4 :
252:4    12 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600N61P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c1/e252/s5 :
252:5    13 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600PX1P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c1/e252/s6 :
252:6    14 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600NJ1P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Drive /c1/e252/s7 :
252:7    15 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600MZ1P6PGN  
Model Number = INTEL SSDSC2BX016T4
Firmware Revision = G2010140
Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz Status: Populated, Enabled Core Enabled: 12
Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz Status: Populated, Enabled Core Enabled: 12
____________________________________________________________________
MemTotal:       65696360 kB


###################################################################################
# 
# mkfs commands
# 
###################################################################################
#!/bin/bash
#
# format 5 partitions on each of 3 RAID 5 volumes with XFS
#
mkfs.xfs -L C0P1 -d su=65536,sw=7 -l su=64k /dev/sdb1
mkfs.xfs -L C1P1 -d su=65536,sw=7 -l su=64k /dev/sdc1
mkfs.xfs -L C2P1 -d su=65536,sw=7 -l su=64k /dev/sdd1
mkfs.xfs -L C0P2 -d su=65536,sw=7 -l su=64k /dev/sdb2
mkfs.xfs -L C1P2 -d su=65536,sw=7 -l su=64k /dev/sdc2
mkfs.xfs -L C2P2 -d su=65536,sw=7 -l su=64k /dev/sdd2
mkfs.xfs -L C0P3 -d su=65536,sw=7 -l su=64k /dev/sdb3
mkfs.xfs -L C1P3 -d su=65536,sw=7 -l su=64k /dev/sdc3
mkfs.xfs -L C2P3 -d su=65536,sw=7 -l su=64k /dev/sdd3
mkfs.xfs -L C0P4 -d su=65536,sw=7 -l su=64k /dev/sdb4
mkfs.xfs -L C1P4 -d su=65536,sw=7 -l su=64k /dev/sdc4
mkfs.xfs -L C2P4 -d su=65536,sw=7 -l su=64k /dev/sdd4
mkfs.xfs -L C0P5 -d su=65536,sw=7 -l su=64k /dev/sdb5
mkfs.xfs -L C1P5 -d su=65536,sw=7 -l su=64k /dev/sdc5
mkfs.xfs -L C2P5 -d su=65536,sw=7 -l su=64k /dev/sdd5
# 
# mdadm -C /dev/md/imsm0 -e imsm -n4 /dev/nvme[0-3]n1 -f --chunk=64
# mdadm -C /dev/md0 /dev/md/imsm0 -n4 -l5 --chunk=64
# parted -a optimal /dev/md0 mkpart NVMe5V0P1 xfs 0% 18%
# parted -a optimal /dev/md0 mkpart NVMe5V0P1 xfs 18% 36%
# parted -a optimal /dev/md0 mkpart NVMe5V0P1 xfs 36% 54%
# parted -a optimal /dev/md0 mkpart NVMe5V0P1 xfs 54% 72%
# parted -a optimal /dev/md0 mkpart NVMe5V0P1 xfs 72% 90%
# 
mkfs.xfs -L NVMR5V0P1 -d su=65536,sw=3 -l su=64k -f /dev/md0p1
mkfs.xfs -L NVMR5V0P2 -d su=65536,sw=3 -l su=64k -f /dev/md0p2
mkfs.xfs -L NVMR5V0P3 -d su=65536,sw=3 -l su=64k -f /dev/md0p3
mkfs.xfs -L NVMR5V0P4 -d su=65536,sw=3 -l su=64k -f /dev/md0p4
mkfs.xfs -L NVMR5V0P5 -d su=65536,sw=3 -l su=64k -f /dev/md0p5
# 
# mdadm -C /dev/md/imsm1 -e imsm -n4 /dev/nvme[4-7]n1 -f --chunk=64
# mdadm -C /dev/md1 /dev/md/imsm1 -n4 -l5 --chunk=64
# parted -a optimal /dev/md1 mkpart NVMe5V1P1 xfs 0% 18%
# parted -a optimal /dev/md1 mkpart NVMe5V1P1 xfs 18% 36%
# parted -a optimal /dev/md1 mkpart NVMe5V1P1 xfs 36% 54%
# parted -a optimal /dev/md1 mkpart NVMe5V1P1 xfs 54% 72%
# parted -a optimal /dev/md1 mkpart NVMe5V1P1 xfs 72% 90%
# 
mkfs.xfs -L NVMR5V1P1 -d su=65536,sw=3 -l su=64k -f /dev/md1p1
mkfs.xfs -L NVMR5V1P2 -d su=65536,sw=3 -l su=64k -f /dev/md1p2
mkfs.xfs -L NVMR5V1P3 -d su=65536,sw=3 -l su=64k -f /dev/md1p3
mkfs.xfs -L NVMR5V1P4 -d su=65536,sw=3 -l su=64k -f /dev/md1p4
mkfs.xfs -L NVMR5V1P5 -d su=65536,sw=3 -l su=64k -f /dev/md1p5



###################################################################################
# 
# /etc/fstab
# kernel parameter: elevator=noop
# 
###################################################################################
LABEL=C0P1		/sr71/C0P1		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P2		/sr71/C0P2		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P3		/sr71/C0P3		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P4		/sr71/C0P4		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P5		/sr71/C0P5		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P1		/sr71/C1P1		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P2		/sr71/C1P2		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P3		/sr71/C1P3		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P4		/sr71/C1P4		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P5		/sr71/C1P5		xfs     sync,noatime,nodiratime,nobarrier 0 0
# LABEL=C2P1		/sr71/C2P1		xfs	sync,noatime,nodiratime,nobarrier 0 0
# LABEL=C2P2		/sr71/C2P2		xfs	sync,noatime,nodiratime,nobarrier 0 0
# LABEL=C2P3		/sr71/C2P3		xfs     sync,noatime,nodiratime,nobarrier 0 0
# LABEL=C2P4		/sr71/C2P4		xfs     sync,noatime,nodiratime,nobarrier 0 0
# LABEL=C2P5		/sr71/C2P5		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V0P1		/sr71/NVMeR5V0P1	xfs	noauto,sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V0P2		/sr71/NVMeR5V0P2	xfs	noauto,sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V0P3		/sr71/NVMeR5V0P3	xfs     noauto,sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V0P4		/sr71/NVMeR5V0P4	xfs     noauto,sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V0P5		/sr71/NVMeR5V0P5	xfs     noauto,sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V1P1		/sr71/NVMeR5V1P1	xfs	noauto,sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V1P2		/sr71/NVMeR5V1P2	xfs	noauto,sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V1P3		/sr71/NVMeR5V1P3	xfs     noauto,sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V1P4		/sr71/NVMeR5V1P4	xfs     noauto,sync,noatime,nodiratime,nobarrier 0 0
LABEL=NVMR5V1P5		/sr71/NVMeR5V1P5	xfs     noauto,sync,noatime,nodiratime,nobarrier 0 0

###################################################################################
# 
# actual mount paramters
# 
###################################################################################
/dev/sda4 on /sr71/C0P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sda5 on /sr71/C0P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sda1 on /sr71/C0P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdb5 on /sr71/C1P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdb1 on /sr71/C1P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdb4 on /sr71/C1P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdb2 on /sr71/C1P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sda2 on /sr71/C0P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdb3 on /sr71/C1P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sda3 on /sr71/C0P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/md125p1 on /sr71/NVMeR5V0P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)
/dev/md125p2 on /sr71/NVMeR5V0P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)
/dev/md125p3 on /sr71/NVMeR5V0P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)
/dev/md125p4 on /sr71/NVMeR5V0P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)
/dev/md125p5 on /sr71/NVMeR5V0P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)
/dev/md124p1 on /sr71/NVMeR5V1P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)
/dev/md124p2 on /sr71/NVMeR5V1P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)
/dev/md124p3 on /sr71/NVMeR5V1P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)
/dev/md124p4 on /sr71/NVMeR5V1P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)
/dev/md124p5 on /sr71/NVMeR5V1P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=384,noquota)


###################################################################################
# 
# fio job file
fio-2.1.10
# 
###################################################################################
60 seconds between fio tests
[global]
ioengine=libaio
direct=1
time_based
ramp_time=${RAMPTIME}
runtime=${RUNTIME}
bs=${BS}
rw=${RW}
rwmixread=${RWMIXREAD}
group_reporting

[SATA-R5-01]
new_group
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C0P1/fio.tst
[SATA-R5-02]
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C0P2/fio.tst
[SATA-R5-03]
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C0P3/fio.tst
[SATA-R5-04]
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C0P4/fio.tst
[SATA-R5-05]
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C0P5/fio.tst

[SATA-R5-11]
new_group
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C1P1/fio.tst
[SATA-R5-12]
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C1P2/fio.tst
[SATA-R5-13]
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C1P3/fio.tst
[SATA-R5-14]
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C1P4/fio.tst
[SATA-R5-15]
size=1536g
iodepth=${IODEPTHS}
numjobs=${NUMJOBSS}
filename=/sr71/C1P5/fio.tst

[NVMe-R5-01]
new_group
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V0P1/fio.tst
[NVMe-R5-02]
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V0P2/fio.tst
[NVMe-R5-03]
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V0P3/fio.tst
[NVMe-R5-04]
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V0P4/fio.tst
[NVMe-R5-05]
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V0P5/fio.tst

[NVMe-R5-11]
new_group
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V1P1/fio.tst
[NVMe-R5-12]
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V1P2/fio.tst
[NVMe-R5-13]
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V1P3/fio.tst
[NVMe-R5-14]
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V1P4/fio.tst
[NVMe-R5-15]
size=200g
iodepth=${IODEPTHN}
numjobs=${NUMJOBSN}
filename=/sr71/NVMeR5V1P5/fio.tst



###################################################################################
# 
# 201512211337 test details
# 
###################################################################################

###################################################################################
# 
# Mon 21 Dec 2015 01:38:16 PM EST
# 8kB randwrite with SATA 2 jobs and 2 I/Os Outstanding
# 8kB randwrite with NVMe 2 jobs and 12 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 15645 IOPS | 125 MBps | 1276 μsec Average Latency 
SATA-R5-1: 15259 IOPS | 122 MBps | 1309 μsec Average Latency 
NVMe-R5-0: 26385 IOPS | 211 MBps | 4546 μsec Average Latency 
NVMe-R5-1: 27439 IOPS | 219 MBps | 4372 μsec Average Latency 

Totals
IOPS: 84728
MBPS: 677
Average Latency (μsec): 2875

SATA-R5-01: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-02: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-03: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-04: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-05: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-12: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-13: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-14: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-15: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
NVMe-R5-02: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
NVMe-R5-03: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
NVMe-R5-04: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
NVMe-R5-05: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
NVMe-R5-11: (g=3): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
NVMe-R5-12: (g=3): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
NVMe-R5-13: (g=3): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
NVMe-R5-14: (g=3): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
NVMe-R5-15: (g=3): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
...
fio-2.1.10
Starting 40 processes

SATA-R5-01: (groupid=0, jobs=10): err= 0: pid=141848: Mon Dec 21 15:43:18 2015
  write: io=880083MB, bw=125167KB/s, iops=15645, runt=7200002msec
    slat (usec): min=2, max=1146, avg=14.69, stdev= 8.09
    clat (usec): min=26, max=13319, avg=1262.03, stdev=278.76
     lat (usec): min=35, max=13327, avg=1276.83, stdev=278.60
    clat percentiles (usec):
     |  1.00th=[  588],  5.00th=[  716], 10.00th=[  860], 20.00th=[ 1032],
     | 30.00th=[ 1144], 40.00th=[ 1224], 50.00th=[ 1288], 60.00th=[ 1352],
     | 70.00th=[ 1432], 80.00th=[ 1496], 90.00th=[ 1592], 95.00th=[ 1672],
     | 99.00th=[ 1800], 99.50th=[ 1864], 99.90th=[ 1976], 99.95th=[ 2008],
     | 99.99th=[ 2064]
    bw (KB  /s): min=    0, max=26880, per=10.01%, avg=12525.15, stdev=134.01
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.07%, 750=6.11%
    lat (usec) : 1000=11.41%
    lat (msec) : 2=82.36%, 4=0.05%, 10=0.01%, 20=0.01%
  cpu          : usr=0.45%, sys=3.02%, ctx=94414970, majf=0, minf=19057
  IO depths    : 1=0.1%, 2=104.2%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=112650626/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=10): err= 0: pid=141858: Mon Dec 21 15:43:18 2015
  write: io=858349MB, bw=122076KB/s, iops=15259, runt=7200002msec
    slat (usec): min=2, max=795, avg=15.78, stdev= 8.92
    clat (usec): min=31, max=12374, avg=1293.22, stdev=263.26
     lat (usec): min=37, max=12403, avg=1309.16, stdev=263.09
    clat percentiles (usec):
     |  1.00th=[  620],  5.00th=[  780], 10.00th=[  916], 20.00th=[ 1096],
     | 30.00th=[ 1192], 40.00th=[ 1256], 50.00th=[ 1320], 60.00th=[ 1368],
     | 70.00th=[ 1432], 80.00th=[ 1512], 90.00th=[ 1608], 95.00th=[ 1688],
     | 99.00th=[ 1816], 99.50th=[ 1864], 99.90th=[ 1976], 99.95th=[ 2008],
     | 99.99th=[ 2064]
    bw (KB  /s): min=    0, max=17968, per=10.01%, avg=12215.95, stdev=130.20
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.08%, 750=4.03%
    lat (usec) : 1000=9.69%
    lat (msec) : 2=86.14%, 4=0.06%, 10=0.01%, 20=0.01%
  cpu          : usr=0.46%, sys=3.16%, ctx=92522598, majf=0, minf=20418
  IO depths    : 1=0.1%, 2=104.2%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=109868631/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=10): err= 0: pid=141868: Mon Dec 21 15:43:18 2015
  write: io=1449.5GB, bw=211086KB/s, iops=26385, runt=7200011msec
    slat (usec): min=2, max=15134, avg=205.42, stdev=347.58
    clat (usec): min=9, max=1817.3M, avg=4338.73, stdev=186500.86
     lat (usec): min=103, max=1817.3M, avg=4546.90, stdev=186501.39
    clat percentiles (usec):
     |  1.00th=[  270],  5.00th=[  524], 10.00th=[  796], 20.00th=[ 1272],
     | 30.00th=[ 1816], 40.00th=[ 2320], 50.00th=[ 2768], 60.00th=[ 3536],
     | 70.00th=[ 5024], 80.00th=[ 7456], 90.00th=[10688], 95.00th=[12608],
     | 99.00th=[15680], 99.50th=[16768], 99.90th=[19072], 99.95th=[19840],
     | 99.99th=[22144]
    bw (KB  /s): min=    0, max=39488, per=10.01%, avg=21120.39, stdev=1750.86
    lat (usec) : 10=0.01%, 50=0.01%, 100=0.01%, 250=0.77%, 500=3.74%
    lat (usec) : 750=4.62%, 1000=5.21%
    lat (msec) : 2=19.13%, 4=30.37%, 10=24.29%, 20=11.81%, 50=0.05%
    lat (msec) : >=2000=0.01%
  cpu          : usr=0.65%, sys=7.75%, ctx=1074396886, majf=0, minf=47697
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=105.6%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=189977750/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=12
NVMe-R5-11: (groupid=3, jobs=10): err= 0: pid=141878: Mon Dec 21 15:43:18 2015
  write: io=1507.4GB, bw=219518KB/s, iops=27439, runt=7200182msec
    slat (usec): min=2, max=16986, avg=207.70, stdev=368.09
    clat (usec): min=11, max=1455.6M, avg=4161.51, stdev=147368.73
     lat (usec): min=55, max=1455.6M, avg=4372.08, stdev=147369.53
    clat percentiles (usec):
     |  1.00th=[  199],  5.00th=[  326], 10.00th=[  466], 20.00th=[  796],
     | 30.00th=[ 1176], 40.00th=[ 1704], 50.00th=[ 2256], 60.00th=[ 2800],
     | 70.00th=[ 4832], 80.00th=[ 8096], 90.00th=[11584], 95.00th=[13376],
     | 99.00th=[16320], 99.50th=[17280], 99.90th=[19584], 99.95th=[20608],
     | 99.99th=[22912]
    bw (KB  /s): min=    0, max=43200, per=10.01%, avg=21964.03, stdev=1963.05
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=2.43%, 500=8.75%
    lat (usec) : 750=7.54%, 1000=6.90%
    lat (msec) : 2=19.80%, 4=21.80%, 10=18.21%, 20=14.49%, 50=0.08%
    lat (msec) : 250=0.01%, >=2000=0.01%
  cpu          : usr=0.70%, sys=8.39%, ctx=1153068657, majf=0, minf=45937
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=105.6%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=197571304/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=12

Run status group 0 (all jobs):
  WRITE: io=880083MB, aggrb=125167KB/s, minb=125167KB/s, maxb=125167KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
  WRITE: io=858349MB, aggrb=122076KB/s, minb=122076KB/s, maxb=122076KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
  WRITE: io=1449.5GB, aggrb=211086KB/s, minb=211086KB/s, maxb=211086KB/s, mint=7200011msec, maxt=7200011msec

Run status group 3 (all jobs):
  WRITE: io=1507.4GB, aggrb=219518KB/s, minb=219518KB/s, maxb=219518KB/s, mint=7200182msec, maxt=7200182msec

Disk stats (read/write):
  sda: ios=0/117404041, merge=0/0, ticks=0/148290068, in_queue=148302324, util=100.00%
  sdb: ios=0/114503933, merge=0/0, ticks=0/148162261, in_queue=148173285, util=100.00%
    md125: ios=0/200980731, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=200561458/200561693, aggrmerge=0/0, aggrticks=283882833/149524444, aggrin_queue=433854883, aggrutil=99.70%
  nvme0n1: ios=200787432/200787676, merge=0/0, ticks=343412756/204920782, in_queue=548921099, util=99.70%
  nvme1n1: ios=200337011/200337237, merge=0/0, ticks=231980547/100737197, in_queue=332997477, util=99.34%
  nvme2n1: ios=200789669/200789923, merge=0/0, ticks=293792867/159188242, in_queue=453434590, util=99.54%
  nvme3n1: ios=200331720/200331939, merge=0/0, ticks=266345164/133251558, in_queue=400066368, util=99.46%
    md124: ios=0/209476652, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=208693573/208693829, aggrmerge=0/0, aggrticks=268187388/147493637, aggrin_queue=415967951, aggrutil=100.00%
  nvme4n1: ios=208782863/208783137, merge=0/0, ticks=125205517/18090254, in_queue=143359872, util=96.65%
  nvme5n1: ios=208618499/208618661, merge=0/0, ticks=126304540/18374401, in_queue=144786252, util=96.78%
  nvme6n1: ios=208756763/208756960, merge=0/0, ticks=127610767/19138318, in_queue=146792035, util=96.85%
  nvme7n1: ios=208616167/208616560, merge=0/0, ticks=693628730/534371575, in_queue=1228933645, util=100.00%


###################################################################################
# 
# Mon 21 Dec 2015 03:44:18 PM EST
# 64kB randwrite with SATA 2 jobs and 2 I/Os Outstanding
# 64kB randwrite with NVMe 2 jobs and 6 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 6228 IOPS | 398 MBps | 3209 μsec Average Latency 
SATA-R5-1: 6242 IOPS | 399 MBps | 3202 μsec Average Latency 
NVMe-R5-0: 3255 IOPS | 208 MBps | 18431 μsec Average Latency 
NVMe-R5-1: 3380 IOPS | 216 MBps | 17745 μsec Average Latency 

Totals
IOPS: 19105
MBPS: 1221
Average Latency (μsec): 10646

SATA-R5-01: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-02: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-03: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-04: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-05: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-12: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-13: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-14: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
SATA-R5-15: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-02: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-03: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-04: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-05: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-11: (g=3): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-12: (g=3): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-13: (g=3): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-14: (g=3): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-15: (g=3): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
fio-2.1.10
Starting 40 processes

SATA-R5-01: (groupid=0, jobs=10): err= 0: pid=147115: Mon Dec 21 17:49:19 2015
  write: io=2737.3GB, bw=398634KB/s, iops=6228, runt=7200005msec
    slat (usec): min=4, max=1887, avg=10.18, stdev= 3.92
    clat (usec): min=46, max=707115, avg=3199.10, stdev=4397.21
     lat (usec): min=60, max=707130, avg=3209.44, stdev=4397.17
    clat percentiles (usec):
     |  1.00th=[  900],  5.00th=[ 1368], 10.00th=[ 1672], 20.00th=[ 2064],
     | 30.00th=[ 2384], 40.00th=[ 2704], 50.00th=[ 3024], 60.00th=[ 3376],
     | 70.00th=[ 3760], 80.00th=[ 4192], 90.00th=[ 4832], 95.00th=[ 5408],
     | 99.00th=[ 6688], 99.50th=[ 7264], 99.90th=[ 8896], 99.95th=[ 9792],
     | 99.99th=[13376]
    bw (KB  /s): min=    0, max=52908, per=10.04%, avg=40039.85, stdev=5670.82
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.05%, 750=0.40%
    lat (usec) : 1000=1.10%
    lat (msec) : 2=16.56%, 4=57.50%, 10=24.36%, 20=0.04%, 50=0.01%
    lat (msec) : 750=0.01%
  cpu          : usr=0.28%, sys=0.79%, ctx=46599650, majf=0, minf=53711
  IO depths    : 1=0.1%, 2=104.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=44846306/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=10): err= 0: pid=147125: Mon Dec 21 17:49:19 2015
  write: io=2743.2GB, bw=399507KB/s, iops=6242, runt=7200005msec
    slat (usec): min=4, max=2195, avg=10.20, stdev= 3.54
    clat (usec): min=41, max=690493, avg=3192.08, stdev=3687.99
     lat (usec): min=55, max=690503, avg=3202.43, stdev=3687.97
    clat percentiles (usec):
     |  1.00th=[  940],  5.00th=[ 1432], 10.00th=[ 1720], 20.00th=[ 2096],
     | 30.00th=[ 2416], 40.00th=[ 2736], 50.00th=[ 3056], 60.00th=[ 3408],
     | 70.00th=[ 3760], 80.00th=[ 4192], 90.00th=[ 4768], 95.00th=[ 5280],
     | 99.00th=[ 6496], 99.50th=[ 7072], 99.90th=[ 8512], 99.95th=[ 9280],
     | 99.99th=[12224]
    bw (KB  /s): min=    0, max=52525, per=10.03%, avg=40083.75, stdev=4893.65
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.03%, 750=0.33%
    lat (usec) : 1000=0.91%
    lat (msec) : 2=15.97%, 4=58.70%, 10=24.04%, 20=0.03%, 50=0.01%
    lat (msec) : 750=0.01%
  cpu          : usr=0.29%, sys=0.80%, ctx=46724016, majf=0, minf=55871
  IO depths    : 1=0.1%, 2=105.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=44944504/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=10): err= 0: pid=147135: Mon Dec 21 17:49:19 2015
  write: io=1430.5GB, bw=208323KB/s, iops=3255, runt=7200025msec
    slat (usec): min=10, max=2340.1M, avg=3061.64, stdev=473282.96
    clat (usec): min=119, max=2442.3M, avg=15362.95, stdev=1295854.97
     lat (usec): min=491, max=2442.3M, avg=18431.46, stdev=1383135.36
    clat percentiles (usec):
     |  1.00th=[ 5472],  5.00th=[ 8032], 10.00th=[ 9280], 20.00th=[10816],
     | 30.00th=[11840], 40.00th=[12864], 50.00th=[14016], 60.00th=[15168],
     | 70.00th=[16512], 80.00th=[18304], 90.00th=[21120], 95.00th=[23680],
     | 99.00th=[28800], 99.50th=[30592], 99.90th=[34560], 99.95th=[36096],
     | 99.99th=[39680]
    bw (KB  /s): min=    0, max=33280, per=10.34%, avg=21551.00, stdev=1734.15
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.03%, 4=0.31%, 10=13.99%, 20=72.54%, 50=13.12%
    lat (msec) : 100=0.01%, >=2000=0.01%
  cpu          : usr=0.16%, sys=12.36%, ctx=1470039816, majf=0, minf=140972
  IO depths    : 1=0.1%, 2=0.1%, 4=104.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=23436413/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
NVMe-R5-11: (groupid=3, jobs=10): err= 0: pid=147145: Mon Dec 21 17:49:19 2015
  write: io=1485.8GB, bw=216365KB/s, iops=3380, runt=7200210msec
    slat (usec): min=12, max=22283, avg=2948.33, stdev=1399.12
    clat (usec): min=111, max=2389.7M, avg=14792.39, stdev=1062368.53
     lat (usec): min=652, max=2389.7M, avg=17745.99, stdev=1062369.76
    clat percentiles (usec):
     |  1.00th=[ 4192],  5.00th=[ 6432], 10.00th=[ 7648], 20.00th=[ 9152],
     | 30.00th=[10560], 40.00th=[11712], 50.00th=[13120], 60.00th=[14656],
     | 70.00th=[16512], 80.00th=[18816], 90.00th=[22400], 95.00th=[25472],
     | 99.00th=[30592], 99.50th=[32384], 99.90th=[36096], 99.95th=[38144],
     | 99.99th=[41728]
    bw (KB  /s): min=    0, max=36918, per=10.01%, avg=21655.96, stdev=1719.85
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.08%, 4=0.75%, 10=25.18%, 20=57.61%, 50=16.35%
    lat (msec) : 100=0.01%, 250=0.01%, >=2000=0.01%
  cpu          : usr=0.16%, sys=13.34%, ctx=1605566046, majf=0, minf=145000
  IO depths    : 1=0.1%, 2=0.1%, 4=104.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=24341696/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6

Run status group 0 (all jobs):
  WRITE: io=2737.3GB, aggrb=398633KB/s, minb=398633KB/s, maxb=398633KB/s, mint=7200005msec, maxt=7200005msec

Run status group 1 (all jobs):
  WRITE: io=2743.2GB, aggrb=399506KB/s, minb=399506KB/s, maxb=399506KB/s, mint=7200005msec, maxt=7200005msec

Run status group 2 (all jobs):
  WRITE: io=1430.5GB, aggrb=208323KB/s, minb=208323KB/s, maxb=208323KB/s, mint=7200025msec, maxt=7200025msec

Run status group 3 (all jobs):
  WRITE: io=1485.8GB, aggrb=216364KB/s, minb=216364KB/s, maxb=216364KB/s, mint=7200210msec, maxt=7200210msec

Disk stats (read/write):
  sda: ios=0/47041404, merge=0/0, ticks=0/149518811, in_queue=149504119, util=100.00%
  sdb: ios=0/47173612, merge=0/0, ticks=0/149504628, in_queue=149490798, util=100.00%
    md125: ios=0/33607218, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=195738348/195738733, aggrmerge=0/0, aggrticks=303125067/164854886, aggrin_queue=468429055, aggrutil=97.87%
  nvme0n1: ios=195781423/195781820, merge=0/0, ticks=357313400/214400133, in_queue=572286648, util=97.87%
  nvme1n1: ios=195718814/195719110, merge=0/0, ticks=251171660/115397016, in_queue=366829590, util=96.60%
  nvme2n1: ios=195768831/195769282, merge=0/0, ticks=323117815/185130240, in_queue=508696743, util=97.45%
  nvme3n1: ios=195684324/195684721, merge=0/0, ticks=280897393/144492158, in_queue=425903242, util=97.05%
    md124: ios=0/36891975, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=203207225/203207634, aggrmerge=0/0, aggrticks=287159521/156070119, aggrin_queue=443529365, aggrutil=100.00%
  nvme4n1: ios=203213794/203214114, merge=0/0, ticks=138415732/23126088, in_queue=161616312, util=90.65%
  nvme5n1: ios=203174455/203174725, merge=0/0, ticks=141759440/24800640, in_queue=166702134, util=90.29%
  nvme6n1: ios=203224080/203224391, merge=0/0, ticks=144662345/26468675, in_queue=171199520, util=90.47%
  nvme7n1: ios=203216573/203217308, merge=0/0, ticks=723800569/549885075, in_queue=1274599497, util=100.00%


###################################################################################
# 
# Mon 21 Dec 2015 05:50:20 PM EST
# 8kB randrw with SATA 3 jobs and 2 I/Os Outstanding
# 8kB randrw with NVMe 4 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 36664 IOPS | 12297 MBps | 794 μsec Average Latency 
SATA-R5-1: 36054 IOPS | 12092 MBps | 804 μsec Average Latency 
NVMe-R5-0: 75640 IOPS | 25371 MBps | 2621 μsec Average Latency 
NVMe-R5-1: 79599 IOPS | 26696 MBps | 2545 μsec Average Latency 

Totals
IOPS: 227957
MBPS: 76456
Average Latency (μsec): 1691

SATA-R5-01: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-02: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-03: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-04: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-05: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-12: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-13: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-14: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-15: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-02: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-03: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-04: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-05: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-12: (g=3): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-13: (g=3): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-14: (g=3): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-15: (g=3): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 70 processes

SATA-R5-01: (groupid=0, jobs=15): err= 0: pid=5767: Mon Dec 21 19:55:22 2015
  read : io=1349.4GB, bw=196508KB/s, iops=24563, runt=7200001msec
    slat (usec): min=2, max=314, avg=10.34, stdev= 6.05
    clat (usec): min=0, max=17989, avg=850.67, stdev=691.10
     lat (usec): min=66, max=17994, avg=861.12, stdev=691.45
    clat percentiles (usec):
     |  1.00th=[  191],  5.00th=[  229], 10.00th=[  253], 20.00th=[  294],
     | 30.00th=[  362], 40.00th=[  486], 50.00th=[  604], 60.00th=[  788],
     | 70.00th=[ 1128], 80.00th=[ 1320], 90.00th=[ 1704], 95.00th=[ 1944],
     | 99.00th=[ 3440], 99.50th=[ 3792], 99.90th=[ 4384], 99.95th=[ 4768],
     | 99.99th=[ 7456]
    bw (KB  /s): min=    0, max=23584, per=6.67%, avg=13108.06, stdev=384.97
  write: io=680726MB, bw=96814KB/s, iops=12101, runt=7200001msec
    slat (usec): min=2, max=307, avg=11.31, stdev= 6.27
    clat (usec): min=0, max=10882, avg=715.68, stdev=473.02
     lat (usec): min=28, max=10887, avg=727.10, stdev=473.78
    clat percentiles (usec):
     |  1.00th=[  124],  5.00th=[  171], 10.00th=[  197], 20.00th=[  235],
     | 30.00th=[  282], 40.00th=[  418], 50.00th=[  636], 60.00th=[  788],
     | 70.00th=[ 1064], 80.00th=[ 1224], 90.00th=[ 1432], 95.00th=[ 1480],
     | 99.00th=[ 1640], 99.50th=[ 1704], 99.90th=[ 1784], 99.95th=[ 1816],
     | 99.99th=[ 1944]
    bw (KB  /s): min=    0, max=11456, per=6.67%, avg=6458.25, stdev=305.36
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.02%
    lat (usec) : 100=0.12%, 250=14.07%, 500=27.60%, 750=16.25%, 1000=8.13%
    lat (msec) : 2=30.60%, 4=3.00%, 10=0.21%, 20=0.01%
  cpu          : usr=0.70%, sys=3.54%, ctx=251694824, majf=0, minf=39541
  IO depths    : 1=0.1%, 2=104.2%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=176856963/w=87132973/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=15): err= 0: pid=5783: Mon Dec 21 19:55:22 2015
  read : io=1326.1GB, bw=193248KB/s, iops=24155, runt=7200003msec
    slat (usec): min=2, max=597, avg= 9.54, stdev= 5.57
    clat (usec): min=1, max=18653, avg=871.73, stdev=712.44
     lat (usec): min=61, max=18658, avg=881.34, stdev=712.63
    clat percentiles (usec):
     |  1.00th=[  187],  5.00th=[  227], 10.00th=[  253], 20.00th=[  302],
     | 30.00th=[  374], 40.00th=[  502], 50.00th=[  628], 60.00th=[  812],
     | 70.00th=[ 1144], 80.00th=[ 1320], 90.00th=[ 1720], 95.00th=[ 2128],
     | 99.00th=[ 3536], 99.50th=[ 3888], 99.90th=[ 4512], 99.95th=[ 5152],
     | 99.99th=[ 7520]
    bw (KB  /s): min=    0, max=23936, per=6.67%, avg=12890.58, stdev=380.73
  write: io=669359MB, bw=95198KB/s, iops=11899, runt=7200003msec
    slat (usec): min=2, max=200, avg=10.56, stdev= 5.79
    clat (usec): min=1, max=11469, avg=717.33, stdev=468.55
     lat (usec): min=29, max=11482, avg=727.96, stdev=469.11
    clat percentiles (usec):
     |  1.00th=[  113],  5.00th=[  167], 10.00th=[  195], 20.00th=[  235],
     | 30.00th=[  290], 40.00th=[  438], 50.00th=[  636], 60.00th=[  804],
     | 70.00th=[ 1064], 80.00th=[ 1208], 90.00th=[ 1416], 95.00th=[ 1480],
     | 99.00th=[ 1624], 99.50th=[ 1688], 99.90th=[ 1784], 99.95th=[ 1816],
     | 99.99th=[ 1928]
    bw (KB  /s): min=    0, max=11184, per=6.67%, avg=6350.35, stdev=303.71
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.03%, 100=0.19%
    lat (usec) : 250=13.68%, 500=26.82%, 750=16.20%, 1000=8.69%
    lat (msec) : 2=30.70%, 4=3.44%, 10=0.25%, 20=0.01%
  cpu          : usr=0.72%, sys=3.25%, ctx=248721080, majf=0, minf=32599
  IO depths    : 1=0.1%, 2=104.2%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=173922917/w=85677941/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=20): err= 0: pid=5798: Mon Dec 21 19:55:22 2015
  read : io=2783.8GB, bw=405400KB/s, iops=50674, runt=7200303msec
    slat (usec): min=1, max=2650, avg= 4.03, stdev= 1.67
    clat (usec): min=0, max=846916K, avg=1063.41, stdev=162469.89
     lat (usec): min=40, max=846916K, avg=1067.53, stdev=162469.89
    clat percentiles (usec):
     |  1.00th=[   78],  5.00th=[   84], 10.00th=[   95], 20.00th=[  109],
     | 30.00th=[  137], 40.00th=[  217], 50.00th=[  386], 60.00th=[  716],
     | 70.00th=[ 1176], 80.00th=[ 1880], 90.00th=[ 2512], 95.00th=[ 3824],
     | 99.00th=[ 6752], 99.50th=[ 7584], 99.90th=[ 9408], 99.95th=[10176],
     | 99.99th=[11968]
    bw (KB  /s): min=    0, max=81696, per=5.44%, avg=22069.62, stdev=6852.56
  write: io=1371.6GB, bw=199730KB/s, iops=24966, runt=7200303msec
    slat (usec): min=2, max=386941K, avg=73.66, stdev=28251.51
    clat (usec): min=0, max=864160K, avg=4102.15, stdev=541254.62
     lat (usec): min=36, max=864160K, avg=4175.37, stdev=542023.57
    clat percentiles (usec):
     |  1.00th=[  167],  5.00th=[  326], 10.00th=[  524], 20.00th=[  956],
     | 30.00th=[ 1464], 40.00th=[ 2008], 50.00th=[ 2448], 60.00th=[ 3024],
     | 70.00th=[ 4384], 80.00th=[ 6496], 90.00th=[ 9280], 95.00th=[11072],
     | 99.00th=[14016], 99.50th=[15040], 99.90th=[17280], 99.95th=[18304],
     | 99.99th=[20352]
    bw (KB  /s): min=    0, max=39232, per=5.44%, avg=10873.03, stdev=3361.62
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=9.60%
    lat (usec) : 250=19.68%, 500=9.96%, 750=6.58%, 1000=6.04%
    lat (msec) : 2=16.19%, 4=18.25%, 10=11.12%, 20=2.58%, 50=0.01%
    lat (msec) : >=2000=0.01%
  cpu          : usr=0.87%, sys=3.57%, ctx=808502272, majf=0, minf=69474
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.4%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=364875238/w=179764064/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=20): err= 0: pid=5818: Mon Dec 21 19:55:22 2015
  read : io=2929.9GB, bw=426633KB/s, iops=53329, runt=7200871msec
    slat (usec): min=1, max=1073, avg= 3.98, stdev= 1.55
    clat (usec): min=0, max=131798K, avg=947.84, stdev=14737.58
     lat (usec): min=36, max=131798K, avg=951.91, stdev=14737.58
    clat percentiles (usec):
     |  1.00th=[   78],  5.00th=[   83], 10.00th=[   94], 20.00th=[  104],
     | 30.00th=[  121], 40.00th=[  155], 50.00th=[  255], 60.00th=[  450],
     | 70.00th=[  852], 80.00th=[ 1608], 90.00th=[ 2416], 95.00th=[ 4256],
     | 99.00th=[ 7328], 99.50th=[ 8256], 99.90th=[ 9920], 99.95th=[10688],
     | 99.99th=[12736]
    bw (KB  /s): min=    0, max=94512, per=5.11%, avg=21783.88, stdev=2669.03
  write: io=1443.3GB, bw=210161KB/s, iops=26270, runt=7200871msec
    slat (usec): min=2, max=17575, avg=118.63, stdev=342.44
    clat (usec): min=1, max=2191.2M, avg=4020.64, stdev=350969.13
     lat (usec): min=30, max=2191.2M, avg=4139.12, stdev=350969.48
    clat percentiles (usec):
     |  1.00th=[  157],  5.00th=[  239], 10.00th=[  350], 20.00th=[  628],
     | 30.00th=[ 1004], 40.00th=[ 1528], 50.00th=[ 2096], 60.00th=[ 2608],
     | 70.00th=[ 4832], 80.00th=[ 7840], 90.00th=[10816], 95.00th=[12480],
     | 99.00th=[15040], 99.50th=[15936], 99.90th=[18048], 99.95th=[19072],
     | 99.99th=[21376]
    bw (KB  /s): min=    0, max=45856, per=5.11%, avg=10731.10, stdev=1271.60
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=10.68%
    lat (usec) : 250=24.41%, 500=11.40%, 750=6.57%, 1000=5.58%
    lat (msec) : 2=13.97%, 4=13.08%, 10=10.05%, 20=4.26%, 50=0.01%
    lat (msec) : 500=0.01%, 750=0.01%, >=2000=0.01%
  cpu          : usr=0.93%, sys=4.45%, ctx=1035498834, majf=0, minf=72004
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=384015933/w=189167797/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=1349.4GB, aggrb=196507KB/s, minb=196507KB/s, maxb=196507KB/s, mint=7200001msec, maxt=7200001msec
  WRITE: io=680726MB, aggrb=96814KB/s, minb=96814KB/s, maxb=96814KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=1326.1GB, aggrb=193247KB/s, minb=193247KB/s, maxb=193247KB/s, mint=7200003msec, maxt=7200003msec
  WRITE: io=669359MB, aggrb=95197KB/s, minb=95197KB/s, maxb=95197KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=2783.8GB, aggrb=405399KB/s, minb=405399KB/s, maxb=405399KB/s, mint=7200303msec, maxt=7200303msec
  WRITE: io=1371.6GB, aggrb=199729KB/s, minb=199729KB/s, maxb=199729KB/s, mint=7200303msec, maxt=7200303msec

Run status group 3 (all jobs):
   READ: io=2929.9GB, aggrb=426632KB/s, minb=426632KB/s, maxb=426632KB/s, mint=7200871msec, maxt=7200871msec
  WRITE: io=1443.3GB, aggrb=210161KB/s, minb=210161KB/s, maxb=210161KB/s, mint=7200871msec, maxt=7200871msec

Disk stats (read/write):
  sda: ios=184321975/90804114, merge=0/0, ticks=157077472/65170643, in_queue=222374645, util=100.00%
  sdb: ios=181267834/89290767, merge=0/0, ticks=158162706/64176564, in_queue=222436337, util=100.00%
    md125: ios=381620597/188001895, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=283023937/187620501, aggrmerge=0/0, aggrticks=341160872/126960285, aggrin_queue=469094103, aggrutil=100.00%
  nvme0n1: ios=283035672/187641115, merge=0/0, ticks=411707921/179359263, in_queue=592297475, util=100.00%
  nvme1n1: ios=283050701/187583489, merge=0/0, ticks=283018520/83475272, in_queue=367157047, util=100.00%
  nvme2n1: ios=283020149/187646041, merge=0/0, ticks=351428895/135205769, in_queue=487634797, util=100.00%
  nvme3n1: ios=282989228/187611362, merge=0/0, ticks=318488153/109800836, in_queue=429287096, util=100.00%
    md124: ios=402171616/198102203, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=297898030/197358194, aggrmerge=0/0, aggrticks=341748550/139941653, aggrin_queue=482533509, aggrutil=100.00%
  nvme4n1: ios=298048150/197460861, merge=0/0, ticks=167313014/15533080, in_queue=183137440, util=99.37%
  nvme5n1: ios=297732205/197261956, merge=0/0, ticks=167993622/15312335, in_queue=183697916, util=99.38%
  nvme6n1: ios=298144765/197437180, merge=0/0, ticks=169949473/16087341, in_queue=186288734, util=99.42%
  nvme7n1: ios=297667003/197272781, merge=0/0, ticks=861738092/512833856, in_queue=1377009947, util=100.00%


###################################################################################
# 
# Mon 21 Dec 2015 07:56:22 PM EST
# 64kB randrw with SATA 2 jobs and 6 I/Os Outstanding
# 64kB randrw with NVMe 2 jobs and 6 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 14797 IOPS | 5517 MBps | 3520 μsec Average Latency 
SATA-R5-1: 14922 IOPS | 5563 MBps | 3493 μsec Average Latency 
NVMe-R5-0: 9506 IOPS | 3542 MBps | 7771 μsec Average Latency 
NVMe-R5-1: 9876 IOPS | 3680 MBps | 7483 μsec Average Latency 

Totals
IOPS: 49101
MBPS: 18302
Average Latency (μsec): 5566

SATA-R5-01: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
SATA-R5-02: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
SATA-R5-03: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
SATA-R5-04: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
SATA-R5-05: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
SATA-R5-11: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
SATA-R5-12: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
SATA-R5-13: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
SATA-R5-14: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
SATA-R5-15: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-01: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-02: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-03: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-04: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-05: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-11: (g=3): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-12: (g=3): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-13: (g=3): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-14: (g=3): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
NVMe-R5-15: (g=3): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
...
fio-2.1.10
Starting 40 processes

SATA-R5-01: (groupid=0, jobs=10): err= 0: pid=11554: Mon Dec 21 22:01:24 2015
  read : io=4357.2GB, bw=634549KB/s, iops=9914, runt=7200007msec
    slat (usec): min=4, max=690, avg=10.95, stdev= 4.21
    clat (usec): min=38, max=722448, avg=5075.32, stdev=16049.63
     lat (usec): min=46, max=722455, avg=5086.49, stdev=16049.61
    clat percentiles (usec):
     |  1.00th=[  972],  5.00th=[ 1656], 10.00th=[ 2128], 20.00th=[ 2800],
     | 30.00th=[ 3376], 40.00th=[ 3920], 50.00th=[ 4448], 60.00th=[ 4960],
     | 70.00th=[ 5600], 80.00th=[ 6368], 90.00th=[ 7584], 95.00th=[ 8640],
     | 99.00th=[11200], 99.50th=[12480], 99.90th=[16768], 99.95th=[651264],
     | 99.99th=[684032]
    bw (KB  /s): min=    0, max=80512, per=10.29%, avg=65320.85, stdev=12088.13
  write: io=2146.7GB, bw=312543KB/s, iops=4883, runt=7200007msec
    slat (usec): min=4, max=1030, avg=13.46, stdev= 5.15
    clat (usec): min=2, max=704281, avg=1940.66, stdev=6123.29
     lat (usec): min=41, max=704292, avg=1954.40, stdev=6123.24
    clat percentiles (usec):
     |  1.00th=[   63],  5.00th=[  175], 10.00th=[  394], 20.00th=[  844],
     | 30.00th=[ 1192], 40.00th=[ 1512], 50.00th=[ 1800], 60.00th=[ 2096],
     | 70.00th=[ 2416], 80.00th=[ 2800], 90.00th=[ 3408], 95.00th=[ 3952],
     | 99.00th=[ 5024], 99.50th=[ 5536], 99.90th=[ 6944], 99.95th=[ 7904],
     | 99.99th=[14400]
    bw (KB  /s): min=    0, max=45056, per=10.30%, avg=32186.03, stdev=6178.38
    lat (usec) : 4=0.01%, 20=0.01%, 50=0.10%, 100=0.71%, 250=1.49%
    lat (usec) : 500=1.77%, 750=2.05%, 1000=2.59%
    lat (msec) : 2=15.80%, 4=35.00%, 10=39.07%, 20=1.38%, 50=0.01%
    lat (msec) : 750=0.04%
  cpu          : usr=0.58%, sys=2.26%, ctx=106110470, majf=0, minf=145879
  IO depths    : 1=0.1%, 2=0.1%, 4=103.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=71386749/w=35161084/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
SATA-R5-11: (groupid=1, jobs=10): err= 0: pid=11564: Mon Dec 21 22:01:24 2015
  read : io=4393.8GB, bw=639881KB/s, iops=9998, runt=7200012msec
    slat (usec): min=4, max=839, avg=10.74, stdev= 4.05
    clat (usec): min=38, max=715810, avg=5027.92, stdev=14749.86
     lat (usec): min=45, max=715820, avg=5038.85, stdev=14749.85
    clat percentiles (usec):
     |  1.00th=[  956],  5.00th=[ 1640], 10.00th=[ 2128], 20.00th=[ 2800],
     | 30.00th=[ 3376], 40.00th=[ 3920], 50.00th=[ 4448], 60.00th=[ 5024],
     | 70.00th=[ 5600], 80.00th=[ 6432], 90.00th=[ 7648], 95.00th=[ 8768],
     | 99.00th=[11200], 99.50th=[12352], 99.90th=[16192], 99.95th=[21888],
     | 99.99th=[684032]
    bw (KB  /s): min=    0, max=85888, per=10.25%, avg=65573.06, stdev=11334.35
  write: io=2164.7GB, bw=315164KB/s, iops=4924, runt=7200012msec
    slat (usec): min=4, max=678, avg=13.07, stdev= 4.99
    clat (usec): min=1, max=695229, avg=1935.39, stdev=5559.64
     lat (usec): min=41, max=695240, avg=1948.70, stdev=5559.61
    clat percentiles (usec):
     |  1.00th=[   59],  5.00th=[  167], 10.00th=[  406], 20.00th=[  868],
     | 30.00th=[ 1224], 40.00th=[ 1512], 50.00th=[ 1800], 60.00th=[ 2096],
     | 70.00th=[ 2416], 80.00th=[ 2800], 90.00th=[ 3408], 95.00th=[ 3920],
     | 99.00th=[ 5024], 99.50th=[ 5472], 99.90th=[ 6752], 99.95th=[ 7584],
     | 99.99th=[12096]
    bw (KB  /s): min=    0, max=45952, per=10.25%, avg=32310.32, stdev=5830.22
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.15%
    lat (usec) : 100=0.76%, 250=1.41%, 500=1.64%, 750=2.00%, 1000=2.58%
    lat (msec) : 2=15.99%, 4=34.96%, 10=39.06%, 20=1.42%, 50=0.01%
    lat (msec) : 750=0.03%
  cpu          : usr=0.59%, sys=2.27%, ctx=107277514, majf=0, minf=147406
  IO depths    : 1=0.1%, 2=0.1%, 4=103.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=71986697/w=35455988/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
NVMe-R5-01: (groupid=2, jobs=10): err= 0: pid=11574: Mon Dec 21 22:01:24 2015
  read : io=2800.7GB, bw=407779KB/s, iops=6371, runt=7200181msec
    slat (usec): min=3, max=834, avg=10.41, stdev= 4.28
    clat (usec): min=17, max=1784.3M, avg=3338.07, stdev=300356.45
     lat (usec): min=114, max=1784.3M, avg=3348.67, stdev=300356.45
    clat percentiles (usec):
     |  1.00th=[  187],  5.00th=[  258], 10.00th=[  362], 20.00th=[  892],
     | 30.00th=[ 1608], 40.00th=[ 2160], 50.00th=[ 2544], 60.00th=[ 3120],
     | 70.00th=[ 3920], 80.00th=[ 5024], 90.00th=[ 6816], 95.00th=[ 8640],
     | 99.00th=[12608], 99.50th=[14272], 99.90th=[17792], 99.95th=[19328],
     | 99.99th=[22912]
    bw (KB  /s): min=    0, max=136448, per=11.96%, avg=48786.47, stdev=13164.75
  write: io=1377.1GB, bw=200666KB/s, iops=3135, runt=7200181msec
    slat (usec): min=8, max=576528K, avg=2082.28, stdev=125126.95
    clat (usec): min=33, max=1785.3M, avg=10135.35, stdev=1555323.48
     lat (usec): min=324, max=1785.3M, avg=12195.75, stdev=1560566.17
    clat percentiles (usec):
     |  1.00th=[ 1004],  5.00th=[ 1928], 10.00th=[ 2512], 20.00th=[ 3312],
     | 30.00th=[ 4128], 40.00th=[ 4960], 50.00th=[ 5856], 60.00th=[ 6944],
     | 70.00th=[ 8256], 80.00th=[10048], 90.00th=[12864], 95.00th=[15424],
     | 99.00th=[20352], 99.50th=[22400], 99.90th=[26752], 99.95th=[28544],
     | 99.99th=[32128]
    bw (KB  /s): min=    0, max=66688, per=11.96%, avg=24007.32, stdev=6296.35
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=2.96%, 500=6.20%
    lat (usec) : 750=2.78%, 1000=2.99%
    lat (msec) : 2=11.38%, 4=30.53%, 10=34.61%, 20=8.14%, 50=0.41%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, >=2000=0.01%
  cpu          : usr=0.32%, sys=8.28%, ctx=922453953, majf=0, minf=123305
  IO depths    : 1=0.1%, 2=0.1%, 4=104.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=45876283/w=22575420/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
NVMe-R5-11: (groupid=3, jobs=10): err= 0: pid=11584: Mon Dec 21 22:01:24 2015
  read : io=2908.2GB, bw=423645KB/s, iops=6619, runt=7200110msec
    slat (usec): min=3, max=966, avg=10.41, stdev= 4.17
    clat (usec): min=0, max=797655K, avg=3232.83, stdev=239250.94
     lat (usec): min=113, max=797655K, avg=3243.39, stdev=239250.94
    clat percentiles (usec):
     |  1.00th=[  181],  5.00th=[  245], 10.00th=[  318], 20.00th=[  708],
     | 30.00th=[ 1352], 40.00th=[ 1960], 50.00th=[ 2352], 60.00th=[ 2864],
     | 70.00th=[ 3728], 80.00th=[ 4960], 90.00th=[ 7072], 95.00th=[ 9024],
     | 99.00th=[13376], 99.50th=[15040], 99.90th=[18560], 99.95th=[20096],
     | 99.99th=[23424]
    bw (KB  /s): min=    0, max=145792, per=11.64%, avg=49330.62, stdev=12053.77
  write: io=1431.7GB, bw=208493KB/s, iops=3257, runt=7200110msec
    slat (usec): min=7, max=40948K, avg=2078.38, stdev=8404.04
    clat (usec): min=38, max=2305.5M, avg=9662.93, stdev=1350767.20
     lat (usec): min=263, max=2305.5M, avg=11724.66, stdev=1350794.01
    clat percentiles (usec):
     |  1.00th=[  612],  5.00th=[ 1400], 10.00th=[ 1960], 20.00th=[ 2672],
     | 30.00th=[ 3344], 40.00th=[ 4192], 50.00th=[ 5216], 60.00th=[ 6560],
     | 70.00th=[ 8384], 80.00th=[10816], 90.00th=[13888], 95.00th=[16512],
     | 99.00th=[21376], 99.50th=[23424], 99.90th=[27520], 99.95th=[29056],
     | 99.99th=[33024]
    bw (KB  /s): min=    0, max=68096, per=11.64%, avg=24277.44, stdev=5716.13
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=3.62%
    lat (usec) : 500=7.46%, 750=3.26%, 1000=3.31%
    lat (msec) : 2=13.19%, 4=30.38%, 10=28.75%, 20=9.46%, 50=0.57%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=0.33%, sys=8.80%, ctx=998532510, majf=0, minf=126449
  IO depths    : 1=0.1%, 2=0.1%, 4=104.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=47660727/w=23455830/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6

Run status group 0 (all jobs):
   READ: io=4357.2GB, aggrb=634548KB/s, minb=634548KB/s, maxb=634548KB/s, mint=7200007msec, maxt=7200007msec
  WRITE: io=2146.7GB, aggrb=312542KB/s, minb=312542KB/s, maxb=312542KB/s, mint=7200007msec, maxt=7200007msec

Run status group 1 (all jobs):
   READ: io=4393.8GB, aggrb=639881KB/s, minb=639881KB/s, maxb=639881KB/s, mint=7200012msec, maxt=7200012msec
  WRITE: io=2164.7GB, aggrb=315163KB/s, minb=315163KB/s, maxb=315163KB/s, mint=7200012msec, maxt=7200012msec

Run status group 2 (all jobs):
   READ: io=2800.7GB, aggrb=407779KB/s, minb=407779KB/s, maxb=407779KB/s, mint=7200181msec, maxt=7200181msec
  WRITE: io=1377.1GB, aggrb=200665KB/s, minb=200665KB/s, maxb=200665KB/s, mint=7200181msec, maxt=7200181msec

Run status group 3 (all jobs):
   READ: io=2908.2GB, aggrb=423644KB/s, minb=423644KB/s, maxb=423644KB/s, mint=7200110msec, maxt=7200110msec
  WRITE: io=1431.7GB, aggrb=208493KB/s, minb=208493KB/s, maxb=208493KB/s, mint=7200110msec, maxt=7200110msec

Disk stats (read/write):
  sda: ios=74164224/36525935, merge=0/0, ticks=377422044/71260105, in_queue=448785887, util=100.00%
  sdb: ios=74781944/36829623, merge=0/0, ticks=376993034/71667695, in_queue=448747969, util=100.00%
    md125: ios=65705801/32331731, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=204877978/188460738, aggrmerge=0/0, aggrticks=319598366/162132177, aggrin_queue=482343718, aggrutil=98.47%
  nvme0n1: ios=204842561/188425745, merge=0/0, ticks=324616577/165246765, in_queue=490477842, util=98.47%
  nvme1n1: ios=204837944/188417304, merge=0/0, ticks=300225607/144356715, in_queue=445016439, util=98.17%
  nvme2n1: ios=204930003/188514213, merge=0/0, ticks=334120381/176374003, in_queue=511089326, util=98.44%
  nvme3n1: ios=204901404/188485692, merge=0/0, ticks=319430901/162551226, in_queue=482791267, util=98.33%
    md124: ios=72315528/35589505, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=213968013/195897826, aggrmerge=0/0, aggrticks=305095551/152599724, aggrin_queue=458047901, aggrutil=100.00%
  nvme4n1: ios=213947327/195873573, merge=0/0, ticks=164269685/31868083, in_queue=196258578, util=94.16%
  nvme5n1: ios=213994332/195928417, merge=0/0, ticks=169128510/35186489, in_queue=204492395, util=93.91%
  nvme6n1: ios=213985970/195910261, merge=0/0, ticks=173029705/37771559, in_queue=210899656, util=94.05%
  nvme7n1: ios=213944425/195879054, merge=0/0, ticks=713954305/505572768, in_queue=1220540977, util=100.00%


###################################################################################
# 
# Mon 21 Dec 2015 10:02:24 PM EST
# 1mB write with SATA 1 jobs and 1 I/Os Outstanding
# 1mB write with NVMe 1 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 1776 IOPS | 1776 MBps | 2811 μsec Average Latency 
SATA-R5-1: 1781 IOPS | 1781 MBps | 2804 μsec Average Latency 
NVMe-R5-0: 897 IOPS | 919 MBps | 22280 μsec Average Latency 
NVMe-R5-1: 992 IOPS | 992 MBps | 20140 μsec Average Latency 

Totals
IOPS: 5446
MBPS: 5468
Average Latency (μsec): 12008

SATA-R5-01: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
SATA-R5-02: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
SATA-R5-03: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
SATA-R5-04: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
SATA-R5-05: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
SATA-R5-11: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
SATA-R5-12: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
SATA-R5-13: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
SATA-R5-14: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
SATA-R5-15: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=1
NVMe-R5-01: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-02: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-03: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-04: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-05: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-11: (g=3): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-12: (g=3): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-13: (g=3): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-14: (g=3): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-15: (g=3): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=17427: Tue Dec 22 00:07:26 2015
  write: io=12492GB, bw=1776.7MB/s, iops=1776, runt=7200001msec
    slat (usec): min=36, max=2914, avg=106.39, stdev=37.83
    clat (usec): min=5, max=29076, avg=2705.13, stdev=1618.81
     lat (usec): min=471, max=29172, avg=2811.71, stdev=1625.44
    clat percentiles (usec):
     |  1.00th=[ 1080],  5.00th=[ 1336], 10.00th=[ 1448], 20.00th=[ 1592],
     | 30.00th=[ 1704], 40.00th=[ 1816], 50.00th=[ 1960], 60.00th=[ 2256],
     | 70.00th=[ 2928], 80.00th=[ 3920], 90.00th=[ 5216], 95.00th=[ 6112],
     | 99.00th=[ 7584], 99.50th=[ 8640], 99.90th=[12608], 99.95th=[13760],
     | 99.99th=[17536]
    bw (KB  /s): min=    3, max=600064, per=20.03%, avg=364330.50, stdev=139973.90
    lat (usec) : 10=0.01%, 20=0.01%, 250=0.01%, 500=0.01%, 750=0.03%
    lat (usec) : 1000=0.50%
    lat (msec) : 2=51.17%, 4=29.18%, 10=18.86%, 20=0.26%, 50=0.01%
  cpu          : usr=1.60%, sys=2.81%, ctx=13203243, majf=0, minf=557735
  IO depths    : 1=103.2%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=12791768/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=17432: Tue Dec 22 00:07:26 2015
  write: io=12525GB, bw=1781.4MB/s, iops=1781, runt=7200002msec
    slat (usec): min=36, max=2018, avg=107.26, stdev=35.98
    clat (usec): min=359, max=26223, avg=2696.89, stdev=1597.21
     lat (usec): min=438, max=26352, avg=2804.30, stdev=1602.92
    clat percentiles (usec):
     |  1.00th=[ 1096],  5.00th=[ 1336], 10.00th=[ 1448], 20.00th=[ 1592],
     | 30.00th=[ 1704], 40.00th=[ 1816], 50.00th=[ 1960], 60.00th=[ 2256],
     | 70.00th=[ 2896], 80.00th=[ 3888], 90.00th=[ 5216], 95.00th=[ 6048],
     | 99.00th=[ 7456], 99.50th=[ 8384], 99.90th=[12224], 99.95th=[13632],
     | 99.99th=[17536]
    bw (KB  /s): min=    3, max=600910, per=20.03%, avg=365282.19, stdev=139843.31
    lat (usec) : 500=0.01%, 750=0.02%, 1000=0.45%
    lat (msec) : 2=51.11%, 4=29.43%, 10=18.76%, 20=0.23%, 50=0.01%
  cpu          : usr=1.58%, sys=2.87%, ctx=13212454, majf=0, minf=556408
  IO depths    : 1=103.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=12825338/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-01: (groupid=2, jobs=5): err= 0: pid=17437: Tue Dec 22 00:07:26 2015
  write: io=6312.9GB, bw=919371KB/s, iops=897, runt=7200009msec
    slat (usec): min=305, max=29847, avg=5663.61, stdev=2163.04
    clat (msec): min=1, max=42768, avg=16.72, stdev=55.36
     lat (msec): min=2, max=42779, avg=22.28, stdev=55.53
    clat percentiles (usec):
     |  1.00th=[ 9024],  5.00th=[10304], 10.00th=[11328], 20.00th=[12736],
     | 30.00th=[13888], 40.00th=[14912], 50.00th=[15936], 60.00th=[17024],
     | 70.00th=[18048], 80.00th=[19584], 90.00th=[21888], 95.00th=[24960],
     | 99.00th=[34560], 99.50th=[37120], 99.90th=[41216], 99.95th=[42752],
     | 99.99th=[46848]
    bw (KB  /s): min=    3, max=327680, per=20.06%, avg=184384.75, stdev=39167.76
    lat (msec) : 2=0.01%, 4=0.01%, 10=3.57%, 20=79.16%, 50=17.26%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.14%, sys=16.91%, ctx=65902081, majf=0, minf=783509
  IO depths    : 1=0.1%, 2=0.1%, 4=102.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=6464317/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=5): err= 0: pid=17442: Tue Dec 22 00:07:26 2015
  write: io=6980.2GB, bw=992.76MB/s, iops=992, runt=7200013msec
    slat (usec): min=211, max=34546, avg=5127.95, stdev=2131.69
    clat (usec): min=702, max=11524K, avg=15120.73, stdev=20757.24
     lat (msec): min=2, max=11539, avg=20.14, stdev=21.19
    clat percentiles (msec):
     |  1.00th=[    9],  5.00th=[   10], 10.00th=[   10], 20.00th=[   12],
     | 30.00th=[   13], 40.00th=[   14], 50.00th=[   15], 60.00th=[   16],
     | 70.00th=[   17], 80.00th=[   19], 90.00th=[   21], 95.00th=[   24],
     | 99.00th=[   34], 99.50th=[   36], 99.90th=[   42], 99.95th=[   44],
     | 99.99th=[  180]
    bw (KB  /s): min=    3, max=350208, per=20.04%, avg=203756.52, stdev=45708.91
    lat (usec) : 750=0.01%
    lat (msec) : 2=0.01%, 4=0.01%, 10=10.42%, 20=78.52%, 50=11.04%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.28%, sys=14.13%, ctx=81451861, majf=0, minf=481813
  IO depths    : 1=0.1%, 2=0.1%, 4=101.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=7147682/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: io=12492GB, aggrb=1776.7MB/s, minb=1776.7MB/s, maxb=1776.7MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=12525GB, aggrb=1781.4MB/s, minb=1781.4MB/s, maxb=1781.4MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
  WRITE: io=6312.9GB, aggrb=919370KB/s, minb=919370KB/s, maxb=919370KB/s, mint=7200009msec, maxt=7200009msec

Run status group 3 (all jobs):
  WRITE: io=6980.2GB, aggrb=992.76MB/s, minb=992.76MB/s, maxb=992.76MB/s, mint=7200013msec, maxt=7200013msec

Disk stats (read/write):
  sda: ios=0/52801177, merge=0/0, ticks=0/131787345, in_queue=131757056, util=100.00%
  sdb: ios=0/52839302, merge=0/0, ticks=0/131952666, in_queue=131921127, util=100.00%
    md125: ios=0/107910235, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=34232473/563121069, aggrmerge=0/0, aggrticks=68879182/506661627, aggrin_queue=575516054, aggrutil=95.41%
  nvme0n1: ios=21987079/563260155, merge=0/0, ticks=46440384/566423103, in_queue=612831879, util=86.49%
  nvme1n1: ios=64391572/562898538, merge=0/0, ticks=130113828/661418366, in_queue=791526317, util=95.41%
  nvme2n1: ios=32048022/563099024, merge=0/0, ticks=62451152/347716203, in_queue=410139041, util=78.11%
  nvme3n1: ios=18503220/563226562, merge=0/0, ticks=36511367/451088836, in_queue=487566981, util=79.53%
    md124: ios=0/119721912, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=39310655/623112282, aggrmerge=0/0, aggrticks=68353848/513877908, aggrin_queue=582186232, aggrutil=94.64%
  nvme4n1: ios=25763466/624281505, merge=0/0, ticks=38625532/433468799, in_queue=472029365, util=83.59%
  nvme5n1: ios=73308020/622590256, merge=0/0, ticks=117902505/538198097, in_queue=656076145, util=94.42%
  nvme6n1: ios=37135167/622928589, merge=0/0, ticks=65303712/649902362, in_queue=715144612, util=94.64%
  nvme7n1: ios=21035970/622648781, merge=0/0, ticks=51583644/433942377, in_queue=485494809, util=80.03%


###################################################################################
# 
# Tue 22 Dec 2015 12:08:26 AM EST
# 1mB read with SATA 1 jobs and 4 I/Os Outstanding
# 1mB read with NVMe 1 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 3622 IOPS | 3622 MBps | 5520 μsec Average Latency 
SATA-R5-1: 3640 IOPS | 3640 MBps | 5492 μsec Average Latency 
NVMe-R5-0: 6056 IOPS | 6056 MBps | 3301 μsec Average Latency 
NVMe-R5-1: 6075 IOPS | 6075 MBps | 3290 μsec Average Latency 

Totals
IOPS: 19393
MBPS: 19393
Average Latency (μsec): 4400

SATA-R5-01: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
SATA-R5-02: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
SATA-R5-03: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
SATA-R5-04: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
SATA-R5-05: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
SATA-R5-11: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
SATA-R5-12: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
SATA-R5-13: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
SATA-R5-14: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
SATA-R5-15: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-01: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-02: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-03: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-04: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-05: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-11: (g=3): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-12: (g=3): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-13: (g=3): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-14: (g=3): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
NVMe-R5-15: (g=3): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=23190: Tue Dec 22 02:13:27 2015
  read : io=25468GB, bw=3622.2MB/s, iops=3622, runt=7200005msec
    slat (usec): min=40, max=2088, avg=117.04, stdev=44.45
    clat (usec): min=325, max=96100, avg=5402.71, stdev=744.67
     lat (usec): min=423, max=96306, avg=5520.04, stdev=744.29
    clat percentiles (usec):
     |  1.00th=[ 4320],  5.00th=[ 4640], 10.00th=[ 4768], 20.00th=[ 4896],
     | 30.00th=[ 5024], 40.00th=[ 5152], 50.00th=[ 5216], 60.00th=[ 5408],
     | 70.00th=[ 5536], 80.00th=[ 5792], 90.00th=[ 6176], 95.00th=[ 6688],
     | 99.00th=[ 8256], 99.50th=[ 8896], 99.90th=[10304], 99.95th=[10944],
     | 99.99th=[14912]
    bw (KB  /s): min=    3, max=909768, per=20.02%, avg=742406.41, stdev=14488.80
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.14%, 10=99.71%, 20=0.15%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.23%, sys=9.26%, ctx=25860205, majf=0, minf=497815
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=26079228/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=23195: Tue Dec 22 02:13:27 2015
  read : io=25597GB, bw=3640.5MB/s, iops=3640, runt=7200005msec
    slat (usec): min=39, max=1550, avg=117.72, stdev=44.85
    clat (usec): min=332, max=25921, avg=5374.37, stdev=805.01
     lat (usec): min=468, max=26122, avg=5492.22, stdev=804.49
    clat percentiles (usec):
     |  1.00th=[ 4320],  5.00th=[ 4576], 10.00th=[ 4704], 20.00th=[ 4832],
     | 30.00th=[ 4960], 40.00th=[ 5024], 50.00th=[ 5152], 60.00th=[ 5280],
     | 70.00th=[ 5536], 80.00th=[ 5792], 90.00th=[ 6240], 95.00th=[ 6816],
     | 99.00th=[ 8512], 99.50th=[ 9152], 99.90th=[10560], 99.95th=[11200],
     | 99.99th=[14400]
    bw (KB  /s): min=    3, max=917716, per=20.02%, avg=746164.52, stdev=13761.26
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.16%, 10=99.64%, 20=0.20%, 50=0.01%
  cpu          : usr=0.23%, sys=9.38%, ctx=26030091, majf=0, minf=475555
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=26211247/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=5): err= 0: pid=23200: Tue Dec 22 02:13:27 2015
  read : io=42582GB, bw=6056.2MB/s, iops=6056, runt=7200003msec
    slat (usec): min=55, max=1234, avg=144.72, stdev=51.90
    clat (usec): min=781, max=14993, avg=3156.35, stdev=598.19
     lat (usec): min=993, max=15090, avg=3301.02, stdev=598.83
    clat percentiles (usec):
     |  1.00th=[ 2096],  5.00th=[ 2320], 10.00th=[ 2480], 20.00th=[ 2640],
     | 30.00th=[ 2800], 40.00th=[ 2928], 50.00th=[ 3088], 60.00th=[ 3216],
     | 70.00th=[ 3408], 80.00th=[ 3632], 90.00th=[ 3952], 95.00th=[ 4256],
     | 99.00th=[ 4896], 99.50th=[ 5088], 99.90th=[ 5600], 99.95th=[ 5856],
     | 99.99th=[ 6368]
    bw (MB  /s): min=    0, max= 1264, per=20.01%, avg=1212.08, stdev=16.60
    lat (usec) : 1000=0.01%
    lat (msec) : 2=0.52%, 4=90.29%, 10=9.19%, 20=0.01%
  cpu          : usr=0.35%, sys=18.85%, ctx=40427691, majf=0, minf=346899
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=43603953/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=5): err= 0: pid=23205: Tue Dec 22 02:13:27 2015
  read : io=42722GB, bw=6075.1MB/s, iops=6075, runt=7200003msec
    slat (usec): min=56, max=1701, avg=147.90, stdev=52.16
    clat (usec): min=880, max=14281, avg=3142.11, stdev=595.58
     lat (usec): min=1027, max=14450, avg=3290.21, stdev=596.19
    clat percentiles (usec):
     |  1.00th=[ 2064],  5.00th=[ 2320], 10.00th=[ 2448], 20.00th=[ 2640],
     | 30.00th=[ 2800], 40.00th=[ 2928], 50.00th=[ 3056], 60.00th=[ 3216],
     | 70.00th=[ 3376], 80.00th=[ 3600], 90.00th=[ 3952], 95.00th=[ 4256],
     | 99.00th=[ 4832], 99.50th=[ 5088], 99.90th=[ 5600], 99.95th=[ 5792],
     | 99.99th=[ 6240]
    bw (MB  /s): min=    0, max= 1262, per=20.01%, avg=1216.06, stdev=15.37
    lat (usec) : 1000=0.01%
    lat (msec) : 2=0.59%, 4=90.58%, 10=8.83%, 20=0.01%
  cpu          : usr=0.36%, sys=19.27%, ctx=40388171, majf=0, minf=376383
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=43746858/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=25468GB, aggrb=3622.2MB/s, minb=3622.2MB/s, maxb=3622.2MB/s, mint=7200005msec, maxt=7200005msec

Run status group 1 (all jobs):
   READ: io=25597GB, aggrb=3640.5MB/s, minb=3640.5MB/s, maxb=3640.5MB/s, mint=7200005msec, maxt=7200005msec

Run status group 2 (all jobs):
   READ: io=42582GB, aggrb=6056.2MB/s, minb=6056.2MB/s, maxb=6056.2MB/s, mint=7200003msec, maxt=7200003msec

Run status group 3 (all jobs):
   READ: io=42722GB, aggrb=6075.1MB/s, minb=6075.1MB/s, maxb=6075.1MB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sda: ios=108672440/10, merge=0/0, ticks=517665683/0, in_queue=517772178, util=100.00%
  sdb: ios=109225694/10, merge=0/0, ticks=503900465/0, in_queue=504020310, util=100.00%
    md125: ios=743753988/10, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=185943120/42, aggrmerge=0/0, aggrticks=258263407/11, aggrin_queue=258438709, aggrutil=100.00%
  nvme0n1: ios=185892981/49, merge=0/0, ticks=258678306/2, in_queue=258857156, util=100.00%
  nvme1n1: ios=186500804/53, merge=0/0, ticks=261025559/44, in_queue=261163782, util=100.00%
  nvme2n1: ios=185777811/18, merge=0/0, ticks=256998938/0, in_queue=257151530, util=100.00%
  nvme3n1: ios=185600887/50, merge=0/0, ticks=256350825/0, in_queue=256582368, util=100.00%
    md124: ios=749752070/10, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=187442719/42, aggrmerge=0/0, aggrticks=259518445/27, aggrin_queue=259660744, aggrutil=100.00%
  nvme4n1: ios=187875773/33, merge=0/0, ticks=256999168/30, in_queue=257158056, util=100.00%
  nvme5n1: ios=187117779/53, merge=0/0, ticks=258086914/37, in_queue=258282917, util=100.00%
  nvme6n1: ios=188207528/18, merge=0/0, ticks=260723392/11, in_queue=260831806, util=100.00%
  nvme7n1: ios=186569799/66, merge=0/0, ticks=262264309/33, in_queue=262370199, util=100.00%


###################################################################################
# 
# Tue 22 Dec 2015 02:14:27 AM EST
# 256kB write with SATA 1 jobs and 1 I/Os Outstanding
# 256kB write with NVMe 1 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 9954 IOPS | 2488 MBps | 500 μsec Average Latency 
SATA-R5-1: 11100 IOPS | 2775 MBps | 448 μsec Average Latency 
NVMe-R5-0: 4662 IOPS | 1165 MBps | 4288 μsec Average Latency 
NVMe-R5-1: 5179 IOPS | 1294 MBps | 3860 μsec Average Latency 

Totals
IOPS: 30895
MBPS: 7722
Average Latency (μsec): 2274

SATA-R5-01: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
SATA-R5-02: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
SATA-R5-03: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
SATA-R5-04: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
SATA-R5-05: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
SATA-R5-11: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
SATA-R5-12: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
SATA-R5-13: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
SATA-R5-14: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
SATA-R5-15: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=1
NVMe-R5-01: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-02: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-03: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-04: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-05: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-11: (g=3): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-12: (g=3): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-13: (g=3): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-14: (g=3): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
NVMe-R5-15: (g=3): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=28393: Tue Dec 22 04:19:28 2015
  write: io=17499GB, bw=2488.8MB/s, iops=9954, runt=7200001msec
    slat (usec): min=10, max=1110, avg=27.34, stdev=13.10
    clat (usec): min=0, max=47226, avg=472.81, stdev=236.21
     lat (usec): min=99, max=47254, avg=500.40, stdev=236.93
    clat percentiles (usec):
     |  1.00th=[  153],  5.00th=[  249], 10.00th=[  302], 20.00th=[  354],
     | 30.00th=[  394], 40.00th=[  430], 50.00th=[  462], 60.00th=[  490],
     | 70.00th=[  524], 80.00th=[  572], 90.00th=[  636], 95.00th=[  700],
     | 99.00th=[  956], 99.50th=[ 1272], 99.90th=[ 2384], 99.95th=[ 3664],
     | 99.99th=[10048]
    bw (KB  /s): min=    0, max=646656, per=20.01%, avg=509933.93, stdev=42154.05
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=5.03%, 500=57.51%, 750=34.70%, 1000=1.85%
    lat (msec) : 2=0.75%, 4=0.11%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=2.07%, sys=6.21%, ctx=74654808, majf=0, minf=123697
  IO depths    : 1=104.1%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=71675373/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=28398: Tue Dec 22 04:19:28 2015
  write: io=19513GB, bw=2775.2MB/s, iops=11100, runt=7200001msec
    slat (usec): min=10, max=1204, avg=27.06, stdev=12.59
    clat (usec): min=0, max=8919, avg=421.31, stdev=126.07
     lat (usec): min=94, max=8948, avg=448.60, stdev=127.37
    clat percentiles (usec):
     |  1.00th=[  133],  5.00th=[  215], 10.00th=[  282], 20.00th=[  322],
     | 30.00th=[  346], 40.00th=[  374], 50.00th=[  410], 60.00th=[  450],
     | 70.00th=[  490], 80.00th=[  532], 90.00th=[  596], 95.00th=[  636],
     | 99.00th=[  700], 99.50th=[  724], 99.90th=[  764], 99.95th=[  788],
     | 99.99th=[  844]
    bw (KB  /s): min=    0, max=646144, per=20.01%, avg=568594.09, stdev=16407.66
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=6.49%, 500=65.93%, 750=27.40%, 1000=0.16%
    lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%
  cpu          : usr=2.27%, sys=6.77%, ctx=83267199, majf=0, minf=113426
  IO depths    : 1=104.2%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=79925110/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
NVMe-R5-01: (groupid=2, jobs=5): err= 0: pid=28403: Tue Dec 22 04:19:28 2015
  write: io=8195.2GB, bw=1165.6MB/s, iops=4662, runt=7200004msec
    slat (usec): min=30, max=9084, avg=905.24, stdev=815.40
    clat (usec): min=170, max=19197K, avg=3382.85, stdev=9565.36
     lat (usec): min=282, max=19198K, avg=4288.72, stdev=9595.40
    clat percentiles (usec):
     |  1.00th=[ 1256],  5.00th=[ 1592], 10.00th=[ 1816], 20.00th=[ 2192],
     | 30.00th=[ 2544], 40.00th=[ 2864], 50.00th=[ 3184], 60.00th=[ 3536],
     | 70.00th=[ 3952], 80.00th=[ 4384], 90.00th=[ 5088], 95.00th=[ 5728],
     | 99.00th=[ 7456], 99.50th=[ 8256], 99.90th=[ 9920], 99.95th=[10688],
     | 99.99th=[12224]
    bw (KB  /s): min=    0, max=416256, per=20.02%, avg=238913.02, stdev=33607.77
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.02%, 1000=0.15%
    lat (msec) : 2=14.90%, 4=56.19%, 10=28.64%, 20=0.09%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.06%, sys=17.55%, ctx=74804168, majf=0, minf=331904
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=33567462/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=5): err= 0: pid=28408: Tue Dec 22 04:19:28 2015
  write: io=9104.1GB, bw=1294.1MB/s, iops=5179, runt=7200004msec
    slat (usec): min=30, max=10599, avg=793.56, stdev=736.21
    clat (usec): min=91, max=15968K, avg=3067.08, stdev=6101.88
     lat (usec): min=294, max=15968K, avg=3860.13, stdev=6141.90
    clat percentiles (usec):
     |  1.00th=[ 1128],  5.00th=[ 1416], 10.00th=[ 1592], 20.00th=[ 1960],
     | 30.00th=[ 2320], 40.00th=[ 2608], 50.00th=[ 2864], 60.00th=[ 3184],
     | 70.00th=[ 3568], 80.00th=[ 3984], 90.00th=[ 4640], 95.00th=[ 5280],
     | 99.00th=[ 7008], 99.50th=[ 7840], 99.90th=[ 9536], 99.95th=[10304],
     | 99.99th=[11712]
    bw (KB  /s): min=    0, max=455281, per=20.02%, avg=265402.73, stdev=36518.00
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.05%, 1000=0.33%
    lat (msec) : 2=20.72%, 4=59.01%, 10=19.82%, 20=0.06%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.01%, sys=13.62%, ctx=96923371, majf=0, minf=227400
  IO depths    : 1=0.1%, 2=0.1%, 4=104.1%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=37293700/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: io=17499GB, aggrb=2488.8MB/s, minb=2488.8MB/s, maxb=2488.8MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=19513GB, aggrb=2775.2MB/s, minb=2775.2MB/s, maxb=2775.2MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=8195.2GB, aggrb=1165.6MB/s, minb=1165.6MB/s, maxb=1165.6MB/s, mint=7200004msec, maxt=7200004msec

Run status group 3 (all jobs):
  WRITE: io=9104.1GB, aggrb=1294.1MB/s, minb=1294.1MB/s, maxb=1294.1MB/s, mint=7200004msec, maxt=7200004msec

Disk stats (read/write):
  sda: ios=0/74651563, merge=0/0, ticks=0/35434101, in_queue=35367732, util=100.00%
  sdb: ios=0/83262661, merge=0/0, ticks=0/35274881, in_queue=35204722, util=100.00%
    md125: ios=0/152927148, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=54540438/748005095, aggrmerge=0/0, aggrticks=86267761/523610324, aggrin_queue=609833899, aggrutil=98.88%
  nvme0n1: ios=50931045/748996641, merge=0/0, ticks=83648407/580705575, in_queue=664313630, util=95.25%
  nvme1n1: ios=90002839/747351543, merge=0/0, ticks=154742704/692694516, in_queue=847415141, util=98.88%
  nvme2n1: ios=43702629/748065856, merge=0/0, ticks=57692711/360174156, in_queue=417807165, util=86.72%
  nvme3n1: ios=33525239/747606343, merge=0/0, ticks=48987222/460867052, in_queue=509799662, util=88.52%
    md124: ios=0/172934239, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=64377658/832385298, aggrmerge=0/0, aggrticks=93237322/533912666, aggrin_queue=627108124, aggrutil=98.27%
  nvme4n1: ios=61464166/833921710, merge=0/0, ticks=83479035/473879267, in_queue=557307137, util=93.53%
  nvme5n1: ios=107006690/830886015, merge=0/0, ticks=155265035/575169577, in_queue=730433525, util=98.16%
  nvme6n1: ios=51532332/831929968, merge=0/0, ticks=81770849/699020831, in_queue=780744097, util=98.27%
  nvme7n1: ios=37507444/832803502, merge=0/0, ticks=52434371/387580990, in_queue=439947739, util=88.21%


###################################################################################
# 
# Tue 22 Dec 2015 04:20:29 AM EST
# 256kB read with SATA 1 jobs and 8 I/Os Outstanding
# 256kB read with NVMe 1 jobs and 6 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 13883 IOPS | 3470 MBps | 2879 μsec Average Latency 
SATA-R5-1: 13783 IOPS | 3445 MBps | 2900 μsec Average Latency 
NVMe-R5-0: 23908 IOPS | 5977 MBps | 1253 μsec Average Latency 
NVMe-R5-1: 23987 IOPS | 5996 MBps | 1249 μsec Average Latency 

Totals
IOPS: 75561
MBPS: 18888
Average Latency (μsec): 2070

SATA-R5-01: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-02: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-03: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-04: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-05: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-11: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-12: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-13: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-14: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
SATA-R5-15: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=8
NVMe-R5-01: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
NVMe-R5-02: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
NVMe-R5-03: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
NVMe-R5-04: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
NVMe-R5-05: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
NVMe-R5-11: (g=3): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
NVMe-R5-12: (g=3): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
NVMe-R5-13: (g=3): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
NVMe-R5-14: (g=3): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
NVMe-R5-15: (g=3): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=6
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=33602: Tue Dec 22 06:25:30 2015
  read : io=24405GB, bw=3470.1MB/s, iops=13883, runt=7200004msec
    slat (usec): min=10, max=880, avg=23.90, stdev= 6.54
    clat (usec): min=141, max=40603, avg=2855.67, stdev=599.22
     lat (usec): min=164, max=40630, avg=2879.73, stdev=599.22
    clat percentiles (usec):
     |  1.00th=[ 1720],  5.00th=[ 2096], 10.00th=[ 2256], 20.00th=[ 2448],
     | 30.00th=[ 2576], 40.00th=[ 2672], 50.00th=[ 2800], 60.00th=[ 2896],
     | 70.00th=[ 3024], 80.00th=[ 3184], 90.00th=[ 3472], 95.00th=[ 3792],
     | 99.00th=[ 4960], 99.50th=[ 5536], 99.90th=[ 6816], 99.95th=[ 7392],
     | 99.99th=[ 9280]
    bw (KB  /s): min=    0, max=1687040, per=20.01%, avg=711207.52, stdev=16537.01
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.06%
    lat (msec) : 2=2.95%, 4=93.59%, 10=3.38%, 20=0.01%, 50=0.01%
  cpu          : usr=0.80%, sys=8.32%, ctx=96743954, majf=0, minf=392026
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.1%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=99963466/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=33607: Tue Dec 22 06:25:30 2015
  read : io=24229GB, bw=3445.1MB/s, iops=13783, runt=7200004msec
    slat (usec): min=10, max=1171, avg=24.85, stdev= 6.56
    clat (usec): min=96, max=22778, avg=2875.53, stdev=799.35
     lat (usec): min=121, max=22804, avg=2900.59, stdev=799.38
    clat percentiles (usec):
     |  1.00th=[ 1208],  5.00th=[ 1816], 10.00th=[ 2096], 20.00th=[ 2352],
     | 30.00th=[ 2512], 40.00th=[ 2672], 50.00th=[ 2800], 60.00th=[ 2928],
     | 70.00th=[ 3088], 80.00th=[ 3280], 90.00th=[ 3664], 95.00th=[ 4192],
     | 99.00th=[ 5856], 99.50th=[ 6496], 99.90th=[ 7904], 99.95th=[ 8512],
     | 99.99th=[10176]
    bw (KB  /s): min=    0, max=845824, per=20.01%, avg=706087.27, stdev=27636.55
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.11%, 1000=0.34%
    lat (msec) : 2=7.60%, 4=85.67%, 10=6.26%, 20=0.01%, 50=0.01%
  cpu          : usr=0.82%, sys=8.64%, ctx=96038650, majf=0, minf=380034
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.2%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=99243690/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=5): err= 0: pid=33612: Tue Dec 22 06:25:30 2015
  read : io=42027GB, bw=5977.2MB/s, iops=23908, runt=7200002msec
    slat (usec): min=13, max=1003, avg=30.83, stdev= 8.45
    clat (usec): min=74, max=12118, avg=1222.70, stdev=514.16
     lat (usec): min=225, max=12147, avg=1253.69, stdev=514.21
    clat percentiles (usec):
     |  1.00th=[  454],  5.00th=[  572], 10.00th=[  660], 20.00th=[  796],
     | 30.00th=[  900], 40.00th=[ 1012], 50.00th=[ 1128], 60.00th=[ 1256],
     | 70.00th=[ 1400], 80.00th=[ 1608], 90.00th=[ 1912], 95.00th=[ 2224],
     | 99.00th=[ 2832], 99.50th=[ 3056], 99.90th=[ 3568], 99.95th=[ 3792],
     | 99.99th=[ 4320]
    bw (MB  /s): min=    0, max= 1306, per=20.01%, avg=1195.77, stdev=25.04
    lat (usec) : 100=0.01%, 250=0.01%, 500=2.16%, 750=14.35%, 1000=22.48%
    lat (msec) : 2=52.71%, 4=8.28%, 10=0.03%, 20=0.01%
  cpu          : usr=1.10%, sys=17.06%, ctx=152386581, majf=0, minf=263233
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=172142880/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
NVMe-R5-11: (groupid=3, jobs=5): err= 0: pid=33617: Tue Dec 22 06:25:30 2015
  read : io=42165GB, bw=5996.9MB/s, iops=23987, runt=7200002msec
    slat (usec): min=14, max=1018, avg=32.08, stdev= 8.36
    clat (usec): min=154, max=12770, avg=1217.30, stdev=507.30
     lat (usec): min=229, max=12817, avg=1249.52, stdev=507.36
    clat percentiles (usec):
     |  1.00th=[  454],  5.00th=[  572], 10.00th=[  660], 20.00th=[  788],
     | 30.00th=[  900], 40.00th=[ 1012], 50.00th=[ 1128], 60.00th=[ 1240],
     | 70.00th=[ 1400], 80.00th=[ 1592], 90.00th=[ 1912], 95.00th=[ 2192],
     | 99.00th=[ 2800], 99.50th=[ 3024], 99.90th=[ 3536], 99.95th=[ 3728],
     | 99.99th=[ 4192]
    bw (MB  /s): min=    0, max= 1275, per=20.01%, avg=1199.68, stdev=23.96
    lat (usec) : 250=0.01%, 500=2.11%, 750=14.38%, 1000=22.73%
    lat (msec) : 2=52.77%, 4=7.99%, 10=0.02%, 20=0.01%
  cpu          : usr=1.15%, sys=17.77%, ctx=151793806, majf=0, minf=259687
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=172708286/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6

Run status group 0 (all jobs):
   READ: io=24405GB, aggrb=3470.1MB/s, minb=3470.1MB/s, maxb=3470.1MB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
   READ: io=24229GB, aggrb=3445.1MB/s, minb=3445.1MB/s, maxb=3445.1MB/s, mint=7200004msec, maxt=7200004msec

Run status group 2 (all jobs):
   READ: io=42027GB, aggrb=5977.2MB/s, minb=5977.2MB/s, maxb=5977.2MB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=42165GB, aggrb=5996.9MB/s, minb=5996.9MB/s, maxb=5996.9MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=104101667/10, merge=0/0, ticks=297507938/0, in_queue=297517371, util=100.00%
  sdb: ios=103382962/10, merge=0/0, ticks=297464570/0, in_queue=297450997, util=100.00%
    md125: ios=784153300/10, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=196041980/42, aggrmerge=0/0, aggrticks=136200317/2, aggrin_queue=136292206, aggrutil=100.00%
  nvme0n1: ios=195853793/33, merge=0/0, ticks=136968612/0, in_queue=137063556, util=100.00%
  nvme1n1: ios=198241890/69, merge=0/0, ticks=137500936/7, in_queue=137582077, util=100.00%
  nvme2n1: ios=195399354/18, merge=0/0, ticks=135678848/0, in_queue=135773986, util=100.00%
  nvme3n1: ios=194672886/50, merge=0/0, ticks=134652874/4, in_queue=134749205, util=100.00%
    md124: ios=801058251/10, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=200267996/42, aggrmerge=0/0, aggrticks=137454283/1, aggrin_queue=137563214, aggrutil=100.00%
  nvme4n1: ios=201967877/33, merge=0/0, ticks=136829753/1, in_queue=136911928, util=100.00%
  nvme5n1: ios=198975899/69, merge=0/0, ticks=137313273/1, in_queue=137475775, util=100.00%
  nvme6n1: ios=203285230/34, merge=0/0, ticks=139144991/0, in_queue=139244898, util=100.00%
  nvme7n1: ios=196842981/34, merge=0/0, ticks=136529118/3, in_queue=136620255, util=100.00%


###################################################################################
# 
# Tue 22 Dec 2015 06:26:30 AM EST
# 64kB write with SATA 1 jobs and 4 I/Os Outstanding
# 64kB write with NVMe 1 jobs and 6 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 37901 IOPS | 2368 MBps | 526 μsec Average Latency 
SATA-R5-1: 39716 IOPS | 2482 MBps | 502 μsec Average Latency 
NVMe-R5-0: 17664 IOPS | 1104 MBps | 1697 μsec Average Latency 
NVMe-R5-1: 17872 IOPS | 1117 MBps | 1677 μsec Average Latency 

Totals
IOPS: 113153
MBPS: 7071
Average Latency (μsec): 1100

SATA-R5-01: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-02: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-03: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-04: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-05: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-11: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-12: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-13: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-14: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
SATA-R5-15: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
NVMe-R5-01: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
NVMe-R5-02: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
NVMe-R5-03: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
NVMe-R5-04: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
NVMe-R5-05: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
NVMe-R5-11: (g=3): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
NVMe-R5-12: (g=3): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
NVMe-R5-13: (g=3): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
NVMe-R5-14: (g=3): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
NVMe-R5-15: (g=3): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=39115: Tue Dec 22 08:31:31 2015
  write: io=16656GB, bw=2368.9MB/s, iops=37901, runt=7200001msec
    slat (usec): min=3, max=3691, avg=12.54, stdev= 6.92
    clat (usec): min=21, max=958258, avg=514.00, stdev=612.44
     lat (usec): min=74, max=958293, avg=526.65, stdev=612.38
    clat percentiles (usec):
     |  1.00th=[  251],  5.00th=[  298], 10.00th=[  338], 20.00th=[  390],
     | 30.00th=[  426], 40.00th=[  458], 50.00th=[  490], 60.00th=[  524],
     | 70.00th=[  580], 80.00th=[  644], 90.00th=[  716], 95.00th=[  780],
     | 99.00th=[  916], 99.50th=[  980], 99.90th=[ 1096], 99.95th=[ 1176],
     | 99.99th=[ 1864]
    bw (KB  /s): min=    0, max=582016, per=20.01%, avg=485486.09, stdev=25964.90
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.97%, 500=52.17%, 750=39.88%
    lat (usec) : 1000=6.60%
    lat (msec) : 2=0.37%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 1000=0.01%
  cpu          : usr=2.60%, sys=11.49%, ctx=133531794, majf=0, minf=144671
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=272892546/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=39120: Tue Dec 22 08:31:31 2015
  write: io=17454GB, bw=2482.3MB/s, iops=39716, runt=7200001msec
    slat (usec): min=3, max=6252, avg=11.65, stdev= 6.25
    clat (usec): min=20, max=8730, avg=490.85, stdev=143.10
     lat (usec): min=79, max=8741, avg=502.57, stdev=142.91
    clat percentiles (usec):
     |  1.00th=[  251],  5.00th=[  298], 10.00th=[  326], 20.00th=[  370],
     | 30.00th=[  406], 40.00th=[  434], 50.00th=[  458], 60.00th=[  490],
     | 70.00th=[  556], 80.00th=[  628], 90.00th=[  692], 95.00th=[  756],
     | 99.00th=[  868], 99.50th=[  940], 99.90th=[ 1032], 99.95th=[ 1048],
     | 99.99th=[ 1112]
    bw (KB  /s): min=    0, max=577024, per=20.01%, avg=508501.78, stdev=13833.98
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.94%, 500=61.20%, 750=32.62%
    lat (usec) : 1000=5.06%
    lat (msec) : 2=0.18%, 4=0.01%, 10=0.01%
  cpu          : usr=2.70%, sys=11.27%, ctx=146848830, majf=0, minf=111643
  IO depths    : 1=0.1%, 2=0.1%, 4=104.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=285959368/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=5): err= 0: pid=39125: Tue Dec 22 08:31:31 2015
  write: io=7762.8GB, bw=1104.3MB/s, iops=17664, runt=7200002msec
    slat (usec): min=6, max=1359, avg=29.29, stdev=15.81
    clat (usec): min=92, max=35513K, avg=1667.94, stdev=9731.45
     lat (usec): min=108, max=35513K, avg=1697.42, stdev=9731.71
    clat percentiles (usec):
     |  1.00th=[  540],  5.00th=[  740], 10.00th=[  852], 20.00th=[  996],
     | 30.00th=[ 1112], 40.00th=[ 1240], 50.00th=[ 1384], 60.00th=[ 1560],
     | 70.00th=[ 1816], 80.00th=[ 2224], 90.00th=[ 2928], 95.00th=[ 3440],
     | 99.00th=[ 4832], 99.50th=[ 5600], 99.90th=[ 7456], 99.95th=[ 8160],
     | 99.99th=[ 9920]
    bw (KB  /s): min=    0, max=465792, per=20.03%, avg=226398.39, stdev=64174.70
    lat (usec) : 100=0.01%, 250=0.02%, 500=0.65%, 750=4.55%, 1000=15.40%
    lat (msec) : 2=54.49%, 4=22.55%, 10=2.31%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.34%, sys=11.13%, ctx=34794190, majf=0, minf=117663
  IO depths    : 1=0.1%, 2=0.1%, 4=103.5%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=127183801/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
NVMe-R5-11: (groupid=3, jobs=5): err= 0: pid=39130: Tue Dec 22 08:31:31 2015
  write: io=7854.4GB, bw=1117.6MB/s, iops=17872, runt=7200003msec
    slat (usec): min=5, max=1111, avg=28.03, stdev=15.52
    clat (usec): min=84, max=27465K, avg=1649.69, stdev=8826.28
     lat (usec): min=101, max=27465K, avg=1677.68, stdev=8826.50
    clat percentiles (usec):
     |  1.00th=[  506],  5.00th=[  692], 10.00th=[  788], 20.00th=[  924],
     | 30.00th=[ 1032], 40.00th=[ 1160], 50.00th=[ 1336], 60.00th=[ 1544],
     | 70.00th=[ 1864], 80.00th=[ 2288], 90.00th=[ 2928], 95.00th=[ 3408],
     | 99.00th=[ 5152], 99.50th=[ 6112], 99.90th=[ 8384], 99.95th=[ 9280],
     | 99.99th=[11712]
    bw (KB  /s): min=    0, max=496128, per=20.04%, avg=229256.57, stdev=74789.50
    lat (usec) : 100=0.01%, 250=0.03%, 500=0.91%, 750=6.73%, 1000=19.25%
    lat (msec) : 2=46.64%, 4=23.96%, 10=2.44%, 20=0.03%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.29%, sys=10.70%, ctx=37747755, majf=0, minf=142829
  IO depths    : 1=0.1%, 2=0.1%, 4=103.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=128684998/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6

Run status group 0 (all jobs):
  WRITE: io=16656GB, aggrb=2368.9MB/s, minb=2368.9MB/s, maxb=2368.9MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=17454GB, aggrb=2482.3MB/s, minb=2482.3MB/s, maxb=2482.3MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=7762.8GB, aggrb=1104.3MB/s, minb=1104.3MB/s, maxb=1104.3MB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
  WRITE: io=7854.4GB, aggrb=1117.6MB/s, minb=1117.6MB/s, maxb=1117.6MB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sda: ios=0/284389859, merge=0/0, ticks=0/142447570, in_queue=142435010, util=100.00%
  sdb: ios=0/298142789, merge=0/0, ticks=0/142950375, in_queue=142935363, util=100.00%
    md125: ios=0/180857193, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=59402667/707608376, aggrmerge=0/0, aggrticks=66245088/312701452, aggrin_queue=378897236, aggrutil=98.05%
  nvme0n1: ios=84414087/710785716, merge=0/0, ticks=97501401/351801331, in_queue=449275209, util=97.30%
  nvme1n1: ios=71669466/706131292, merge=0/0, ticks=85977765/406894902, in_queue=492829259, util=98.05%
  nvme2n1: ios=39215968/707230017, merge=0/0, ticks=37145066/215411364, in_queue=252487241, util=84.39%
  nvme3n1: ios=42311147/706286482, merge=0/0, ticks=44356122/276698213, in_queue=320997236, util=89.30%
    md124: ios=0/192756503, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=73059745/720501372, aggrmerge=0/0, aggrticks=74394663/293912928, aggrin_queue=368259330, aggrutil=97.09%
  nvme4n1: ios=102037563/724189795, merge=0/0, ticks=99702062/264278862, in_queue=363942452, util=95.89%
  nvme5n1: ios=88618672/718823240, merge=0/0, ticks=90719716/311023298, in_queue=401708836, util=96.74%
  nvme6n1: ios=50114179/719887517, merge=0/0, ticks=56499690/383405874, in_queue=439850290, util=97.09%
  nvme7n1: ios=51468568/719104936, merge=0/0, ticks=50657187/216943680, in_queue=267535745, util=88.15%


###################################################################################
# 
# Tue 22 Dec 2015 08:32:31 AM EST
# 64kB read with SATA 1 jobs and 16 I/Os Outstanding
# 64kB read with NVMe 1 jobs and 12 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 52699 IOPS | 3293 MBps | 1517 μsec Average Latency 
SATA-R5-1: 53730 IOPS | 3358 MBps | 1488 μsec Average Latency 
NVMe-R5-0: 93879 IOPS | 5867 MBps | 638 μsec Average Latency 
NVMe-R5-1: 94016 IOPS | 5876 MBps | 637 μsec Average Latency 

Totals
IOPS: 294324
MBPS: 18394
Average Latency (μsec): 1070

SATA-R5-01: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
SATA-R5-02: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
SATA-R5-03: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
SATA-R5-04: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
SATA-R5-05: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
SATA-R5-11: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
SATA-R5-12: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
SATA-R5-13: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
SATA-R5-14: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
SATA-R5-15: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
NVMe-R5-01: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
NVMe-R5-02: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
NVMe-R5-03: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
NVMe-R5-04: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
NVMe-R5-05: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
NVMe-R5-11: (g=3): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
NVMe-R5-12: (g=3): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
NVMe-R5-13: (g=3): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
NVMe-R5-14: (g=3): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
NVMe-R5-15: (g=3): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=12
fio-2.1.10
Starting 20 processes

SATA-R5-01: (groupid=0, jobs=5): err= 0: pid=44809: Tue Dec 22 10:37:33 2015
  read : io=23159GB, bw=3293.8MB/s, iops=52699, runt=7200002msec
    slat (usec): min=4, max=1263, avg= 9.16, stdev= 3.14
    clat (usec): min=70, max=38873, avg=1507.87, stdev=565.37
     lat (usec): min=81, max=38882, avg=1517.18, stdev=565.40
    clat percentiles (usec):
     |  1.00th=[  564],  5.00th=[  700], 10.00th=[  788], 20.00th=[  956],
     | 30.00th=[ 1160], 40.00th=[ 1336], 50.00th=[ 1496], 60.00th=[ 1640],
     | 70.00th=[ 1784], 80.00th=[ 1976], 90.00th=[ 2224], 95.00th=[ 2480],
     | 99.00th=[ 2928], 99.50th=[ 3120], 99.90th=[ 3760], 99.95th=[ 4192],
     | 99.99th=[ 5472]
    bw (KB  /s): min=    0, max=1545472, per=20.00%, avg=674624.70, stdev=50458.03
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.35%, 750=7.29%, 1000=14.53%
    lat (msec) : 2=59.09%, 4=18.67%, 10=0.07%, 20=0.01%, 50=0.01%
  cpu          : usr=2.37%, sys=13.11%, ctx=337790555, majf=0, minf=186281
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=104.2%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=379434980/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
SATA-R5-11: (groupid=1, jobs=5): err= 0: pid=44814: Tue Dec 22 10:37:33 2015
  read : io=23612GB, bw=3358.2MB/s, iops=53730, runt=7200003msec
    slat (usec): min=4, max=1049, avg= 9.63, stdev= 3.50
    clat (usec): min=41, max=23063, avg=1478.23, stdev=643.75
     lat (usec): min=47, max=23075, avg=1488.02, stdev=643.77
    clat percentiles (usec):
     |  1.00th=[  596],  5.00th=[  716], 10.00th=[  788], 20.00th=[  892],
     | 30.00th=[  980], 40.00th=[ 1096], 50.00th=[ 1352], 60.00th=[ 1608],
     | 70.00th=[ 1832], 80.00th=[ 2040], 90.00th=[ 2352], 95.00th=[ 2608],
     | 99.00th=[ 3152], 99.50th=[ 3408], 99.90th=[ 4448], 99.95th=[ 5024],
     | 99.99th=[ 6432]
    bw (KB  /s): min=    0, max=880256, per=20.00%, avg=687826.63, stdev=50877.91
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.22%, 750=6.69%
    lat (usec) : 1000=24.85%
    lat (msec) : 2=46.26%, 4=21.81%, 10=0.17%, 20=0.01%, 50=0.01%
  cpu          : usr=2.46%, sys=14.05%, ctx=339955507, majf=0, minf=186622
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=104.1%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=386862521/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-01: (groupid=2, jobs=5): err= 0: pid=44819: Tue Dec 22 10:37:33 2015
  read : io=41255GB, bw=5867.5MB/s, iops=93879, runt=7200001msec
    slat (usec): min=3, max=1047, avg= 8.93, stdev= 3.30
    clat (usec): min=34, max=11823, avg=629.33, stdev=382.29
     lat (usec): min=134, max=11828, avg=638.35, stdev=382.39
    clat percentiles (usec):
     |  1.00th=[  211],  5.00th=[  258], 10.00th=[  290], 20.00th=[  338],
     | 30.00th=[  390], 40.00th=[  446], 50.00th=[  516], 60.00th=[  596],
     | 70.00th=[  708], 80.00th=[  860], 90.00th=[ 1128], 95.00th=[ 1400],
     | 99.00th=[ 2008], 99.50th=[ 2256], 99.90th=[ 2832], 99.95th=[ 3056],
     | 99.99th=[ 3600]
    bw (MB  /s): min=    0, max= 1308, per=20.00%, avg=1173.46, stdev=62.83
    lat (usec) : 50=0.01%, 100=0.01%, 250=4.15%, 500=44.08%, 750=25.03%
    lat (usec) : 1000=12.75%
    lat (msec) : 2=12.95%, 4=1.03%, 10=0.01%, 20=0.01%
  cpu          : usr=3.28%, sys=22.18%, ctx=532324099, majf=0, minf=135670
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.2%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=675929885/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=12
NVMe-R5-11: (groupid=3, jobs=5): err= 0: pid=44824: Tue Dec 22 10:37:33 2015
  read : io=41316GB, bw=5876.3MB/s, iops=94016, runt=7200002msec
    slat (usec): min=3, max=959, avg= 7.72, stdev= 2.75
    clat (usec): min=24, max=15219, avg=629.68, stdev=382.39
     lat (usec): min=124, max=15228, avg=637.50, stdev=382.46
    clat percentiles (usec):
     |  1.00th=[  209],  5.00th=[  255], 10.00th=[  286], 20.00th=[  338],
     | 30.00th=[  390], 40.00th=[  446], 50.00th=[  516], 60.00th=[  596],
     | 70.00th=[  708], 80.00th=[  860], 90.00th=[ 1128], 95.00th=[ 1400],
     | 99.00th=[ 2008], 99.50th=[ 2256], 99.90th=[ 2832], 99.95th=[ 3056],
     | 99.99th=[ 3568]
    bw (MB  /s): min=    0, max= 1323, per=20.00%, avg=1175.20, stdev=64.85
    lat (usec) : 50=0.01%, 100=0.01%, 250=4.43%, 500=43.59%, 750=25.13%
    lat (usec) : 1000=12.84%
    lat (msec) : 2=12.97%, 4=1.03%, 10=0.01%, 20=0.01%
  cpu          : usr=3.12%, sys=19.12%, ctx=552912900, majf=0, minf=136290
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.2%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=676917747/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=12

Run status group 0 (all jobs):
   READ: io=23159GB, aggrb=3293.8MB/s, minb=3293.8MB/s, maxb=3293.8MB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=23612GB, aggrb=3358.2MB/s, minb=3358.2MB/s, maxb=3358.2MB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=41255GB, aggrb=5867.5MB/s, minb=5867.5MB/s, maxb=5867.5MB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=41316GB, aggrb=5876.3MB/s, minb=5876.3MB/s, maxb=5876.3MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=395303895/10, merge=0/0, ticks=595874671/0, in_queue=596489631, util=100.00%
  sdb: ios=402649301/10, merge=0/0, ticks=595560177/0, in_queue=596513352, util=100.00%
    md125: ios=966746330/10, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=241688195/42, aggrmerge=0/0, aggrticks=131037473/0, aggrin_queue=131172842, aggrutil=100.00%
  nvme0n1: ios=241688234/17, merge=0/0, ticks=131458780/0, in_queue=131587372, util=100.00%
  nvme1n1: ios=241688051/37, merge=0/0, ticks=130797034/1, in_queue=130906871, util=100.00%
  nvme2n1: ios=241688077/82, merge=0/0, ticks=130823802/1, in_queue=130951029, util=100.00%
  nvme3n1: ios=241688420/34, merge=0/0, ticks=131070278/0, in_queue=131246097, util=100.00%
    md124: ios=1023882948/10, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=255971933/42, aggrmerge=0/0, aggrticks=135425252/1, aggrin_queue=135492538, aggrutil=100.00%
  nvme4n1: ios=255971794/49, merge=0/0, ticks=135361361/0, in_queue=135450718, util=100.00%
  nvme5n1: ios=255972131/21, merge=0/0, ticks=135275296/3, in_queue=135395760, util=100.00%
  nvme6n1: ios=255971975/50, merge=0/0, ticks=135520588/4, in_queue=135548025, util=100.00%
  nvme7n1: ios=255971835/50, merge=0/0, ticks=135543766/0, in_queue=135575652, util=100.00%


###################################################################################
# 
# Tue 22 Dec 2015 10:38:33 AM EST
# 64kB randread with SATA 2 jobs and 4 I/Os Outstanding
# 64kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 45247 IOPS | 2827 MBps | 882 μsec Average Latency 
SATA-R5-1: 45587 IOPS | 2849 MBps | 876 μsec Average Latency 
NVMe-R5-0: 93741 IOPS | 5858 MBps | 852 μsec Average Latency 
NVMe-R5-1: 93968 IOPS | 5873 MBps | 850 μsec Average Latency 

Totals
IOPS: 278543
MBPS: 17407
Average Latency (μsec): 865

SATA-R5-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-02: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-03: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-04: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-05: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-12: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-13: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-14: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-15: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-02: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-03: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-04: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-05: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-12: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-13: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-14: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-15: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 60 processes

SATA-R5-01: (groupid=0, jobs=10): err= 0: pid=49966: Tue Dec 22 12:43:35 2015
  read : io=19884GB, bw=2827.1MB/s, iops=45247, runt=7200001msec
    slat (usec): min=3, max=994, avg= 9.32, stdev= 3.36
    clat (usec): min=1, max=38934, avg=873.31, stdev=359.58
     lat (usec): min=50, max=38940, avg=882.76, stdev=359.60
    clat percentiles (usec):
     |  1.00th=[  318],  5.00th=[  410], 10.00th=[  482], 20.00th=[  580],
     | 30.00th=[  660], 40.00th=[  732], 50.00th=[  812], 60.00th=[  900],
     | 70.00th=[ 1004], 80.00th=[ 1128], 90.00th=[ 1336], 95.00th=[ 1544],
     | 99.00th=[ 1960], 99.50th=[ 2128], 99.90th=[ 2640], 99.95th=[ 2928],
     | 99.99th=[ 3664]
    bw (KB  /s): min=    0, max=356352, per=10.00%, avg=289663.41, stdev=4031.38
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.05%
    lat (usec) : 500=11.57%, 750=30.44%, 1000=27.82%
    lat (msec) : 2=29.26%, 4=0.85%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.25%, sys=5.85%, ctx=321982746, majf=0, minf=107138
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=325780802/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=10): err= 0: pid=49976: Tue Dec 22 12:43:35 2015
  read : io=20033GB, bw=2849.3MB/s, iops=45587, runt=7200002msec
    slat (usec): min=3, max=994, avg= 9.48, stdev= 3.37
    clat (usec): min=11, max=18334, avg=866.48, stdev=379.59
     lat (usec): min=53, max=18340, avg=876.11, stdev=379.61
    clat percentiles (usec):
     |  1.00th=[  302],  5.00th=[  382], 10.00th=[  450], 20.00th=[  556],
     | 30.00th=[  636], 40.00th=[  716], 50.00th=[  796], 60.00th=[  892],
     | 70.00th=[  996], 80.00th=[ 1144], 90.00th=[ 1368], 95.00th=[ 1576],
     | 99.00th=[ 2040], 99.50th=[ 2256], 99.90th=[ 2928], 99.95th=[ 3248],
     | 99.99th=[ 4256]
    bw (KB  /s): min=    0, max=310528, per=10.00%, avg=291837.57, stdev=4495.76
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.09%, 500=14.21%
    lat (usec) : 750=29.75%, 1000=26.26%
    lat (msec) : 2=28.52%, 4=1.16%, 10=0.01%, 20=0.01%
  cpu          : usr=1.34%, sys=6.01%, ctx=323634328, majf=0, minf=103827
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=328228642/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=20): err= 0: pid=49986: Tue Dec 22 12:43:35 2015
  read : io=41195GB, bw=5858.9MB/s, iops=93741, runt=7200002msec
    slat (usec): min=3, max=973, avg= 8.19, stdev= 2.95
    clat (usec): min=0, max=12506, avg=844.06, stdev=574.62
     lat (usec): min=129, max=12514, avg=852.35, stdev=574.75
    clat percentiles (usec):
     |  1.00th=[  207],  5.00th=[  262], 10.00th=[  302], 20.00th=[  366],
     | 30.00th=[  446], 40.00th=[  540], 50.00th=[  668], 60.00th=[  828],
     | 70.00th=[ 1020], 80.00th=[ 1256], 90.00th=[ 1640], 95.00th=[ 1992],
     | 99.00th=[ 2736], 99.50th=[ 3056], 99.90th=[ 3696], 99.95th=[ 3984],
     | 99.99th=[ 4576]
    bw (KB  /s): min=    0, max=364928, per=5.00%, avg=300053.61, stdev=5417.93
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=3.84%, 500=32.12%, 750=19.36%, 1000=13.87%
    lat (msec) : 2=25.93%, 4=4.84%, 10=0.05%, 20=0.01%
  cpu          : usr=1.01%, sys=5.25%, ctx=668719079, majf=0, minf=200328
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=674940509/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=20): err= 0: pid=50006: Tue Dec 22 12:43:35 2015
  read : io=41295GB, bw=5873.4MB/s, iops=93968, runt=7200002msec
    slat (usec): min=3, max=1280, avg= 8.72, stdev= 3.14
    clat (usec): min=0, max=12528, avg=841.41, stdev=565.19
     lat (usec): min=122, max=12534, avg=850.23, stdev=565.33
    clat percentiles (usec):
     |  1.00th=[  207],  5.00th=[  262], 10.00th=[  302], 20.00th=[  370],
     | 30.00th=[  450], 40.00th=[  548], 50.00th=[  676], 60.00th=[  828],
     | 70.00th=[ 1012], 80.00th=[ 1256], 90.00th=[ 1624], 95.00th=[ 1960],
     | 99.00th=[ 2704], 99.50th=[ 3024], 99.90th=[ 3664], 99.95th=[ 3920],
     | 99.99th=[ 4512]
    bw (KB  /s): min=    0, max=380416, per=5.00%, avg=300779.52, stdev=5259.74
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=3.94%
    lat (usec) : 500=31.40%, 750=19.68%, 1000=14.31%
    lat (msec) : 2=26.04%, 4=4.60%, 10=0.04%, 20=0.01%
  cpu          : usr=1.07%, sys=5.60%, ctx=668215573, majf=0, minf=199951
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=676573558/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=19884GB, aggrb=2827.1MB/s, minb=2827.1MB/s, maxb=2827.1MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=20033GB, aggrb=2849.3MB/s, minb=2849.3MB/s, maxb=2849.3MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=41195GB, aggrb=5858.9MB/s, minb=5858.9MB/s, maxb=5858.9MB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=41295GB, aggrb=5873.4MB/s, minb=5873.4MB/s, maxb=5873.4MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=339365116/0, merge=0/0, ticks=296478917/0, in_queue=296724142, util=100.00%
  sdb: ios=341919711/0, merge=0/0, ticks=296326798/0, in_queue=296598730, util=100.00%
    md125: ios=965140988/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=241288247/0, aggrmerge=0/0, aggrticks=171700527/0, aggrin_queue=171859153, aggrutil=100.00%
  nvme0n1: ios=241289623/0, merge=0/0, ticks=171496316/0, in_queue=171653709, util=100.00%
  nvme1n1: ios=241281774/0, merge=0/0, ticks=172264901/0, in_queue=172407192, util=100.00%
  nvme2n1: ios=241299001/0, merge=0/0, ticks=171268937/0, in_queue=171427438, util=100.00%
  nvme3n1: ios=241282590/0, merge=0/0, ticks=171771956/0, in_queue=171948275, util=100.00%
    md124: ios=1023450463/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=255864978/0, aggrmerge=0/0, aggrticks=176376774/0, aggrin_queue=176573551, aggrutil=100.00%
  nvme4n1: ios=255864290/0, merge=0/0, ticks=175957608/0, in_queue=176118739, util=100.00%
  nvme5n1: ios=255859208/0, merge=0/0, ticks=176040135/0, in_queue=176303763, util=100.00%
  nvme6n1: ios=255874005/0, merge=0/0, ticks=176346257/0, in_queue=176540123, util=100.00%
  nvme7n1: ios=255862409/0, merge=0/0, ticks=177163098/0, in_queue=177331580, util=100.00%


###################################################################################
# 
# Tue 22 Dec 2015 12:44:35 PM EST
# 64kB randread with SATA 5 jobs and 4 I/Os Outstanding
# 64kB randread with NVMe 2 jobs and 10 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 54004 IOPS | 3375 MBps | 1850 μsec Average Latency 
SATA-R5-1: 54330 IOPS | 3395 MBps | 1839 μsec Average Latency 
NVMe-R5-0: 95348 IOPS | 5959 MBps | 1047 μsec Average Latency 
NVMe-R5-1: 95595 IOPS | 5974 MBps | 1045 μsec Average Latency 

Totals
IOPS: 299277
MBPS: 18703
Average Latency (μsec): 1445

SATA-R5-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-02: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-03: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-04: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-05: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-12: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-13: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-14: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
SATA-R5-15: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
NVMe-R5-02: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
NVMe-R5-03: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
NVMe-R5-04: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
NVMe-R5-05: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
NVMe-R5-11: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
NVMe-R5-12: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
NVMe-R5-13: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
NVMe-R5-14: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
NVMe-R5-15: (g=3): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
fio-2.1.10
Starting 70 processes

SATA-R5-01: (groupid=0, jobs=25): err= 0: pid=55178: Tue Dec 22 14:49:37 2015
  read : io=23733GB, bw=3375.4MB/s, iops=54004, runt=7200003msec
    slat (usec): min=3, max=4364, avg= 9.92, stdev= 3.86
    clat (usec): min=0, max=37249, avg=1840.32, stdev=699.88
     lat (usec): min=55, max=37257, avg=1850.37, stdev=699.90
    clat percentiles (usec):
     |  1.00th=[  532],  5.00th=[  796], 10.00th=[  972], 20.00th=[ 1240],
     | 30.00th=[ 1432], 40.00th=[ 1624], 50.00th=[ 1800], 60.00th=[ 1976],
     | 70.00th=[ 2160], 80.00th=[ 2384], 90.00th=[ 2736], 95.00th=[ 3056],
     | 99.00th=[ 3728], 99.50th=[ 3984], 99.90th=[ 4640], 99.95th=[ 4960],
     | 99.99th=[ 5984]
    bw (KB  /s): min=    0, max=206464, per=4.00%, avg=138331.10, stdev=2403.01
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.75%, 750=3.33%, 1000=6.75%
    lat (msec) : 2=50.75%, 4=37.93%, 10=0.48%, 20=0.01%, 50=0.01%
  cpu          : usr=0.62%, sys=2.95%, ctx=394885877, majf=0, minf=267505
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=388834762/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=25): err= 0: pid=55203: Tue Dec 22 14:49:37 2015
  read : io=23876GB, bw=3395.7MB/s, iops=54330, runt=7200003msec
    slat (usec): min=3, max=6565, avg= 9.76, stdev= 4.14
    clat (usec): min=0, max=22510, avg=1829.35, stdev=728.82
     lat (usec): min=56, max=22531, avg=1839.24, stdev=728.82
    clat percentiles (usec):
     |  1.00th=[  486],  5.00th=[  740], 10.00th=[  924], 20.00th=[ 1192],
     | 30.00th=[ 1416], 40.00th=[ 1592], 50.00th=[ 1784], 60.00th=[ 1960],
     | 70.00th=[ 2160], 80.00th=[ 2416], 90.00th=[ 2768], 95.00th=[ 3088],
     | 99.00th=[ 3824], 99.50th=[ 4128], 99.90th=[ 4960], 99.95th=[ 5408],
     | 99.99th=[ 6816]
    bw (KB  /s): min=    0, max=172160, per=4.00%, avg=139165.97, stdev=3092.63
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=1.12%, 750=4.04%, 1000=7.29%
    lat (msec) : 2=49.54%, 4=37.34%, 10=0.66%, 20=0.01%, 50=0.01%
  cpu          : usr=0.63%, sys=2.94%, ctx=397298052, majf=0, minf=269637
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=391182287/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=10): err= 0: pid=55228: Tue Dec 22 14:49:37 2015
  read : io=41901GB, bw=5959.4MB/s, iops=95348, runt=7200003msec
    slat (usec): min=3, max=8007, avg= 8.01, stdev= 4.23
    clat (usec): min=0, max=16079, avg=1039.64, stdev=738.13
     lat (usec): min=128, max=16087, avg=1047.76, stdev=738.31
    clat percentiles (usec):
     |  1.00th=[  211],  5.00th=[  270], 10.00th=[  310], 20.00th=[  386],
     | 30.00th=[  486], 40.00th=[  636], 50.00th=[  828], 60.00th=[ 1064],
     | 70.00th=[ 1320], 80.00th=[ 1640], 90.00th=[ 2096], 95.00th=[ 2480],
     | 99.00th=[ 3280], 99.50th=[ 3600], 99.90th=[ 4320], 99.95th=[ 4576],
     | 99.99th=[ 5216]
    bw (KB  /s): min=    0, max=731520, per=10.00%, avg=610295.30, stdev=10180.98
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=3.39%, 500=27.60%, 750=15.18%, 1000=11.41%
    lat (msec) : 2=30.78%, 4=11.43%, 10=0.21%, 20=0.01%
  cpu          : usr=1.99%, sys=10.29%, ctx=634161284, majf=0, minf=229638
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.1%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=686512247/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
NVMe-R5-11: (groupid=3, jobs=10): err= 0: pid=55238: Tue Dec 22 14:49:37 2015
  read : io=42010GB, bw=5974.8MB/s, iops=95595, runt=7200003msec
    slat (usec): min=3, max=1987, avg= 7.92, stdev= 2.72
    clat (usec): min=1, max=15399, avg=1037.09, stdev=721.52
     lat (usec): min=123, max=15405, avg=1045.09, stdev=721.69
    clat percentiles (usec):
     |  1.00th=[  209],  5.00th=[  270], 10.00th=[  310], 20.00th=[  394],
     | 30.00th=[  498], 40.00th=[  652], 50.00th=[  844], 60.00th=[ 1064],
     | 70.00th=[ 1320], 80.00th=[ 1624], 90.00th=[ 2064], 95.00th=[ 2448],
     | 99.00th=[ 3216], 99.50th=[ 3536], 99.90th=[ 4256], 99.95th=[ 4512],
     | 99.99th=[ 5088]
    bw (KB  /s): min=    0, max=859008, per=10.00%, avg=611877.88, stdev=9515.48
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=3.48%
    lat (usec) : 500=26.61%, 750=15.23%, 1000=11.80%
    lat (msec) : 2=31.83%, 4=10.86%, 10=0.18%, 20=0.01%
  cpu          : usr=1.95%, sys=10.13%, ctx=637361863, majf=0, minf=227765
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.2%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=688288236/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10

Run status group 0 (all jobs):
   READ: io=23733GB, aggrb=3375.4MB/s, minb=3375.4MB/s, maxb=3375.4MB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=23876GB, aggrb=3395.7MB/s, minb=3395.7MB/s, maxb=3395.7MB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=41901GB, aggrb=5959.4MB/s, minb=5959.4MB/s, maxb=5959.4MB/s, mint=7200003msec, maxt=7200003msec

Run status group 3 (all jobs):
   READ: io=42010GB, aggrb=5974.8MB/s, minb=5974.8MB/s, maxb=5974.8MB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sda: ios=405065280/0, merge=0/0, ticks=745816583/0, in_queue=746900247, util=100.00%
  sdb: ios=407492600/0, merge=0/0, ticks=745775168/0, in_queue=746841729, util=100.00%
    md125: ios=981538164/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=245387162/0, aggrmerge=0/0, aggrticks=213352560/0, aggrin_queue=213566581, aggrutil=100.00%
  nvme0n1: ios=245387632/0, merge=0/0, ticks=213425730/0, in_queue=213641442, util=100.00%
  nvme1n1: ios=245385220/0, merge=0/0, ticks=214119421/0, in_queue=214308505, util=100.00%
  nvme2n1: ios=245389965/0, merge=0/0, ticks=212092946/0, in_queue=212307747, util=100.00%
  nvme3n1: ios=245385831/0, merge=0/0, ticks=213772146/0, in_queue=214008630, util=100.00%
    md124: ios=1041141404/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=260287377/0, aggrmerge=0/0, aggrticks=218990373/0, aggrin_queue=219205087, aggrutil=100.00%
  nvme4n1: ios=260285758/0, merge=0/0, ticks=218585832/0, in_queue=218758793, util=100.00%
  nvme5n1: ios=260284937/0, merge=0/0, ticks=218052927/0, in_queue=218352279, util=100.00%
  nvme6n1: ios=260292468/0, merge=0/0, ticks=219537716/0, in_queue=219730652, util=100.00%
  nvme7n1: ios=260286348/0, merge=0/0, ticks=219785017/0, in_queue=219978624, util=100.00%


###################################################################################
# 
# Tue 22 Dec 2015 02:50:37 PM EST
# 32kB randread with SATA 2 jobs and 4 I/Os Outstanding
# 32kB randread with NVMe 4 jobs and 4 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 81485 IOPS | 2546 MBps | 489 μsec Average Latency 
SATA-R5-1: 82529 IOPS | 2579 MBps | 483 μsec Average Latency 
NVMe-R5-0: 179769 IOPS | 5617 MBps | 444 μsec Average Latency 
NVMe-R5-1: 179933 IOPS | 5622 MBps | 443 μsec Average Latency 

Totals
IOPS: 523716
MBPS: 16364
Average Latency (μsec): 464

SATA-R5-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
SATA-R5-02: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
SATA-R5-03: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
SATA-R5-04: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
SATA-R5-05: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
SATA-R5-12: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
SATA-R5-13: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
SATA-R5-14: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
SATA-R5-15: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-02: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-03: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-04: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-05: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-11: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-12: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-13: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-14: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
NVMe-R5-15: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 60 processes

SATA-R5-01: (groupid=0, jobs=10): err= 0: pid=60465: Tue Dec 22 16:55:39 2015
  read : io=17904GB, bw=2546.5MB/s, iops=81485, runt=7200001msec
    slat (usec): min=2, max=4521, avg= 6.90, stdev= 2.75
    clat (usec): min=0, max=36630, avg=482.78, stdev=208.14
     lat (usec): min=38, max=36648, avg=489.80, stdev=208.16
    clat percentiles (usec):
     |  1.00th=[  201],  5.00th=[  237], 10.00th=[  274], 20.00th=[  326],
     | 30.00th=[  370], 40.00th=[  410], 50.00th=[  450], 60.00th=[  494],
     | 70.00th=[  548], 80.00th=[  612], 90.00th=[  724], 95.00th=[  828],
     | 99.00th=[ 1064], 99.50th=[ 1192], 99.90th=[ 2512], 99.95th=[ 2896],
     | 99.99th=[ 3280]
    bw (KB  /s): min=    0, max=301056, per=10.00%, avg=260784.62, stdev=3348.77
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=6.65%
    lat (usec) : 500=54.57%, 750=30.51%, 1000=6.71%
    lat (msec) : 2=1.39%, 4=0.16%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=2.01%, sys=7.99%, ctx=566216356, majf=0, minf=61875
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=586693604/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=10): err= 0: pid=60475: Tue Dec 22 16:55:39 2015
  read : io=18134GB, bw=2579.4MB/s, iops=82529, runt=7200001msec
    slat (usec): min=2, max=2952, avg= 6.59, stdev= 2.51
    clat (usec): min=8, max=16071, avg=476.92, stdev=208.46
     lat (usec): min=39, max=16075, avg=483.59, stdev=208.48
    clat percentiles (usec):
     |  1.00th=[  201],  5.00th=[  235], 10.00th=[  266], 20.00th=[  318],
     | 30.00th=[  362], 40.00th=[  402], 50.00th=[  442], 60.00th=[  486],
     | 70.00th=[  540], 80.00th=[  612], 90.00th=[  716], 95.00th=[  828],
     | 99.00th=[ 1080], 99.50th=[ 1224], 99.90th=[ 2576], 99.95th=[ 2896],
     | 99.99th=[ 3280]
    bw (KB  /s): min=    0, max=299456, per=10.00%, avg=264124.34, stdev=3546.58
    lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=7.38%
    lat (usec) : 500=55.08%, 750=29.42%, 1000=6.47%
    lat (msec) : 2=1.47%, 4=0.17%, 10=0.01%, 20=0.01%
  cpu          : usr=2.05%, sys=7.79%, ctx=574150849, majf=0, minf=61833
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=594210077/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=20): err= 0: pid=60485: Tue Dec 22 16:55:39 2015
  read : io=39500GB, bw=5617.9MB/s, iops=179769, runt=7200002msec
    slat (usec): min=2, max=9095, avg= 5.38, stdev= 2.36
    clat (usec): min=0, max=17151, avg=438.55, stdev=314.28
     lat (usec): min=99, max=17162, avg=444.02, stdev=314.29
    clat percentiles (usec):
     |  1.00th=[  151],  5.00th=[  169], 10.00th=[  187], 20.00th=[  219],
     | 30.00th=[  249], 40.00th=[  286], 50.00th=[  334], 60.00th=[  394],
     | 70.00th=[  478], 80.00th=[  596], 90.00th=[  828], 95.00th=[ 1064],
     | 99.00th=[ 1656], 99.50th=[ 1912], 99.90th=[ 2512], 99.95th=[ 2736],
     | 99.99th=[ 3248]
    bw (KB  /s): min=    0, max=346048, per=5.00%, avg=287664.46, stdev=4198.15
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=30.04%, 500=42.21%, 750=15.23%, 1000=6.51%
    lat (msec) : 2=5.60%, 4=0.40%, 10=0.01%, 20=0.01%
  cpu          : usr=1.85%, sys=7.15%, ctx=1252148547, majf=0, minf=117504
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1294341549/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-11: (groupid=3, jobs=20): err= 0: pid=60505: Tue Dec 22 16:55:39 2015
  read : io=39536GB, bw=5622.1MB/s, iops=179933, runt=7200001msec
    slat (usec): min=2, max=3099, avg= 5.39, stdev= 2.02
    clat (usec): min=0, max=16795, avg=438.13, stdev=312.53
     lat (usec): min=96, max=16799, avg=443.61, stdev=312.54
    clat percentiles (usec):
     |  1.00th=[  151],  5.00th=[  169], 10.00th=[  187], 20.00th=[  219],
     | 30.00th=[  249], 40.00th=[  286], 50.00th=[  334], 60.00th=[  394],
     | 70.00th=[  478], 80.00th=[  596], 90.00th=[  820], 95.00th=[ 1064],
     | 99.00th=[ 1640], 99.50th=[ 1896], 99.90th=[ 2480], 99.95th=[ 2704],
     | 99.99th=[ 3216]
    bw (KB  /s): min=    0, max=329472, per=5.00%, avg=287925.90, stdev=4046.04
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=30.06%, 500=42.16%, 750=15.28%, 1000=6.52%
    lat (msec) : 2=5.60%, 4=0.38%, 10=0.01%, 20=0.01%
  cpu          : usr=1.88%, sys=7.16%, ctx=1253044300, majf=0, minf=115026
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1295519717/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=17904GB, aggrb=2546.5MB/s, minb=2546.5MB/s, maxb=2546.5MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=18134GB, aggrb=2579.4MB/s, minb=2579.4MB/s, maxb=2579.4MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=39500GB, aggrb=5617.9MB/s, minb=5617.9MB/s, maxb=5617.9MB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=39536GB, aggrb=5622.1MB/s, minb=5622.1MB/s, maxb=5622.1MB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sda: ios=611132064/0, merge=0/0, ticks=294984460/0, in_queue=295367406, util=100.00%
  sdb: ios=619088849/0, merge=0/0, ticks=295016151/0, in_queue=295359046, util=100.00%
    md125: ios=1364364699/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=341091489/0, aggrmerge=0/0, aggrticks=147962470/0, aggrin_queue=148136730, aggrutil=100.00%
  nvme0n1: ios=341090039/0, merge=0/0, ticks=147959898/0, in_queue=148125229, util=100.00%
  nvme1n1: ios=341080766/0, merge=0/0, ticks=147989344/0, in_queue=148153617, util=100.00%
  nvme2n1: ios=341135667/0, merge=0/0, ticks=147943372/0, in_queue=148110462, util=100.00%
  nvme3n1: ios=341059484/0, merge=0/0, ticks=147957268/0, in_queue=148157615, util=100.00%
    md124: ios=1378262450/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=344565893/0, aggrmerge=0/0, aggrticks=148549913/0, aggrin_queue=148730947, aggrutil=100.00%
  nvme4n1: ios=344562647/0, merge=0/0, ticks=148199633/0, in_queue=148346443, util=100.00%
  nvme5n1: ios=344522693/0, merge=0/0, ticks=148322952/0, in_queue=148605351, util=100.00%
  nvme6n1: ios=344640222/0, merge=0/0, ticks=148792795/0, in_queue=148928809, util=100.00%
  nvme7n1: ios=344538012/0, merge=0/0, ticks=148884274/0, in_queue=149043188, util=100.00%


###################################################################################
# 
# Tue 22 Dec 2015 04:56:39 PM EST
# 32kB randread with SATA 4 jobs and 6 I/Os Outstanding
# 32kB randread with NVMe 4 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 102283 IOPS | 3196 MBps | 1172 μsec Average Latency 
SATA-R5-1: 102936 IOPS | 3216 MBps | 1164 μsec Average Latency 
NVMe-R5-0: 195190 IOPS | 6099 MBps | 818 μsec Average Latency 
NVMe-R5-1: 195329 IOPS | 6104 MBps | 818 μsec Average Latency 

Totals
IOPS: 595738
MBPS: 18615
Average Latency (μsec): 993

SATA-R5-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
SATA-R5-02: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
SATA-R5-03: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
SATA-R5-04: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
SATA-R5-05: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
SATA-R5-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
SATA-R5-12: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
SATA-R5-13: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
SATA-R5-14: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
SATA-R5-15: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
NVMe-R5-01: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-02: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-03: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-04: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-05: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-12: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-13: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-14: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
NVMe-R5-15: (g=3): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 80 processes

SATA-R5-01: (groupid=0, jobs=20): err= 0: pid=66143: Tue Dec 22 19:01:41 2015
  read : io=22474GB, bw=3196.4MB/s, iops=102283, runt=7200003msec
    slat (usec): min=3, max=19964, avg= 9.68, stdev= 5.13
    clat (usec): min=1, max=38254, avg=1162.28, stdev=560.40
     lat (usec): min=45, max=38259, avg=1172.06, stdev=560.44
    clat percentiles (usec):
     |  1.00th=[  334],  5.00th=[  462], 10.00th=[  548], 20.00th=[  684],
     | 30.00th=[  804], 40.00th=[  924], 50.00th=[ 1048], 60.00th=[ 1192],
     | 70.00th=[ 1368], 80.00th=[ 1592], 90.00th=[ 1928], 95.00th=[ 2224],
     | 99.00th=[ 2832], 99.50th=[ 3056], 99.90th=[ 3568], 99.95th=[ 3792],
     | 99.99th=[ 4448]
    bw (KB  /s): min=    0, max=281344, per=5.00%, avg=163695.81, stdev=2278.22
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.11%
    lat (usec) : 500=6.89%, 750=18.46%, 1000=20.43%
    lat (msec) : 2=45.53%, 4=8.56%, 10=0.03%, 20=0.01%, 50=0.01%
  cpu          : usr=1.22%, sys=6.80%, ctx=703092981, majf=0, minf=172139
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=736442697/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
SATA-R5-11: (groupid=1, jobs=20): err= 0: pid=66163: Tue Dec 22 19:01:41 2015
  read : io=22618GB, bw=3216.8MB/s, iops=102936, runt=7200002msec
    slat (usec): min=2, max=6602, avg=10.31, stdev= 5.56
    clat (usec): min=0, max=19286, avg=1154.20, stdev=542.69
     lat (usec): min=50, max=19290, avg=1164.60, stdev=542.73
    clat percentiles (usec):
     |  1.00th=[  338],  5.00th=[  462], 10.00th=[  548], 20.00th=[  684],
     | 30.00th=[  804], 40.00th=[  924], 50.00th=[ 1048], 60.00th=[ 1192],
     | 70.00th=[ 1368], 80.00th=[ 1576], 90.00th=[ 1896], 95.00th=[ 2192],
     | 99.00th=[ 2768], 99.50th=[ 2992], 99.90th=[ 3536], 99.95th=[ 3792],
     | 99.99th=[ 4704]
    bw (KB  /s): min=    0, max=215616, per=5.00%, avg=164740.52, stdev=2342.04
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.09%, 500=6.80%, 750=18.41%, 1000=20.56%
    lat (msec) : 2=46.28%, 4=7.84%, 10=0.03%, 20=0.01%
  cpu          : usr=1.24%, sys=7.24%, ctx=702063776, majf=0, minf=173162
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=741145165/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
NVMe-R5-01: (groupid=2, jobs=20): err= 0: pid=66183: Tue Dec 22 19:01:41 2015
  read : io=42888GB, bw=6099.8MB/s, iops=195190, runt=7200002msec
    slat (usec): min=2, max=11352, avg= 5.53, stdev= 3.02
    clat (usec): min=0, max=14486, avg=813.09, stdev=710.90
     lat (usec): min=97, max=14491, avg=818.71, stdev=710.92
    clat percentiles (usec):
     |  1.00th=[  153],  5.00th=[  175], 10.00th=[  197], 20.00th=[  241],
     | 30.00th=[  302], 40.00th=[  390], 50.00th=[  532], 60.00th=[  740],
     | 70.00th=[ 1004], 80.00th=[ 1352], 90.00th=[ 1848], 95.00th=[ 2256],
     | 99.00th=[ 3120], 99.50th=[ 3472], 99.90th=[ 4192], 99.95th=[ 4512],
     | 99.99th=[ 5152]
    bw (KB  /s): min=    0, max=347200, per=5.00%, avg=312337.31, stdev=5218.30
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=21.60%, 500=26.35%, 750=12.40%, 1000=9.38%
    lat (msec) : 2=22.37%, 4=7.74%, 10=0.16%, 20=0.01%
  cpu          : usr=2.03%, sys=7.98%, ctx=1330449537, majf=0, minf=201699
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.2%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1405369115/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=20): err= 0: pid=66203: Tue Dec 22 19:01:41 2015
  read : io=42919GB, bw=6104.7MB/s, iops=195329, runt=7200002msec
    slat (usec): min=2, max=5768, avg= 5.70, stdev= 2.67
    clat (usec): min=1, max=12965, avg=812.25, stdev=707.02
     lat (usec): min=86, max=12970, avg=818.05, stdev=707.04
    clat percentiles (usec):
     |  1.00th=[  153],  5.00th=[  175], 10.00th=[  197], 20.00th=[  241],
     | 30.00th=[  302], 40.00th=[  394], 50.00th=[  540], 60.00th=[  748],
     | 70.00th=[ 1012], 80.00th=[ 1352], 90.00th=[ 1848], 95.00th=[ 2256],
     | 99.00th=[ 3120], 99.50th=[ 3440], 99.90th=[ 4192], 99.95th=[ 4512],
     | 99.99th=[ 5152]
    bw (KB  /s): min=    0, max=382208, per=5.00%, avg=312563.66, stdev=5021.51
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=21.55%, 500=26.25%, 750=12.44%, 1000=9.46%
    lat (msec) : 2=22.52%, 4=7.63%, 10=0.15%, 20=0.01%
  cpu          : usr=2.13%, sys=8.22%, ctx=1326389495, majf=0, minf=201188
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.2%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1406375691/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=22474GB, aggrb=3196.4MB/s, minb=3196.4MB/s, maxb=3196.4MB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=22618GB, aggrb=3216.8MB/s, minb=3216.8MB/s, maxb=3216.8MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=42888GB, aggrb=6099.8MB/s, minb=6099.8MB/s, maxb=6099.8MB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=42919GB, aggrb=6104.7MB/s, minb=6104.7MB/s, maxb=6104.7MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=767169821/0, merge=0/0, ticks=892388713/0, in_queue=894522079, util=100.00%
  sdb: ios=771997865/0, merge=0/0, ticks=891957158/0, in_queue=894151751, util=100.00%
    md125: ios=1481324093/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=370333813/0, aggrmerge=0/0, aggrticks=298139622/0, aggrin_queue=298565554, aggrutil=100.00%
  nvme0n1: ios=370334483/0, merge=0/0, ticks=295621605/0, in_queue=296049881, util=100.00%
  nvme1n1: ios=370340432/0, merge=0/0, ticks=298769001/0, in_queue=299159717, util=100.00%
  nvme2n1: ios=370345995/0, merge=0/0, ticks=299362385/0, in_queue=299785078, util=100.00%
  nvme3n1: ios=370314344/0, merge=0/0, ticks=298805500/0, in_queue=299267540, util=100.00%
    md124: ios=1496197055/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=374052606/0, aggrmerge=0/0, aggrticks=299090133/0, aggrin_queue=299567802, aggrutil=100.00%
  nvme4n1: ios=374063275/0, merge=0/0, ticks=297222384/0, in_queue=297601338, util=100.00%
  nvme5n1: ios=374013964/0, merge=0/0, ticks=296513766/0, in_queue=297203345, util=100.00%
  nvme6n1: ios=374119129/0, merge=0/0, ticks=301890136/0, in_queue=302307250, util=100.00%
  nvme7n1: ios=374014058/0, merge=0/0, ticks=300734249/0, in_queue=301159278, util=100.00%


###################################################################################
# 
# Tue 22 Dec 2015 07:02:41 PM EST
# 16kB randread with SATA 2 jobs and 8 I/Os Outstanding
# 16kB randread with NVMe 4 jobs and 16 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 152074 IOPS | 2376 MBps | 524 μsec Average Latency 
SATA-R5-1: 143473 IOPS | 2241 MBps | 556 μsec Average Latency 
NVMe-R5-0: 391370 IOPS | 6115 MBps | 816 μsec Average Latency 
NVMe-R5-1: 390849 IOPS | 6107 MBps | 817 μsec Average Latency 

Totals
IOPS: 1077766
MBPS: 16839
Average Latency (μsec): 678

SATA-R5-01: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
SATA-R5-02: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
SATA-R5-03: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
SATA-R5-04: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
SATA-R5-05: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
SATA-R5-11: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
SATA-R5-12: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
SATA-R5-13: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
SATA-R5-14: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
SATA-R5-15: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
NVMe-R5-01: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-02: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-03: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-04: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-05: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-11: (g=3): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-12: (g=3): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-13: (g=3): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-14: (g=3): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
NVMe-R5-15: (g=3): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=16
...
fio-2.1.10
Starting 60 processes

SATA-R5-01: (groupid=0, jobs=10): err= 0: pid=71947: Tue Dec 22 21:07:43 2015
  read : io=16707GB, bw=2376.2MB/s, iops=152074, runt=7200001msec
    slat (usec): min=2, max=7767, avg=14.88, stdev=10.97
    clat (usec): min=11, max=35098, avg=509.88, stdev=135.95
     lat (usec): min=119, max=35104, avg=524.93, stdev=135.66
    clat percentiles (usec):
     |  1.00th=[  314],  5.00th=[  362], 10.00th=[  386], 20.00th=[  414],
     | 30.00th=[  442], 40.00th=[  470], 50.00th=[  494], 60.00th=[  516],
     | 70.00th=[  548], 80.00th=[  588], 90.00th=[  652], 95.00th=[  708],
     | 99.00th=[  852], 99.50th=[  916], 99.90th=[ 1624], 99.95th=[ 2544],
     | 99.99th=[ 3184]
    bw (KB  /s): min=    0, max=420512, per=10.00%, avg=243328.86, stdev=3149.23
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.02%, 500=53.03%
    lat (usec) : 750=43.79%, 1000=2.90%
    lat (msec) : 2=0.19%, 4=0.08%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=2.68%, sys=27.81%, ctx=661441967, majf=0, minf=63356
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.2%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1094937307/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
SATA-R5-11: (groupid=1, jobs=10): err= 0: pid=71957: Tue Dec 22 21:07:43 2015
  read : io=15763GB, bw=2241.8MB/s, iops=143473, runt=7200001msec
    slat (usec): min=2, max=18396, avg=14.95, stdev=11.25
    clat (usec): min=15, max=19918, avg=541.35, stdev=114.86
     lat (usec): min=119, max=19930, avg=556.43, stdev=114.57
    clat percentiles (usec):
     |  1.00th=[  354],  5.00th=[  406], 10.00th=[  426], 20.00th=[  454],
     | 30.00th=[  486], 40.00th=[  510], 50.00th=[  532], 60.00th=[  548],
     | 70.00th=[  572], 80.00th=[  620], 90.00th=[  668], 95.00th=[  724],
     | 99.00th=[  844], 99.50th=[  900], 99.90th=[ 1208], 99.95th=[ 2192],
     | 99.99th=[ 3120]
    bw (KB  /s): min=    0, max=239680, per=10.00%, avg=229569.17, stdev=2192.77
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=35.78%
    lat (usec) : 750=60.74%, 1000=3.28%
    lat (msec) : 2=0.14%, 4=0.06%, 10=0.01%, 20=0.01%
  cpu          : usr=2.54%, sys=26.29%, ctx=621218113, majf=0, minf=66689
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=104.2%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1033012380/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-01: (groupid=2, jobs=20): err= 0: pid=71967: Tue Dec 22 21:07:43 2015
  read : io=42997GB, bw=6115.2MB/s, iops=391370, runt=7200004msec
    slat (usec): min=2, max=8932, avg= 5.51, stdev= 6.84
    clat (usec): min=0, max=19741, avg=810.87, stdev=780.64
     lat (usec): min=67, max=19775, avg=816.50, stdev=780.69
    clat percentiles (usec):
     |  1.00th=[  120],  5.00th=[  141], 10.00th=[  157], 20.00th=[  191],
     | 30.00th=[  237], 40.00th=[  318], 50.00th=[  474], 60.00th=[  716],
     | 70.00th=[ 1020], 80.00th=[ 1432], 90.00th=[ 1992], 95.00th=[ 2448],
     | 99.00th=[ 3248], 99.50th=[ 3600], 99.90th=[ 4384], 99.95th=[ 4704],
     | 99.99th=[ 5344]
    bw (KB  /s): min=    0, max=491904, per=5.00%, avg=313104.21, stdev=4862.18
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.02%, 250=32.10%, 500=19.06%, 750=10.11%, 1000=8.02%
    lat (msec) : 2=20.75%, 4=9.72%, 10=0.22%, 20=0.01%
  cpu          : usr=4.04%, sys=16.39%, ctx=2297507123, majf=0, minf=232355
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=104.2%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=2817870562/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-11: (groupid=3, jobs=20): err= 0: pid=71987: Tue Dec 22 21:07:43 2015
  read : io=42940GB, bw=6107.3MB/s, iops=390849, runt=7200004msec
    slat (usec): min=2, max=7756, avg= 5.29, stdev= 4.56
    clat (usec): min=0, max=21617, avg=812.22, stdev=798.39
     lat (usec): min=57, max=21621, avg=817.61, stdev=798.42
    clat percentiles (usec):
     |  1.00th=[  119],  5.00th=[  139], 10.00th=[  155], 20.00th=[  187],
     | 30.00th=[  233], 40.00th=[  310], 50.00th=[  458], 60.00th=[  692],
     | 70.00th=[ 1012], 80.00th=[ 1432], 90.00th=[ 2040], 95.00th=[ 2480],
     | 99.00th=[ 3312], 99.50th=[ 3664], 99.90th=[ 4448], 99.95th=[ 4768],
     | 99.99th=[ 5472]
    bw (KB  /s): min=    0, max=479520, per=5.00%, avg=312686.26, stdev=4802.44
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.02%
    lat (usec) : 250=32.84%, 500=19.13%, 750=10.02%, 1000=7.74%
    lat (msec) : 2=19.68%, 4=10.31%, 10=0.25%, 20=0.01%, 50=0.01%
  cpu          : usr=3.94%, sys=15.96%, ctx=2298455414, majf=0, minf=204509
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=104.2%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=2814116492/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16

Run status group 0 (all jobs):
   READ: io=16707GB, aggrb=2376.2MB/s, minb=2376.2MB/s, maxb=2376.2MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=15763GB, aggrb=2241.8MB/s, minb=2241.8MB/s, maxb=2241.8MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=42997GB, aggrb=6115.2MB/s, minb=6115.2MB/s, maxb=6115.2MB/s, mint=7200004msec, maxt=7200004msec

Run status group 3 (all jobs):
   READ: io=42940GB, aggrb=6107.3MB/s, minb=6107.3MB/s, maxb=6107.3MB/s, mint=7200004msec, maxt=7200004msec

Disk stats (read/write):
  sda: ios=1140567496/0, merge=0/0, ticks=576965480/0, in_queue=579126408, util=100.00%
  sdb: ios=1076150393/0, merge=0/0, ticks=577946696/0, in_queue=579689549, util=100.00%
    md125: ios=2953000615/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=738266895/0, aggrmerge=0/0, aggrticks=591359984/0, aggrin_queue=593141738, aggrutil=100.00%
  nvme0n1: ios=738257583/0, merge=0/0, ticks=573047660/0, in_queue=574571275, util=100.00%
  nvme1n1: ios=738310947/0, merge=0/0, ticks=575743015/0, in_queue=577623860, util=100.00%
  nvme2n1: ios=738272452/0, merge=0/0, ticks=612512489/0, in_queue=614064064, util=100.00%
  nvme3n1: ios=738226598/0, merge=0/0, ticks=604136772/0, in_queue=606307754, util=100.00%
    md124: ios=2962852328/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=740732088/0, aggrmerge=0/0, aggrticks=593828360/0, aggrin_queue=595999812, aggrutil=100.00%
  nvme4n1: ios=740792732/0, merge=0/0, ticks=572496478/0, in_queue=574240598, util=100.00%
  nvme5n1: ios=740639134/0, merge=0/0, ticks=562183428/0, in_queue=564955479, util=100.00%
  nvme6n1: ios=740917899/0, merge=0/0, ticks=641264470/0, in_queue=643373164, util=100.00%
  nvme7n1: ios=740578587/0, merge=0/0, ticks=599369067/0, in_queue=601430009, util=100.00%


###################################################################################
# 
# Tue 22 Dec 2015 09:08:43 PM EST
# 8kB randread with SATA 2 jobs and 2 I/Os Outstanding
# 8kB randread with NVMe 4 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 116060 IOPS | 928 MBps | 171 μsec Average Latency 
SATA-R5-1: 114607 IOPS | 916 MBps | 173 μsec Average Latency 
NVMe-R5-0: 703950 IOPS | 5499 MBps | 226 μsec Average Latency 
NVMe-R5-1: 702976 IOPS | 5492 MBps | 226 μsec Average Latency 

Totals
IOPS: 1637593
MBPS: 12835
Average Latency (μsec): 199

SATA-R5-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-02: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-03: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-04: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-05: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-12: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-13: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-14: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
SATA-R5-15: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-02: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-03: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-04: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-05: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-12: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-13: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-14: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-15: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 60 processes

SATA-R5-01: (groupid=0, jobs=10): err= 0: pid=77127: Tue Dec 22 23:13:46 2015
  read : io=6375.5GB, bw=928484KB/s, iops=116060, runt=7200001msec
    slat (usec): min=2, max=1098, avg= 7.94, stdev= 4.15
    clat (usec): min=1, max=34544, avg=163.03, stdev=64.83
     lat (usec): min=35, max=34551, avg=171.08, stdev=64.95
    clat percentiles (usec):
     |  1.00th=[  120],  5.00th=[  127], 10.00th=[  133], 20.00th=[  141],
     | 30.00th=[  145], 40.00th=[  151], 50.00th=[  157], 60.00th=[  163],
     | 70.00th=[  169], 80.00th=[  181], 90.00th=[  199], 95.00th=[  217],
     | 99.00th=[  255], 99.50th=[  274], 99.90th=[  620], 99.95th=[ 1640],
     | 99.99th=[ 2672]
    bw (KB  /s): min=    0, max=104192, per=10.00%, avg=92852.44, stdev=1395.66
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=98.80%, 500=1.08%, 750=0.03%, 1000=0.02%
    lat (msec) : 2=0.03%, 4=0.04%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=2.86%, sys=13.22%, ctx=800311764, majf=0, minf=23078
  IO depths    : 1=0.1%, 2=104.2%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=835635457/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=10): err= 0: pid=77137: Tue Dec 22 23:13:46 2015
  read : io=6295.6GB, bw=916861KB/s, iops=114607, runt=7200001msec
    slat (usec): min=2, max=2620, avg= 7.05, stdev= 3.81
    clat (usec): min=0, max=13197, avg=166.12, stdev=54.06
     lat (usec): min=40, max=13202, avg=173.25, stdev=54.16
    clat percentiles (usec):
     |  1.00th=[  122],  5.00th=[  131], 10.00th=[  137], 20.00th=[  143],
     | 30.00th=[  149], 40.00th=[  155], 50.00th=[  161], 60.00th=[  167],
     | 70.00th=[  173], 80.00th=[  183], 90.00th=[  201], 95.00th=[  219],
     | 99.00th=[  253], 99.50th=[  270], 99.90th=[  390], 99.95th=[ 1208],
     | 99.99th=[ 2608]
    bw (KB  /s): min=    0, max=104176, per=10.00%, avg=91690.11, stdev=1309.01
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=98.86%, 500=1.05%, 750=0.02%, 1000=0.01%
    lat (msec) : 2=0.03%, 4=0.03%, 10=0.01%, 20=0.01%
  cpu          : usr=2.86%, sys=11.95%, ctx=795766970, majf=0, minf=21954
  IO depths    : 1=0.1%, 2=104.1%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=825174822/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=20): err= 0: pid=77147: Tue Dec 22 23:13:46 2015
  read : io=38669GB, bw=5499.7MB/s, iops=703950, runt=7200001msec
    slat (usec): min=1, max=36057, avg= 4.25, stdev= 8.19
    clat (usec): min=0, max=16846, avg=221.94, stdev=155.93
     lat (usec): min=40, max=16851, avg=226.27, stdev=155.94
    clat percentiles (usec):
     |  1.00th=[   91],  5.00th=[  104], 10.00th=[  112], 20.00th=[  125],
     | 30.00th=[  139], 40.00th=[  153], 50.00th=[  171], 60.00th=[  195],
     | 70.00th=[  229], 80.00th=[  286], 90.00th=[  390], 95.00th=[  516],
     | 99.00th=[  844], 99.50th=[  996], 99.90th=[ 1480], 99.95th=[ 1816],
     | 99.99th=[ 2448]
    bw (KB  /s): min=    0, max=390016, per=5.00%, avg=281579.54, stdev=3590.12
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=2.80%
    lat (usec) : 250=71.57%, 500=20.29%, 750=3.83%, 1000=1.02%
    lat (msec) : 2=0.46%, 4=0.04%, 10=0.01%, 20=0.01%
  cpu          : usr=6.88%, sys=23.06%, ctx=3723643101, majf=0, minf=171294
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=19.4%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=18.6%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=18.6%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5068443781/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=20): err= 0: pid=77167: Tue Dec 22 23:13:46 2015
  read : io=38616GB, bw=5492.7MB/s, iops=702976, runt=7200001msec
    slat (usec): min=1, max=10484, avg= 4.43, stdev= 2.94
    clat (usec): min=0, max=15284, avg=222.01, stdev=153.02
     lat (usec): min=40, max=15286, avg=226.54, stdev=153.04
    clat percentiles (usec):
     |  1.00th=[   92],  5.00th=[  104], 10.00th=[  112], 20.00th=[  126],
     | 30.00th=[  139], 40.00th=[  155], 50.00th=[  171], 60.00th=[  195],
     | 70.00th=[  231], 80.00th=[  286], 90.00th=[  390], 95.00th=[  510],
     | 99.00th=[  836], 99.50th=[  988], 99.90th=[ 1416], 99.95th=[ 1656],
     | 99.99th=[ 2320]
    bw (KB  /s): min=    0, max=382288, per=5.00%, avg=281192.13, stdev=3853.32
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=2.74%, 250=71.52%, 500=20.44%, 750=3.82%, 1000=1.01%
    lat (msec) : 2=0.45%, 4=0.02%, 10=0.01%, 20=0.01%
  cpu          : usr=6.86%, sys=23.81%, ctx=3639975119, majf=0, minf=74593
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=19.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=18.6%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=18.6%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5061434483/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=6375.5GB, aggrb=928483KB/s, minb=928483KB/s, maxb=928483KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=6295.6GB, aggrb=916860KB/s, minb=916860KB/s, maxb=916860KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=38669GB, aggrb=5499.7MB/s, minb=5499.7MB/s, maxb=5499.7MB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=38616GB, aggrb=5492.7MB/s, minb=5492.7MB/s, maxb=5492.7MB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sda: ios=870570458/0, merge=0/0, ticks=142898775/0, in_queue=143193390, util=100.00%
  sdb: ios=859376329/0, merge=0/0, ticks=143153943/0, in_queue=143297146, util=100.00%
    md125: ios=994620390/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=248674559/0, aggrmerge=0/0, aggrticks=287367725/0, aggrin_queue=288670562, aggrutil=100.00%
  nvme0n1: ios=1322407994/0, merge=0/0, ticks=287294572/0, in_queue=288524951, util=100.00%
  nvme1n1: ios=1322434987/0, merge=0/0, ticks=287332215/0, in_queue=288541578, util=100.00%
  nvme2n1: ios=1322411082/0, merge=0/0, ticks=287403160/0, in_queue=288679298, util=100.00%
  nvme3n1: ios=1322411472/0, merge=0/0, ticks=287440954/0, in_queue=288936423, util=100.00%
    md124: ios=999050749/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=249763439/0, aggrmerge=0/0, aggrticks=286767994/0, aggrin_queue=288159703, aggrutil=100.00%
  nvme4n1: ios=1323523743/0, merge=0/0, ticks=286132263/0, in_queue=287194409, util=100.00%
  nvme5n1: ios=1323488921/0, merge=0/0, ticks=285863554/0, in_queue=287763947, util=100.00%
  nvme6n1: ios=1323543998/0, merge=0/0, ticks=287623598/0, in_queue=288917657, util=100.00%
  nvme7n1: ios=1323464393/0, merge=0/0, ticks=287452562/0, in_queue=288762801, util=100.00%


###################################################################################
# 
# Tue 22 Dec 2015 11:14:46 PM EST
# 8kB randread with SATA 4 jobs and 4 I/Os Outstanding
# 8kB randread with NVMe 6 jobs and 10 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 160472 IOPS | 1253 MBps | 497 μsec Average Latency 
SATA-R5-1: 149556 IOPS | 1168 MBps | 533 μsec Average Latency 
NVMe-R5-0: 768138 IOPS | 6001 MBps | 389 μsec Average Latency 
NVMe-R5-1: 765696 IOPS | 5982 MBps | 390 μsec Average Latency 

Totals
IOPS: 1843862
MBPS: 14404
Average Latency (μsec): 452

SATA-R5-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-02: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-03: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-04: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-05: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-12: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-13: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-14: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-15: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
NVMe-R5-02: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
NVMe-R5-03: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
NVMe-R5-04: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
NVMe-R5-05: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
NVMe-R5-11: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
NVMe-R5-12: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
NVMe-R5-13: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
NVMe-R5-14: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
NVMe-R5-15: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=10
...
fio-2.1.10
Starting 100 processes

SATA-R5-01: (groupid=0, jobs=20): err= 0: pid=82346: Wed Dec 23 01:19:49 2015
  read : io=8815.0GB, bw=1253.8MB/s, iops=160472, runt=7200001msec
    slat (usec): min=2, max=12208, avg=18.50, stdev=14.83
    clat (usec): min=0, max=35079, avg=478.45, stdev=82.04
     lat (usec): min=111, max=35114, avg=497.07, stdev=81.88
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  394], 10.00th=[  414], 20.00th=[  438],
     | 30.00th=[  450], 40.00th=[  462], 50.00th=[  474], 60.00th=[  486],
     | 70.00th=[  498], 80.00th=[  516], 90.00th=[  548], 95.00th=[  572],
     | 99.00th=[  628], 99.50th=[  660], 99.90th=[  860], 99.95th=[ 1448],
     | 99.99th=[ 2928]
    bw (KB  /s): min=    0, max=85024, per=5.00%, avg=64195.55, stdev=673.31
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=70.73%, 750=29.10%, 1000=0.09%
    lat (msec) : 2=0.04%, 4=0.04%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.65%, sys=17.35%, ctx=912321356, majf=0, minf=64426
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1155399651/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=20): err= 0: pid=82366: Wed Dec 23 01:19:49 2015
  read : io=8215.5GB, bw=1168.5MB/s, iops=149556, runt=7200001msec
    slat (usec): min=2, max=13225, avg=19.02, stdev=14.18
    clat (usec): min=0, max=17940, avg=514.26, stdev=71.44
     lat (usec): min=111, max=17959, avg=533.43, stdev=71.60
    clat percentiles (usec):
     |  1.00th=[  406],  5.00th=[  434], 10.00th=[  458], 20.00th=[  478],
     | 30.00th=[  490], 40.00th=[  502], 50.00th=[  510], 60.00th=[  524],
     | 70.00th=[  532], 80.00th=[  548], 90.00th=[  572], 95.00th=[  596],
     | 99.00th=[  652], 99.50th=[  676], 99.90th=[  948], 99.95th=[ 1848],
     | 99.99th=[ 2992]
    bw (KB  /s): min=    0, max=85136, per=5.00%, avg=59829.89, stdev=683.52
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=39.72%, 750=60.10%, 1000=0.08%
    lat (msec) : 2=0.05%, 4=0.04%, 10=0.01%, 20=0.01%
  cpu          : usr=1.59%, sys=17.17%, ctx=851606110, majf=0, minf=66472
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1076809378/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=30): err= 0: pid=82386: Wed Dec 23 01:19:49 2015
  read : io=42195GB, bw=6001.9MB/s, iops=768138, runt=7200002msec
    slat (usec): min=1, max=19383, avg= 5.15, stdev= 7.75
    clat (usec): min=0, max=20115, avg=384.06, stdev=350.39
     lat (usec): min=41, max=20119, avg=389.34, stdev=350.45
    clat percentiles (usec):
     |  1.00th=[   98],  5.00th=[  113], 10.00th=[  126], 20.00th=[  149],
     | 30.00th=[  175], 40.00th=[  205], 50.00th=[  249], 60.00th=[  314],
     | 70.00th=[  414], 80.00th=[  564], 90.00th=[  820], 95.00th=[ 1080],
     | 99.00th=[ 1768], 99.50th=[ 2064], 99.90th=[ 2672], 99.95th=[ 2928],
     | 99.99th=[ 3472]
    bw (KB  /s): min=    0, max=351200, per=3.33%, avg=204839.99, stdev=3263.92
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=1.22%, 250=49.01%, 500=26.01%, 750=11.73%, 1000=5.83%
    lat (msec) : 2=5.62%, 4=0.57%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=4.47%, sys=18.15%, ctx=3392652291, majf=0, minf=147206
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=26.5%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=25.5%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=25.5%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5530600888/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
NVMe-R5-11: (groupid=3, jobs=30): err= 0: pid=82416: Wed Dec 23 01:19:49 2015
  read : io=42061GB, bw=5982.7MB/s, iops=765696, runt=7200002msec
    slat (usec): min=1, max=15690, avg= 5.11, stdev= 7.21
    clat (usec): min=0, max=18652, avg=385.37, stdev=356.17
     lat (usec): min=39, max=18655, avg=390.58, stdev=356.21
    clat percentiles (usec):
     |  1.00th=[   97],  5.00th=[  112], 10.00th=[  124], 20.00th=[  147],
     | 30.00th=[  171], 40.00th=[  201], 50.00th=[  243], 60.00th=[  310],
     | 70.00th=[  414], 80.00th=[  572], 90.00th=[  836], 95.00th=[ 1096],
     | 99.00th=[ 1768], 99.50th=[ 2064], 99.90th=[ 2672], 99.95th=[ 2928],
     | 99.99th=[ 3472]
    bw (KB  /s): min=    0, max=440608, per=3.33%, avg=204188.12, stdev=3478.78
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=1.36%, 250=49.87%, 500=24.55%, 750=11.50%, 1000=6.24%
    lat (msec) : 2=5.89%, 4=0.58%, 10=0.01%, 20=0.01%
  cpu          : usr=4.70%, sys=18.21%, ctx=3541735035, majf=0, minf=128011
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=26.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=25.2%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=25.2%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5513018834/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10

Run status group 0 (all jobs):
   READ: io=8815.0GB, aggrb=1253.8MB/s, minb=1253.8MB/s, maxb=1253.8MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=8215.5GB, aggrb=1168.5MB/s, minb=1168.5MB/s, maxb=1168.5MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=42195GB, aggrb=6001.9MB/s, minb=6001.9MB/s, maxb=6001.9MB/s, mint=7200002msec, maxt=7200002msec

Run status group 3 (all jobs):
   READ: io=42061GB, aggrb=5982.7MB/s, minb=5982.7MB/s, maxb=5982.7MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=1203614027/0, merge=0/0, ticks=569167028/0, in_queue=571419300, util=100.00%
  sdb: ios=1121823594/0, merge=0/0, ticks=572964221/0, in_queue=574928930, util=100.00%
    md125: ios=1479387996/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=369852511/0, aggrmerge=0/0, aggrticks=523754138/0, aggrin_queue=526776964, aggrutil=100.00%
  nvme0n1: ios=1443543318/0, merge=0/0, ticks=513680225/0, in_queue=516563186, util=100.00%
  nvme1n1: ios=1443847591/0, merge=0/0, ticks=512322000/0, in_queue=514978189, util=100.00%
  nvme2n1: ios=1443834297/0, merge=0/0, ticks=540359402/0, in_queue=543372982, util=100.00%
  nvme3n1: ios=1443152136/0, merge=0/0, ticks=528654926/0, in_queue=532193500, util=100.00%
    md124: ios=1470541959/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=367639722/0, aggrmerge=0/0, aggrticks=528440664/0, aggrin_queue=531429316, aggrutil=100.00%
  nvme4n1: ios=1441851103/0, merge=0/0, ticks=510578821/0, in_queue=512869794, util=100.00%
  nvme5n1: ios=1440609052/0, merge=0/0, ticks=502283904/0, in_queue=505964067, util=100.00%
  nvme6n1: ios=1442905384/0, merge=0/0, ticks=569138565/0, in_queue=572169414, util=100.00%
  nvme7n1: ios=1440160646/0, merge=0/0, ticks=531761368/0, in_queue=534713989, util=100.00%


###################################################################################
# 
# Wed 23 Dec 2015 01:20:49 AM EST
# 4kB randread with SATA 2 jobs and 2 I/Os Outstanding
# 4kB randread with NVMe 4 jobs and 12 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 126526 IOPS | 506 MBps | 156 μsec Average Latency 
SATA-R5-1: 123836 IOPS | 495 MBps | 160 μsec Average Latency 
NVMe-R5-0: 1246000 IOPS | 4867 MBps | 191 μsec Average Latency 
NVMe-R5-1: 1262000 IOPS | 4932 MBps | 189 μsec Average Latency 

Totals
IOPS: 2758362
MBPS: 10800
Average Latency (μsec): 174

SATA-R5-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-02: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-03: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-04: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-05: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-12: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-13: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-14: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
SATA-R5-15: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=2
...
NVMe-R5-01: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
NVMe-R5-02: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
NVMe-R5-03: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
NVMe-R5-04: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
NVMe-R5-05: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
NVMe-R5-11: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
NVMe-R5-12: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
NVMe-R5-13: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
NVMe-R5-14: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
NVMe-R5-15: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=12
...
fio-2.1.10
Starting 60 processes

SATA-R5-01: (groupid=0, jobs=10): err= 0: pid=88227: Wed Dec 23 03:25:52 2015
  read : io=3475.2GB, bw=506107KB/s, iops=126526, runt=7200001msec
    slat (usec): min=2, max=15075, avg= 8.52, stdev= 5.27
    clat (usec): min=0, max=33337, avg=148.01, stdev=72.32
     lat (usec): min=38, max=33344, avg=156.67, stdev=72.43
    clat percentiles (usec):
     |  1.00th=[  112],  5.00th=[  120], 10.00th=[  124], 20.00th=[  131],
     | 30.00th=[  135], 40.00th=[  139], 50.00th=[  143], 60.00th=[  149],
     | 70.00th=[  153], 80.00th=[  159], 90.00th=[  169], 95.00th=[  179],
     | 99.00th=[  213], 99.50th=[  239], 99.90th=[ 1224], 99.95th=[ 2040],
     | 99.99th=[ 2704]
    bw (KB  /s): min=    0, max=56888, per=10.00%, avg=50612.34, stdev=844.90
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.02%, 250=99.55%, 500=0.24%, 750=0.04%, 1000=0.02%
    lat (msec) : 2=0.06%, 4=0.05%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=3.22%, sys=15.61%, ctx=849886920, majf=0, minf=18526
  IO depths    : 1=0.1%, 2=104.2%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=910992068/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
SATA-R5-11: (groupid=1, jobs=10): err= 0: pid=88237: Wed Dec 23 03:25:52 2015
  read : io=3401.3GB, bw=495347KB/s, iops=123836, runt=7200005msec
    slat (usec): min=2, max=3136, avg= 9.18, stdev= 5.41
    clat (usec): min=0, max=14038, avg=150.82, stdev=50.71
     lat (usec): min=44, max=14045, avg=160.11, stdev=50.84
    clat percentiles (usec):
     |  1.00th=[  115],  5.00th=[  123], 10.00th=[  129], 20.00th=[  135],
     | 30.00th=[  139], 40.00th=[  143], 50.00th=[  149], 60.00th=[  153],
     | 70.00th=[  157], 80.00th=[  163], 90.00th=[  173], 95.00th=[  183],
     | 99.00th=[  207], 99.50th=[  221], 99.90th=[  438], 99.95th=[ 1240],
     | 99.99th=[ 2576]
    bw (KB  /s): min=    0, max=53928, per=10.00%, avg=49536.50, stdev=740.65
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=99.80%, 500=0.09%, 750=0.02%, 1000=0.01%
    lat (msec) : 2=0.03%, 4=0.03%, 10=0.01%, 20=0.01%
  cpu          : usr=3.16%, sys=16.24%, ctx=828769500, majf=0, minf=17851
  IO depths    : 1=0.1%, 2=104.2%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=891625514/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
NVMe-R5-01: (groupid=2, jobs=20): err= 0: pid=88247: Wed Dec 23 03:25:52 2015
  read : io=34224GB, bw=4867.4MB/s, iops=1246.6K, runt=7200001msec
    slat (usec): min=1, max=46655, avg= 4.95, stdev=12.78
    clat (usec): min=0, max=74327, avg=186.46, stdev=127.41
     lat (usec): min=14, max=75418, avg=191.55, stdev=128.23
    clat percentiles (usec):
     |  1.00th=[   78],  5.00th=[   90], 10.00th=[   99], 20.00th=[  112],
     | 30.00th=[  125], 40.00th=[  139], 50.00th=[  155], 60.00th=[  175],
     | 70.00th=[  201], 80.00th=[  239], 90.00th=[  310], 95.00th=[  382],
     | 99.00th=[  588], 99.50th=[  700], 99.90th=[ 1160], 99.95th=[ 1784],
     | 99.99th=[ 2512]
    bw (KB  /s): min=    0, max=337120, per=5.00%, avg=249210.89, stdev=5742.19
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.03%
    lat (usec) : 100=10.55%, 250=23.77%, 500=15.85%, 750=1.53%, 1000=0.24%
    lat (msec) : 2=0.10%, 4=0.04%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=8.93%, sys=34.99%, ctx=3349806023, majf=0, minf=147418
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=8.5%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=8.2%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=8.2%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=8971575091/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=12
NVMe-R5-11: (groupid=3, jobs=20): err= 0: pid=88267: Wed Dec 23 03:25:52 2015
  read : io=34683GB, bw=4932.8MB/s, iops=1262.8K, runt=7200001msec
    slat (usec): min=1, max=40155, avg= 4.55, stdev= 5.57
    clat (usec): min=0, max=23927, avg=184.43, stdev=115.31
     lat (usec): min=29, max=23929, avg=189.06, stdev=115.38
    clat percentiles (usec):
     |  1.00th=[   77],  5.00th=[   88], 10.00th=[   96], 20.00th=[  109],
     | 30.00th=[  122], 40.00th=[  135], 50.00th=[  151], 60.00th=[  171],
     | 70.00th=[  197], 80.00th=[  237], 90.00th=[  310], 95.00th=[  390],
     | 99.00th=[  604], 99.50th=[  708], 99.90th=[ 1064], 99.95th=[ 1448],
     | 99.99th=[ 2352]
    bw (KB  /s): min=    0, max=355352, per=5.00%, avg=252554.21, stdev=5474.05
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=12.36%, 250=22.68%, 500=15.64%, 750=1.67%, 1000=0.27%
    lat (msec) : 2=0.10%, 4=0.02%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=9.95%, sys=36.38%, ctx=3782571566, majf=0, minf=63866
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=9.7%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=9.3%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=9.3%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=9092000703/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=12

Run status group 0 (all jobs):
   READ: io=3475.2GB, aggrb=506106KB/s, minb=506106KB/s, maxb=506106KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=3401.3GB, aggrb=495347KB/s, minb=495347KB/s, maxb=495347KB/s, mint=7200005msec, maxt=7200005msec

Run status group 2 (all jobs):
   READ: io=34224GB, aggrb=4867.4MB/s, minb=4867.4MB/s, maxb=4867.4MB/s, mint=7200001msec, maxt=7200001msec

Run status group 3 (all jobs):
   READ: io=34683GB, aggrb=4932.8MB/s, minb=4932.8MB/s, maxb=4932.8MB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sda: ios=949371479/0, merge=0/0, ticks=140835604/0, in_queue=141154716, util=100.00%
  sdb: ios=929144881/0, merge=0/0, ticks=140500009/0, in_queue=140841330, util=100.00%
    md125: ios=762154049/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=190553592/0, aggrmerge=0/0, aggrticks=388233392/0, aggrin_queue=391684795, aggrutil=100.00%
  nvme0n1: ios=2337597278/0, merge=0/0, ticks=388329887/0, in_queue=391723931, util=100.00%
  nvme1n1: ios=2337573259/0, merge=0/0, ticks=387187628/0, in_queue=390363362, util=100.00%
  nvme2n1: ios=2339002852/0, merge=0/0, ticks=388153523/0, in_queue=391418961, util=100.00%
  nvme3n1: ios=2337975572/0, merge=0/0, ticks=389262533/0, in_queue=393232927, util=100.00%
    md124: ios=877982701/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=219511970/0, aggrmerge=0/0, aggrticks=404133079/0, aggrin_queue=407868730, aggrutil=100.00%
  nvme4n1: ios=2367007797/0, merge=0/0, ticks=402029984/0, in_queue=404779830, util=100.00%
  nvme5n1: ios=2366991337/0, merge=0/0, ticks=400870644/0, in_queue=405549416, util=100.00%
  nvme6n1: ios=2367005796/0, merge=0/0, ticks=405149956/0, in_queue=408849103, util=100.00%
  nvme7n1: ios=2366977543/0, merge=0/0, ticks=408481733/0, in_queue=412296571, util=100.00%


###################################################################################
# 
# Wed 23 Dec 2015 03:26:52 AM EST
# 4kB randread with SATA 4 jobs and 6 I/Os Outstanding
# 4kB randread with NVMe 10 jobs and 16 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 163946 IOPS | 655 MBps | 730 μsec Average Latency 
SATA-R5-1: 156686 IOPS | 626 MBps | 764 μsec Average Latency 
NVMe-R5-0: 1345000 IOPS | 5255 MBps | 592 μsec Average Latency 
NVMe-R5-1: 1330000 IOPS | 5196 MBps | 599 μsec Average Latency 

Totals
IOPS: 2995632
MBPS: 11732
Average Latency (μsec): 671

SATA-R5-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
SATA-R5-02: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
SATA-R5-03: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
SATA-R5-04: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
SATA-R5-05: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
SATA-R5-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
SATA-R5-12: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
SATA-R5-13: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
SATA-R5-14: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
SATA-R5-15: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
NVMe-R5-01: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-02: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-03: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-04: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-05: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-11: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-12: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-13: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-14: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
NVMe-R5-15: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=16
...
fio-2.1.10
Starting 140 processes

SATA-R5-01: (groupid=0, jobs=20): err= 0: pid=93856: Wed Dec 23 05:31:56 2015
  read : io=4502.1GB, bw=655785KB/s, iops=163946, runt=7200002msec
    slat (usec): min=1, max=671930, avg=12.01, stdev=38.36
    clat (usec): min=5, max=9999.4K, avg=718.43, stdev=5400.75
     lat (usec): min=29, max=9999.4K, avg=730.53, stdev=5400.98
    clat percentiles (usec):
     |  1.00th=[  588],  5.00th=[  644], 10.00th=[  652], 20.00th=[  668],
     | 30.00th=[  676], 40.00th=[  692], 50.00th=[  708], 60.00th=[  716],
     | 70.00th=[  732], 80.00th=[  748], 90.00th=[  772], 95.00th=[  788],
     | 99.00th=[  916], 99.50th=[ 1224], 99.90th=[ 2896], 99.95th=[ 3248],
     | 99.99th=[ 7712]
    bw (KB  /s): min=    0, max=104432, per=5.02%, avg=32942.64, stdev=2119.05
    lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.23%
    lat (usec) : 500=0.24%, 750=81.42%, 1000=17.33%
    lat (msec) : 2=0.53%, 4=0.22%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.66%, sys=13.24%, ctx=862982101, majf=0, minf=58508
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1180413338/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
SATA-R5-11: (groupid=1, jobs=20): err= 0: pid=93876: Wed Dec 23 05:31:56 2015
  read : io=4303.6GB, bw=626745KB/s, iops=156686, runt=7200001msec
    slat (usec): min=2, max=24073, avg=13.66, stdev=10.31
    clat (usec): min=7, max=27786, avg=750.64, stdev=115.13
     lat (usec): min=94, max=27912, avg=764.44, stdev=114.41
    clat percentiles (usec):
     |  1.00th=[  660],  5.00th=[  684], 10.00th=[  692], 20.00th=[  700],
     | 30.00th=[  716], 40.00th=[  732], 50.00th=[  740], 60.00th=[  756],
     | 70.00th=[  772], 80.00th=[  788], 90.00th=[  804], 95.00th=[  820],
     | 99.00th=[  876], 99.50th=[ 1144], 99.90th=[ 2768], 99.95th=[ 3120],
     | 99.99th=[ 3472]
    bw (KB  /s): min=    0, max=75592, per=5.00%, avg=31342.31, stdev=378.31
    lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.02%, 750=55.46%, 1000=43.88%
    lat (msec) : 2=0.42%, 4=0.21%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.63%, sys=14.45%, ctx=832803764, majf=0, minf=55805
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1128141148/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
NVMe-R5-01: (groupid=2, jobs=50): err= 0: pid=93896: Wed Dec 23 05:31:56 2015
  read : io=36956GB, bw=5255.1MB/s, iops=1345.6K, runt=7200004msec
    slat (usec): min=1, max=10192K, avg= 8.45, stdev=142.03
    clat (usec): min=0, max=10521K, avg=584.23, stdev=1257.31
     lat (usec): min=29, max=10521K, avg=592.89, stdev=1267.75
    clat percentiles (usec):
     |  1.00th=[   86],  5.00th=[  109], 10.00th=[  131], 20.00th=[  173],
     | 30.00th=[  217], 40.00th=[  282], 50.00th=[  362], 60.00th=[  478],
     | 70.00th=[  660], 80.00th=[  860], 90.00th=[ 1208], 95.00th=[ 1656],
     | 99.00th=[ 3312], 99.50th=[ 4192], 99.90th=[ 6368], 99.95th=[ 7840],
     | 99.99th=[26752]
    bw (KB  /s): min=    0, max=319408, per=2.00%, avg=107709.88, stdev=8585.79
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.04%, 50=0.12%
    lat (usec) : 100=2.85%, 250=32.37%, 500=26.20%, 750=13.25%, 1000=10.34%
    lat (msec) : 2=11.51%, 4=2.73%, 10=0.54%, 20=0.02%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=3.52%, sys=14.30%, ctx=3319108219, majf=0, minf=249510
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=15.6%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=15.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=15.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=9687687555/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
NVMe-R5-11: (groupid=3, jobs=50): err= 0: pid=93946: Wed Dec 23 05:31:56 2015
  read : io=36536GB, bw=5196.3MB/s, iops=1330.3K, runt=7200006msec
    slat (usec): min=1, max=28945, avg= 7.16, stdev=24.22
    clat (usec): min=0, max=39546, avg=592.62, stdev=843.47
     lat (usec): min=29, max=39652, avg=599.94, stdev=843.47
    clat percentiles (usec):
     |  1.00th=[   84],  5.00th=[  105], 10.00th=[  123], 20.00th=[  159],
     | 30.00th=[  195], 40.00th=[  239], 50.00th=[  302], 60.00th=[  398],
     | 70.00th=[  572], 80.00th=[  828], 90.00th=[ 1240], 95.00th=[ 1848],
     | 99.00th=[ 4832], 99.50th=[ 6048], 99.90th=[ 7840], 99.95th=[ 8384],
     | 99.99th=[ 9408]
    bw (KB  /s): min=    0, max=322520, per=2.00%, avg=106426.00, stdev=3867.24
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.03%, 50=0.10%
    lat (usec) : 100=3.39%, 250=38.25%, 500=24.89%, 750=10.04%, 1000=8.58%
    lat (msec) : 2=10.29%, 4=2.95%, 10=1.47%, 20=0.01%, 50=0.01%
  cpu          : usr=3.74%, sys=14.61%, ctx=3705929642, majf=0, minf=199957
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=14.6%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=14.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=14.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=9577646544/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16

Run status group 0 (all jobs):
   READ: io=4502.1GB, aggrb=655785KB/s, minb=655785KB/s, maxb=655785KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=4303.6GB, aggrb=626745KB/s, minb=626745KB/s, maxb=626745KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=36956GB, aggrb=5255.1MB/s, minb=5255.1MB/s, maxb=5255.1MB/s, mint=7200004msec, maxt=7200004msec

Run status group 3 (all jobs):
   READ: io=36536GB, aggrb=5196.3MB/s, minb=5196.3MB/s, maxb=5196.3MB/s, mint=7200006msec, maxt=7200006msec

Disk stats (read/write):
  sda: ios=1229985399/0, merge=0/0, ticks=869020738/0, in_queue=873978632, util=100.00%
  sdb: ios=1175285835/0, merge=0/0, ticks=870377117/0, in_queue=874611250, util=100.00%
    md125: ios=1510592612/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=377675012/0, aggrmerge=0/0, aggrticks=67452739/0, aggrin_queue=79035522, aggrutil=100.00%
  nvme0n1: ios=2524285694/0, merge=0/0, ticks=926104807/0, in_queue=934916568, util=100.00%
  nvme1n1: ios=2522887223/0, merge=0/0, ticks=940890789/0, in_queue=949423490, util=100.00%
  nvme2n1: ios=2530516107/0, merge=0/0, ticks=1437129131/0, in_queue=1451149981, util=100.00%
  nvme3n1: ios=2522945616/0, merge=0/0, ticks=1260653526/0, in_queue=1275619348, util=100.00%
    md124: ios=1399410898/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=349873913/0, aggrmerge=0/0, aggrticks=193001695/0, aggrin_queue=207948037, aggrutil=100.00%
  nvme4n1: ios=2495314323/0, merge=0/0, ticks=861978774/0, in_queue=869188700, util=100.00%
  nvme5n1: ios=2491964778/0, merge=0/0, ticks=783233449/0, in_queue=793993971, util=100.00%
  nvme6n1: ios=2504260924/0, merge=0/0, ticks=1992922499/0, in_queue=2017698857, util=100.00%
  nvme7n1: ios=2497890220/0, merge=0/0, ticks=1428839356/0, in_queue=1445877918, util=100.00%


###################################################################################
# 
# Wed 23 Dec 2015 01:56:32 PM EST
# 8kB randread with SATA 6 jobs and 4 I/Os Outstanding
# 8kB randread with NVMe 16 jobs and 8 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 163518 IOPS | 1277 MBps | 732 μsec Average Latency 
SATA-R5-1: 160981 IOPS | 1257 MBps | 743 μsec Average Latency 
NVMe-R5-0: 771547 IOPS | 6027 MBps | 828 μsec Average Latency 
NVMe-R5-1: 765613 IOPS | 5981 MBps | 834 μsec Average Latency 

Totals
IOPS: 1861659
MBPS: 14542
Average Latency (μsec): 784

SATA-R5-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-02: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-03: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-04: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-05: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-12: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-13: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-14: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
SATA-R5-15: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-02: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-03: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-04: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-05: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-11: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-12: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-13: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-14: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
NVMe-R5-15: (g=3): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 220 processes

SATA-R5-01: (groupid=0, jobs=30): err= 0: pid=9319: Wed Dec 23 16:01:36 2015
  read : io=8982.4GB, bw=1277.6MB/s, iops=163518, runt=7200003msec
    slat (usec): min=2, max=266745, avg=14.73, stdev=16.05
    clat (usec): min=0, max=442174, avg=717.61, stdev=133.19
     lat (usec): min=110, max=442183, avg=732.37, stdev=133.94
    clat percentiles (usec):
     |  1.00th=[  604],  5.00th=[  636], 10.00th=[  652], 20.00th=[  668],
     | 30.00th=[  684], 40.00th=[  692], 50.00th=[  708], 60.00th=[  724],
     | 70.00th=[  740], 80.00th=[  756], 90.00th=[  780], 95.00th=[  812],
     | 99.00th=[  876], 99.50th=[  932], 99.90th=[ 2320], 99.95th=[ 3024],
     | 99.99th=[ 6048]
    bw (KB  /s): min=    0, max=116752, per=3.33%, avg=43614.37, stdev=1692.23
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.09%, 500=0.10%, 750=76.34%, 1000=23.08%
    lat (msec) : 2=0.24%, 4=0.12%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%
  cpu          : usr=1.19%, sys=9.62%, ctx=983751928, majf=0, minf=97822
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1177336902/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=30): err= 0: pid=9349: Wed Dec 23 16:01:36 2015
  read : io=8842.2GB, bw=1257.7MB/s, iops=160981, runt=7200003msec
    slat (usec): min=2, max=15642, avg=17.06, stdev=13.71
    clat (usec): min=0, max=20499, avg=726.75, stdev=99.56
     lat (usec): min=111, max=20537, avg=743.92, stdev=99.39
    clat percentiles (usec):
     |  1.00th=[  612],  5.00th=[  644], 10.00th=[  660], 20.00th=[  676],
     | 30.00th=[  692], 40.00th=[  708], 50.00th=[  716], 60.00th=[  732],
     | 70.00th=[  748], 80.00th=[  764], 90.00th=[  796], 95.00th=[  820],
     | 99.00th=[  892], 99.50th=[  948], 99.90th=[ 2160], 99.95th=[ 2800],
     | 99.99th=[ 3376]
    bw (KB  /s): min=    0, max=100464, per=3.33%, avg=42937.72, stdev=562.88
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.02%, 750=71.25%, 1000=28.33%
    lat (msec) : 2=0.26%, 4=0.12%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.18%, sys=10.68%, ctx=963830672, majf=0, minf=109876
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1159067219/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=80): err= 0: pid=9379: Wed Dec 23 16:01:36 2015
  read : io=42382GB, bw=6027.8MB/s, iops=771547, runt=7200004msec
    slat (usec): min=1, max=544886, avg= 5.20, stdev=31.85
    clat (usec): min=0, max=889195, avg=822.85, stdev=1046.61
     lat (usec): min=42, max=889200, avg=828.16, stdev=1047.25
    clat percentiles (usec):
     |  1.00th=[  100],  5.00th=[  118], 10.00th=[  135], 20.00th=[  167],
     | 30.00th=[  211], 40.00th=[  286], 50.00th=[  414], 60.00th=[  636],
     | 70.00th=[  980], 80.00th=[ 1336], 90.00th=[ 1896], 95.00th=[ 2576],
     | 99.00th=[ 4960], 99.50th=[ 6560], 99.90th=[10048], 99.95th=[11072],
     | 99.99th=[12864]
    bw (KB  /s): min=    0, max=242960, per=1.25%, avg=77164.73, stdev=2049.01
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.90%, 250=34.94%, 500=18.64%, 750=9.45%, 1000=6.64%
    lat (msec) : 2=20.44%, 4=7.30%, 10=1.58%, 20=0.10%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
  cpu          : usr=2.02%, sys=7.44%, ctx=4705904020, majf=0, minf=540706
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=26.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=25.8%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=25.8%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5555145848/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
NVMe-R5-11: (groupid=3, jobs=80): err= 0: pid=9459: Wed Dec 23 16:01:36 2015
  read : io=42056GB, bw=5981.4MB/s, iops=765613, runt=7200005msec
    slat (usec): min=1, max=16699, avg= 5.06, stdev= 5.61
    clat (usec): min=0, max=24403, avg=829.43, stdev=1103.16
     lat (usec): min=42, max=24410, avg=834.59, stdev=1103.12
    clat percentiles (usec):
     |  1.00th=[   99],  5.00th=[  115], 10.00th=[  129], 20.00th=[  157],
     | 30.00th=[  195], 40.00th=[  255], 50.00th=[  362], 60.00th=[  548],
     | 70.00th=[  900], 80.00th=[ 1368], 90.00th=[ 2064], 95.00th=[ 2832],
     | 99.00th=[ 5344], 99.50th=[ 6752], 99.90th=[ 9792], 99.95th=[10688],
     | 99.99th=[12352]
    bw (KB  /s): min=    0, max=305520, per=1.25%, avg=76570.33, stdev=2049.92
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=1.13%, 250=38.27%, 500=18.51%, 750=8.56%, 1000=5.52%
    lat (msec) : 2=17.47%, 4=8.40%, 10=2.05%, 20=0.09%, 50=0.01%
  cpu          : usr=2.13%, sys=7.40%, ctx=4862947328, majf=0, minf=309668
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=26.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=25.3%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=25.3%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=5512424206/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=8982.4GB, aggrb=1277.6MB/s, minb=1277.6MB/s, maxb=1277.6MB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=8842.2GB, aggrb=1257.7MB/s, minb=1257.7MB/s, maxb=1257.7MB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=42382GB, aggrb=6027.8MB/s, minb=6027.8MB/s, maxb=6027.8MB/s, mint=7200004msec, maxt=7200004msec

Run status group 3 (all jobs):
   READ: io=42056GB, aggrb=5981.4MB/s, minb=5981.4MB/s, maxb=5981.4MB/s, mint=7200005msec, maxt=7200005msec

Disk stats (read/write):
  sda: ios=1226585008/0, merge=0/0, ticks=862790837/0, in_queue=865834886, util=100.00%
  sdb: ios=1207479350/0, merge=0/0, ticks=858953864/0, in_queue=862692103, util=100.00%
    md125: ios=1506541843/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=376653323/0, aggrmerge=0/0, aggrticks=91719067/0, aggrin_queue=100091343, aggrutil=100.00%
  nvme0n1: ios=1449917183/0, merge=0/0, ticks=943806766/0, in_queue=952265067, util=100.00%
  nvme1n1: ios=1449822019/0, merge=0/0, ticks=955737713/0, in_queue=961183380, util=100.00%
  nvme2n1: ios=1451895048/0, merge=0/0, ticks=1478107178/0, in_queue=1490201268, util=100.00%
  nvme3n1: ios=1449946340/0, merge=0/0, ticks=1284191909/0, in_queue=1291682953, util=100.00%
    md124: ios=1472902336/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=368240126/0, aggrmerge=0/0, aggrticks=99984030/0, aggrin_queue=105040209, aggrutil=100.00%
  nvme4n1: ios=1441343968/0, merge=0/0, ticks=926460170/0, in_queue=931248251, util=100.00%
  nvme5n1: ios=1440283142/0, merge=0/0, ticks=887745575/0, in_queue=892377741, util=100.00%
  nvme6n1: ios=1443953447/0, merge=0/0, ticks=1598797417/0, in_queue=1603821655, util=100.00%
  nvme7n1: ios=1442347243/0, merge=0/0, ticks=1281900257/0, in_queue=1287680485, util=100.00%


###################################################################################
# 
# Wed 23 Dec 2015 04:02:36 PM EST
# 4kB randread with SATA 6 jobs and 4 I/Os Outstanding
# 4kB randread with NVMe 24 jobs and 10 I/Os Outstanding
# CentOS Linux release 7.2.1511  Linux 3.10.0-327.3.1.el7.x86_64
# 
###################################################################################

Summary
SATA-R5-0: 167840 IOPS | 671 MBps | 713 μsec Average Latency 
SATA-R5-1: 167904 IOPS | 671 MBps | 713 μsec Average Latency 
NVMe-R5-0: 1348000 IOPS | 5268 MBps | 887 μsec Average Latency 
NVMe-R5-1: 1334000 IOPS | 5214 MBps | 897 μsec Average Latency 

Totals
IOPS: 3017744
MBPS: 11824
Average Latency (μsec): 802

SATA-R5-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
SATA-R5-02: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
SATA-R5-03: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
SATA-R5-04: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
SATA-R5-05: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
SATA-R5-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
SATA-R5-12: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
SATA-R5-13: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
SATA-R5-14: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
SATA-R5-15: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
NVMe-R5-01: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
NVMe-R5-02: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
NVMe-R5-03: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
NVMe-R5-04: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
NVMe-R5-05: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
NVMe-R5-11: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
NVMe-R5-12: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
NVMe-R5-13: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
NVMe-R5-14: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
NVMe-R5-15: (g=3): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=10
...
fio-2.1.10
Starting 300 processes

SATA-R5-01: (groupid=0, jobs=30): err= 0: pid=15279: Wed Dec 23 18:07:43 2015
  read : io=4609.9GB, bw=671362KB/s, iops=167840, runt=7200002msec
    slat (usec): min=2, max=228714, avg=10.12, stdev=15.23
    clat (usec): min=0, max=9728.1K, avg=703.33, stdev=1000.30
     lat (usec): min=103, max=9728.1K, avg=713.50, stdev=1000.55
    clat percentiles (usec):
     |  1.00th=[  564],  5.00th=[  628], 10.00th=[  644], 20.00th=[  652],
     | 30.00th=[  660], 40.00th=[  676], 50.00th=[  684], 60.00th=[  700],
     | 70.00th=[  716], 80.00th=[  740], 90.00th=[  756], 95.00th=[  772],
     | 99.00th=[  956], 99.50th=[ 1528], 99.90th=[ 3088], 99.95th=[ 3664],
     | 99.99th=[ 7648]
    bw (KB  /s): min=    0, max=83664, per=3.33%, avg=22387.80, stdev=1573.32
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.09%, 500=0.11%, 750=86.57%, 1000=12.30%
    lat (msec) : 2=0.62%, 4=0.28%, 10=0.04%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, >=2000=0.01%
  cpu          : usr=1.22%, sys=7.29%, ctx=961105477, majf=0, minf=70722
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1208451752/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
SATA-R5-11: (groupid=1, jobs=30): err= 0: pid=15309: Wed Dec 23 18:07:43 2015
  read : io=4611.7GB, bw=671617KB/s, iops=167904, runt=7200002msec
    slat (usec): min=2, max=80119, avg= 8.61, stdev=13.10
    clat (usec): min=0, max=111193, avg=704.65, stdev=188.62
     lat (usec): min=103, max=111324, avg=713.29, stdev=190.02
    clat percentiles (usec):
     |  1.00th=[  548],  5.00th=[  596], 10.00th=[  636], 20.00th=[  652],
     | 30.00th=[  660], 40.00th=[  668], 50.00th=[  684], 60.00th=[  700],
     | 70.00th=[  716], 80.00th=[  740], 90.00th=[  756], 95.00th=[  780],
     | 99.00th=[ 1352], 99.50th=[ 1784], 99.90th=[ 3056], 99.95th=[ 3536],
     | 99.99th=[ 5664]
    bw (KB  /s): min=    0, max=95208, per=3.33%, avg=22393.17, stdev=2421.67
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.02%, 500=0.27%, 750=86.78%, 1000=11.01%
    lat (msec) : 2=1.54%, 4=0.34%, 10=0.03%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.18%, sys=6.74%, ctx=955063009, majf=0, minf=70632
  IO depths    : 1=0.1%, 2=0.1%, 4=104.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1208911140/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
NVMe-R5-01: (groupid=2, jobs=120): err= 0: pid=15339: Wed Dec 23 18:07:43 2015
  read : io=37048GB, bw=5268.2MB/s, iops=1348.9K, runt=7200005msec
    slat (usec): min=1, max=9920.1K, avg= 8.25, stdev=179.18
    clat (usec): min=0, max=10043K, avg=879.41, stdev=1628.48
     lat (usec): min=36, max=10043K, avg=887.90, stdev=1641.00
    clat percentiles (usec):
     |  1.00th=[   87],  5.00th=[  113], 10.00th=[  141], 20.00th=[  195],
     | 30.00th=[  266], 40.00th=[  366], 50.00th=[  502], 60.00th=[  684],
     | 70.00th=[  868], 80.00th=[ 1144], 90.00th=[ 1736], 95.00th=[ 2704],
     | 99.00th=[ 7456], 99.50th=[ 9920], 99.90th=[13376], 99.95th=[14528],
     | 99.99th=[30848]
    bw (KB  /s): min=    0, max=205312, per=0.83%, avg=44988.65, stdev=4971.27
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.04%, 20=0.08%, 50=0.07%
    lat (usec) : 100=2.33%, 250=25.29%, 500=22.06%, 750=13.73%, 1000=11.88%
    lat (msec) : 2=16.55%, 4=5.24%, 10=2.24%, 20=0.47%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2000=0.01%, >=2000=0.01%
  cpu          : usr=1.81%, sys=6.31%, ctx=4549503674, majf=0, minf=354296
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=15.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=15.2%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=15.2%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=9711824682/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
NVMe-R5-11: (groupid=3, jobs=120): err= 0: pid=15459: Wed Dec 23 18:07:43 2015
  read : io=36666GB, bw=5214.8MB/s, iops=1334.1K, runt=7200012msec
    slat (usec): min=1, max=223902, avg= 7.94, stdev=50.79
    clat (usec): min=0, max=224706, avg=889.00, stdev=1664.72
     lat (usec): min=34, max=224709, avg=897.22, stdev=1665.01
    clat percentiles (usec):
     |  1.00th=[   86],  5.00th=[  109], 10.00th=[  131], 20.00th=[  175],
     | 30.00th=[  223], 40.00th=[  294], 50.00th=[  402], 60.00th=[  564],
     | 70.00th=[  772], 80.00th=[ 1032], 90.00th=[ 1672], 95.00th=[ 3024],
     | 99.00th=[10048], 99.50th=[12224], 99.90th=[15040], 99.95th=[15936],
     | 99.99th=[17792]
    bw (KB  /s): min=    0, max=226032, per=0.83%, avg=44512.62, stdev=2926.90
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.03%, 20=0.07%, 50=0.10%
    lat (usec) : 100=2.73%, 250=30.89%, 500=22.76%, 750=12.37%, 1000=10.09%
    lat (msec) : 2=13.02%, 4=4.24%, 10=2.66%, 20=1.03%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=1.83%, sys=6.33%, ctx=4728997178, majf=0, minf=439921
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=14.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=14.3%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=14.3%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=9611743921/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10

Run status group 0 (all jobs):
   READ: io=4609.9GB, aggrb=671361KB/s, minb=671361KB/s, maxb=671361KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=4611.7GB, aggrb=671617KB/s, minb=671617KB/s, maxb=671617KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=37048GB, aggrb=5268.2MB/s, minb=5268.2MB/s, maxb=5268.2MB/s, mint=7200005msec, maxt=7200005msec

Run status group 3 (all jobs):
   READ: io=36666GB, aggrb=5214.8MB/s, minb=5214.8MB/s, maxb=5214.8MB/s, mint=7200012msec, maxt=7200012msec

Disk stats (read/write):
  sda: ios=1259225092/0, merge=0/0, ticks=854892712/0, in_queue=858950016, util=100.00%
  sdb: ios=1259530832/0, merge=0/0, ticks=845593887/0, in_queue=850527416, util=100.00%
    md125: ios=1534791585/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=383757313/0, aggrmerge=0/0, aggrticks=548680173/0, aggrin_queue=571059480, aggrutil=100.00%
  nvme0n1: ios=2532948744/0, merge=0/0, ticks=1228142613/0, in_queue=1249731789, util=100.00%
  nvme1n1: ios=2528860887/0, merge=0/0, ticks=1156917428/0, in_queue=1168344485, util=100.00%
  nvme2n1: ios=2545876071/0, merge=0/0, ticks=2673737196/0, in_queue=2715593462, util=100.00%
  nvme3n1: ios=2517278143/0, merge=0/0, ticks=1430890754/0, in_queue=1445535480, util=100.00%
    md124: ios=1433347452/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=358338109/0, aggrmerge=0/0, aggrticks=677289023/0, aggrin_queue=693152131, aggrutil=100.00%
  nvme4n1: ios=2510594559/0, merge=0/0, ticks=979175234/0, in_queue=989144523, util=100.00%
  nvme5n1: ios=2488367523/0, merge=0/0, ticks=884852681/0, in_queue=894189901, util=100.00%
  nvme6n1: ios=2537832424/0, merge=0/0, ticks=3332478999/0, in_queue=3360526887, util=100.00%
  nvme7n1: ios=2486492525/0, merge=0/0, ticks=1807616476/0, in_queue=1823714512, util=100.00%