###################################################################################
# ION SR-71mach5 
# with (24) Intel DC S3610 1600GB SSDs in (3) RAID 5 arrays of 8.75TiB each
# each with (5) 1.75TiB partitions
# All Partitions on each array are xfs
# format and mount options listed below
###################################################################################

2015 December 04, 12:44
CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
600 s ramp + 7200 s run, per test
Block
Size
 
Test
 Num
Jobs
 ## I/Os
Queued
 
IOPS
 
MBps
Latency (μsec)
8kB randwrite 1 1 45248 352 329
64kB randwrite 1 1 23038 1439 647
8kB randrw 2 2 104713 816 1074
64kB randrw 2 2 39651 2475 2339
1mB write 1 2 5335 5335 5627
1mB read 1 4 10851 10851 5526
256kB write 1 2 33062 8264 904
256kB read 1 4 35237 8808 1700
256kB read 1 14 43297 10823 4848
64kB write 1 4 118831 7425 503
64kB read 1 4 92764 5796 645
64kB read 1 16 154264 9640 1554
64kB randread 1 8 128025 8000 935
64kB randread 4 4 153082 9566 1566
32kB randread 2 8 286734 8959 835
32kB randread 4 8 309483 9670 1549
16kB randread 2 8 448881 7013 533
8kB randread 4 2 455924 3560 261
8kB randread 4 4 474122 3703 504
4kB randread 2 4 474274 1851 251
4kB randread 4 6 487307 1902 737

###################################################################################
# 
# SR-71mach5 Configuration
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################


###################################################################################
# 
# mkfs commands
# 
###################################################################################
#!/bin/bash
#
# format 5 partitions on each of 3 RAID 5 volumes with XFS
#
mkfs.xfs -L C0P1 -d su=65536,sw=7 -l su=64k /dev/sdb1
mkfs.xfs -L C1P1 -d su=65536,sw=7 -l su=64k /dev/sdc1
mkfs.xfs -L C2P1 -d su=65536,sw=7 -l su=64k /dev/sdd1
mkfs.xfs -L C0P2 -d su=65536,sw=7 -l su=64k /dev/sdb2
mkfs.xfs -L C1P2 -d su=65536,sw=7 -l su=64k /dev/sdc2
mkfs.xfs -L C2P2 -d su=65536,sw=7 -l su=64k /dev/sdd2
mkfs.xfs -L C0P3 -d su=65536,sw=7 -l su=64k /dev/sdb3
mkfs.xfs -L C1P3 -d su=65536,sw=7 -l su=64k /dev/sdc3
mkfs.xfs -L C2P3 -d su=65536,sw=7 -l su=64k /dev/sdd3
mkfs.xfs -L C0P4 -d su=65536,sw=7 -l su=64k /dev/sdb4
mkfs.xfs -L C1P4 -d su=65536,sw=7 -l su=64k /dev/sdc4
mkfs.xfs -L C2P4 -d su=65536,sw=7 -l su=64k /dev/sdd4
mkfs.xfs -L C0P5 -d su=65536,sw=7 -l su=64k /dev/sdb5
mkfs.xfs -L C1P5 -d su=65536,sw=7 -l su=64k /dev/sdc5
mkfs.xfs -L C2P5 -d su=65536,sw=7 -l su=64k /dev/sdd5


###################################################################################
# 
# /etc/fstab
# kernel parameter: elevator=deadline
# 
###################################################################################
LABEL=C0P1		/sr71/C0P1		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P2		/sr71/C0P2		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P3		/sr71/C0P3		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P4		/sr71/C0P4		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P5		/sr71/C0P5		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P1		/sr71/C1P1		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P2		/sr71/C1P2		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P3		/sr71/C1P3		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P4		/sr71/C1P4		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P5		/sr71/C1P5		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P1		/sr71/C2P1		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P2		/sr71/C2P2		xfs	sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P3		/sr71/C2P3		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P4		/sr71/C2P4		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P5		/sr71/C2P5		xfs     sync,noatime,nodiratime,nobarrier 0 0

###################################################################################
# 
# actual mount paramters
# 
###################################################################################
/dev/sdb2 on /sr71/C1P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdb5 on /sr71/C1P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sda5 on /sr71/C0P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdb1 on /sr71/C1P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sda1 on /sr71/C0P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sda4 on /sr71/C0P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sda2 on /sr71/C0P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sda3 on /sr71/C0P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdb3 on /sr71/C1P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdb4 on /sr71/C1P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdc2 on /sr71/C2P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdc5 on /sr71/C2P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdc3 on /sr71/C2P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdc4 on /sr71/C2P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)
/dev/sdc1 on /sr71/C2P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota)


###################################################################################
# 
# fio job file
fio-2.1.10
# 
###################################################################################
60 seconds between fio tests
[global]
ioengine=libaio
size=1536g
direct=1
time_based
ramp_time=${RAMPTIME}
runtime=${RUNTIME}
iodepth=${IODEPTH}
bs=${BS}
rw=${RW}
rwmixread=${RWMIXREAD}
numjobs=${NUMJOBS}
group_reporting

# disable some measurements to reduce system overhead
# cannot: breaks ALL latency reporting
# disable_clat=1
# disable_slat=1
# disable_lat=1
# HOWTO lists disable_bw but fio-2.1.10 exits with global error
# disable_bw=1

# write log options seem to keep fio from completing
# write_bw_log=mach5
# write_iops_log=mach5
# write_lat_log=mach5

[xfs-01]
new_group
filename=/sr71/C0P1/fio.tst
[xfs-02]
filename=/sr71/C0P2/fio.tst
[xfs-03]
filename=/sr71/C0P3/fio.tst
[xfs-04]
filename=/sr71/C0P4/fio.tst
[xfs-05]
filename=/sr71/C0P5/fio.tst


[xfs-11]
new_group
filename=/sr71/C1P1/fio.tst
[xfs-12]
filename=/sr71/C1P2/fio.tst
[xfs-13]
filename=/sr71/C1P3/fio.tst
[xfs-14]
filename=/sr71/C1P4/fio.tst
[xfs-15]
filename=/sr71/C1P5/fio.tst

[xfs-21]
new_group
filename=/sr71/C2P1/fio.tst
[xfs-22]
filename=/sr71/C2P2/fio.tst
[xfs-23]
filename=/sr71/C2P3/fio.tst
[xfs-24]
filename=/sr71/C2P4/fio.tst
[xfs-25]
filename=/sr71/C2P5/fio.tst




###################################################################################
# 
# 201512041244 test details
# 
###################################################################################

###################################################################################
# 
# Fri 04 Dec 2015 12:45:05 PM EST
# 8kB randwrite with 1 jobs and 1 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  write: io=854340MB, bw=121506KB/s, iops=15188, runt=7200002msec
     lat (usec): min=32, max=13076, avg=327.11, stdev=392.08
xfs
  write: io=845663MB, bw=120272KB/s, iops=15034, runt=7200001msec
     lat (usec): min=34, max=4047, avg=330.63, stdev=392.45
xfs
  write: io=845258MB, bw=120214KB/s, iops=15026, runt=7200002msec
     lat (usec): min=33, max=11412, avg=330.84, stdev=431.24

Totals
IOPS: 45248
MBPS: 352
Average Latency (μsec): 329

xfs-01: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-02: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-03: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-04: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-05: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-11: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-12: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-13: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-14: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-15: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-21: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-22: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-23: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-24: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
xfs-25: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=63516: Fri Dec  4 14:55:07 2015
  write: io=854340MB, bw=121506KB/s, iops=15188, runt=7200002msec
    slat (usec): min=3, max=10186, avg=13.56, stdev= 5.65
    clat (usec): min=0, max=13063, avg=313.57, stdev=391.98
     lat (usec): min=32, max=13076, avg=327.11, stdev=392.08
    clat percentiles (usec):
     |  1.00th=[   43],  5.00th=[   57], 10.00th=[   63], 20.00th=[   68],
     | 30.00th=[   72], 40.00th=[   75], 50.00th=[   82], 60.00th=[   97],
     | 70.00th=[  302], 80.00th=[  828], 90.00th=[  988], 95.00th=[ 1176],
     | 99.00th=[ 1352], 99.50th=[ 1384], 99.90th=[ 1432], 99.95th=[ 1448],
     | 99.99th=[ 1496]
    bw (KB  /s): min=    0, max=34208, per=20.01%, avg=24315.28, stdev=385.79
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=2.26%
    lat (usec) : 100=58.34%, 250=7.99%, 500=8.70%, 750=1.94%, 1000=11.24%
    lat (msec) : 2=9.53%, 4=0.01%, 10=0.01%, 20=0.01%
  cpu          : usr=1.53%, sys=6.85%, ctx=118514539, majf=0, minf=8609
  IO depths    : 1=108.4%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=109355464/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
xfs-11: (groupid=1, jobs=5): err= 0: pid=63521: Fri Dec  4 14:55:07 2015
  write: io=845663MB, bw=120272KB/s, iops=15034, runt=7200001msec
    slat (usec): min=3, max=2163, avg=13.06, stdev= 5.33
    clat (usec): min=0, max=4036, avg=317.47, stdev=392.26
     lat (usec): min=34, max=4047, avg=330.63, stdev=392.45
    clat percentiles (usec):
     |  1.00th=[   45],  5.00th=[   58], 10.00th=[   64], 20.00th=[   69],
     | 30.00th=[   73], 40.00th=[   77], 50.00th=[   84], 60.00th=[  103],
     | 70.00th=[  310], 80.00th=[  828], 90.00th=[  996], 95.00th=[ 1176],
     | 99.00th=[ 1352], 99.50th=[ 1384], 99.90th=[ 1432], 99.95th=[ 1464],
     | 99.99th=[ 1496]
    bw (KB  /s): min=    0, max=26112, per=20.01%, avg=24068.63, stdev=363.98
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=1.82%
    lat (usec) : 100=57.29%, 250=9.12%, 500=8.69%, 750=2.13%, 1000=11.23%
    lat (msec) : 2=9.72%, 4=0.01%, 10=0.01%
  cpu          : usr=1.46%, sys=6.61%, ctx=117348025, majf=0, minf=9634
  IO depths    : 1=108.4%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=108244857/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
xfs-21: (groupid=2, jobs=5): err= 0: pid=63526: Fri Dec  4 14:55:07 2015
  write: io=845258MB, bw=120214KB/s, iops=15026, runt=7200002msec
    slat (usec): min=3, max=2181, avg=13.20, stdev= 5.46
    clat (usec): min=0, max=11399, avg=317.61, stdev=431.12
     lat (usec): min=33, max=11412, avg=330.84, stdev=431.24
    clat percentiles (usec):
     |  1.00th=[   43],  5.00th=[   56], 10.00th=[   62], 20.00th=[   68],
     | 30.00th=[   71], 40.00th=[   74], 50.00th=[   79], 60.00th=[   87],
     | 70.00th=[  135], 80.00th=[  756], 90.00th=[ 1128], 95.00th=[ 1224],
     | 99.00th=[ 1512], 99.50th=[ 1528], 99.90th=[ 1576], 99.95th=[ 1592],
     | 99.99th=[ 1688]
    bw (KB  /s): min=    0, max=40144, per=20.01%, avg=24057.30, stdev=439.07
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=2.39%
    lat (usec) : 100=64.01%, 250=5.93%, 500=3.71%, 750=3.75%, 1000=5.40%
    lat (msec) : 2=14.79%, 4=0.01%, 10=0.01%, 20=0.01%
  cpu          : usr=1.42%, sys=6.66%, ctx=117267690, majf=0, minf=8989
  IO depths    : 1=108.4%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=108192965/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: io=854340MB, aggrb=121506KB/s, minb=121506KB/s, maxb=121506KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
  WRITE: io=845663MB, aggrb=120272KB/s, minb=120272KB/s, maxb=120272KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=845258MB, aggrb=120214KB/s, minb=120214KB/s, maxb=120214KB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=0/118489725, merge=0/0, ticks=0/37045995, in_queue=36997313, util=100.00%
  sdb: ios=0/117324147, merge=0/0, ticks=0/37084981, in_queue=37042708, util=100.00%
  sdc: ios=0/117243486, merge=0/0, ticks=0/37070433, in_queue=37027547, util=100.00%


###################################################################################
# 
# Fri 04 Dec 2015 02:56:07 PM EST
# 64kB randwrite with 1 jobs and 1 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  write: io=3357.2GB, bw=488918KB/s, iops=7639, runt=7200001msec
     lat (usec): min=61, max=694853, avg=650.94, stdev=3159.15
xfs
  write: io=3362.1GB, bw=489762KB/s, iops=7652, runt=7200001msec
     lat (usec): min=59, max=921558, avg=649.89, stdev=3359.52
xfs
  write: io=3404.6GB, bw=495816KB/s, iops=7747, runt=7200001msec
     lat (usec): min=56, max=1131.7K, avg=642.06, stdev=4402.89

Totals
IOPS: 23038
MBPS: 1439
Average Latency (μsec): 647

xfs-01: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-02: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-03: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-04: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-05: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-11: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-12: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-13: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-14: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-15: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-21: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-22: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-23: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-24: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
xfs-25: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=69156: Fri Dec  4 17:06:08 2015
  write: io=3357.2GB, bw=488918KB/s, iops=7639, runt=7200001msec
    slat (usec): min=7, max=1158, avg=25.65, stdev= 9.19
    clat (usec): min=1, max=694800, avg=624.91, stdev=3159.10
     lat (usec): min=61, max=694853, avg=650.94, stdev=3159.15
    clat percentiles (usec):
     |  1.00th=[  103],  5.00th=[  147], 10.00th=[  185], 20.00th=[  270],
     | 30.00th=[  342], 40.00th=[  422], 50.00th=[  510], 60.00th=[  612],
     | 70.00th=[  732], 80.00th=[  892], 90.00th=[ 1160], 95.00th=[ 1416],
     | 99.00th=[ 2040], 99.50th=[ 2320], 99.90th=[ 3024], 99.95th=[ 3344],
     | 99.99th=[ 4576]
    bw (KB  /s): min=    0, max=120576, per=20.21%, avg=98827.62, stdev=12100.67
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.84%, 250=16.86%, 500=31.37%, 750=22.21%, 1000=13.62%
    lat (msec) : 2=14.00%, 4=1.09%, 10=0.01%, 20=0.01%, 750=0.01%
  cpu          : usr=1.70%, sys=5.65%, ctx=59723779, majf=0, minf=22546
  IO depths    : 1=108.5%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=55003240/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
xfs-11: (groupid=1, jobs=5): err= 0: pid=69161: Fri Dec  4 17:06:08 2015
  write: io=3362.1GB, bw=489762KB/s, iops=7652, runt=7200001msec
    slat (usec): min=7, max=1158, avg=25.19, stdev= 8.93
    clat (usec): min=1, max=921537, avg=624.34, stdev=3359.47
     lat (usec): min=59, max=921558, avg=649.89, stdev=3359.52
    clat percentiles (usec):
     |  1.00th=[  101],  5.00th=[  145], 10.00th=[  183], 20.00th=[  270],
     | 30.00th=[  342], 40.00th=[  422], 50.00th=[  510], 60.00th=[  612],
     | 70.00th=[  732], 80.00th=[  892], 90.00th=[ 1144], 95.00th=[ 1400],
     | 99.00th=[ 2024], 99.50th=[ 2288], 99.90th=[ 2928], 99.95th=[ 3248],
     | 99.99th=[ 4448]
    bw (KB  /s): min=    0, max=120192, per=20.24%, avg=99120.29, stdev=12077.31
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.91%, 250=16.84%, 500=31.22%, 750=22.42%, 1000=13.75%
    lat (msec) : 2=13.82%, 4=1.03%, 10=0.01%, 20=0.01%, 750=0.01%
    lat (msec) : 1000=0.01%
  cpu          : usr=1.67%, sys=5.53%, ctx=59708804, majf=0, minf=23357
  IO depths    : 1=108.3%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=55098228/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
xfs-21: (groupid=2, jobs=5): err= 0: pid=69166: Fri Dec  4 17:06:08 2015
  write: io=3404.6GB, bw=495816KB/s, iops=7747, runt=7200001msec
    slat (usec): min=7, max=1175, avg=25.47, stdev= 9.32
    clat (usec): min=1, max=1131.7K, avg=616.23, stdev=4402.68
     lat (usec): min=56, max=1131.7K, avg=642.06, stdev=4402.89
    clat percentiles (usec):
     |  1.00th=[   92],  5.00th=[  124], 10.00th=[  159], 20.00th=[  223],
     | 30.00th=[  302], 40.00th=[  370], 50.00th=[  454], 60.00th=[  556],
     | 70.00th=[  692], 80.00th=[  868], 90.00th=[ 1176], 95.00th=[ 1448],
     | 99.00th=[ 2064], 99.50th=[ 2416], 99.90th=[ 3408], 99.95th=[ 4080],
     | 99.99th=[12224]
    bw (KB  /s): min=    0, max=189696, per=20.28%, avg=100543.08, stdev=17925.36
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=1.92%, 250=21.11%, 500=31.67%, 750=19.14%, 1000=11.35%
    lat (msec) : 2=13.61%, 4=1.15%, 10=0.04%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%, 2000=0.01%
  cpu          : usr=1.64%, sys=5.66%, ctx=60550031, majf=0, minf=26261
  IO depths    : 1=108.5%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=55779323/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: io=3357.2GB, aggrb=488917KB/s, minb=488917KB/s, maxb=488917KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=3362.1GB, aggrb=489761KB/s, minb=489761KB/s, maxb=489761KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=3404.6GB, aggrb=495816KB/s, minb=495816KB/s, maxb=495816KB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sda: ios=0/59691755, merge=0/0, ticks=0/37145570, in_queue=37095400, util=100.00%
  sdb: ios=0/59677509, merge=0/0, ticks=0/37191957, in_queue=37144806, util=100.00%
  sdc: ios=0/60516706, merge=0/0, ticks=0/37236819, in_queue=37192550, util=100.00%


###################################################################################
# 
# Fri 04 Dec 2015 05:07:09 PM EST
# 8kB randrw with 2 jobs and 2 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=1287.9GB, bw=187560KB/s, iops=23444, runt=7200002msec
     lat (usec): min=38, max=18029, avg=634.13, stdev=647.99
  write: io=649827MB, bw=92420KB/s, iops=11552, runt=7200002msec
     lat (usec): min=28, max=10605, avg=438.90, stdev=433.80
xfs
  read : io=1289.9GB, bw=187848KB/s, iops=23480, runt=7200003msec
     lat (usec): min=36, max=16767, avg=624.64, stdev=621.12
  write: io=650824MB, bw=92562KB/s, iops=11570, runt=7200003msec
     lat (usec): min=29, max=12197, avg=455.54, stdev=440.62
xfs
  read : io=1275.8GB, bw=185794KB/s, iops=23224, runt=7200003msec
     lat (usec): min=40, max=21893, avg=649.26, stdev=707.86
  write: io=643686MB, bw=91546KB/s, iops=11443, runt=7200003msec
     lat (usec): min=28, max=12979, avg=424.72, stdev=448.09

Totals
IOPS: 104713
MBPS: 816
Average Latency (μsec): 1074

xfs-01: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-02: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-03: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-04: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-05: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-11: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-12: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-13: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-14: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-15: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-21: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-22: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-23: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-24: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-25: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 30 processes

xfs-01: (groupid=0, jobs=10): err= 0: pid=74995: Fri Dec  4 19:17:11 2015
  read : io=1287.9GB, bw=187560KB/s, iops=23444, runt=7200002msec
    slat (usec): min=2, max=6557, avg=16.44, stdev=11.01
    clat (usec): min=0, max=18021, avg=617.56, stdev=648.04
     lat (usec): min=38, max=18029, avg=634.13, stdev=647.99
    clat percentiles (usec):
     |  1.00th=[  135],  5.00th=[  155], 10.00th=[  169], 20.00th=[  191],
     | 30.00th=[  215], 40.00th=[  247], 50.00th=[  314], 60.00th=[  450],
     | 70.00th=[  676], 80.00th=[ 1064], 90.00th=[ 1336], 95.00th=[ 1752],
     | 99.00th=[ 3248], 99.50th=[ 3504], 99.90th=[ 4128], 99.95th=[ 4576],
     | 99.99th=[ 6624]
    bw (KB  /s): min=    0, max=27760, per=10.01%, avg=18765.68, stdev=570.66
  write: io=649827MB, bw=92420KB/s, iops=11552, runt=7200002msec
    slat (usec): min=2, max=8909, avg=18.11, stdev=11.75
    clat (usec): min=0, max=10590, avg=420.65, stdev=433.17
     lat (usec): min=28, max=10605, avg=438.90, stdev=433.80
    clat percentiles (usec):
     |  1.00th=[   48],  5.00th=[   69], 10.00th=[   83], 20.00th=[  103],
     | 30.00th=[  120], 40.00th=[  139], 50.00th=[  163], 60.00th=[  258],
     | 70.00th=[  532], 80.00th=[  932], 90.00th=[ 1208], 95.00th=[ 1336],
     | 99.00th=[ 1448], 99.50th=[ 1496], 99.90th=[ 1560], 99.95th=[ 1576],
     | 99.99th=[ 1640]
    bw (KB  /s): min=    0, max=12672, per=10.01%, avg=9246.75, stdev=432.04
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.36%
    lat (usec) : 100=5.64%, 250=40.91%, 500=18.12%, 750=9.25%, 1000=5.17%
    lat (msec) : 2=17.79%, 4=2.67%, 10=0.09%, 20=0.01%
  cpu          : usr=1.40%, sys=9.22%, ctx=237898032, majf=0, minf=39880
  IO depths    : 1=0.1%, 2=108.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=168803785/w=83177855/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=10): err= 0: pid=75005: Fri Dec  4 19:17:11 2015
  read : io=1289.9GB, bw=187848KB/s, iops=23480, runt=7200003msec
    slat (usec): min=2, max=8260, avg=17.12, stdev=11.85
    clat (usec): min=0, max=16752, avg=607.44, stdev=621.17
     lat (usec): min=36, max=16767, avg=624.64, stdev=621.12
    clat percentiles (usec):
     |  1.00th=[  137],  5.00th=[  159], 10.00th=[  173], 20.00th=[  195],
     | 30.00th=[  217], 40.00th=[  247], 50.00th=[  310], 60.00th=[  446],
     | 70.00th=[  684], 80.00th=[ 1064], 90.00th=[ 1336], 95.00th=[ 1688],
     | 99.00th=[ 3152], 99.50th=[ 3440], 99.90th=[ 4016], 99.95th=[ 4384],
     | 99.99th=[ 6240]
    bw (KB  /s): min=    0, max=32240, per=10.01%, avg=18794.58, stdev=552.32
  write: io=650824MB, bw=92562KB/s, iops=11570, runt=7200003msec
    slat (usec): min=2, max=11077, avg=18.75, stdev=12.56
    clat (usec): min=0, max=10735, avg=436.69, stdev=440.11
     lat (usec): min=29, max=12197, avg=455.54, stdev=440.62
    clat percentiles (usec):
     |  1.00th=[   53],  5.00th=[   75], 10.00th=[   89], 20.00th=[  109],
     | 30.00th=[  126], 40.00th=[  145], 50.00th=[  171], 60.00th=[  290],
     | 70.00th=[  548], 80.00th=[  964], 90.00th=[ 1240], 95.00th=[ 1352],
     | 99.00th=[ 1464], 99.50th=[ 1496], 99.90th=[ 1560], 99.95th=[ 1592],
     | 99.99th=[ 1656]
    bw (KB  /s): min=    0, max=15168, per=10.01%, avg=9261.11, stdev=418.51
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.24%
    lat (usec) : 100=4.68%, 250=41.56%, 500=18.47%, 750=8.85%, 1000=5.08%
    lat (msec) : 2=18.80%, 4=2.24%, 10=0.07%, 20=0.01%
  cpu          : usr=1.38%, sys=9.51%, ctx=236356371, majf=0, minf=49767
  IO depths    : 1=0.1%, 2=108.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=169062974/w=83305494/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=10): err= 0: pid=75015: Fri Dec  4 19:17:11 2015
  read : io=1275.8GB, bw=185794KB/s, iops=23224, runt=7200003msec
    slat (usec): min=2, max=7545, avg=16.65, stdev=11.50
    clat (usec): min=0, max=21870, avg=632.49, stdev=707.85
     lat (usec): min=40, max=21893, avg=649.26, stdev=707.86
    clat percentiles (usec):
     |  1.00th=[  135],  5.00th=[  153], 10.00th=[  167], 20.00th=[  185],
     | 30.00th=[  205], 40.00th=[  227], 50.00th=[  266], 60.00th=[  398],
     | 70.00th=[  740], 80.00th=[ 1096], 90.00th=[ 1432], 95.00th=[ 2024],
     | 99.00th=[ 3344], 99.50th=[ 3696], 99.90th=[ 4640], 99.95th=[ 5216],
     | 99.99th=[10688]
    bw (KB  /s): min=    0, max=25104, per=10.01%, avg=18588.83, stdev=592.25
  write: io=643686MB, bw=91546KB/s, iops=11443, runt=7200003msec
    slat (usec): min=2, max=7477, avg=18.25, stdev=12.22
    clat (usec): min=0, max=12970, avg=406.34, stdev=447.49
     lat (usec): min=28, max=12979, avg=424.72, stdev=448.09
    clat percentiles (usec):
     |  1.00th=[   50],  5.00th=[   70], 10.00th=[   82], 20.00th=[  100],
     | 30.00th=[  115], 40.00th=[  131], 50.00th=[  149], 60.00th=[  181],
     | 70.00th=[  434], 80.00th=[  964], 90.00th=[ 1176], 95.00th=[ 1288],
     | 99.00th=[ 1592], 99.50th=[ 1624], 99.90th=[ 1688], 99.95th=[ 1704],
     | 99.99th=[ 1736]
    bw (KB  /s): min=    0, max=12752, per=10.01%, avg=9159.77, stdev=433.27
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.30%
    lat (usec) : 100=6.22%, 250=46.44%, 500=13.67%, 750=5.61%, 1000=5.75%
    lat (msec) : 2=18.23%, 4=3.56%, 10=0.20%, 20=0.01%, 50=0.01%
  cpu          : usr=1.35%, sys=9.16%, ctx=235131167, majf=0, minf=47171
  IO depths    : 1=0.1%, 2=108.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=167215087/w=82391864/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=1287.9GB, aggrb=187559KB/s, minb=187559KB/s, maxb=187559KB/s, mint=7200002msec, maxt=7200002msec
  WRITE: io=649827MB, aggrb=92419KB/s, minb=92419KB/s, maxb=92419KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=1289.9GB, aggrb=187847KB/s, minb=187847KB/s, maxb=187847KB/s, mint=7200003msec, maxt=7200003msec
  WRITE: io=650824MB, aggrb=92561KB/s, minb=92561KB/s, maxb=92561KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=1275.8GB, aggrb=185794KB/s, minb=185794KB/s, maxb=185794KB/s, mint=7200003msec, maxt=7200003msec
  WRITE: io=643686MB, aggrb=91546KB/s, minb=91546KB/s, maxb=91546KB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sda: ios=183088677/90211370, merge=0/0, ticks=112675266/37995568, in_queue=150765789, util=100.00%
  sdb: ios=183392830/90361511, merge=0/0, ticks=110949968/39542142, in_queue=150616787, util=100.00%
  sdc: ios=181415930/89383765, merge=0/0, ticks=114329148/36355294, in_queue=150894772, util=100.00%


###################################################################################
# 
# Fri 04 Dec 2015 07:18:11 PM EST
# 64kB randrw with 2 jobs and 2 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=3910.5GB, bw=569495KB/s, iops=8898, runt=7200003msec
     lat (usec): min=55, max=713591, avg=2159.30, stdev=7649.18
  write: io=1926.2GB, bw=280516KB/s, iops=4383, runt=7200003msec
     lat (usec): min=42, max=9870, avg=171.19, stdev=208.13
xfs
  read : io=3921.3GB, bw=571076KB/s, iops=8923, runt=7200012msec
     lat (usec): min=53, max=713479, avg=2154.64, stdev=8015.81
  write: io=1931.6GB, bw=281294KB/s, iops=4395, runt=7200012msec
     lat (usec): min=44, max=12299, avg=167.95, stdev=206.82
xfs
  read : io=3843.2GB, bw=559695KB/s, iops=8745, runt=7200003msec
     lat (usec): min=53, max=676938, avg=2202.07, stdev=7465.60
  write: io=1892.9GB, bw=275662KB/s, iops=4307, runt=7200003msec
     lat (usec): min=41, max=20762, avg=164.23, stdev=208.56

Totals
IOPS: 39651
MBPS: 2475
Average Latency (μsec): 2339

xfs-01: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-02: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-03: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-04: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-05: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-11: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-12: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-13: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-14: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-15: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-21: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-22: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-23: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-24: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
xfs-25: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 30 processes

xfs-01: (groupid=0, jobs=10): err= 0: pid=80368: Fri Dec  4 21:28:13 2015
  read : io=3910.5GB, bw=569495KB/s, iops=8898, runt=7200003msec
    slat (usec): min=4, max=1230, avg=20.35, stdev= 8.61
    clat (usec): min=1, max=713573, avg=2138.75, stdev=7649.07
     lat (usec): min=55, max=713591, avg=2159.30, stdev=7649.18
    clat percentiles (usec):
     |  1.00th=[  386],  5.00th=[  474], 10.00th=[  556], 20.00th=[  748],
     | 30.00th=[ 1080], 40.00th=[ 1368], 50.00th=[ 1720], 60.00th=[ 2160],
     | 70.00th=[ 2672], 80.00th=[ 3216], 90.00th=[ 3920], 95.00th=[ 4640],
     | 99.00th=[ 6240], 99.50th=[ 7008], 99.90th=[ 9792], 99.95th=[12224],
     | 99.99th=[659456]
    bw (KB  /s): min=    0, max=70898, per=10.17%, avg=57933.71, stdev=7938.91
  write: io=1926.2GB, bw=280516KB/s, iops=4383, runt=7200003msec
    slat (usec): min=5, max=1682, avg=24.23, stdev=10.55
    clat (usec): min=0, max=9856, avg=146.72, stdev=206.97
     lat (usec): min=42, max=9870, avg=171.19, stdev=208.13
    clat percentiles (usec):
     |  1.00th=[   45],  5.00th=[   50], 10.00th=[   52], 20.00th=[   58],
     | 30.00th=[   64], 40.00th=[   71], 50.00th=[   80], 60.00th=[   92],
     | 70.00th=[  112], 80.00th=[  155], 90.00th=[  278], 95.00th=[  402],
     | 99.00th=[ 1176], 99.50th=[ 1240], 99.90th=[ 1352], 99.95th=[ 1384],
     | 99.99th=[ 1448]
    bw (KB  /s): min=    0, max=41088, per=10.18%, avg=28542.59, stdev=4410.13
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=1.64%
    lat (usec) : 100=19.59%, 250=7.91%, 500=6.99%, 750=9.05%, 1000=5.00%
    lat (msec) : 2=20.76%, 4=22.79%, 10=6.20%, 20=0.06%, 50=0.01%
    lat (msec) : 750=0.01%
  cpu          : usr=0.95%, sys=4.22%, ctx=97455723, majf=0, minf=336171
  IO depths    : 1=0.1%, 2=108.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=64068179/w=31558061/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=10): err= 0: pid=80378: Fri Dec  4 21:28:13 2015
  read : io=3921.3GB, bw=571076KB/s, iops=8923, runt=7200012msec
    slat (usec): min=4, max=1143, avg=20.46, stdev= 8.70
    clat (usec): min=1, max=713460, avg=2133.98, stdev=8015.71
     lat (usec): min=53, max=713479, avg=2154.64, stdev=8015.81
    clat percentiles (usec):
     |  1.00th=[  386],  5.00th=[  474], 10.00th=[  548], 20.00th=[  740],
     | 30.00th=[ 1080], 40.00th=[ 1352], 50.00th=[ 1704], 60.00th=[ 2128],
     | 70.00th=[ 2640], 80.00th=[ 3184], 90.00th=[ 3920], 95.00th=[ 4640],
     | 99.00th=[ 6240], 99.50th=[ 7008], 99.90th=[ 9664], 99.95th=[11840],
     | 99.99th=[659456]
    bw (KB  /s): min=    0, max=71394, per=10.19%, avg=58194.58, stdev=8417.04
  write: io=1931.6GB, bw=281294KB/s, iops=4395, runt=7200012msec
    slat (usec): min=5, max=1296, avg=24.30, stdev=10.59
    clat (usec): min=0, max=12285, avg=143.41, stdev=205.59
     lat (usec): min=44, max=12299, avg=167.95, stdev=206.82
    clat percentiles (usec):
     |  1.00th=[   44],  5.00th=[   49], 10.00th=[   52], 20.00th=[   57],
     | 30.00th=[   63], 40.00th=[   69], 50.00th=[   78], 60.00th=[   90],
     | 70.00th=[  108], 80.00th=[  149], 90.00th=[  274], 95.00th=[  390],
     | 99.00th=[ 1192], 99.50th=[ 1240], 99.90th=[ 1352], 99.95th=[ 1384],
     | 99.99th=[ 1448]
    bw (KB  /s): min=    0, max=41644, per=10.19%, avg=28668.90, stdev=4627.90
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=1.95%
    lat (usec) : 100=19.85%, 250=7.45%, 500=6.98%, 750=9.40%, 1000=4.97%
    lat (msec) : 2=20.58%, 4=22.66%, 10=6.11%, 20=0.05%, 50=0.01%
    lat (msec) : 750=0.01%
  cpu          : usr=0.96%, sys=4.25%, ctx=97792369, majf=0, minf=363307
  IO depths    : 1=0.1%, 2=108.1%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=64246201/w=31645624/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=10): err= 0: pid=80388: Fri Dec  4 21:28:13 2015
  read : io=3843.2GB, bw=559695KB/s, iops=8745, runt=7200003msec
    slat (usec): min=4, max=1353, avg=20.24, stdev= 8.46
    clat (usec): min=29, max=676924, avg=2181.63, stdev=7465.50
     lat (usec): min=53, max=676938, avg=2202.07, stdev=7465.60
    clat percentiles (usec):
     |  1.00th=[  386],  5.00th=[  470], 10.00th=[  548], 20.00th=[  708],
     | 30.00th=[ 1032], 40.00th=[ 1336], 50.00th=[ 1672], 60.00th=[ 2128],
     | 70.00th=[ 2672], 80.00th=[ 3280], 90.00th=[ 4128], 95.00th=[ 4960],
     | 99.00th=[ 6944], 99.50th=[ 7840], 99.90th=[13376], 99.95th=[18048],
     | 99.99th=[610304]
    bw (KB  /s): min=    0, max=72960, per=10.16%, avg=56851.97, stdev=8280.85
  write: io=1892.9GB, bw=275662KB/s, iops=4307, runt=7200003msec
    slat (usec): min=4, max=1292, avg=24.19, stdev=10.46
    clat (usec): min=0, max=20735, avg=139.79, stdev=207.38
     lat (usec): min=41, max=20762, avg=164.23, stdev=208.56
    clat percentiles (usec):
     |  1.00th=[   44],  5.00th=[   48], 10.00th=[   51], 20.00th=[   55],
     | 30.00th=[   60], 40.00th=[   67], 50.00th=[   75], 60.00th=[   85],
     | 70.00th=[  103], 80.00th=[  141], 90.00th=[  274], 95.00th=[  386],
     | 99.00th=[ 1224], 99.50th=[ 1288], 99.90th=[ 1400], 99.95th=[ 1448],
     | 99.99th=[ 1528]
    bw (KB  /s): min=    0, max=40448, per=10.16%, avg=28005.47, stdev=4558.37
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=2.56%
    lat (usec) : 100=20.06%, 250=6.68%, 500=7.17%, 750=10.08%, 1000=5.01%
    lat (msec) : 2=19.84%, 4=21.15%, 10=7.34%, 20=0.09%, 50=0.01%
    lat (msec) : 750=0.01%
  cpu          : usr=0.94%, sys=4.13%, ctx=95970987, majf=0, minf=292201
  IO depths    : 1=0.1%, 2=108.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=62965689/w=31011963/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=3910.5GB, aggrb=569494KB/s, minb=569494KB/s, maxb=569494KB/s, mint=7200003msec, maxt=7200003msec
  WRITE: io=1926.2GB, aggrb=280515KB/s, minb=280515KB/s, maxb=280515KB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=3921.3GB, aggrb=571076KB/s, minb=571076KB/s, maxb=571076KB/s, mint=7200012msec, maxt=7200012msec
  WRITE: io=1931.6GB, aggrb=281293KB/s, minb=281293KB/s, maxb=281293KB/s, mint=7200012msec, maxt=7200012msec

Run status group 2 (all jobs):
   READ: io=3843.2GB, aggrb=559694KB/s, minb=559694KB/s, maxb=559694KB/s, mint=7200003msec, maxt=7200003msec
  WRITE: io=1892.9GB, aggrb=275661KB/s, minb=275661KB/s, maxb=275661KB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sda: ios=69227566/34096523, merge=0/0, ticks=148196321/4998934, in_queue=153195776, util=100.00%
  sdb: ios=69462141/34212385, merge=0/0, ticks=148271385/4897222, in_queue=153174257, util=100.00%
  sdc: ios=68008284/33493261, merge=0/0, ticks=148557138/4678283, in_queue=153261444, util=100.00%


###################################################################################
# 
# Fri 04 Dec 2015 09:29:13 PM EST
# 1mB write with 1 jobs and 2 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  write: io=12898GB, bw=1834.4MB/s, iops=1834, runt=7200004msec
     lat (msec): min=1, max=57, avg= 5.45, stdev= 2.84
xfs
  write: io=12772GB, bw=1816.5MB/s, iops=1816, runt=7200002msec
     lat (usec): min=476, max=32149, avg=5502.10, stdev=2933.37
xfs
  write: io=11848GB, bw=1685.9MB/s, iops=1685, runt=7200002msec
     lat (msec): min=1, max=46, avg= 5.93, stdev= 3.03

Totals
IOPS: 5335
MBPS: 5335
Average Latency (μsec): 5627

xfs-01: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-02: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-03: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-04: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-05: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-11: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-12: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-13: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-14: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-15: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-21: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-22: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-23: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-24: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-25: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=86628: Fri Dec  4 23:39:14 2015
  write: io=12898GB, bw=1834.4MB/s, iops=1834, runt=7200004msec
    slat (usec): min=41, max=6546, avg=128.72, stdev=19.12
    clat (msec): min=1, max=57, avg= 5.32, stdev= 2.83
     lat (msec): min=1, max=57, avg= 5.45, stdev= 2.84
    clat percentiles (usec):
     |  1.00th=[ 2928],  5.00th=[ 3152], 10.00th=[ 3216], 20.00th=[ 3344],
     | 30.00th=[ 3440], 40.00th=[ 3536], 50.00th=[ 3696], 60.00th=[ 4384],
     | 70.00th=[ 5984], 80.00th=[ 7712], 90.00th=[ 9792], 95.00th=[11328],
     | 99.00th=[14016], 99.50th=[15040], 99.90th=[18304], 99.95th=[19840],
     | 99.99th=[23168]
    bw (KB  /s): min=    1, max=587776, per=20.03%, avg=376203.82, stdev=137817.52
    lat (msec) : 2=0.01%, 4=55.35%, 10=35.81%, 20=8.79%, 50=0.05%
    lat (msec) : 100=0.01%
  cpu          : usr=1.78%, sys=3.89%, ctx=13871510, majf=0, minf=488950
  IO depths    : 1=0.1%, 2=105.6%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=13207356/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=5): err= 0: pid=86633: Fri Dec  4 23:39:14 2015
  write: io=12772GB, bw=1816.5MB/s, iops=1816, runt=7200002msec
    slat (usec): min=39, max=4570, avg=127.81, stdev=20.22
    clat (usec): min=379, max=32019, avg=5373.87, stdev=2929.20
     lat (usec): min=476, max=32149, avg=5502.10, stdev=2933.37
    clat percentiles (usec):
     |  1.00th=[ 2960],  5.00th=[ 3152], 10.00th=[ 3216], 20.00th=[ 3344],
     | 30.00th=[ 3408], 40.00th=[ 3504], 50.00th=[ 3664], 60.00th=[ 4384],
     | 70.00th=[ 6048], 80.00th=[ 7840], 90.00th=[ 9920], 95.00th=[11840],
     | 99.00th=[14400], 99.50th=[15424], 99.90th=[18048], 99.95th=[19584],
     | 99.99th=[23168]
    bw (KB  /s): min=    1, max=587776, per=20.03%, avg=372545.09, stdev=142384.55
    lat (usec) : 500=0.01%
    lat (msec) : 2=0.01%, 4=55.69%, 10=34.67%, 20=9.60%, 50=0.04%
  cpu          : usr=1.73%, sys=3.84%, ctx=13732695, majf=0, minf=498068
  IO depths    : 1=0.1%, 2=105.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=13078105/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=5): err= 0: pid=86638: Fri Dec  4 23:39:14 2015
  write: io=11848GB, bw=1685.9MB/s, iops=1685, runt=7200002msec
    slat (usec): min=48, max=2826, avg=139.77, stdev=26.11
    clat (msec): min=1, max=46, avg= 5.79, stdev= 3.04
     lat (msec): min=1, max=46, avg= 5.93, stdev= 3.03
    clat percentiles (usec):
     |  1.00th=[ 2672],  5.00th=[ 3024], 10.00th=[ 3216], 20.00th=[ 3440],
     | 30.00th=[ 3632], 40.00th=[ 4128], 50.00th=[ 4640], 60.00th=[ 5536],
     | 70.00th=[ 6432], 80.00th=[ 7968], 90.00th=[10176], 95.00th=[11968],
     | 99.00th=[15680], 99.50th=[17280], 99.90th=[21376], 99.95th=[27264],
     | 99.99th=[31872]
    bw (KB  /s): min=    1, max=585728, per=20.03%, avg=345576.51, stdev=112438.00
    lat (msec) : 2=0.08%, 4=37.47%, 10=51.68%, 20=10.63%, 50=0.15%
  cpu          : usr=1.69%, sys=3.99%, ctx=12403319, majf=0, minf=1009342
  IO depths    : 1=0.1%, 2=106.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=12132575/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=12898GB, aggrb=1834.4MB/s, minb=1834.4MB/s, maxb=1834.4MB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
  WRITE: io=12772GB, aggrb=1816.5MB/s, minb=1816.5MB/s, maxb=1816.5MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
  WRITE: io=11848GB, aggrb=1685.9MB/s, minb=1685.9MB/s, maxb=1685.9MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=0/55769369, merge=0/0, ticks=0/291770089, in_queue=291734637, util=100.00%
  sdb: ios=0/55187184, merge=0/0, ticks=0/292238435, in_queue=292203265, util=100.00%
  sdc: ios=0/51426189, merge=0/0, ticks=0/290113425, in_queue=290088876, util=100.00%


###################################################################################
# 
# Fri 04 Dec 2015 11:40:14 PM EST
# 1mB read with 1 jobs and 4 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=25315GB, bw=3600.5MB/s, iops=3600, runt=7200006msec
     lat (usec): min=429, max=27870, avg=5553.09, stdev=665.86
xfs
  read : io=25403GB, bw=3612.9MB/s, iops=3612, runt=7200006msec
     lat (usec): min=446, max=25573, avg=5534.01, stdev=683.13
xfs
  read : io=25590GB, bw=3639.5MB/s, iops=3639, runt=7200008msec
     lat (usec): min=496, max=27456, avg=5493.46, stdev=764.88

Totals
IOPS: 10851
MBPS: 10851
Average Latency (μsec): 5526

xfs-01: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-02: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-03: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-04: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-05: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-11: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-12: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-13: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-14: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-15: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-21: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-22: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-23: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-24: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-25: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=91904: Sat Dec  5 01:50:15 2015
  read : io=25315GB, bw=3600.5MB/s, iops=3600, runt=7200006msec
    slat (usec): min=37, max=4234, avg=137.38, stdev=44.16
    clat (usec): min=359, max=27733, avg=5415.48, stdev=666.48
     lat (usec): min=429, max=27870, avg=5553.09, stdev=665.86
    clat percentiles (usec):
     |  1.00th=[ 4384],  5.00th=[ 4640], 10.00th=[ 4768], 20.00th=[ 4896],
     | 30.00th=[ 5024], 40.00th=[ 5152], 50.00th=[ 5280], 60.00th=[ 5472],
     | 70.00th=[ 5664], 80.00th=[ 5856], 90.00th=[ 6176], 95.00th=[ 6496],
     | 99.00th=[ 7776], 99.50th=[ 8384], 99.90th=[ 9664], 99.95th=[10304],
     | 99.99th=[13504]
    bw (KB  /s): min=    1, max=915456, per=20.02%, avg=737960.57, stdev=14442.96
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.07%, 10=99.86%, 20=0.07%, 50=0.01%
  cpu          : usr=0.29%, sys=11.32%, ctx=26160329, majf=0, minf=1074372
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=25922977/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-11: (groupid=1, jobs=5): err= 0: pid=91909: Sat Dec  5 01:50:15 2015
  read : io=25403GB, bw=3612.9MB/s, iops=3612, runt=7200006msec
    slat (usec): min=37, max=3669, avg=137.29, stdev=43.88
    clat (usec): min=326, max=25406, avg=5396.16, stdev=683.95
     lat (usec): min=446, max=25573, avg=5534.01, stdev=683.13
    clat percentiles (usec):
     |  1.00th=[ 4320],  5.00th=[ 4640], 10.00th=[ 4768], 20.00th=[ 4896],
     | 30.00th=[ 5024], 40.00th=[ 5088], 50.00th=[ 5216], 60.00th=[ 5408],
     | 70.00th=[ 5600], 80.00th=[ 5856], 90.00th=[ 6176], 95.00th=[ 6560],
     | 99.00th=[ 7776], 99.50th=[ 8384], 99.90th=[ 9664], 99.95th=[10304],
     | 99.99th=[13248]
    bw (KB  /s): min=    1, max=909540, per=20.02%, avg=740505.70, stdev=12952.57
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.08%, 10=99.85%, 20=0.07%, 50=0.01%
  cpu          : usr=0.29%, sys=11.34%, ctx=26278784, majf=0, minf=1071255
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=26012366/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-21: (groupid=2, jobs=5): err= 0: pid=91914: Sat Dec  5 01:50:15 2015
  read : io=25590GB, bw=3639.5MB/s, iops=3639, runt=7200008msec
    slat (usec): min=38, max=3179, avg=134.28, stdev=43.28
    clat (usec): min=398, max=27322, avg=5359.06, stdev=765.62
     lat (usec): min=496, max=27456, avg=5493.46, stdev=764.88
    clat percentiles (usec):
     |  1.00th=[ 4320],  5.00th=[ 4576], 10.00th=[ 4704], 20.00th=[ 4832],
     | 30.00th=[ 4960], 40.00th=[ 5024], 50.00th=[ 5152], 60.00th=[ 5280],
     | 70.00th=[ 5536], 80.00th=[ 5792], 90.00th=[ 6176], 95.00th=[ 6624],
     | 99.00th=[ 8384], 99.50th=[ 9024], 99.90th=[10944], 99.95th=[11840],
     | 99.99th=[15808]
    bw (KB  /s): min=    1, max=915672, per=20.02%, avg=745966.10, stdev=14214.33
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.12%, 10=99.66%, 20=0.21%, 50=0.01%
  cpu          : usr=0.29%, sys=11.18%, ctx=26688458, majf=0, minf=693092
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=26204275/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=25315GB, aggrb=3600.5MB/s, minb=3600.5MB/s, maxb=3600.5MB/s, mint=7200006msec, maxt=7200006msec

Run status group 1 (all jobs):
   READ: io=25403GB, aggrb=3612.9MB/s, minb=3612.9MB/s, maxb=3612.9MB/s, mint=7200006msec, maxt=7200006msec

Run status group 2 (all jobs):
   READ: io=25590GB, aggrb=3639.5MB/s, minb=3639.5MB/s, maxb=3639.5MB/s, mint=7200008msec, maxt=7200008msec

Disk stats (read/write):
  sda: ios=112336427/10, merge=0/0, ticks=541986477/0, in_queue=542105457, util=100.00%
  sdb: ios=112721466/10, merge=0/0, ticks=538006282/0, in_queue=538106524, util=100.00%
  sdc: ios=113563500/10, merge=0/0, ticks=532693019/0, in_queue=532839116, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 01:51:16 AM EST
# 256kB write with 1 jobs and 2 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  write: io=19384GB, bw=2756.9MB/s, iops=11027, runt=7200001msec
     lat (usec): min=182, max=21450, avg=904.86, stdev=470.39
xfs
  write: io=19367GB, bw=2754.5MB/s, iops=11017, runt=7200001msec
     lat (usec): min=176, max=23138, avg=905.65, stdev=475.23
xfs
  write: io=19368GB, bw=2754.7MB/s, iops=11018, runt=7200001msec
     lat (usec): min=238, max=11955, avg=905.52, stdev=165.48

Totals
IOPS: 33062
MBPS: 8264
Average Latency (μsec): 904

xfs-01: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-02: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-03: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-04: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-05: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-11: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-12: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-13: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-14: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-15: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-21: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-22: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-23: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-24: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-25: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=98037: Sat Dec  5 04:01:17 2015
  write: io=19384GB, bw=2756.9MB/s, iops=11027, runt=7200001msec
    slat (usec): min=11, max=1861, avg=36.71, stdev= 8.59
    clat (usec): min=131, max=21422, avg=867.87, stdev=470.46
     lat (usec): min=182, max=21450, avg=904.86, stdev=470.39
    clat percentiles (usec):
     |  1.00th=[  516],  5.00th=[  596], 10.00th=[  644], 20.00th=[  692],
     | 30.00th=[  732], 40.00th=[  772], 50.00th=[  812], 60.00th=[  860],
     | 70.00th=[  908], 80.00th=[  980], 90.00th=[ 1096], 95.00th=[ 1256],
     | 99.00th=[ 1528], 99.50th=[ 1640], 99.90th=[10944], 99.95th=[11456],
     | 99.99th=[11968]
    bw (KB  /s): min=    0, max=694784, per=20.01%, avg=564909.75, stdev=17982.50
    lat (usec) : 250=0.01%, 500=0.76%, 750=33.94%, 1000=47.92%
    lat (msec) : 2=17.03%, 4=0.04%, 10=0.17%, 20=0.12%, 50=0.01%
  cpu          : usr=3.02%, sys=8.91%, ctx=73338528, majf=0, minf=204388
  IO depths    : 1=0.1%, 2=108.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=79395693/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=5): err= 0: pid=98042: Sat Dec  5 04:01:17 2015
  write: io=19367GB, bw=2754.5MB/s, iops=11017, runt=7200001msec
    slat (usec): min=11, max=1427, avg=36.65, stdev= 8.48
    clat (usec): min=43, max=23078, avg=868.69, stdev=475.32
     lat (usec): min=176, max=23138, avg=905.65, stdev=475.23
    clat percentiles (usec):
     |  1.00th=[  524],  5.00th=[  596], 10.00th=[  636], 20.00th=[  684],
     | 30.00th=[  724], 40.00th=[  764], 50.00th=[  812], 60.00th=[  852],
     | 70.00th=[  908], 80.00th=[  980], 90.00th=[ 1112], 95.00th=[ 1272],
     | 99.00th=[ 1528], 99.50th=[ 1656], 99.90th=[10688], 99.95th=[11328],
     | 99.99th=[11712]
    bw (KB  /s): min=    0, max=680622, per=20.01%, avg=564425.62, stdev=16624.18
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.52%, 750=35.29%
    lat (usec) : 1000=46.21%
    lat (msec) : 2=17.62%, 4=0.05%, 10=0.19%, 20=0.12%, 50=0.01%
  cpu          : usr=3.01%, sys=8.87%, ctx=73658813, majf=0, minf=198905
  IO depths    : 1=0.1%, 2=108.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=79327664/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=5): err= 0: pid=98047: Sat Dec  5 04:01:17 2015
  write: io=19368GB, bw=2754.7MB/s, iops=11018, runt=7200001msec
    slat (usec): min=11, max=1202, avg=38.01, stdev= 9.05
    clat (usec): min=63, max=11904, avg=867.20, stdev=166.03
     lat (usec): min=238, max=11955, avg=905.52, stdev=165.48
    clat percentiles (usec):
     |  1.00th=[  516],  5.00th=[  620], 10.00th=[  668], 20.00th=[  732],
     | 30.00th=[  780], 40.00th=[  812], 50.00th=[  852], 60.00th=[  900],
     | 70.00th=[  940], 80.00th=[  996], 90.00th=[ 1080], 95.00th=[ 1176],
     | 99.00th=[ 1304], 99.50th=[ 1352], 99.90th=[ 1448], 99.95th=[ 1480],
     | 99.99th=[ 1576]
    bw (KB  /s): min=    0, max=678400, per=20.01%, avg=564469.34, stdev=16099.59
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.84%, 750=23.07%, 1000=56.58%
    lat (msec) : 2=19.51%, 4=0.01%, 10=0.01%, 20=0.01%
  cpu          : usr=3.04%, sys=9.28%, ctx=72008670, majf=0, minf=234313
  IO depths    : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=79333053/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=19384GB, aggrb=2756.9MB/s, minb=2756.9MB/s, maxb=2756.9MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=19367GB, aggrb=2754.5MB/s, minb=2754.5MB/s, maxb=2754.5MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=19368GB, aggrb=2754.7MB/s, minb=2754.7MB/s, maxb=2754.7MB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sda: ios=0/86027599, merge=0/0, ticks=0/74362562, in_queue=74274463, util=100.00%
  sdb: ios=0/85975680, merge=0/0, ticks=0/74401843, in_queue=74317527, util=100.00%
  sdc: ios=0/85952495, merge=0/0, ticks=0/74245750, in_queue=74169219, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 04:02:17 AM EST
# 256kB read with 1 jobs and 4 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=20649GB, bw=2936.8MB/s, iops=11746, runt=7200003msec
     lat (usec): min=115, max=22142, avg=1701.05, stdev=471.42
xfs
  read : io=20655GB, bw=2937.7MB/s, iops=11750, runt=7200002msec
     lat (usec): min=109, max=20650, avg=1700.55, stdev=487.26
xfs
  read : io=20640GB, bw=2935.6MB/s, iops=11741, runt=7200002msec
     lat (usec): min=111, max=28130, avg=1701.78, stdev=557.87

Totals
IOPS: 35237
MBPS: 8808
Average Latency (μsec): 1700

xfs-01: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-02: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-03: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-04: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-05: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-11: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-12: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-13: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-14: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-15: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-21: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-22: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-23: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-24: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
xfs-25: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=104284: Sat Dec  5 06:12:18 2015
  read : io=20649GB, bw=2936.8MB/s, iops=11746, runt=7200003msec
    slat (usec): min=10, max=1027, avg=31.44, stdev= 6.86
    clat (usec): min=88, max=22102, avg=1669.38, stdev=471.34
     lat (usec): min=115, max=22142, avg=1701.05, stdev=471.42
    clat percentiles (usec):
     |  1.00th=[  956],  5.00th=[ 1144], 10.00th=[ 1224], 20.00th=[ 1352],
     | 30.00th=[ 1432], 40.00th=[ 1512], 50.00th=[ 1592], 60.00th=[ 1672],
     | 70.00th=[ 1784], 80.00th=[ 1912], 90.00th=[ 2128], 95.00th=[ 2384],
     | 99.00th=[ 3568], 99.50th=[ 3984], 99.90th=[ 5152], 99.95th=[ 5728],
     | 99.99th=[ 7072]
    bw (KB  /s): min=    0, max=820736, per=20.01%, avg=601785.06, stdev=14045.83
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.09%, 1000=1.38%
    lat (msec) : 2=83.63%, 4=14.41%, 10=0.49%, 20=0.01%, 50=0.01%
  cpu          : usr=0.87%, sys=9.82%, ctx=82972598, majf=0, minf=307002
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=84578115/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-11: (groupid=1, jobs=5): err= 0: pid=104289: Sat Dec  5 06:12:18 2015
  read : io=20655GB, bw=2937.7MB/s, iops=11750, runt=7200002msec
    slat (usec): min=10, max=1174, avg=31.71, stdev= 6.89
    clat (usec): min=86, max=20617, avg=1668.64, stdev=487.19
     lat (usec): min=109, max=20650, avg=1700.55, stdev=487.26
    clat percentiles (usec):
     |  1.00th=[  932],  5.00th=[ 1128], 10.00th=[ 1224], 20.00th=[ 1336],
     | 30.00th=[ 1432], 40.00th=[ 1512], 50.00th=[ 1592], 60.00th=[ 1672],
     | 70.00th=[ 1768], 80.00th=[ 1912], 90.00th=[ 2128], 95.00th=[ 2448],
     | 99.00th=[ 3632], 99.50th=[ 4080], 99.90th=[ 5280], 99.95th=[ 5856],
     | 99.99th=[ 7136]
    bw (KB  /s): min=    0, max=798208, per=20.01%, avg=601961.91, stdev=14916.16
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.12%, 1000=1.63%
    lat (msec) : 2=83.29%, 4=14.39%, 10=0.56%, 20=0.01%, 50=0.01%
  cpu          : usr=0.87%, sys=9.88%, ctx=82407848, majf=0, minf=307397
  IO depths    : 1=0.1%, 2=0.1%, 4=108.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=84602835/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-21: (groupid=2, jobs=5): err= 0: pid=104294: Sat Dec  5 06:12:18 2015
  read : io=20640GB, bw=2935.6MB/s, iops=11741, runt=7200002msec
    slat (usec): min=10, max=1241, avg=31.54, stdev= 6.86
    clat (usec): min=83, max=28095, avg=1670.04, stdev=557.81
     lat (usec): min=111, max=28130, avg=1701.78, stdev=557.87
    clat percentiles (usec):
     |  1.00th=[  916],  5.00th=[ 1112], 10.00th=[ 1208], 20.00th=[ 1336],
     | 30.00th=[ 1416], 40.00th=[ 1496], 50.00th=[ 1576], 60.00th=[ 1656],
     | 70.00th=[ 1752], 80.00th=[ 1880], 90.00th=[ 2128], 95.00th=[ 2448],
     | 99.00th=[ 4016], 99.50th=[ 4896], 99.90th=[ 6432], 99.95th=[ 7072],
     | 99.99th=[ 9024]
    bw (KB  /s): min=    0, max=714240, per=20.01%, avg=601533.55, stdev=17788.54
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.15%, 1000=1.83%
    lat (msec) : 2=84.09%, 4=12.90%, 10=1.02%, 20=0.01%, 50=0.01%
  cpu          : usr=0.87%, sys=9.82%, ctx=82353801, majf=0, minf=292899
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=84542359/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=20649GB, aggrb=2936.8MB/s, minb=2936.8MB/s, maxb=2936.8MB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=20655GB, aggrb=2937.7MB/s, minb=2937.7MB/s, maxb=2937.7MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=20640GB, aggrb=2935.6MB/s, minb=2935.6MB/s, maxb=2935.6MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=91717644/10, merge=0/0, ticks=152886672/0, in_queue=152840387, util=100.00%
  sdb: ios=91574306/10, merge=0/0, ticks=152835763/0, in_queue=152788927, util=100.00%
  sdc: ios=91531358/10, merge=0/0, ticks=152862762/0, in_queue=152824206, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 06:13:18 AM EST
# 256kB read with 1 jobs and 14 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=25267GB, bw=3593.6MB/s, iops=14374, runt=7200006msec
     lat (usec): min=154, max=27007, avg=4868.35, stdev=891.17
xfs
  read : io=25328GB, bw=3602.3MB/s, iops=14409, runt=7200006msec
     lat (usec): min=159, max=25889, avg=4856.60, stdev=1015.66
xfs
  read : io=25514GB, bw=3628.7MB/s, iops=14514, runt=7200006msec
     lat (usec): min=146, max=30376, avg=4821.19, stdev=1049.62

Totals
IOPS: 43297
MBPS: 10823
Average Latency (μsec): 4848

xfs-01: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-02: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-03: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-04: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-05: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-11: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-12: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-13: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-14: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-15: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-21: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-22: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-23: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-24: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
xfs-25: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=109403: Sat Dec  5 08:23:19 2015
  read : io=25267GB, bw=3593.6MB/s, iops=14374, runt=7200006msec
    slat (usec): min=10, max=2159, avg=30.07, stdev= 7.17
    clat (usec): min=127, max=26979, avg=4838.10, stdev=891.08
     lat (usec): min=154, max=27007, avg=4868.35, stdev=891.17
    clat percentiles (usec):
     |  1.00th=[ 2704],  5.00th=[ 3216], 10.00th=[ 3568], 20.00th=[ 4192],
     | 30.00th=[ 4576], 40.00th=[ 4768], 50.00th=[ 4896], 60.00th=[ 5088],
     | 70.00th=[ 5216], 80.00th=[ 5472], 90.00th=[ 5792], 95.00th=[ 6176],
     | 99.00th=[ 7008], 99.50th=[ 7520], 99.90th=[ 8896], 99.95th=[ 9536],
     | 99.99th=[11328]
    bw (KB  /s): min=    0, max=962048, per=20.01%, avg=736315.97, stdev=48130.76
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.09%, 4=16.94%, 10=82.95%, 20=0.03%, 50=0.01%
  cpu          : usr=1.04%, sys=11.50%, ctx=101223845, majf=0, minf=880074
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=103493859/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=14
xfs-11: (groupid=1, jobs=5): err= 0: pid=109408: Sat Dec  5 08:23:19 2015
  read : io=25328GB, bw=3602.3MB/s, iops=14409, runt=7200006msec
    slat (usec): min=10, max=2357, avg=29.83, stdev= 7.06
    clat (usec): min=133, max=25843, avg=4826.61, stdev=1015.57
     lat (usec): min=159, max=25889, avg=4856.60, stdev=1015.66
    clat percentiles (usec):
     |  1.00th=[ 2576],  5.00th=[ 3120], 10.00th=[ 3440], 20.00th=[ 3952],
     | 30.00th=[ 4384], 40.00th=[ 4704], 50.00th=[ 4896], 60.00th=[ 5088],
     | 70.00th=[ 5280], 80.00th=[ 5536], 90.00th=[ 5984], 95.00th=[ 6368],
     | 99.00th=[ 7584], 99.50th=[ 8160], 99.90th=[ 9536], 99.95th=[10176],
     | 99.99th=[12480]
    bw (KB  /s): min=    0, max=914432, per=20.01%, avg=738100.26, stdev=43264.81
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.16%, 4=20.85%, 10=78.92%, 20=0.06%, 50=0.01%
  cpu          : usr=1.02%, sys=11.43%, ctx=101591216, majf=0, minf=785621
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.4%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=103744977/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=14
xfs-21: (groupid=2, jobs=5): err= 0: pid=109413: Sat Dec  5 08:23:19 2015
  read : io=25514GB, bw=3628.7MB/s, iops=14514, runt=7200006msec
    slat (usec): min=10, max=2842, avg=30.09, stdev= 7.07
    clat (usec): min=123, max=30347, avg=4790.88, stdev=1049.53
     lat (usec): min=146, max=30376, avg=4821.19, stdev=1049.62
    clat percentiles (usec):
     |  1.00th=[ 2480],  5.00th=[ 3088], 10.00th=[ 3408], 20.00th=[ 3888],
     | 30.00th=[ 4320], 40.00th=[ 4640], 50.00th=[ 4896], 60.00th=[ 5088],
     | 70.00th=[ 5280], 80.00th=[ 5536], 90.00th=[ 5920], 95.00th=[ 6368],
     | 99.00th=[ 7712], 99.50th=[ 8384], 99.90th=[10176], 99.95th=[10944],
     | 99.99th=[13888]
    bw (KB  /s): min=    0, max=897024, per=20.01%, avg=743515.38, stdev=44147.22
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.27%, 4=22.16%, 10=77.44%, 20=0.11%, 50=0.01%
  cpu          : usr=1.05%, sys=11.59%, ctx=101750101, majf=0, minf=881410
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=104506094/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=14

Run status group 0 (all jobs):
   READ: io=25267GB, aggrb=3593.6MB/s, minb=3593.6MB/s, maxb=3593.6MB/s, mint=7200006msec, maxt=7200006msec

Run status group 1 (all jobs):
   READ: io=25328GB, aggrb=3602.3MB/s, minb=3602.3MB/s, maxb=3602.3MB/s, mint=7200006msec, maxt=7200006msec

Run status group 2 (all jobs):
   READ: io=25514GB, aggrb=3628.7MB/s, minb=3628.7MB/s, maxb=3628.7MB/s, mint=7200006msec, maxt=7200006msec

Disk stats (read/write):
  sda: ios=112107031/0, merge=0/0, ticks=542431494/0, in_queue=542569402, util=100.00%
  sdb: ios=112422879/0, merge=0/0, ticks=542457206/0, in_queue=542586233, util=100.00%
  sdc: ios=113219903/0, merge=0/0, ticks=542411187/0, in_queue=542579483, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 08:24:20 AM EST
# 64kB write with 1 jobs and 4 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  write: io=17338GB, bw=2465.9MB/s, iops=39453, runt=7200001msec
     lat (usec): min=80, max=15197, avg=506.16, stdev=140.38
xfs
  write: io=17491GB, bw=2487.6MB/s, iops=39801, runt=7200001msec
     lat (usec): min=75, max=15048, avg=501.73, stdev=139.81
xfs
  write: io=17393GB, bw=2473.7MB/s, iops=39577, runt=7200001msec
     lat (usec): min=116, max=14836, avg=504.54, stdev=133.76

Totals
IOPS: 118831
MBPS: 7425
Average Latency (μsec): 503

xfs-01: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-02: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-03: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-04: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-05: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-11: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-12: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-13: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-14: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-15: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-21: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-22: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-23: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-24: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-25: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=114517: Sat Dec  5 10:34:21 2015
  write: io=17338GB, bw=2465.9MB/s, iops=39453, runt=7200001msec
    slat (usec): min=3, max=3527, avg=12.64, stdev= 6.04
    clat (usec): min=28, max=15186, avg=493.41, stdev=140.58
     lat (usec): min=80, max=15197, avg=506.16, stdev=140.38
    clat percentiles (usec):
     |  1.00th=[  258],  5.00th=[  302], 10.00th=[  330], 20.00th=[  374],
     | 30.00th=[  410], 40.00th=[  438], 50.00th=[  466], 60.00th=[  498],
     | 70.00th=[  556], 80.00th=[  628], 90.00th=[  692], 95.00th=[  748],
     | 99.00th=[  844], 99.50th=[  916], 99.90th=[ 1020], 99.95th=[ 1048],
     | 99.99th=[ 1096]
    bw (KB  /s): min=    0, max=585472, per=20.00%, avg=505118.06, stdev=20984.49
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.68%, 500=59.85%, 750=34.51%
    lat (usec) : 1000=4.81%
    lat (msec) : 2=0.15%, 4=0.01%, 10=0.01%, 20=0.01%
  cpu          : usr=2.62%, sys=12.23%, ctx=159039477, majf=0, minf=73175
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=284061880/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-11: (groupid=1, jobs=5): err= 0: pid=114522: Sat Dec  5 10:34:21 2015
  write: io=17491GB, bw=2487.6MB/s, iops=39801, runt=7200001msec
    slat (usec): min=3, max=14770, avg=12.89, stdev= 6.29
    clat (usec): min=28, max=15024, avg=488.83, stdev=140.02
     lat (usec): min=75, max=15048, avg=501.73, stdev=139.81
    clat percentiles (usec):
     |  1.00th=[  253],  5.00th=[  298], 10.00th=[  326], 20.00th=[  370],
     | 30.00th=[  406], 40.00th=[  434], 50.00th=[  458], 60.00th=[  490],
     | 70.00th=[  556], 80.00th=[  628], 90.00th=[  684], 95.00th=[  748],
     | 99.00th=[  844], 99.50th=[  908], 99.90th=[  988], 99.95th=[ 1012],
     | 99.99th=[ 1064]
    bw (KB  /s): min=    0, max=580480, per=20.00%, avg=509575.59, stdev=20127.36
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.83%, 500=61.31%, 750=33.15%
    lat (usec) : 1000=4.63%
    lat (msec) : 2=0.08%, 4=0.01%, 10=0.01%, 20=0.01%
  cpu          : usr=2.67%, sys=12.56%, ctx=161002374, majf=0, minf=71320
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=286569590/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-21: (groupid=2, jobs=5): err= 0: pid=114527: Sat Dec  5 10:34:21 2015
  write: io=17393GB, bw=2473.7MB/s, iops=39577, runt=7200001msec
    slat (usec): min=3, max=7859, avg=13.78, stdev= 6.71
    clat (usec): min=26, max=14825, avg=490.72, stdev=134.01
     lat (usec): min=116, max=14836, avg=504.54, stdev=133.76
    clat percentiles (usec):
     |  1.00th=[  266],  5.00th=[  306], 10.00th=[  334], 20.00th=[  378],
     | 30.00th=[  410], 40.00th=[  434], 50.00th=[  462], 60.00th=[  494],
     | 70.00th=[  556], 80.00th=[  620], 90.00th=[  692], 95.00th=[  740],
     | 99.00th=[  812], 99.50th=[  836], 99.90th=[  876], 99.95th=[  900],
     | 99.99th=[  948]
    bw (KB  /s): min=    0, max=574080, per=20.00%, avg=506713.12, stdev=23846.71
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.42%, 500=60.92%, 750=34.19%
    lat (usec) : 1000=4.47%
    lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%
  cpu          : usr=2.73%, sys=13.44%, ctx=157987006, majf=0, minf=73279
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=284958752/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: io=17338GB, aggrb=2465.9MB/s, minb=2465.9MB/s, maxb=2465.9MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=17491GB, aggrb=2487.6MB/s, minb=2487.6MB/s, maxb=2487.6MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=17393GB, aggrb=2473.7MB/s, minb=2473.7MB/s, maxb=2473.7MB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sda: ios=0/307967225, merge=0/0, ticks=0/148747512, in_queue=148720028, util=100.00%
  sdb: ios=0/310568776, merge=0/0, ticks=0/148538126, in_queue=148523944, util=100.00%
  sdc: ios=0/308764826, merge=0/0, ticks=0/147774741, in_queue=147807990, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 10:35:21 AM EST
# 64kB read with 1 jobs and 4 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=13955GB, bw=1984.8MB/s, iops=31755, runt=7200001msec
     lat (usec): min=42, max=15674, avg=628.94, stdev=140.47
xfs
  read : io=13311GB, bw=1893.2MB/s, iops=30290, runt=7200001msec
     lat (usec): min=38, max=14631, avg=659.41, stdev=145.91
xfs
  read : io=13500GB, bw=1919.1MB/s, iops=30719, runt=7200001msec
     lat (usec): min=44, max=17190, avg=650.17, stdev=161.06

Totals
IOPS: 92764
MBPS: 5796
Average Latency (μsec): 645

xfs-01: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-02: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-03: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-04: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-05: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-11: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-12: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-13: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-14: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-15: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-21: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-22: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-23: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-24: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
xfs-25: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=120706: Sat Dec  5 12:45:22 2015
  read : io=13955GB, bw=1984.8MB/s, iops=31755, runt=7200001msec
    slat (usec): min=3, max=843, avg=10.32, stdev= 3.79
    clat (usec): min=2, max=15667, avg=618.53, stdev=140.44
     lat (usec): min=42, max=15674, avg=628.94, stdev=140.47
    clat percentiles (usec):
     |  1.00th=[  386],  5.00th=[  438], 10.00th=[  466], 20.00th=[  510],
     | 30.00th=[  540], 40.00th=[  572], 50.00th=[  604], 60.00th=[  636],
     | 70.00th=[  676], 80.00th=[  716], 90.00th=[  780], 95.00th=[  844],
     | 99.00th=[  988], 99.50th=[ 1064], 99.90th=[ 1384], 99.95th=[ 2288],
     | 99.99th=[ 3248]
    bw (KB  /s): min=    0, max=531584, per=20.00%, avg=406554.33, stdev=18993.37
    lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=17.34%
    lat (usec) : 750=68.33%, 1000=13.44%
    lat (msec) : 2=0.83%, 4=0.06%, 10=0.01%, 20=0.01%
  cpu          : usr=1.43%, sys=9.79%, ctx=223355172, majf=0, minf=53631
  IO depths    : 1=0.1%, 2=0.1%, 4=108.1%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=228640616/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-11: (groupid=1, jobs=5): err= 0: pid=120711: Sat Dec  5 12:45:22 2015
  read : io=13311GB, bw=1893.2MB/s, iops=30290, runt=7200001msec
    slat (usec): min=3, max=1248, avg=10.35, stdev= 3.86
    clat (usec): min=28, max=14624, avg=648.92, stdev=145.84
     lat (usec): min=38, max=14631, avg=659.41, stdev=145.91
    clat percentiles (usec):
     |  1.00th=[  394],  5.00th=[  450], 10.00th=[  486], 20.00th=[  532],
     | 30.00th=[  572], 40.00th=[  604], 50.00th=[  636], 60.00th=[  676],
     | 70.00th=[  708], 80.00th=[  756], 90.00th=[  820], 95.00th=[  876],
     | 99.00th=[ 1032], 99.50th=[ 1096], 99.90th=[ 1416], 99.95th=[ 2288],
     | 99.99th=[ 3280]
    bw (KB  /s): min=    0, max=472960, per=20.00%, avg=387798.47, stdev=16770.30
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=12.64%, 750=66.58%
    lat (usec) : 1000=19.46%
    lat (msec) : 2=1.26%, 4=0.06%, 10=0.01%, 20=0.01%
  cpu          : usr=1.37%, sys=9.39%, ctx=213329175, majf=0, minf=58365
  IO depths    : 1=0.1%, 2=0.1%, 4=108.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=218089251/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-21: (groupid=2, jobs=5): err= 0: pid=120716: Sat Dec  5 12:45:22 2015
  read : io=13500GB, bw=1919.1MB/s, iops=30719, runt=7200001msec
    slat (usec): min=3, max=1280, avg=10.50, stdev= 3.87
    clat (usec): min=23, max=17180, avg=639.59, stdev=160.97
     lat (usec): min=44, max=17190, avg=650.17, stdev=161.06
    clat percentiles (usec):
     |  1.00th=[  398],  5.00th=[  450], 10.00th=[  482], 20.00th=[  524],
     | 30.00th=[  564], 40.00th=[  596], 50.00th=[  620], 60.00th=[  660],
     | 70.00th=[  700], 80.00th=[  740], 90.00th=[  804], 95.00th=[  868],
     | 99.00th=[ 1012], 99.50th=[ 1096], 99.90th=[ 1896], 99.95th=[ 2992],
     | 99.99th=[ 5152]
    bw (KB  /s): min=    0, max=480256, per=20.00%, avg=393294.65, stdev=21035.12
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=13.93%, 750=67.41%
    lat (usec) : 1000=17.53%
    lat (msec) : 2=1.04%, 4=0.07%, 10=0.03%, 20=0.01%
  cpu          : usr=1.40%, sys=9.67%, ctx=214858681, majf=0, minf=55109
  IO depths    : 1=0.1%, 2=0.1%, 4=108.5%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=221179893/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=13955GB, aggrb=1984.8MB/s, minb=1984.8MB/s, maxb=1984.8MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=13311GB, aggrb=1893.2MB/s, minb=1893.2MB/s, maxb=1893.2MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=13500GB, aggrb=1919.1MB/s, minb=1919.1MB/s, maxb=1919.1MB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sda: ios=247133874/10, merge=0/0, ticks=152881473/0, in_queue=152874571, util=100.00%
  sdb: ios=236014034/10, merge=0/0, ticks=152994259/0, in_queue=152984442, util=100.00%
  sdc: ios=240052216/10, merge=0/0, ticks=152879602/0, in_queue=152877382, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 12:46:23 PM EST
# 64kB read with 1 jobs and 16 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=22589GB, bw=3212.8MB/s, iops=51403, runt=7200002msec
     lat (usec): min=43, max=20613, avg=1555.57, stdev=398.21
xfs
  read : io=22342GB, bw=3177.6MB/s, iops=50840, runt=7200002msec
     lat (usec): min=51, max=20199, avg=1572.78, stdev=413.40
xfs
  read : io=22861GB, bw=3251.4MB/s, iops=52021, runt=7200002msec
     lat (usec): min=53, max=23660, avg=1537.09, stdev=398.30

Totals
IOPS: 154264
MBPS: 9640
Average Latency (μsec): 1554

xfs-01: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-02: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-03: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-04: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-05: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-11: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-12: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-13: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-14: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-15: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-21: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-22: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-23: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-24: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-25: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=125833: Sat Dec  5 14:56:24 2015
  read : io=22589GB, bw=3212.8MB/s, iops=51403, runt=7200002msec
    slat (usec): min=3, max=7232, avg=11.42, stdev= 4.78
    clat (usec): min=36, max=20593, avg=1543.95, stdev=398.15
     lat (usec): min=43, max=20613, avg=1555.57, stdev=398.21
    clat percentiles (usec):
     |  1.00th=[  796],  5.00th=[  972], 10.00th=[ 1080], 20.00th=[ 1208],
     | 30.00th=[ 1304], 40.00th=[ 1400], 50.00th=[ 1496], 60.00th=[ 1608],
     | 70.00th=[ 1720], 80.00th=[ 1864], 90.00th=[ 2064], 95.00th=[ 2256],
     | 99.00th=[ 2640], 99.50th=[ 2800], 99.90th=[ 3216], 99.95th=[ 3504],
     | 99.99th=[ 4512]
    bw (KB  /s): min=    0, max=755328, per=20.00%, avg=658044.94, stdev=21662.44
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.55%
    lat (usec) : 1000=5.48%
    lat (msec) : 2=81.27%, 4=12.67%, 10=0.02%, 20=0.01%, 50=0.01%
  cpu          : usr=2.16%, sys=16.73%, ctx=325458800, majf=0, minf=201665
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=108.3%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=370103487/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
xfs-11: (groupid=1, jobs=5): err= 0: pid=125838: Sat Dec  5 14:56:24 2015
  read : io=22342GB, bw=3177.6MB/s, iops=50840, runt=7200002msec
    slat (usec): min=3, max=4595, avg=10.96, stdev= 4.51
    clat (usec): min=42, max=20192, avg=1561.62, stdev=413.34
     lat (usec): min=51, max=20199, avg=1572.78, stdev=413.40
    clat percentiles (usec):
     |  1.00th=[  780],  5.00th=[  964], 10.00th=[ 1064], 20.00th=[ 1208],
     | 30.00th=[ 1320], 40.00th=[ 1416], 50.00th=[ 1512], 60.00th=[ 1624],
     | 70.00th=[ 1752], 80.00th=[ 1896], 90.00th=[ 2128], 95.00th=[ 2320],
     | 99.00th=[ 2672], 99.50th=[ 2832], 99.90th=[ 3184], 99.95th=[ 3376],
     | 99.99th=[ 4192]
    bw (KB  /s): min=    0, max=758016, per=20.00%, avg=650839.39, stdev=24440.60
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.68%
    lat (usec) : 1000=5.71%
    lat (msec) : 2=79.11%, 4=14.47%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=2.09%, sys=16.03%, ctx=326942072, majf=0, minf=194705
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=108.3%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=366051519/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
xfs-21: (groupid=2, jobs=5): err= 0: pid=125843: Sat Dec  5 14:56:24 2015
  read : io=22861GB, bw=3251.4MB/s, iops=52021, runt=7200002msec
    slat (usec): min=3, max=1389, avg=11.72, stdev= 4.80
    clat (usec): min=46, max=23653, avg=1525.31, stdev=398.22
     lat (usec): min=53, max=23660, avg=1537.09, stdev=398.30
    clat percentiles (usec):
     |  1.00th=[  780],  5.00th=[  964], 10.00th=[ 1064], 20.00th=[ 1192],
     | 30.00th=[ 1304], 40.00th=[ 1384], 50.00th=[ 1480], 60.00th=[ 1576],
     | 70.00th=[ 1688], 80.00th=[ 1832], 90.00th=[ 2040], 95.00th=[ 2224],
     | 99.00th=[ 2608], 99.50th=[ 2768], 99.90th=[ 3408], 99.95th=[ 3984],
     | 99.99th=[ 5792]
    bw (KB  /s): min=    0, max=812544, per=20.00%, avg=665949.97, stdev=25269.16
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.69%
    lat (usec) : 1000=5.84%
    lat (msec) : 2=81.93%, 4=11.47%, 10=0.05%, 20=0.01%, 50=0.01%
  cpu          : usr=2.19%, sys=17.39%, ctx=325225711, majf=0, minf=194181
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=108.4%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=374552488/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16

Run status group 0 (all jobs):
   READ: io=22589GB, aggrb=3212.8MB/s, minb=3212.8MB/s, maxb=3212.8MB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=22342GB, aggrb=3177.6MB/s, minb=3177.6MB/s, maxb=3177.6MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=22861GB, aggrb=3251.4MB/s, minb=3251.4MB/s, maxb=3251.4MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=400850907/0, merge=0/0, ticks=618705538/0, in_queue=619747050, util=100.00%
  sdb: ios=396507984/0, merge=0/0, ticks=618943630/0, in_queue=619868954, util=100.00%
  sdc: ios=405819749/0, merge=0/0, ticks=618381965/0, in_queue=619797095, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 02:57:24 PM EST
# 64kB randread with 1 jobs and 8 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=18918GB, bw=2690.6MB/s, iops=43048, runt=7200001msec
     lat (usec): min=55, max=16889, avg=928.05, stdev=318.21
xfs
  read : io=18560GB, bw=2639.7MB/s, iops=42234, runt=7200002msec
     lat (usec): min=53, max=17563, avg=945.97, stdev=318.39
xfs
  read : io=18784GB, bw=2671.5MB/s, iops=42743, runt=7200002msec
     lat (usec): min=56, max=19439, avg=934.67, stdev=324.62

Totals
IOPS: 128025
MBPS: 8000
Average Latency (μsec): 935

xfs-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-02: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-03: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-04: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-05: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-12: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-13: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-14: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-15: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-21: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-22: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-23: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-24: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-25: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=131304: Sat Dec  5 17:07:25 2015
  read : io=18918GB, bw=2690.6MB/s, iops=43048, runt=7200001msec
    slat (usec): min=3, max=1817, avg=12.19, stdev= 4.58
    clat (usec): min=47, max=16871, avg=915.74, stdev=318.15
     lat (usec): min=55, max=16889, avg=928.05, stdev=318.21
    clat percentiles (usec):
     |  1.00th=[  446],  5.00th=[  524], 10.00th=[  580], 20.00th=[  652],
     | 30.00th=[  724], 40.00th=[  788], 50.00th=[  860], 60.00th=[  932],
     | 70.00th=[ 1032], 80.00th=[ 1144], 90.00th=[ 1336], 95.00th=[ 1496],
     | 99.00th=[ 1896], 99.50th=[ 2064], 99.90th=[ 2768], 99.95th=[ 3184],
     | 99.99th=[ 4128]
    bw (KB  /s): min=    0, max=571904, per=20.00%, avg=551101.50, stdev=8752.00
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=3.37%, 750=30.96%
    lat (usec) : 1000=33.01%
    lat (msec) : 2=32.00%, 4=0.64%, 10=0.01%, 20=0.01%
  cpu          : usr=2.33%, sys=15.20%, ctx=282021092, majf=0, minf=107845
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=309952629/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-11: (groupid=1, jobs=5): err= 0: pid=131309: Sat Dec  5 17:07:25 2015
  read : io=18560GB, bw=2639.7MB/s, iops=42234, runt=7200002msec
    slat (usec): min=3, max=1411, avg=11.10, stdev= 3.88
    clat (usec): min=44, max=17549, avg=934.76, stdev=318.37
     lat (usec): min=53, max=17563, avg=945.97, stdev=318.39
    clat percentiles (usec):
     |  1.00th=[  454],  5.00th=[  540], 10.00th=[  596], 20.00th=[  676],
     | 30.00th=[  740], 40.00th=[  804], 50.00th=[  876], 60.00th=[  948],
     | 70.00th=[ 1048], 80.00th=[ 1160], 90.00th=[ 1352], 95.00th=[ 1528],
     | 99.00th=[ 1912], 99.50th=[ 2096], 99.90th=[ 2640], 99.95th=[ 3024],
     | 99.99th=[ 3920]
    bw (KB  /s): min=    0, max=571264, per=20.00%, avg=540679.71, stdev=10657.28
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=2.71%, 750=28.54%
    lat (usec) : 1000=34.30%
    lat (msec) : 2=33.73%, 4=0.71%, 10=0.01%, 20=0.01%
  cpu          : usr=2.31%, sys=13.71%, ctx=279241666, majf=0, minf=108698
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=304089879/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-21: (groupid=2, jobs=5): err= 0: pid=131314: Sat Dec  5 17:07:25 2015
  read : io=18784GB, bw=2671.5MB/s, iops=42743, runt=7200002msec
    slat (usec): min=3, max=2799, avg=12.47, stdev= 4.73
    clat (usec): min=32, max=19426, avg=922.09, stdev=324.58
     lat (usec): min=56, max=19439, avg=934.67, stdev=324.62
    clat percentiles (usec):
     |  1.00th=[  450],  5.00th=[  524], 10.00th=[  580], 20.00th=[  660],
     | 30.00th=[  724], 40.00th=[  788], 50.00th=[  860], 60.00th=[  940],
     | 70.00th=[ 1032], 80.00th=[ 1144], 90.00th=[ 1336], 95.00th=[ 1512],
     | 99.00th=[ 1912], 99.50th=[ 2096], 99.90th=[ 2800], 99.95th=[ 3408],
     | 99.99th=[ 5344]
    bw (KB  /s): min=    0, max=573824, per=20.00%, avg=547185.28, stdev=10714.09
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=3.13%, 750=30.39%
    lat (usec) : 1000=33.37%
    lat (msec) : 2=32.41%, 4=0.67%, 10=0.03%, 20=0.01%
  cpu          : usr=2.26%, sys=15.52%, ctx=282144899, majf=0, minf=106837
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=307750857/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=18918GB, aggrb=2690.6MB/s, minb=2690.6MB/s, maxb=2690.6MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=18560GB, aggrb=2639.7MB/s, minb=2639.7MB/s, maxb=2639.7MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=18784GB, aggrb=2671.5MB/s, minb=2671.5MB/s, maxb=2671.5MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=335781722/0, merge=0/0, ticks=307161541/0, in_queue=307530180, util=100.00%
  sdb: ios=329450646/0, merge=0/0, ticks=307397699/0, in_queue=307615986, util=100.00%
  sdc: ios=333370262/0, merge=0/0, ticks=307082670/0, in_queue=307621500, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 05:08:25 PM EST
# 64kB randread with 4 jobs and 4 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=22449GB, bw=3192.8MB/s, iops=51083, runt=7200003msec
     lat (usec): min=59, max=21974, avg=1564.54, stdev=557.47
xfs
  read : io=22252GB, bw=3164.8MB/s, iops=50635, runt=7200003msec
     lat (usec): min=56, max=20757, avg=1578.51, stdev=573.45
xfs
  read : io=22572GB, bw=3210.3MB/s, iops=51364, runt=7200002msec
     lat (usec): min=56, max=23413, avg=1556.03, stdev=573.67

Totals
IOPS: 153082
MBPS: 9566
Average Latency (μsec): 1566

xfs-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-02: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-03: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-04: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-05: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-12: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-13: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-14: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-15: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-21: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-22: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-23: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-24: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
xfs-25: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 60 processes

xfs-01: (groupid=0, jobs=20): err= 0: pid=136970: Sat Dec  5 19:18:27 2015
  read : io=22449GB, bw=3192.8MB/s, iops=51083, runt=7200003msec
    slat (usec): min=3, max=4695, avg=14.22, stdev= 5.59
    clat (usec): min=27, max=21967, avg=1550.17, stdev=557.42
     lat (usec): min=59, max=21974, avg=1564.54, stdev=557.47
    clat percentiles (usec):
     |  1.00th=[  580],  5.00th=[  748], 10.00th=[  876], 20.00th=[ 1048],
     | 30.00th=[ 1208], 40.00th=[ 1352], 50.00th=[ 1496], 60.00th=[ 1640],
     | 70.00th=[ 1816], 80.00th=[ 2008], 90.00th=[ 2288], 95.00th=[ 2544],
     | 99.00th=[ 3056], 99.50th=[ 3248], 99.90th=[ 3760], 99.95th=[ 4016],
     | 99.99th=[ 4960]
    bw (KB  /s): min=    0, max=172032, per=5.00%, avg=163549.87, stdev=2445.15
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.24%, 750=4.78%
    lat (usec) : 1000=11.67%
    lat (msec) : 2=62.94%, 4=20.31%, 10=0.05%, 20=0.01%, 50=0.01%
  cpu          : usr=0.84%, sys=5.37%, ctx=380462846, majf=0, minf=283402
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=367802203/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-11: (groupid=1, jobs=20): err= 0: pid=136990: Sat Dec  5 19:18:27 2015
  read : io=22252GB, bw=3164.8MB/s, iops=50635, runt=7200003msec
    slat (usec): min=3, max=6719, avg=13.17, stdev= 5.68
    clat (usec): min=0, max=20747, avg=1565.23, stdev=573.40
     lat (usec): min=56, max=20757, avg=1578.51, stdev=573.45
    clat percentiles (usec):
     |  1.00th=[  588],  5.00th=[  748], 10.00th=[  868], 20.00th=[ 1048],
     | 30.00th=[ 1208], 40.00th=[ 1352], 50.00th=[ 1496], 60.00th=[ 1656],
     | 70.00th=[ 1832], 80.00th=[ 2040], 90.00th=[ 2320], 95.00th=[ 2608],
     | 99.00th=[ 3120], 99.50th=[ 3344], 99.90th=[ 3856], 99.95th=[ 4128],
     | 99.99th=[ 5024]
    bw (KB  /s): min=    0, max=171008, per=5.00%, avg=162115.65, stdev=2628.87
    lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.23%
    lat (usec) : 750=4.72%, 1000=11.84%
    lat (msec) : 2=61.64%, 4=21.50%, 10=0.07%, 20=0.01%, 50=0.01%
  cpu          : usr=0.77%, sys=4.91%, ctx=379181628, majf=0, minf=263657
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=364575306/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-21: (groupid=2, jobs=20): err= 0: pid=137010: Sat Dec  5 19:18:27 2015
  read : io=22572GB, bw=3210.3MB/s, iops=51364, runt=7200002msec
    slat (usec): min=3, max=8547, avg=13.70, stdev= 5.74
    clat (usec): min=2, max=23405, avg=1542.19, stdev=573.62
     lat (usec): min=56, max=23413, avg=1556.03, stdev=573.67
    clat percentiles (usec):
     |  1.00th=[  572],  5.00th=[  740], 10.00th=[  852], 20.00th=[ 1032],
     | 30.00th=[ 1192], 40.00th=[ 1336], 50.00th=[ 1480], 60.00th=[ 1624],
     | 70.00th=[ 1800], 80.00th=[ 2008], 90.00th=[ 2320], 95.00th=[ 2576],
     | 99.00th=[ 3120], 99.50th=[ 3344], 99.90th=[ 3984], 99.95th=[ 4448],
     | 99.99th=[ 6176]
    bw (KB  /s): min=    0, max=241792, per=5.00%, avg=164449.98, stdev=3227.01
    lat (usec) : 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.26%, 750=5.23%, 1000=12.32%
    lat (msec) : 2=61.92%, 4=20.17%, 10=0.09%, 20=0.01%, 50=0.01%
  cpu          : usr=0.81%, sys=5.18%, ctx=383351656, majf=0, minf=276915
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=369824535/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=22449GB, aggrb=3192.8MB/s, minb=3192.8MB/s, maxb=3192.8MB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=22252GB, aggrb=3164.8MB/s, minb=3164.8MB/s, maxb=3164.8MB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=22572GB, aggrb=3210.3MB/s, minb=3210.3MB/s, maxb=3210.3MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=398463529/0, merge=0/0, ticks=618189134/0, in_queue=619463660, util=100.00%
  sdb: ios=394975502/0, merge=0/0, ticks=618589768/0, in_queue=619863110, util=100.00%
  sdc: ios=400625314/0, merge=0/0, ticks=618317711/0, in_queue=619989711, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 07:19:27 PM EST
# 32kB randread with 2 jobs and 8 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=20978GB, bw=2983.6MB/s, iops=95472, runt=7200002msec
     lat (usec): min=48, max=22253, avg=836.99, stdev=326.11
xfs
  read : io=20878GB, bw=2969.3MB/s, iops=95017, runt=7200002msec
     lat (usec): min=46, max=18472, avg=841.01, stdev=329.03
xfs
  read : io=21148GB, bw=3007.7MB/s, iops=96245, runt=7200002msec
     lat (usec): min=37, max=19087, avg=830.24, stdev=320.65

Totals
IOPS: 286734
MBPS: 8959
Average Latency (μsec): 835

xfs-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-02: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-03: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-04: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-05: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-12: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-13: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-14: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-15: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-21: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-22: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-23: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-24: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-25: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 30 processes

xfs-01: (groupid=0, jobs=10): err= 0: pid=142168: Sat Dec  5 21:29:29 2015
  read : io=20978GB, bw=2983.6MB/s, iops=95472, runt=7200002msec
    slat (usec): min=2, max=9945, avg= 8.63, stdev= 4.59
    clat (usec): min=12, max=22240, avg=828.26, stdev=326.07
     lat (usec): min=48, max=22253, avg=836.99, stdev=326.11
    clat percentiles (usec):
     |  1.00th=[  346],  5.00th=[  418], 10.00th=[  470], 20.00th=[  556],
     | 30.00th=[  628], 40.00th=[  692], 50.00th=[  772], 60.00th=[  852],
     | 70.00th=[  948], 80.00th=[ 1064], 90.00th=[ 1256], 95.00th=[ 1448],
     | 99.00th=[ 1848], 99.50th=[ 2008], 99.90th=[ 2416], 99.95th=[ 2640],
     | 99.99th=[ 3312]
    bw (KB  /s): min=    0, max=329152, per=10.00%, avg=305544.51, stdev=3184.57
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.02%, 500=13.23%
    lat (usec) : 750=34.17%, 1000=27.46%
    lat (msec) : 2=24.60%, 4=0.51%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=2.14%, sys=12.18%, ctx=650991552, majf=0, minf=111481
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=687399976/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-11: (groupid=1, jobs=10): err= 0: pid=142178: Sat Dec  5 21:29:29 2015
  read : io=20878GB, bw=2969.3MB/s, iops=95017, runt=7200002msec
    slat (usec): min=2, max=7937, avg= 9.20, stdev= 5.05
    clat (usec): min=11, max=18465, avg=831.72, stdev=328.97
     lat (usec): min=46, max=18472, avg=841.01, stdev=329.03
    clat percentiles (usec):
     |  1.00th=[  342],  5.00th=[  422], 10.00th=[  474], 20.00th=[  556],
     | 30.00th=[  628], 40.00th=[  700], 50.00th=[  772], 60.00th=[  852],
     | 70.00th=[  948], 80.00th=[ 1064], 90.00th=[ 1272], 95.00th=[ 1448],
     | 99.00th=[ 1848], 99.50th=[ 2024], 99.90th=[ 2512], 99.95th=[ 2864],
     | 99.99th=[ 3504]
    bw (KB  /s): min=    0, max=313472, per=10.00%, avg=304087.29, stdev=3467.33
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.03%, 500=12.69%
    lat (usec) : 750=34.47%, 1000=27.48%
    lat (msec) : 2=24.78%, 4=0.55%, 10=0.01%, 20=0.01%
  cpu          : usr=2.12%, sys=12.84%, ctx=646443755, majf=0, minf=111121
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=684122614/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-21: (groupid=2, jobs=10): err= 0: pid=142188: Sat Dec  5 21:29:29 2015
  read : io=21148GB, bw=3007.7MB/s, iops=96245, runt=7200002msec
    slat (usec): min=2, max=10319, avg= 9.16, stdev= 5.03
    clat (usec): min=1, max=19078, avg=820.98, stdev=320.59
     lat (usec): min=37, max=19087, avg=830.24, stdev=320.65
    clat percentiles (usec):
     |  1.00th=[  342],  5.00th=[  418], 10.00th=[  470], 20.00th=[  556],
     | 30.00th=[  620], 40.00th=[  692], 50.00th=[  764], 60.00th=[  844],
     | 70.00th=[  940], 80.00th=[ 1064], 90.00th=[ 1240], 95.00th=[ 1416],
     | 99.00th=[ 1800], 99.50th=[ 1960], 99.90th=[ 2384], 99.95th=[ 2704],
     | 99.99th=[ 3920]
    bw (KB  /s): min=    0, max=326912, per=10.00%, avg=308019.20, stdev=3228.89
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.02%, 500=13.37%, 750=34.58%, 1000=27.71%
    lat (msec) : 2=23.88%, 4=0.42%, 10=0.01%, 20=0.01%
  cpu          : usr=2.15%, sys=12.97%, ctx=654044003, majf=0, minf=109891
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=692967654/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=20978GB, aggrb=2983.6MB/s, minb=2983.6MB/s, maxb=2983.6MB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=20878GB, aggrb=2969.3MB/s, minb=2969.3MB/s, maxb=2969.3MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=21148GB, aggrb=3007.7MB/s, minb=3007.7MB/s, maxb=3007.7MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=744711869/0, merge=0/0, ticks=616581194/0, in_queue=617904697, util=100.00%
  sdb: ios=741201820/0, merge=0/0, ticks=616343610/0, in_queue=618052537, util=100.00%
  sdc: ios=750763322/0, merge=0/0, ticks=616207757/0, in_queue=618621595, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 09:30:29 PM EST
# 32kB randread with 4 jobs and 8 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=22510GB, bw=3201.5MB/s, iops=102445, runt=7200004msec
     lat (usec): min=46, max=30707, avg=1560.75, stdev=764.75
xfs
  read : io=22533GB, bw=3204.8MB/s, iops=102550, runt=7200003msec
     lat (usec): min=50, max=23074, avg=1559.27, stdev=762.17
xfs
  read : io=22959GB, bw=3265.3MB/s, iops=104488, runt=7200003msec
     lat (usec): min=49, max=24971, avg=1530.35, stdev=733.18

Totals
IOPS: 309483
MBPS: 9670
Average Latency (μsec): 1549

xfs-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-02: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-03: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-04: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-05: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-12: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-13: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-14: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-15: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-21: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-22: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-23: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-24: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
xfs-25: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 60 processes

xfs-01: (groupid=0, jobs=20): err= 0: pid=1239: Sat Dec  5 23:40:31 2015
  read : io=22510GB, bw=3201.5MB/s, iops=102445, runt=7200004msec
    slat (usec): min=2, max=15103, avg=53.02, stdev=260.70
    clat (usec): min=1, max=21933, avg=1507.64, stdev=732.05
     lat (usec): min=46, max=30707, avg=1560.75, stdev=764.75
    clat percentiles (usec):
     |  1.00th=[  486],  5.00th=[  620], 10.00th=[  724], 20.00th=[  884],
     | 30.00th=[ 1032], 40.00th=[ 1176], 50.00th=[ 1336], 60.00th=[ 1544],
     | 70.00th=[ 1784], 80.00th=[ 2096], 90.00th=[ 2544], 95.00th=[ 2928],
     | 99.00th=[ 3728], 99.50th=[ 4048], 99.90th=[ 4704], 99.95th=[ 4960],
     | 99.99th=[ 5600]
    bw (KB  /s): min=    0, max=227328, per=5.00%, avg=163975.38, stdev=2188.22
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=1.27%, 750=10.38%, 1000=16.60%
    lat (msec) : 2=49.36%, 4=21.85%, 10=0.54%, 20=0.01%, 50=0.01%
  cpu          : usr=1.07%, sys=6.95%, ctx=599608608, majf=0, minf=273144
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.4%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=737610048/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-11: (groupid=1, jobs=20): err= 0: pid=1259: Sat Dec  5 23:40:31 2015
  read : io=22533GB, bw=3204.8MB/s, iops=102550, runt=7200003msec
    slat (usec): min=2, max=12487, avg=52.55, stdev=263.80
    clat (usec): min=9, max=22215, avg=1506.67, stdev=728.33
     lat (usec): min=50, max=23074, avg=1559.27, stdev=762.17
    clat percentiles (usec):
     |  1.00th=[  482],  5.00th=[  620], 10.00th=[  724], 20.00th=[  884],
     | 30.00th=[ 1032], 40.00th=[ 1176], 50.00th=[ 1352], 60.00th=[ 1544],
     | 70.00th=[ 1768], 80.00th=[ 2064], 90.00th=[ 2512], 95.00th=[ 2928],
     | 99.00th=[ 3696], 99.50th=[ 4016], 99.90th=[ 4704], 99.95th=[ 4960],
     | 99.99th=[ 5600]
    bw (KB  /s): min=    0, max=254080, per=5.00%, avg=164142.91, stdev=2184.54
    lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=1.30%, 750=10.28%, 1000=16.52%
    lat (msec) : 2=49.64%, 4=21.74%, 10=0.52%, 20=0.01%, 50=0.01%
  cpu          : usr=0.98%, sys=6.01%, ctx=603692430, majf=0, minf=249669
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=738366057/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-21: (groupid=2, jobs=20): err= 0: pid=1282: Sat Dec  5 23:40:31 2015
  read : io=22959GB, bw=3265.3MB/s, iops=104488, runt=7200003msec
    slat (usec): min=2, max=18575, avg=51.62, stdev=259.48
    clat (usec): min=22, max=24962, avg=1478.63, stdev=699.27
     lat (usec): min=49, max=24971, avg=1530.35, stdev=733.18
    clat percentiles (usec):
     |  1.00th=[  482],  5.00th=[  620], 10.00th=[  724], 20.00th=[  884],
     | 30.00th=[ 1032], 40.00th=[ 1176], 50.00th=[ 1320], 60.00th=[ 1512],
     | 70.00th=[ 1736], 80.00th=[ 2024], 90.00th=[ 2448], 95.00th=[ 2832],
     | 99.00th=[ 3600], 99.50th=[ 3888], 99.90th=[ 4576], 99.95th=[ 4832],
     | 99.99th=[ 5600]
    bw (KB  /s): min=    0, max=244800, per=5.00%, avg=167245.06, stdev=2220.63
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=1.36%, 750=10.15%
    lat (usec) : 1000=16.65%
    lat (msec) : 2=51.26%, 4=20.18%, 10=0.39%, 20=0.01%, 50=0.01%
  cpu          : usr=0.98%, sys=5.91%, ctx=616648050, majf=0, minf=263588
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=752316632/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=22510GB, aggrb=3201.5MB/s, minb=3201.5MB/s, maxb=3201.5MB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
   READ: io=22533GB, aggrb=3204.8MB/s, minb=3204.8MB/s, maxb=3204.8MB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=22959GB, aggrb=3265.3MB/s, minb=3265.3MB/s, maxb=3265.3MB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sda: ios=799206083/0, merge=0/0, ticks=1074999612/0, in_queue=1076680252, util=100.00%
  sdb: ios=800001829/0, merge=0/0, ticks=1072401462/0, in_queue=1073717368, util=100.00%
  sdc: ios=815088337/0, merge=0/0, ticks=1071027161/0, in_queue=1072785766, util=100.00%


###################################################################################
# 
# Sat 05 Dec 2015 11:41:31 PM EST
# 16kB randread with 2 jobs and 8 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=16420GB, bw=2335.4MB/s, iops=149462, runt=7200001msec
     lat (usec): min=72, max=15692, avg=534.37, stdev=119.41
xfs
  read : io=16377GB, bw=2329.2MB/s, iops=149067, runt=7200001msec
     lat (usec): min=119, max=13972, avg=535.72, stdev=120.43
xfs
  read : io=16518GB, bw=2349.3MB/s, iops=150352, runt=7200003msec
     lat (usec): min=83, max=16383, avg=531.36, stdev=122.17

Totals
IOPS: 448881
MBPS: 7013
Average Latency (μsec): 533

xfs-01: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-02: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-03: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-04: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-05: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-11: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-12: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-13: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-14: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-15: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-21: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-22: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-23: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-24: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
xfs-25: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 30 processes

xfs-01: (groupid=0, jobs=10): err= 0: pid=6572: Sun Dec  6 01:51:33 2015
  read : io=16420GB, bw=2335.4MB/s, iops=149462, runt=7200001msec
    slat (usec): min=2, max=10218, avg=14.25, stdev=10.46
    clat (usec): min=10, max=15685, avg=520.07, stdev=119.95
     lat (usec): min=72, max=15692, avg=534.37, stdev=119.41
    clat percentiles (usec):
     |  1.00th=[  342],  5.00th=[  378], 10.00th=[  398], 20.00th=[  426],
     | 30.00th=[  454], 40.00th=[  482], 50.00th=[  502], 60.00th=[  524],
     | 70.00th=[  556], 80.00th=[  604], 90.00th=[  660], 95.00th=[  716],
     | 99.00th=[  844], 99.50th=[  908], 99.90th=[ 1144], 99.95th=[ 2024],
     | 99.99th=[ 3088]
    bw (KB  /s): min=    0, max=274112, per=10.00%, avg=239147.56, stdev=2215.59
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=48.56%
    lat (usec) : 750=48.07%, 1000=3.16%
    lat (msec) : 2=0.15%, 4=0.05%, 10=0.01%, 20=0.01%
  cpu          : usr=2.22%, sys=27.99%, ctx=676556949, majf=0, minf=63859
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1076129027/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-11: (groupid=1, jobs=10): err= 0: pid=6582: Sun Dec  6 01:51:33 2015
  read : io=16377GB, bw=2329.2MB/s, iops=149067, runt=7200001msec
    slat (usec): min=2, max=12084, avg= 8.36, stdev= 7.12
    clat (usec): min=1, max=13964, avg=527.23, stdev=120.46
     lat (usec): min=119, max=13972, avg=535.72, stdev=120.43
    clat percentiles (usec):
     |  1.00th=[  342],  5.00th=[  382], 10.00th=[  402], 20.00th=[  430],
     | 30.00th=[  458], 40.00th=[  490], 50.00th=[  510], 60.00th=[  532],
     | 70.00th=[  572], 80.00th=[  612], 90.00th=[  676], 95.00th=[  732],
     | 99.00th=[  860], 99.50th=[  924], 99.90th=[ 1112], 99.95th=[ 1544],
     | 99.99th=[ 2992]
    bw (KB  /s): min=    0, max=378560, per=10.00%, avg=238519.46, stdev=2316.59
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=45.06%, 750=50.96%, 1000=3.76%
    lat (msec) : 2=0.18%, 4=0.04%, 10=0.01%, 20=0.01%
  cpu          : usr=2.25%, sys=17.78%, ctx=747390396, majf=0, minf=61430
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1073288239/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-21: (groupid=2, jobs=10): err= 0: pid=6592: Sun Dec  6 01:51:33 2015
  read : io=16518GB, bw=2349.3MB/s, iops=150352, runt=7200003msec
    slat (usec): min=2, max=9384, avg=10.41, stdev= 7.97
    clat (usec): min=6, max=16379, avg=520.90, stdev=122.52
     lat (usec): min=83, max=16383, avg=531.36, stdev=122.17
    clat percentiles (usec):
     |  1.00th=[  338],  5.00th=[  378], 10.00th=[  398], 20.00th=[  426],
     | 30.00th=[  454], 40.00th=[  482], 50.00th=[  506], 60.00th=[  532],
     | 70.00th=[  564], 80.00th=[  604], 90.00th=[  668], 95.00th=[  724],
     | 99.00th=[  852], 99.50th=[  908], 99.90th=[ 1080], 99.95th=[ 1352],
     | 99.99th=[ 3408]
    bw (KB  /s): min=    0, max=333984, per=10.00%, avg=240575.18, stdev=2486.16
    lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=48.10%, 750=48.33%, 1000=3.37%
    lat (msec) : 2=0.15%, 4=0.03%, 10=0.01%, 20=0.01%
  cpu          : usr=2.28%, sys=21.19%, ctx=711576031, majf=0, minf=61426
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1082540444/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=16420GB, aggrb=2335.4MB/s, minb=2335.4MB/s, maxb=2335.4MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=16377GB, aggrb=2329.2MB/s, minb=2329.2MB/s, maxb=2329.2MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=16518GB, aggrb=2349.3MB/s, minb=2349.3MB/s, maxb=2349.3MB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sda: ios=1165803932/0, merge=0/0, ticks=599638672/0, in_queue=601844367, util=100.00%
  sdb: ios=1162722583/0, merge=0/0, ticks=608706711/0, in_queue=610038394, util=100.00%
  sdc: ios=1172687119/0, merge=0/0, ticks=605566261/0, in_queue=607515969, util=100.00%


###################################################################################
# 
# Sun 06 Dec 2015 01:52:33 AM EST
# 8kB randread with 4 jobs and 2 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=8400.7GB, bw=1194.8MB/s, iops=152928, runt=7200003msec
     lat (usec): min=55, max=18120, avg=260.53, stdev=62.74
xfs
  read : io=8265.9GB, bw=1175.6MB/s, iops=150474, runt=7200001msec
     lat (usec): min=97, max=18233, avg=264.75, stdev=60.77
xfs
  read : io=8378.4GB, bw=1191.6MB/s, iops=152522, runt=7200002msec
     lat (usec): min=82, max=15482, avg=261.20, stdev=71.33

Totals
IOPS: 455924
MBPS: 3560
Average Latency (μsec): 261

xfs-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-02: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-03: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-04: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-05: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-12: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-13: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-14: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-15: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-21: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-22: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-23: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-24: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-25: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 60 processes

xfs-01: (groupid=0, jobs=20): err= 0: pid=12731: Sun Dec  6 04:02:36 2015
  read : io=8400.7GB, bw=1194.8MB/s, iops=152928, runt=7200003msec
    slat (usec): min=2, max=8649, avg=14.86, stdev=10.58
    clat (usec): min=0, max=18110, avg=245.58, stdev=62.75
     lat (usec): min=55, max=18120, avg=260.53, stdev=62.74
    clat percentiles (usec):
     |  1.00th=[  173],  5.00th=[  191], 10.00th=[  201], 20.00th=[  213],
     | 30.00th=[  221], 40.00th=[  231], 50.00th=[  239], 60.00th=[  249],
     | 70.00th=[  262], 80.00th=[  274], 90.00th=[  298], 95.00th=[  318],
     | 99.00th=[  358], 99.50th=[  382], 99.90th=[  628], 99.95th=[ 1384],
     | 99.99th=[ 2704]
    bw (KB  /s): min=    0, max=99360, per=5.00%, avg=61176.92, stdev=696.75
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=61.28%, 500=38.60%, 750=0.04%, 1000=0.02%
    lat (msec) : 2=0.03%, 4=0.03%, 10=0.01%, 20=0.01%
  cpu          : usr=1.51%, sys=15.98%, ctx=1053655116, majf=0, minf=62406
  IO depths    : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1101082588/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=20): err= 0: pid=12751: Sun Dec  6 04:02:36 2015
  read : io=8265.9GB, bw=1175.6MB/s, iops=150474, runt=7200001msec
    slat (usec): min=1, max=12613, avg=13.79, stdev=10.53
    clat (usec): min=0, max=18194, avg=251.01, stdev=60.75
     lat (usec): min=97, max=18233, avg=264.75, stdev=60.77
    clat percentiles (usec):
     |  1.00th=[  175],  5.00th=[  193], 10.00th=[  203], 20.00th=[  215],
     | 30.00th=[  225], 40.00th=[  235], 50.00th=[  245], 60.00th=[  255],
     | 70.00th=[  270], 80.00th=[  282], 90.00th=[  306], 95.00th=[  326],
     | 99.00th=[  370], 99.50th=[  390], 99.90th=[  494], 99.95th=[ 1004],
     | 99.99th=[ 2640]
    bw (KB  /s): min=    0, max=93296, per=5.00%, avg=60195.86, stdev=626.36
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=55.22%, 500=44.68%, 750=0.03%, 1000=0.02%
    lat (msec) : 2=0.02%, 4=0.03%, 10=0.01%, 20=0.01%
  cpu          : usr=1.46%, sys=14.59%, ctx=1040422347, majf=0, minf=51149
  IO depths    : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1083420079/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=20): err= 0: pid=12771: Sun Dec  6 04:02:36 2015
  read : io=8378.4GB, bw=1191.6MB/s, iops=152522, runt=7200002msec
    slat (usec): min=1, max=9509, avg=14.56, stdev=10.37
    clat (usec): min=0, max=15462, avg=246.52, stdev=71.33
     lat (usec): min=82, max=15482, avg=261.20, stdev=71.33
    clat percentiles (usec):
     |  1.00th=[  173],  5.00th=[  191], 10.00th=[  201], 20.00th=[  213],
     | 30.00th=[  221], 40.00th=[  231], 50.00th=[  239], 60.00th=[  251],
     | 70.00th=[  262], 80.00th=[  278], 90.00th=[  302], 95.00th=[  322],
     | 99.00th=[  362], 99.50th=[  382], 99.90th=[  458], 99.95th=[  828],
     | 99.99th=[ 3792]
    bw (KB  /s): min=    0, max=79216, per=5.00%, avg=61014.75, stdev=669.27
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=59.80%, 500=40.12%, 750=0.03%, 1000=0.01%
    lat (msec) : 2=0.02%, 4=0.02%, 10=0.01%, 20=0.01%
  cpu          : usr=1.51%, sys=15.59%, ctx=1052933241, majf=0, minf=60930
  IO depths    : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1098160327/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=8400.7GB, aggrb=1194.8MB/s, minb=1194.8MB/s, maxb=1194.8MB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=8265.9GB, aggrb=1175.6MB/s, minb=1175.6MB/s, maxb=1175.6MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=8378.4GB, aggrb=1191.6MB/s, minb=1191.6MB/s, maxb=1191.6MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sda: ios=1192841716/0, merge=0/0, ticks=295150901/0, in_queue=296284610, util=100.00%
  sdb: ios=1173803604/0, merge=0/0, ticks=295957657/0, in_queue=297241601, util=100.00%
  sdc: ios=1189706915/0, merge=0/0, ticks=295344166/0, in_queue=297018649, util=100.00%


###################################################################################
# 
# Sun 06 Dec 2015 04:03:36 AM EST
# 8kB randread with 4 jobs and 4 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=8708.3GB, bw=1238.5MB/s, iops=158524, runt=7200002msec
     lat (usec): min=72, max=25028, avg=503.49, stdev=77.03
xfs
  read : io=8635.3GB, bw=1228.2MB/s, iops=157199, runt=7200003msec
     lat (usec): min=110, max=22678, avg=507.74, stdev=92.75
xfs
  read : io=8701.2GB, bw=1237.6MB/s, iops=158399, runt=7200005msec
     lat (usec): min=106, max=21424, avg=503.89, stdev=196.33

Totals
IOPS: 474122
MBPS: 3703
Average Latency (μsec): 504

xfs-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-02: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-03: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-04: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-05: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-12: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-13: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-14: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-15: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-21: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-22: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-23: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-24: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
xfs-25: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 60 processes

xfs-01: (groupid=0, jobs=20): err= 0: pid=18896: Sun Dec  6 06:13:40 2015
  read : io=8708.3GB, bw=1238.5MB/s, iops=158524, runt=7200002msec
    slat (usec): min=1, max=15258, avg=18.94, stdev=16.90
    clat (usec): min=0, max=21201, avg=484.47, stdev=77.17
     lat (usec): min=72, max=25028, avg=503.49, stdev=77.03
    clat percentiles (usec):
     |  1.00th=[  374],  5.00th=[  398], 10.00th=[  418], 20.00th=[  442],
     | 30.00th=[  458], 40.00th=[  470], 50.00th=[  478], 60.00th=[  490],
     | 70.00th=[  506], 80.00th=[  524], 90.00th=[  556], 95.00th=[  580],
     | 99.00th=[  628], 99.50th=[  652], 99.90th=[ 1020], 99.95th=[ 1912],
     | 99.99th=[ 2992]
    bw (KB  /s): min=    0, max=168624, per=5.00%, avg=63416.19, stdev=724.26
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.03%, 500=66.60%, 750=33.21%, 1000=0.06%
    lat (msec) : 2=0.06%, 4=0.05%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.33%, sys=19.79%, ctx=929147375, majf=0, minf=88672
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1141378045/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-11: (groupid=1, jobs=20): err= 0: pid=18916: Sun Dec  6 06:13:40 2015
  read : io=8635.3GB, bw=1228.2MB/s, iops=157199, runt=7200003msec
    slat (usec): min=1, max=14677, avg=19.70, stdev=17.39
    clat (usec): min=0, max=18853, avg=488.03, stdev=92.98
     lat (usec): min=110, max=22678, avg=507.74, stdev=92.75
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  398], 10.00th=[  418], 20.00th=[  442],
     | 30.00th=[  458], 40.00th=[  470], 50.00th=[  482], 60.00th=[  494],
     | 70.00th=[  510], 80.00th=[  532], 90.00th=[  556], 95.00th=[  580],
     | 99.00th=[  636], 99.50th=[  676], 99.90th=[ 1784], 99.95th=[ 2576],
     | 99.99th=[ 3120]
    bw (KB  /s): min=    0, max=101984, per=5.00%, avg=62886.24, stdev=693.50
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.03%, 500=64.05%, 750=35.66%, 1000=0.09%
    lat (msec) : 2=0.09%, 4=0.09%, 10=0.01%, 20=0.01%
  cpu          : usr=1.33%, sys=20.37%, ctx=924791451, majf=0, minf=99194
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1131835638/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-21: (groupid=2, jobs=20): err= 0: pid=18936: Sun Dec  6 06:13:40 2015
  read : io=8701.2GB, bw=1237.6MB/s, iops=158399, runt=7200005msec
    slat (usec): min=1, max=16978, avg=18.50, stdev=16.54
    clat (usec): min=0, max=21406, avg=485.28, stdev=196.40
     lat (usec): min=106, max=21424, avg=503.89, stdev=196.33
    clat percentiles (usec):
     |  1.00th=[  342],  5.00th=[  382], 10.00th=[  398], 20.00th=[  426],
     | 30.00th=[  446], 40.00th=[  462], 50.00th=[  474], 60.00th=[  486],
     | 70.00th=[  502], 80.00th=[  524], 90.00th=[  548], 95.00th=[  580],
     | 99.00th=[  652], 99.50th=[ 1012], 99.90th=[ 4048], 99.95th=[ 4832],
     | 99.99th=[ 5536]
    bw (KB  /s): min=    0, max=134416, per=5.00%, avg=63366.63, stdev=782.20
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.05%, 500=69.70%, 750=29.61%, 1000=0.13%
    lat (msec) : 2=0.20%, 4=0.20%, 10=0.10%, 20=0.01%, 50=0.01%
  cpu          : usr=1.34%, sys=19.39%, ctx=934831188, majf=0, minf=95101
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1140477810/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=8708.3GB, aggrb=1238.5MB/s, minb=1238.5MB/s, maxb=1238.5MB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=8635.3GB, aggrb=1228.2MB/s, minb=1228.2MB/s, maxb=1228.2MB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=8701.2GB, aggrb=1237.6MB/s, minb=1237.6MB/s, maxb=1237.6MB/s, mint=7200005msec, maxt=7200005msec

Disk stats (read/write):
  sda: ios=1236636585/0, merge=0/0, ticks=596780002/0, in_queue=599497089, util=100.00%
  sdb: ios=1226218027/0, merge=0/0, ticks=596550341/0, in_queue=599459546, util=100.00%
  sdc: ios=1235663496/0, merge=0/0, ticks=597239403/0, in_queue=600815571, util=100.00%


###################################################################################
# 
# Sun 06 Dec 2015 06:14:40 AM EST
# 4kB randread with 2 jobs and 4 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=4359.9GB, bw=634941KB/s, iops=158735, runt=7200001msec
     lat (usec): min=50, max=14828, avg=251.22, stdev=94.94
xfs
  read : io=4335.6GB, bw=631407KB/s, iops=157851, runt=7200001msec
     lat (usec): min=44, max=15200, avg=252.53, stdev=113.85
xfs
  read : io=4331.5GB, bw=630754KB/s, iops=157688, runt=7200001msec
     lat (usec): min=43, max=17838, avg=252.76, stdev=202.34

Totals
IOPS: 474274
MBPS: 1851
Average Latency (μsec): 251

xfs-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-02: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-03: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-04: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-05: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-12: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-13: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-14: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-15: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-21: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-22: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-23: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-24: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
xfs-25: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
fio-2.1.10
Starting 30 processes

xfs-01: (groupid=0, jobs=10): err= 0: pid=24063: Sun Dec  6 08:24:43 2015
  read : io=4359.9GB, bw=634941KB/s, iops=158735, runt=7200001msec
    slat (usec): min=1, max=12739, avg=11.77, stdev= 8.90
    clat (usec): min=0, max=14821, avg=239.31, stdev=95.19
     lat (usec): min=50, max=14828, avg=251.22, stdev=94.94
    clat percentiles (usec):
     |  1.00th=[  169],  5.00th=[  187], 10.00th=[  197], 20.00th=[  209],
     | 30.00th=[  217], 40.00th=[  225], 50.00th=[  233], 60.00th=[  241],
     | 70.00th=[  249], 80.00th=[  262], 90.00th=[  278], 95.00th=[  294],
     | 99.00th=[  334], 99.50th=[  374], 99.90th=[ 2064], 99.95th=[ 2512],
     | 99.99th=[ 2896]
    bw (KB  /s): min=    0, max=94728, per=10.00%, avg=63495.53, stdev=967.08
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=70.14%, 500=29.48%, 750=0.09%, 1000=0.06%
    lat (msec) : 2=0.12%, 4=0.11%, 10=0.01%, 20=0.01%
  cpu          : usr=2.48%, sys=25.59%, ctx=801738863, majf=0, minf=20014
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1142894261/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-11: (groupid=1, jobs=10): err= 0: pid=24073: Sun Dec  6 08:24:43 2015
  read : io=4335.6GB, bw=631407KB/s, iops=157851, runt=7200001msec
    slat (usec): min=1, max=7979, avg=12.20, stdev= 9.68
    clat (usec): min=0, max=15196, avg=240.21, stdev=114.07
     lat (usec): min=44, max=15200, avg=252.53, stdev=113.85
    clat percentiles (usec):
     |  1.00th=[  165],  5.00th=[  185], 10.00th=[  195], 20.00th=[  207],
     | 30.00th=[  215], 40.00th=[  223], 50.00th=[  231], 60.00th=[  239],
     | 70.00th=[  249], 80.00th=[  262], 90.00th=[  278], 95.00th=[  294],
     | 99.00th=[  342], 99.50th=[  564], 99.90th=[ 2352], 99.95th=[ 2640],
     | 99.99th=[ 2960]
    bw (KB  /s): min=    0, max=103328, per=10.00%, avg=63142.32, stdev=755.16
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=70.97%, 500=28.49%, 750=0.12%, 1000=0.08%
    lat (msec) : 2=0.17%, 4=0.16%, 10=0.01%, 20=0.01%
  cpu          : usr=2.39%, sys=26.06%, ctx=799663717, majf=0, minf=21301
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1136533497/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-21: (groupid=2, jobs=10): err= 0: pid=24083: Sun Dec  6 08:24:43 2015
  read : io=4331.5GB, bw=630754KB/s, iops=157688, runt=7200001msec
    slat (usec): min=1, max=12610, avg=11.10, stdev= 9.03
    clat (usec): min=0, max=17831, avg=241.64, stdev=202.42
     lat (usec): min=43, max=17838, avg=252.76, stdev=202.34
    clat percentiles (usec):
     |  1.00th=[  151],  5.00th=[  175], 10.00th=[  187], 20.00th=[  201],
     | 30.00th=[  211], 40.00th=[  219], 50.00th=[  227], 60.00th=[  235],
     | 70.00th=[  245], 80.00th=[  258], 90.00th=[  274], 95.00th=[  294],
     | 99.00th=[  346], 99.50th=[  908], 99.90th=[ 4016], 99.95th=[ 4640],
     | 99.99th=[ 5280]
    bw (KB  /s): min=    0, max=98728, per=10.00%, avg=63077.00, stdev=946.73
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=73.94%, 500=25.41%, 750=0.09%, 1000=0.07%
    lat (msec) : 2=0.16%, 4=0.22%, 10=0.10%, 20=0.01%
  cpu          : usr=2.43%, sys=24.04%, ctx=820290138, majf=0, minf=21481
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1135357230/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=4359.9GB, aggrb=634941KB/s, minb=634941KB/s, maxb=634941KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
   READ: io=4335.6GB, aggrb=631407KB/s, minb=631407KB/s, maxb=631407KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
   READ: io=4331.5GB, aggrb=630753KB/s, minb=630753KB/s, maxb=630753KB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sda: ios=1238206338/0, merge=0/0, ticks=291747883/0, in_queue=292809466, util=100.00%
  sdb: ios=1231420594/0, merge=0/0, ticks=291347602/0, in_queue=292621697, util=100.00%
  sdc: ios=1229587889/0, merge=0/0, ticks=292919564/0, in_queue=294408732, util=100.00%


###################################################################################
# 
# Sun 06 Dec 2015 08:25:43 AM EST
# 4kB randread with 4 jobs and 6 I/Os outstanding
# CentOS Linux release 7.1.1503  Linux 3.10.0-229.20.1.el7.x86_64
# 
###################################################################################

Summary
xfs
  read : io=4460.5GB, bw=649596KB/s, iops=162398, runt=7200002msec
     lat (usec): min=59, max=20090, avg=737.81, stdev=114.41
xfs
  read : io=4454.3GB, bw=648695KB/s, iops=162173, runt=7200002msec
     lat (usec): min=101, max=19830, avg=738.77, stdev=106.55
xfs
  read : io=4469.8GB, bw=650948KB/s, iops=162736, runt=7200001msec
     lat (usec): min=84, max=19163, avg=736.28, stdev=153.74

Totals
IOPS: 487307
MBPS: 1902
Average Latency (μsec): 737

xfs-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-02: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-03: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-04: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-05: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-12: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-13: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-14: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-15: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-21: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-22: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-23: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-24: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
xfs-25: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6
...
fio-2.1.10
Starting 60 processes

xfs-01: (groupid=0, jobs=20): err= 0: pid=29737: Sun Dec  6 10:35:47 2015
  read : io=4460.5GB, bw=649596KB/s, iops=162398, runt=7200002msec
    slat (usec): min=1, max=18269, avg=18.03, stdev=14.95
    clat (usec): min=0, max=18725, avg=719.64, stdev=114.96
     lat (usec): min=59, max=20090, avg=737.81, stdev=114.41
    clat percentiles (usec):
     |  1.00th=[  628],  5.00th=[  652], 10.00th=[  660], 20.00th=[  676],
     | 30.00th=[  684], 40.00th=[  700], 50.00th=[  716], 60.00th=[  724],
     | 70.00th=[  740], 80.00th=[  756], 90.00th=[  772], 95.00th=[  788],
     | 99.00th=[  836], 99.50th=[  996], 99.90th=[ 2736], 99.95th=[ 3088],
     | 99.99th=[ 3440]
    bw (KB  /s): min=    0, max=71400, per=5.00%, avg=32482.91, stdev=427.29
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.11%, 750=78.42%, 1000=20.96%
    lat (msec) : 2=0.29%, 4=0.21%, 10=0.01%, 20=0.01%
  cpu          : usr=1.37%, sys=19.56%, ctx=933590115, majf=0, minf=75015
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1169272162/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
xfs-11: (groupid=1, jobs=20): err= 0: pid=29757: Sun Dec  6 10:35:47 2015
  read : io=4454.3GB, bw=648695KB/s, iops=162173, runt=7200002msec
    slat (usec): min=1, max=15681, avg=14.69, stdev=14.27
    clat (usec): min=0, max=19822, avg=724.43, stdev=106.83
     lat (usec): min=101, max=19830, avg=738.77, stdev=106.55
    clat percentiles (usec):
     |  1.00th=[  636],  5.00th=[  660], 10.00th=[  668], 20.00th=[  684],
     | 30.00th=[  692], 40.00th=[  708], 50.00th=[  716], 60.00th=[  732],
     | 70.00th=[  740], 80.00th=[  756], 90.00th=[  780], 95.00th=[  796],
     | 99.00th=[  836], 99.50th=[  900], 99.90th=[ 2640], 99.95th=[ 3024],
     | 99.99th=[ 3408]
    bw (KB  /s): min=    0, max=47928, per=5.00%, avg=32438.44, stdev=328.11
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.05%, 750=74.95%, 1000=24.57%
    lat (msec) : 2=0.23%, 4=0.18%, 10=0.01%, 20=0.01%
  cpu          : usr=1.35%, sys=16.20%, ctx=940028653, majf=0, minf=65737
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1167651532/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
xfs-21: (groupid=2, jobs=20): err= 0: pid=29777: Sun Dec  6 10:35:47 2015
  read : io=4469.8GB, bw=650948KB/s, iops=162736, runt=7200001msec
    slat (usec): min=1, max=17704, avg=18.33, stdev=14.91
    clat (usec): min=0, max=19158, avg=717.87, stdev=154.12
     lat (usec): min=84, max=19163, avg=736.28, stdev=153.74
    clat percentiles (usec):
     |  1.00th=[  612],  5.00th=[  644], 10.00th=[  660], 20.00th=[  668],
     | 30.00th=[  684], 40.00th=[  700], 50.00th=[  708], 60.00th=[  724],
     | 70.00th=[  732], 80.00th=[  748], 90.00th=[  772], 95.00th=[  788],
     | 99.00th=[  828], 99.50th=[  876], 99.90th=[ 3440], 99.95th=[ 4576],
     | 99.99th=[ 5600]
    bw (KB  /s): min=    0, max=52912, per=5.00%, avg=32550.51, stdev=361.27
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=0.09%, 750=79.65%, 1000=19.88%
    lat (msec) : 2=0.18%, 4=0.13%, 10=0.07%, 20=0.01%
  cpu          : usr=1.36%, sys=19.84%, ctx=931888168, majf=0, minf=71240
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1171705652/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6

Run status group 0 (all jobs):
   READ: io=4460.5GB, aggrb=649595KB/s, minb=649595KB/s, maxb=649595KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=4454.3GB, aggrb=648695KB/s, minb=648695KB/s, maxb=648695KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=4469.8GB, aggrb=650947KB/s, minb=650947KB/s, maxb=650947KB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sda: ios=1266941472/0, merge=0/0, ticks=909758076/0, in_queue=914856859, util=100.00%
  sdb: ios=1265058418/0, merge=0/0, ticks=912564127/0, in_queue=917307956, util=100.00%
  sdc: ios=1269508963/0, merge=0/0, ticks=909313832/0, in_queue=915774464, util=100.00%