###################################################################################
# ION SR-71mach5 
# with (24) Intel DC S3610 1600GB SSDs in (3) RAID 5 arrays of 8.75TiB each
# each with (5) 1.75TiB partitions
# All Partitions on each array are xfs
# format and mount options listed below
###################################################################################
2015 November 28, 11:16
CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
600 s ramp + 7200 s run, per test

1mB write with 1 jobs and 2 I/Os outstanding 
IOPS: 6777
MBPS: 6777
Average Latency (μsec): 4436

1mB read with 1 jobs and 4 I/Os outstanding 
IOPS: 10852
MBPS: 10852
Average Latency (μsec): 5527

256kB write with 1 jobs and 2 I/Os outstanding 
IOPS: 33182
MBPS: 8294
Average Latency (μsec): 902

256kB read with 1 jobs and 10 I/Os outstanding 
IOPS: 42675
MBPS: 10668
Average Latency (μsec): 3513

64kB write with 1 jobs and 6 I/Os outstanding 
IOPS: 122069
MBPS: 7628
Average Latency (μsec): 735

64kB read with 1 jobs and 16 I/Os outstanding 
IOPS: 154330
MBPS: 9644
Average Latency (μsec): 1553

8kB randwrite with 1 jobs and 2 I/Os outstanding 
IOPS: 47825
MBPS: 373
Average Latency (μsec): 625

64kB randwrite with 1 jobs and 2 I/Os outstanding 
IOPS: 21566
MBPS: 1346
Average Latency (μsec): 1390

64kB randread with 2 jobs and 10 I/Os outstanding 
IOPS: 156995
MBPS: 9812
Average Latency (μsec): 1909

32kB randread with 4 jobs and 6 I/Os outstanding 
IOPS: 305826
MBPS: 9556
Average Latency (μsec): 1175

16kB randread with 2 jobs and 14 I/Os outstanding 
IOPS: 463977
MBPS: 7249
Average Latency (μsec): 903

8kB randread with 3 jobs and 6 I/Os outstanding 
IOPS: 287162
MBPS: 2242
Average Latency (μsec): 948

4kB randread with 3 jobs and 8 I/Os outstanding 
IOPS: 322970
MBPS: 1260
Average Latency (μsec): 1118

8kB randrw with 4 jobs and 2 I/Os outstanding 
IOPS: 112630
MBPS: 877
Average Latency (μsec): 2076

64kB randrw with 1 jobs and 8 I/Os outstanding 
IOPS: 44919
MBPS: 2805
Average Latency (μsec): 4142


###################################################################################
# 
# SR-71mach5 Configuration
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################
Product Name = Intel(R) Integrated RAID Module RMS3CC080
FW Package Build = 24.9.0-0024
Driver Name = megaraid_sas
Driver Version = 06.809.18.00
Product Name = Intel(R) RAID Controller RS3DC080
FW Package Build = 24.9.0-0024
Driver Name = megaraid_sas
Driver Version = 06.809.18.00
Product Name = Intel(R) RAID Controller RS3DC080
FW Package Build = 24.9.0-0024
Driver Name = megaraid_sas
Driver Version = 06.809.18.00
Drive /c0/e252/s0 :
252:0    10 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601DZ1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c0/e252/s1 :
252:1    11 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601GG1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c0/e252/s2 :
252:2     8 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601G51P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c0/e252/s3 :
252:3     9 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601GK1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c0/e252/s4 :
252:4    14 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601AW1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c0/e252/s5 :
252:5    13 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601C71P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c0/e252/s6 :
252:6    12 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601C81P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c0/e252/s7 :
252:7    15 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC452601GW1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c1/e252/s0 :
252:0     8 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600MS1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c1/e252/s1 :
252:1     9 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600T21P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c1/e252/s2 :
252:2    10 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600Q01P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c1/e252/s3 :
252:3    11 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600N51P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c1/e252/s4 :
252:4    12 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600N61P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c1/e252/s5 :
252:5    13 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600PX1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c1/e252/s6 :
252:6    14 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600NJ1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c1/e252/s7 :
252:7    15 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600MZ1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010140
Drive /c2/e252/s0 :
252:0     8 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC506601AU1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010110
Drive /c2/e252/s1 :
252:1     9 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600N01P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010110
Drive /c2/e252/s2 :
252:2    10 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600R01P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010110
Drive /c2/e252/s3 :
252:3    11 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600N71P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010110
Drive /c2/e252/s4 :
252:4    12 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600TE1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010110
Drive /c2/e252/s5 :
252:5    13 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC506601951P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010110
Drive /c2/e252/s6 :
252:6    14 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600NH1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010110
Drive /c2/e252/s7 :
252:7    15 Onln   0 1.249 TB SATA SSD N   N  512B INTEL SSDSC2BX016T4 U  
SN = BTHC505600PL1P6PGN  
Model Number = INTEL SSDSC2BX01
Firmware Revision = G2010110
Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz Status: Populated, Enabled Core Enabled: 12
Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz Status: Populated, Enabled Core Enabled: 12
DIMM_A1: 8192 MB | 64/72 bits | 2133 MHz | Micron | 18ASF1G72PZ-2G1A2 | SN: 0D06B48F
DIMM_B1: 8192 MB | 64/72 bits | 2133 MHz | Micron | 18ASF1G72PZ-2G1A2 | SN: 0D06B415
DIMM_C1: 8192 MB | 64/72 bits | 2133 MHz | Micron | 18ASF1G72PZ-2G1A2 | SN: 0D06B46E
DIMM_D1: 8192 MB | 64/72 bits | 2133 MHz | Micron | 18ASF1G72PZ-2G1A2 | SN: 0D06B412
DIMM_E1: 8192 MB | 64/72 bits | 2133 MHz | Micron | 18ASF1G72PZ-2G1A2 | SN: 0D06B411
DIMM_F1: 8192 MB | 64/72 bits | 2133 MHz | Micron | 18ASF1G72PZ-2G1A2 | SN: 0D06B464
DIMM_G1: 8192 MB | 64/72 bits | 2133 MHz | Micron | 18ASF1G72PZ-2G1A2 | SN: 0D06B434
DIMM_H1: 8192 MB | 64/72 bits | 2133 MHz | Micron | 18ASF1G72PZ-2G1A2 | SN: 0D06B492
____________________________________________________________________
MemTotal:       65860608 kB


###################################################################################
# 
# mkfs commands
# 
###################################################################################
#!/bin/bash
#
# format 5partitions on each of 3 RAID 5 volumes with XFS
#
# P1 & P2 on each were previously XFS, do not re-format
#
# mkfs.xfs -L C0P1 -d su=65536,sw=7 -l su=64k /dev/sdb1
# mkfs.xfs -L C1P1 -d su=65536,sw=7 -l su=64k /dev/sdc1
# mkfs.xfs -L C2P1 -d su=65536,sw=7 -l su=64k /dev/sdd1
# mkfs.xfs -L C0P2 -d su=65536,sw=7 -l su=64k /dev/sdb2
# mkfs.xfs -L C1P2 -d su=65536,sw=7 -l su=64k /dev/sdc2
# mkfs.xfs -L C2P2 -d su=65536,sw=7 -l su=64k /dev/sdd2
#
mkfs.xfs -L C0P3 -d su=65536,sw=7 -l su=64k /dev/sdb3
mkfs.xfs -L C1P3 -d su=65536,sw=7 -l su=64k /dev/sdc3
mkfs.xfs -L C2P3 -d su=65536,sw=7 -l su=64k /dev/sdd3
mkfs.xfs -L C0P4 -d su=65536,sw=7 -l su=64k /dev/sdb4
mkfs.xfs -L C1P4 -d su=65536,sw=7 -l su=64k /dev/sdc4
mkfs.xfs -L C2P4 -d su=65536,sw=7 -l su=64k /dev/sdd4
mkfs.xfs -L C0P5 -d su=65536,sw=7 -l su=64k /dev/sdb5
mkfs.xfs -L C1P5 -d su=65536,sw=7 -l su=64k /dev/sdc5
mkfs.xfs -L C2P5 -d su=65536,sw=7 -l su=64k /dev/sdd5



###################################################################################
# 
# /etc/fstab
# kernel parameter: elevator=noop
# 
###################################################################################
LABEL=C0P1		/sr71/C0P1		xfs	    sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P2		/sr71/C0P2		xfs	    sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P3		/sr71/C0P3		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P4		/sr71/C0P4		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C0P5		/sr71/C0P5		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P1		/sr71/C1P1		xfs	    sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P2		/sr71/C1P2		xfs	    sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P3		/sr71/C1P3		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P4		/sr71/C1P4		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C1P5		/sr71/C1P5		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P1		/sr71/C2P1		xfs	    sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P2		/sr71/C2P2		xfs	    sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P3		/sr71/C2P3		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P4		/sr71/C2P4		xfs     sync,noatime,nodiratime,nobarrier 0 0
LABEL=C2P5		/sr71/C2P5		xfs     sync,noatime,nodiratime,nobarrier 0 0

###################################################################################
# 
# actual mount paramters
# 
###################################################################################
/dev/sdb1 on /sr71/C0P1 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdb2 on /sr71/C0P2 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdb3 on /sr71/C0P3 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdb4 on /sr71/C0P4 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdb5 on /sr71/C0P5 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdc1 on /sr71/C1P1 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdc2 on /sr71/C1P2 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdc3 on /sr71/C1P3 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdc4 on /sr71/C1P4 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdc5 on /sr71/C1P5 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdd1 on /sr71/C2P1 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdd2 on /sr71/C2P2 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdd3 on /sr71/C2P3 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdd4 on /sr71/C2P4 type xfs (rw,sync,noatime,nodiratime,nobarrier)
/dev/sdd5 on /sr71/C2P5 type xfs (rw,sync,noatime,nodiratime,nobarrier)


###################################################################################
# 
# fio job file
fio-2.1.10
# 
###################################################################################
60 seconds between fio tests
[global]
ioengine=libaio
size=1536g
direct=1
time_based
ramp_time=${RAMPTIME}
runtime=${RUNTIME}
iodepth=${IODEPTH}
bs=${BS}
rw=${RW}
rwmixread=${RWMIXREAD}
numjobs=${NUMJOBS}
group_reporting

# disable some measurements to reduce system overhead
# cannot: breaks ALL latency reporting
# disable_clat=1
# disable_slat=1
# disable_lat=1
# HOWTO lists disable_bw but fio-2.1.10 exits with global error
# disable_bw=1

# write log options seem to keep fio from completing
# write_bw_log=mach5
# write_iops_log=mach5
# write_lat_log=mach5

[xfs-01]
new_group
filename=/sr71/C0P1/fio.tst
[xfs-02]
filename=/sr71/C0P2/fio.tst
[xfs-03]
filename=/sr71/C0P3/fio.tst
[xfs-04]
filename=/sr71/C0P4/fio.tst
[xfs-05]
filename=/sr71/C0P5/fio.tst


[xfs-11]
new_group
filename=/sr71/C1P1/fio.tst
[xfs-12]
filename=/sr71/C1P2/fio.tst
[xfs-13]
filename=/sr71/C1P3/fio.tst
[xfs-14]
filename=/sr71/C1P4/fio.tst
[xfs-15]
filename=/sr71/C1P5/fio.tst

[xfs-21]
new_group
filename=/sr71/C2P1/fio.tst
[xfs-22]
filename=/sr71/C2P2/fio.tst
[xfs-23]
filename=/sr71/C2P3/fio.tst
[xfs-24]
filename=/sr71/C2P4/fio.tst
[xfs-25]
filename=/sr71/C2P5/fio.tst




###################################################################################
# 
# 201511281116 test details
# 
###################################################################################

###################################################################################
# 
# Sat 28 Nov 2015 11:17:44 AM EST
# 1mB write with 1 jobs and 2 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  write: io=16556GB, bw=2354.7MB/s, iops=2354, runt=7200004msec
     lat (msec): min=1, max=28, avg= 4.25, stdev= 1.73
xfs
  write: io=16169GB, bw=2299.6MB/s, iops=2299, runt=7200003msec
     lat (msec): min=1, max=29, avg= 4.35, stdev= 1.83
xfs
  write: io=14935GB, bw=2124.2MB/s, iops=2124, runt=7200005msec
     lat (msec): min=1, max=36, avg= 4.71, stdev= 1.83

Totals
IOPS: 6777
MBPS: 6777
Average Latency (μsec): 4436

xfs-01: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-02: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-03: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-04: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-05: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-11: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-12: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-13: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-14: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-15: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-21: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-22: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-23: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-24: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
xfs-25: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=20276: Sat Nov 28 13:27:45 2015
  write: io=16556GB, bw=2354.7MB/s, iops=2354, runt=7200004msec
    slat (usec): min=56, max=1114, avg=146.26, stdev=52.05
    clat (msec): min=1, max=27, avg= 4.10, stdev= 1.73
     lat (msec): min=1, max=28, avg= 4.25, stdev= 1.73
    clat percentiles (usec):
     |  1.00th=[ 2896],  5.00th=[ 3088], 10.00th=[ 3152], 20.00th=[ 3248],
     | 30.00th=[ 3312], 40.00th=[ 3376], 50.00th=[ 3440], 60.00th=[ 3504],
     | 70.00th=[ 3632], 80.00th=[ 4256], 90.00th=[ 6496], 95.00th=[ 8160],
     | 99.00th=[10816], 99.50th=[12608], 99.90th=[14272], 99.95th=[15168],
     | 99.99th=[18560]
    bw (KB  /s): min=    1, max=596822, per=20.02%, avg=482734.67, stdev=104030.98
    lat (msec) : 2=0.01%, 4=77.73%, 10=20.51%, 20=1.76%, 50=0.01%
  cpu          : usr=5.70%, sys=2.23%, ctx=19724584, majf=0, minf=170
  IO depths    : 1=0.1%, 2=109.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=16953577/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=5): err= 0: pid=20281: Sat Nov 28 13:27:45 2015
  write: io=16169GB, bw=2299.6MB/s, iops=2299, runt=7200003msec
    slat (usec): min=57, max=1091, avg=161.25, stdev=56.61
    clat (msec): min=1, max=28, avg= 4.19, stdev= 1.82
     lat (msec): min=1, max=29, avg= 4.35, stdev= 1.83
    clat percentiles (usec):
     |  1.00th=[ 2864],  5.00th=[ 3056], 10.00th=[ 3152], 20.00th=[ 3248],
     | 30.00th=[ 3312], 40.00th=[ 3376], 50.00th=[ 3440], 60.00th=[ 3504],
     | 70.00th=[ 3664], 80.00th=[ 4512], 90.00th=[ 6944], 95.00th=[ 8640],
     | 99.00th=[10816], 99.50th=[12352], 99.90th=[14016], 99.95th=[14656],
     | 99.99th=[18560]
    bw (KB  /s): min=    1, max=593593, per=20.02%, avg=471448.04, stdev=109374.96
    lat (msec) : 2=0.01%, 4=75.19%, 10=22.86%, 20=1.94%, 50=0.01%
  cpu          : usr=6.44%, sys=2.07%, ctx=19392348, majf=0, minf=174
  IO depths    : 1=0.1%, 2=110.1%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=16556877/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=5): err= 0: pid=20286: Sat Nov 28 13:27:45 2015
  write: io=14935GB, bw=2124.2MB/s, iops=2124, runt=7200005msec
    slat (usec): min=55, max=1070, avg=170.93, stdev=64.77
    clat (msec): min=1, max=36, avg= 4.54, stdev= 1.83
     lat (msec): min=1, max=36, avg= 4.71, stdev= 1.83
    clat percentiles (usec):
     |  1.00th=[ 2480],  5.00th=[ 2896], 10.00th=[ 3088], 20.00th=[ 3280],
     | 30.00th=[ 3408], 40.00th=[ 3536], 50.00th=[ 3824], 60.00th=[ 4320],
     | 70.00th=[ 4832], 80.00th=[ 5600], 90.00th=[ 6880], 95.00th=[ 8384],
     | 99.00th=[10816], 99.50th=[11712], 99.90th=[13888], 99.95th=[16320],
     | 99.99th=[27776]
    bw (KB  /s): min=    1, max=618071, per=20.02%, avg=435451.82, stdev=73765.00
    lat (msec) : 2=0.16%, 4=53.28%, 10=44.47%, 20=2.05%, 50=0.04%
  cpu          : usr=5.87%, sys=2.52%, ctx=17364617, majf=0, minf=176
  IO depths    : 1=0.1%, 2=109.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=15293850/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=16556GB, aggrb=2354.7MB/s, minb=2354.7MB/s, maxb=2354.7MB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
  WRITE: io=16169GB, aggrb=2299.6MB/s, minb=2299.6MB/s, maxb=2299.6MB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
  WRITE: io=14935GB, aggrb=2124.2MB/s, minb=2124.2MB/s, maxb=2124.2MB/s, mint=7200005msec, maxt=7200005msec

Disk stats (read/write):
  sdb: ios=0/74476898, merge=0/0, ticks=0/284739896, in_queue=284696393, util=100.00%
  sdc: ios=0/72920019, merge=0/0, ticks=0/284409511, in_queue=284365180, util=100.00%
  sdd: ios=0/66972457, merge=0/0, ticks=0/277945967, in_queue=277891539, util=100.00%


###################################################################################
# 
# Sat 28 Nov 2015 01:28:45 PM EST
# 1mB read with 1 jobs and 4 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=25308GB, bw=3599.5MB/s, iops=3599, runt=7200005msec
     lat (usec): min=314, max=27402, avg=5555.46, stdev=658.38
xfs
  read : io=25383GB, bw=3610.4MB/s, iops=3610, runt=7200005msec
     lat (usec): min=320, max=27866, avg=5539.14, stdev=679.58
xfs
  read : io=25622GB, bw=3643.2MB/s, iops=3643, runt=7200007msec
     lat (usec): min=363, max=28179, avg=5487.51, stdev=728.13

Totals
IOPS: 10852
MBPS: 10852
Average Latency (μsec): 5527

xfs-01: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-02: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-03: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-04: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-05: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-11: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-12: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-13: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-14: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-15: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-21: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-22: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-23: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-24: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
xfs-25: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=21996: Sat Nov 28 15:38:46 2015
  read : io=25308GB, bw=3599.5MB/s, iops=3599, runt=7200005msec
    slat (usec): min=30, max=1161, avg=76.65, stdev=40.19
    clat (usec): min=260, max=27344, avg=5478.39, stdev=658.51
     lat (usec): min=314, max=27402, avg=5555.46, stdev=658.38
    clat percentiles (usec):
     |  1.00th=[ 4448],  5.00th=[ 4704], 10.00th=[ 4832], 20.00th=[ 4960],
     | 30.00th=[ 5088], 40.00th=[ 5216], 50.00th=[ 5344], 60.00th=[ 5536],
     | 70.00th=[ 5728], 80.00th=[ 5920], 90.00th=[ 6240], 95.00th=[ 6560],
     | 99.00th=[ 7776], 99.50th=[ 8384], 99.90th=[ 9792], 99.95th=[10304],
     | 99.99th=[13376]
    bw (KB  /s): min=    1, max=929792, per=20.02%, avg=737754.61, stdev=15436.95
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.04%, 10=99.88%, 20=0.07%, 50=0.01%
  cpu          : usr=0.16%, sys=6.27%, ctx=28711105, majf=0, minf=2737
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=25915710/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-11: (groupid=1, jobs=5): err= 0: pid=22001: Sat Nov 28 15:38:46 2015
  read : io=25383GB, bw=3610.4MB/s, iops=3610, runt=7200005msec
    slat (usec): min=30, max=1335, avg=73.73, stdev=37.87
    clat (usec): min=218, max=27805, avg=5465.35, stdev=679.80
     lat (usec): min=320, max=27866, avg=5539.14, stdev=679.58
    clat percentiles (usec):
     |  1.00th=[ 4448],  5.00th=[ 4704], 10.00th=[ 4832], 20.00th=[ 4960],
     | 30.00th=[ 5088], 40.00th=[ 5152], 50.00th=[ 5280], 60.00th=[ 5472],
     | 70.00th=[ 5664], 80.00th=[ 5920], 90.00th=[ 6240], 95.00th=[ 6624],
     | 99.00th=[ 7840], 99.50th=[ 8512], 99.90th=[ 9792], 99.95th=[10304],
     | 99.99th=[13632]
    bw (KB  /s): min=    1, max=915456, per=20.02%, avg=739929.51, stdev=12954.19
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.04%, 10=99.88%, 20=0.07%, 50=0.01%
  cpu          : usr=0.15%, sys=6.05%, ctx=28788399, majf=0, minf=2741
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=25992272/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4
xfs-21: (groupid=2, jobs=5): err= 0: pid=22006: Sat Nov 28 15:38:46 2015
  read : io=25622GB, bw=3643.2MB/s, iops=3643, runt=7200007msec
    slat (usec): min=29, max=1163, avg=74.28, stdev=38.70
    clat (usec): min=236, max=28100, avg=5412.71, stdev=728.51
     lat (usec): min=363, max=28179, avg=5487.51, stdev=728.13
    clat percentiles (usec):
     |  1.00th=[ 4384],  5.00th=[ 4640], 10.00th=[ 4768], 20.00th=[ 4960],
     | 30.00th=[ 5024], 40.00th=[ 5152], 50.00th=[ 5216], 60.00th=[ 5344],
     | 70.00th=[ 5600], 80.00th=[ 5792], 90.00th=[ 6176], 95.00th=[ 6624],
     | 99.00th=[ 8256], 99.50th=[ 8896], 99.90th=[10688], 99.95th=[11584],
     | 99.99th=[15552]
    bw (KB  /s): min=    1, max=927744, per=20.02%, avg=746888.74, stdev=15149.55
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.06%, 10=99.75%, 20=0.18%, 50=0.01%
  cpu          : usr=0.15%, sys=6.15%, ctx=29141288, majf=0, minf=2742
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=26236708/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=25308GB, aggrb=3599.5MB/s, minb=3599.5MB/s, maxb=3599.5MB/s, mint=7200005msec, maxt=7200005msec

Run status group 1 (all jobs):
   READ: io=25383GB, aggrb=3610.4MB/s, minb=3610.4MB/s, maxb=3610.4MB/s, mint=7200005msec, maxt=7200005msec

Run status group 2 (all jobs):
   READ: io=25622GB, aggrb=3643.2MB/s, minb=3643.2MB/s, maxb=3643.2MB/s, mint=7200007msec, maxt=7200007msec

Disk stats (read/write):
  sdb: ios=112344821/25, merge=0/0, ticks=545525654/0, in_queue=545461906, util=100.00%
  sdc: ios=112603248/25, merge=0/0, ticks=539695533/0, in_queue=539633877, util=100.00%
  sdd: ios=113694261/25, merge=0/0, ticks=535971849/0, in_queue=535908115, util=100.00%


###################################################################################
# 
# Sat 28 Nov 2015 03:39:47 PM EST
# 256kB write with 1 jobs and 2 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  write: io=19427GB, bw=2763.2MB/s, iops=11052, runt=7200001msec
     lat (usec): min=148, max=25756, avg=903.75, stdev=466.98
xfs
  write: io=19483GB, bw=2770.1MB/s, iops=11083, runt=7200002msec
     lat (usec): min=149, max=29519, avg=901.19, stdev=445.87
xfs
  write: io=19420GB, bw=2761.2MB/s, iops=11047, runt=7200001msec
     lat (usec): min=173, max=17202, avg=904.01, stdev=179.76

Totals
IOPS: 33182
MBPS: 8294
Average Latency (μsec): 902

xfs-01: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-02: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-03: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-04: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-05: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-11: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-12: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-13: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-14: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-15: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-21: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-22: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-23: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-24: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
xfs-25: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=23672: Sat Nov 28 17:49:48 2015
  write: io=19427GB, bw=2763.2MB/s, iops=11052, runt=7200001msec
    slat (usec): min=14, max=738, avg=30.26, stdev=11.09
    clat (usec): min=114, max=25734, avg=873.58, stdev=466.64
     lat (usec): min=148, max=25756, avg=903.75, stdev=466.98
    clat percentiles (usec):
     |  1.00th=[  486],  5.00th=[  580], 10.00th=[  628], 20.00th=[  692],
     | 30.00th=[  732], 40.00th=[  772], 50.00th=[  812], 60.00th=[  868],
     | 70.00th=[  924], 80.00th=[  996], 90.00th=[ 1128], 95.00th=[ 1272],
     | 99.00th=[ 1528], 99.50th=[ 1640], 99.90th=[10432], 99.95th=[11200],
     | 99.99th=[11712]
    bw (KB  /s): min=    0, max=695296, per=20.01%, avg=566167.93, stdev=17242.34
    lat (usec) : 250=0.01%, 500=1.28%, 750=33.23%, 1000=45.65%
    lat (msec) : 2=19.48%, 4=0.04%, 10=0.20%, 20=0.11%, 50=0.01%
  cpu          : usr=4.38%, sys=4.46%, ctx=78581010, majf=0, minf=166
  IO depths    : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=79574877/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=5): err= 0: pid=23677: Sat Nov 28 17:49:48 2015
  write: io=19483GB, bw=2770.1MB/s, iops=11083, runt=7200002msec
    slat (usec): min=15, max=883, avg=29.00, stdev=10.67
    clat (usec): min=45, max=29492, avg=871.92, stdev=445.45
     lat (usec): min=149, max=29519, avg=901.19, stdev=445.87
    clat percentiles (usec):
     |  1.00th=[  486],  5.00th=[  580], 10.00th=[  628], 20.00th=[  692],
     | 30.00th=[  732], 40.00th=[  772], 50.00th=[  812], 60.00th=[  860],
     | 70.00th=[  924], 80.00th=[ 1004], 90.00th=[ 1144], 95.00th=[ 1288],
     | 99.00th=[ 1528], 99.50th=[ 1656], 99.90th=[ 9408], 99.95th=[10816],
     | 99.99th=[11456]
    bw (KB  /s): min=    0, max=698880, per=20.01%, avg=567785.98, stdev=15782.34
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=1.27%, 750=33.82%
    lat (usec) : 1000=44.87%
    lat (msec) : 2=19.69%, 4=0.06%, 10=0.20%, 20=0.09%, 50=0.01%
  cpu          : usr=4.22%, sys=4.30%, ctx=78265717, majf=0, minf=171
  IO depths    : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=79802324/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=5): err= 0: pid=23682: Sat Nov 28 17:49:48 2015
  write: io=19420GB, bw=2761.2MB/s, iops=11047, runt=7200001msec
    slat (usec): min=15, max=758, avg=32.00, stdev=11.57
    clat (usec): min=68, max=17181, avg=871.71, stdev=177.98
     lat (usec): min=173, max=17202, avg=904.01, stdev=179.76
    clat percentiles (usec):
     |  1.00th=[  510],  5.00th=[  604], 10.00th=[  652], 20.00th=[  724],
     | 30.00th=[  772], 40.00th=[  812], 50.00th=[  852], 60.00th=[  900],
     | 70.00th=[  956], 80.00th=[ 1020], 90.00th=[ 1112], 95.00th=[ 1192],
     | 99.00th=[ 1352], 99.50th=[ 1400], 99.90th=[ 1480], 99.95th=[ 1512],
     | 99.99th=[ 1624]
    bw (KB  /s): min=    0, max=692736, per=20.01%, avg=565964.71, stdev=15689.04
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.77%, 750=23.70%, 1000=52.86%
    lat (msec) : 2=22.66%, 4=0.01%, 10=0.01%, 20=0.01%
  cpu          : usr=4.61%, sys=4.74%, ctx=75805478, majf=0, minf=172
  IO depths    : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=79545156/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=19427GB, aggrb=2763.2MB/s, minb=2763.2MB/s, maxb=2763.2MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=19483GB, aggrb=2770.1MB/s, minb=2770.1MB/s, maxb=2770.1MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
  WRITE: io=19420GB, aggrb=2761.2MB/s, minb=2761.2MB/s, maxb=2761.2MB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sdb: ios=0/86216078, merge=0/0, ticks=0/75608371, in_queue=75540780, util=100.00%
  sdc: ios=0/86444686, merge=0/0, ticks=0/75357473, in_queue=75288538, util=100.00%
  sdd: ios=0/86182311, merge=0/0, ticks=0/75274292, in_queue=75194253, util=100.00%


###################################################################################
# 
# Sat 28 Nov 2015 05:50:48 PM EST
# 256kB read with 1 jobs and 10 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=24920GB, bw=3544.2MB/s, iops=14176, runt=7200004msec
     lat (usec): min=118, max=25598, avg=3525.99, stdev=671.47
xfs
  read : io=24994GB, bw=3554.8MB/s, iops=14218, runt=7200004msec
     lat (usec): min=114, max=24813, avg=3515.53, stdev=671.38
xfs
  read : io=25104GB, bw=3570.5MB/s, iops=14281, runt=7200004msec
     lat (usec): min=129, max=25894, avg=3500.14, stdev=737.22

Totals
IOPS: 42675
MBPS: 10668
Average Latency (μsec): 3513

xfs-01: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-02: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-03: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-04: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-05: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-11: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-12: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-13: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-14: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-15: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-21: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-22: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-23: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-24: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
xfs-25: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=10
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=25395: Sat Nov 28 20:00:49 2015
  read : io=24920GB, bw=3544.2MB/s, iops=14176, runt=7200004msec
    slat (usec): min=8, max=934, avg=34.66, stdev=16.97
    clat (usec): min=98, max=25548, avg=3490.68, stdev=671.47
     lat (usec): min=118, max=25598, avg=3525.99, stdev=671.47
    clat percentiles (usec):
     |  1.00th=[ 2064],  5.00th=[ 2640], 10.00th=[ 2832], 20.00th=[ 3056],
     | 30.00th=[ 3184], 40.00th=[ 3312], 50.00th=[ 3408], 60.00th=[ 3536],
     | 70.00th=[ 3664], 80.00th=[ 3856], 90.00th=[ 4192], 95.00th=[ 4576],
     | 99.00th=[ 5920], 99.50th=[ 6496], 99.90th=[ 7840], 99.95th=[ 8384],
     | 99.99th=[10304]
    bw (KB  /s): min=    0, max=915968, per=20.01%, avg=726195.45, stdev=15581.02
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.02%
    lat (msec) : 2=0.78%, 4=84.65%, 10=14.53%, 20=0.01%, 50=0.01%
  cpu          : usr=0.80%, sys=11.71%, ctx=102238626, majf=0, minf=816
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.4%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=102071788/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
xfs-11: (groupid=1, jobs=5): err= 0: pid=25400: Sat Nov 28 20:00:49 2015
  read : io=24994GB, bw=3554.8MB/s, iops=14218, runt=7200004msec
    slat (usec): min=8, max=1025, avg=32.40, stdev=16.55
    clat (usec): min=93, max=24783, avg=3482.99, stdev=671.35
     lat (usec): min=114, max=24813, avg=3515.53, stdev=671.38
    clat percentiles (usec):
     |  1.00th=[ 2064],  5.00th=[ 2640], 10.00th=[ 2832], 20.00th=[ 3056],
     | 30.00th=[ 3184], 40.00th=[ 3312], 50.00th=[ 3408], 60.00th=[ 3536],
     | 70.00th=[ 3664], 80.00th=[ 3856], 90.00th=[ 4192], 95.00th=[ 4576],
     | 99.00th=[ 5920], 99.50th=[ 6496], 99.90th=[ 7776], 99.95th=[ 8384],
     | 99.99th=[10176]
    bw (KB  /s): min=    0, max=918528, per=20.01%, avg=728365.88, stdev=15342.79
    lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.02%
    lat (msec) : 2=0.80%, 4=84.94%, 10=14.24%, 20=0.01%, 50=0.01%
  cpu          : usr=0.72%, sys=11.07%, ctx=102597379, majf=0, minf=821
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=102376606/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
xfs-21: (groupid=2, jobs=5): err= 0: pid=25405: Sat Nov 28 20:00:49 2015
  read : io=25104GB, bw=3570.5MB/s, iops=14281, runt=7200004msec
    slat (usec): min=8, max=803, avg=31.70, stdev=16.65
    clat (usec): min=108, max=25872, avg=3468.82, stdev=737.19
     lat (usec): min=129, max=25894, avg=3500.14, stdev=737.22
    clat percentiles (usec):
     |  1.00th=[ 1912],  5.00th=[ 2544], 10.00th=[ 2800], 20.00th=[ 3024],
     | 30.00th=[ 3152], 40.00th=[ 3280], 50.00th=[ 3376], 60.00th=[ 3504],
     | 70.00th=[ 3632], 80.00th=[ 3824], 90.00th=[ 4192], 95.00th=[ 4640],
     | 99.00th=[ 6176], 99.50th=[ 6880], 99.90th=[ 8512], 99.95th=[ 9408],
     | 99.99th=[11968]
    bw (KB  /s): min=    0, max=835072, per=20.01%, avg=731573.15, stdev=17190.46
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.04%
    lat (msec) : 2=1.22%, 4=84.56%, 10=14.15%, 20=0.03%, 50=0.01%
  cpu          : usr=0.71%, sys=10.88%, ctx=103555071, majf=0, minf=822
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=102827787/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10

Run status group 0 (all jobs):
   READ: io=24920GB, aggrb=3544.2MB/s, minb=3544.2MB/s, maxb=3544.2MB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
   READ: io=24994GB, aggrb=3554.8MB/s, minb=3554.8MB/s, maxb=3554.8MB/s, mint=7200004msec, maxt=7200004msec

Run status group 2 (all jobs):
   READ: io=25104GB, aggrb=3570.5MB/s, minb=3570.5MB/s, maxb=3570.5MB/s, mint=7200004msec, maxt=7200004msec

Disk stats (read/write):
  sdb: ios=110603748/25, merge=0/0, ticks=386079543/0, in_queue=385974995, util=100.00%
  sdc: ios=110906082/25, merge=0/0, ticks=386037921/0, in_queue=385949603, util=100.00%
  sdd: ios=111381719/25, merge=0/0, ticks=386430907/0, in_queue=386333824, util=100.00%


###################################################################################
# 
# Sat 28 Nov 2015 08:01:49 PM EST
# 64kB write with 1 jobs and 6 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  write: io=17754GB, bw=2524.1MB/s, iops=40399, runt=7200001msec
     lat (usec): min=80, max=2242, avg=741.64, stdev=191.83
xfs
  write: io=17967GB, bw=2555.4MB/s, iops=40885, runt=7200001msec
     lat (usec): min=94, max=3646, avg=732.86, stdev=189.40
xfs
  write: io=17923GB, bw=2549.1MB/s, iops=40785, runt=7200001msec
     lat (usec): min=184, max=2119, avg=734.64, stdev=175.56

Totals
IOPS: 122069
MBPS: 7628
Average Latency (μsec): 735

xfs-01: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-02: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-03: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-04: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-05: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-11: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-12: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-13: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-14: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-15: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-21: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-22: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-23: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-24: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
xfs-25: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=6
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=27060: Sat Nov 28 22:11:51 2015
  write: io=17754GB, bw=2524.1MB/s, iops=40399, runt=7200001msec
    slat (usec): min=4, max=983, avg=17.90, stdev= 9.82
    clat (usec): min=44, max=2225, avg=723.56, stdev=192.45
     lat (usec): min=80, max=2242, avg=741.64, stdev=191.83
    clat percentiles (usec):
     |  1.00th=[  374],  5.00th=[  450], 10.00th=[  494], 20.00th=[  548],
     | 30.00th=[  596], 40.00th=[  644], 50.00th=[  700], 60.00th=[  764],
     | 70.00th=[  828], 80.00th=[  908], 90.00th=[  988], 95.00th=[ 1048],
     | 99.00th=[ 1208], 99.50th=[ 1304], 99.90th=[ 1400], 99.95th=[ 1432],
     | 99.99th=[ 1496]
    bw (KB  /s): min=    0, max=584960, per=20.01%, avg=517256.47, stdev=15414.10
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.02%, 500=11.05%, 750=47.12%
    lat (usec) : 1000=32.84%
    lat (msec) : 2=8.97%, 4=0.01%
  cpu          : usr=6.64%, sys=15.60%, ctx=121936494, majf=0, minf=165
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=290873785/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
xfs-11: (groupid=1, jobs=5): err= 0: pid=27065: Sat Nov 28 22:11:51 2015
  write: io=17967GB, bw=2555.4MB/s, iops=40885, runt=7200001msec
    slat (usec): min=4, max=944, avg=16.95, stdev= 9.67
    clat (usec): min=51, max=3633, avg=715.77, stdev=189.98
     lat (usec): min=94, max=3646, avg=732.86, stdev=189.40
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  446], 10.00th=[  486], 20.00th=[  548],
     | 30.00th=[  588], 40.00th=[  636], 50.00th=[  692], 60.00th=[  756],
     | 70.00th=[  820], 80.00th=[  892], 90.00th=[  980], 95.00th=[ 1032],
     | 99.00th=[ 1192], 99.50th=[ 1272], 99.90th=[ 1384], 99.95th=[ 1416],
     | 99.99th=[ 1480]
    bw (KB  /s): min=    0, max=590336, per=20.01%, avg=523491.55, stdev=13516.46
    lat (usec) : 100=0.01%, 250=0.02%, 500=11.93%, 750=46.92%, 1000=33.44%
    lat (msec) : 2=7.69%, 4=0.01%
  cpu          : usr=6.45%, sys=14.88%, ctx=122555251, majf=0, minf=170
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=294378328/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
xfs-21: (groupid=2, jobs=5): err= 0: pid=27070: Sat Nov 28 22:11:51 2015
  write: io=17923GB, bw=2549.1MB/s, iops=40785, runt=7200001msec
    slat (usec): min=4, max=891, avg=17.27, stdev= 9.71
    clat (usec): min=80, max=2107, avg=717.23, stdev=176.14
     lat (usec): min=184, max=2119, avg=734.64, stdev=175.56
    clat percentiles (usec):
     |  1.00th=[  402],  5.00th=[  462], 10.00th=[  498], 20.00th=[  556],
     | 30.00th=[  596], 40.00th=[  644], 50.00th=[  700], 60.00th=[  756],
     | 70.00th=[  820], 80.00th=[  892], 90.00th=[  972], 95.00th=[ 1020],
     | 99.00th=[ 1096], 99.50th=[ 1128], 99.90th=[ 1192], 99.95th=[ 1224],
     | 99.99th=[ 1288]
    bw (KB  /s): min=    0, max=589568, per=20.01%, avg=522202.74, stdev=15001.38
    lat (usec) : 100=0.01%, 250=0.01%, 500=10.13%, 750=48.55%, 1000=34.53%
    lat (msec) : 2=6.78%, 4=0.01%
  cpu          : usr=6.53%, sys=15.09%, ctx=123128541, majf=0, minf=170
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=293655425/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6

Run status group 0 (all jobs):
  WRITE: io=17754GB, aggrb=2524.1MB/s, minb=2524.1MB/s, maxb=2524.1MB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=17967GB, aggrb=2555.4MB/s, minb=2555.4MB/s, maxb=2555.4MB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=17923GB, aggrb=2549.1MB/s, minb=2549.1MB/s, maxb=2549.1MB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sdb: ios=0/314940979, merge=0/373865, ticks=0/218651966, in_queue=218368118, util=100.00%
  sdc: ios=0/318687038, merge=0/371053, ticks=0/219194692, in_queue=218906921, util=100.00%
  sdd: ios=0/317919578, merge=0/345624, ticks=0/219402281, in_queue=219104706, util=100.00%


###################################################################################
# 
# Sat 28 Nov 2015 10:12:51 PM EST
# 64kB read with 1 jobs and 16 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=22531GB, bw=3204.5MB/s, iops=51270, runt=7200002msec
     lat (usec): min=44, max=23025, avg=1559.27, stdev=423.82
xfs
  read : io=22497GB, bw=3199.6MB/s, iops=51192, runt=7200002msec
     lat (usec): min=42, max=21389, avg=1561.59, stdev=414.76
xfs
  read : io=22794GB, bw=3241.8MB/s, iops=51868, runt=7200002msec
     lat (usec): min=56, max=22005, avg=1541.30, stdev=412.36

Totals
IOPS: 154330
MBPS: 9644
Average Latency (μsec): 1553

xfs-01: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-02: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-03: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-04: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-05: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-11: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-12: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-13: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-14: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-15: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-21: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-22: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-23: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-24: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
xfs-25: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=28772: Sun Nov 29 00:22:52 2015
  read : io=22531GB, bw=3204.5MB/s, iops=51270, runt=7200002msec
    slat (usec): min=2, max=911, avg=13.20, stdev= 5.96
    clat (usec): min=38, max=23019, avg=1545.93, stdev=423.78
     lat (usec): min=44, max=23025, avg=1559.27, stdev=423.82
    clat percentiles (usec):
     |  1.00th=[  796],  5.00th=[  964], 10.00th=[ 1064], 20.00th=[ 1192],
     | 30.00th=[ 1304], 40.00th=[ 1400], 50.00th=[ 1496], 60.00th=[ 1592],
     | 70.00th=[ 1720], 80.00th=[ 1864], 90.00th=[ 2096], 95.00th=[ 2288],
     | 99.00th=[ 2736], 99.50th=[ 2992], 99.90th=[ 3920], 99.95th=[ 4384],
     | 99.99th=[ 5600]
    bw (KB  /s): min=    0, max=762624, per=20.00%, avg=656345.34, stdev=19345.06
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.58%
    lat (usec) : 1000=5.85%
    lat (msec) : 2=80.43%, 4=13.05%, 10=0.09%, 20=0.01%, 50=0.01%
  cpu          : usr=3.20%, sys=19.46%, ctx=317177738, majf=0, minf=1451
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=108.4%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=369149713/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
xfs-11: (groupid=1, jobs=5): err= 0: pid=28777: Sun Nov 29 00:22:52 2015
  read : io=22497GB, bw=3199.6MB/s, iops=51192, runt=7200002msec
    slat (usec): min=2, max=856, avg=13.87, stdev= 5.40
    clat (usec): min=37, max=21378, avg=1547.49, stdev=414.72
     lat (usec): min=42, max=21389, avg=1561.59, stdev=414.76
    clat percentiles (usec):
     |  1.00th=[  788],  5.00th=[  964], 10.00th=[ 1064], 20.00th=[ 1208],
     | 30.00th=[ 1304], 40.00th=[ 1400], 50.00th=[ 1496], 60.00th=[ 1608],
     | 70.00th=[ 1720], 80.00th=[ 1864], 90.00th=[ 2096], 95.00th=[ 2288],
     | 99.00th=[ 2704], 99.50th=[ 2896], 99.90th=[ 3728], 99.95th=[ 4192],
     | 99.99th=[ 5408]
    bw (KB  /s): min=    0, max=769024, per=20.00%, avg=655339.26, stdev=28339.83
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.64%
    lat (usec) : 1000=5.81%
    lat (msec) : 2=80.43%, 4=13.04%, 10=0.06%, 20=0.01%, 50=0.01%
  cpu          : usr=3.64%, sys=20.35%, ctx=312690758, majf=0, minf=1456
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=108.4%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=368584343/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16
xfs-21: (groupid=2, jobs=5): err= 0: pid=28782: Sun Nov 29 00:22:52 2015
  read : io=22794GB, bw=3241.8MB/s, iops=51868, runt=7200002msec
    slat (usec): min=2, max=927, avg=13.36, stdev= 5.88
    clat (usec): min=46, max=21993, avg=1527.81, stdev=412.27
     lat (usec): min=56, max=22005, avg=1541.30, stdev=412.36
    clat percentiles (usec):
     |  1.00th=[  780],  5.00th=[  956], 10.00th=[ 1048], 20.00th=[ 1192],
     | 30.00th=[ 1288], 40.00th=[ 1384], 50.00th=[ 1480], 60.00th=[ 1576],
     | 70.00th=[ 1704], 80.00th=[ 1848], 90.00th=[ 2064], 95.00th=[ 2256],
     | 99.00th=[ 2672], 99.50th=[ 2832], 99.90th=[ 3664], 99.95th=[ 4256],
     | 99.99th=[ 6048]
    bw (KB  /s): min=    0, max=788992, per=20.00%, avg=663990.55, stdev=26193.32
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.73%
    lat (usec) : 1000=6.25%
    lat (msec) : 2=80.85%, 4=12.10%, 10=0.07%, 20=0.01%, 50=0.01%
  cpu          : usr=3.22%, sys=19.91%, ctx=318570526, majf=0, minf=1457
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=108.3%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=373449999/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=16

Run status group 0 (all jobs):
   READ: io=22531GB, aggrb=3204.5MB/s, minb=3204.5MB/s, maxb=3204.5MB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=22497GB, aggrb=3199.6MB/s, minb=3199.6MB/s, maxb=3199.6MB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=22794GB, aggrb=3241.8MB/s, minb=3241.8MB/s, maxb=3241.8MB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sdb: ios=399896499/25, merge=131643/0, ticks=617373868/0, in_queue=617042827, util=100.00%
  sdc: ios=399355499/25, merge=68818/0, ticks=617029842/0, in_queue=616664956, util=100.00%
  sdd: ios=404154800/25, merge=133860/0, ticks=617228607/0, in_queue=616890442, util=100.00%


###################################################################################
# 
# Sun 29 Nov 2015 12:23:52 AM EST
# 8kB randwrite with 1 jobs and 2 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  write: io=902007MB, bw=128285KB/s, iops=16035, runt=7200001msec
     lat (usec): min=29, max=5415, avg=622.30, stdev=430.10
xfs
  write: io=893351MB, bw=127054KB/s, iops=15881, runt=7200001msec
     lat (usec): min=31, max=4057, avg=628.35, stdev=427.67
xfs
  write: io=894896MB, bw=127274KB/s, iops=15909, runt=7200002msec
     lat (usec): min=30, max=3258, avg=627.31, stdev=492.63

Totals
IOPS: 47825
MBPS: 373
Average Latency (μsec): 625

xfs-01: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-02: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-03: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-04: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-05: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-11: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-12: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-13: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-14: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-15: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-21: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-22: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-23: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-24: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
xfs-25: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=30431: Sun Nov 29 02:33:54 2015
  write: io=902007MB, bw=128285KB/s, iops=16035, runt=7200001msec
    slat (usec): min=1, max=770, avg= 9.72, stdev= 5.37
    clat (usec): min=0, max=5408, avg=612.47, stdev=430.19
     lat (usec): min=29, max=5415, avg=622.30, stdev=430.10
    clat percentiles (usec):
     |  1.00th=[   78],  5.00th=[  104], 10.00th=[  115], 20.00th=[  137],
     | 30.00th=[  215], 40.00th=[  398], 50.00th=[  482], 60.00th=[  900],
     | 70.00th=[  964], 80.00th=[ 1032], 90.00th=[ 1208], 95.00th=[ 1320],
     | 99.00th=[ 1432], 99.50th=[ 1448], 99.90th=[ 1496], 99.95th=[ 1528],
     | 99.99th=[ 1560]
    bw (KB  /s): min=    0, max=29680, per=20.01%, avg=25672.81, stdev=399.41
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.07%, 100=3.69%
    lat (usec) : 250=29.59%, 500=18.31%, 750=5.24%, 1000=18.54%
    lat (msec) : 2=24.56%, 4=0.01%, 10=0.01%
  cpu          : usr=1.41%, sys=4.73%, ctx=87802348, majf=0, minf=159
  IO depths    : 1=0.1%, 2=108.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=115456830/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=5): err= 0: pid=30436: Sun Nov 29 02:33:54 2015
  write: io=893351MB, bw=127054KB/s, iops=15881, runt=7200001msec
    slat (usec): min=1, max=763, avg= 9.74, stdev= 5.36
    clat (usec): min=0, max=4040, avg=618.48, stdev=427.77
     lat (usec): min=31, max=4057, avg=628.35, stdev=427.67
    clat percentiles (usec):
     |  1.00th=[   81],  5.00th=[  107], 10.00th=[  118], 20.00th=[  143],
     | 30.00th=[  223], 40.00th=[  402], 50.00th=[  490], 60.00th=[  900],
     | 70.00th=[  964], 80.00th=[ 1032], 90.00th=[ 1208], 95.00th=[ 1320],
     | 99.00th=[ 1432], 99.50th=[ 1448], 99.90th=[ 1496], 99.95th=[ 1528],
     | 99.99th=[ 1560]
    bw (KB  /s): min=    0, max=29056, per=20.01%, avg=25426.23, stdev=385.17
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.06%, 100=3.11%
    lat (usec) : 250=29.23%, 500=18.39%, 750=5.62%, 1000=18.83%
    lat (msec) : 2=24.76%, 4=0.01%, 10=0.01%
  cpu          : usr=1.38%, sys=4.73%, ctx=87038303, majf=0, minf=165
  IO depths    : 1=0.1%, 2=108.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=114348985/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=5): err= 0: pid=30441: Sun Nov 29 02:33:54 2015
  write: io=894896MB, bw=127274KB/s, iops=15909, runt=7200002msec
    slat (usec): min=1, max=719, avg= 9.37, stdev= 5.41
    clat (usec): min=0, max=3248, avg=617.78, stdev=492.82
     lat (usec): min=30, max=3258, avg=627.31, stdev=492.63
    clat percentiles (usec):
     |  1.00th=[   79],  5.00th=[  102], 10.00th=[  112], 20.00th=[  123],
     | 30.00th=[  137], 40.00th=[  247], 50.00th=[  418], 60.00th=[  836],
     | 70.00th=[ 1004], 80.00th=[ 1176], 90.00th=[ 1272], 95.00th=[ 1480],
     | 99.00th=[ 1592], 99.50th=[ 1608], 99.90th=[ 1656], 99.95th=[ 1672],
     | 99.99th=[ 1784]
    bw (KB  /s): min=    0, max=33776, per=20.01%, avg=25470.24, stdev=432.88
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.06%, 100=4.04%
    lat (usec) : 250=35.95%, 500=11.06%, 750=5.30%, 1000=13.57%
    lat (msec) : 2=30.00%, 4=0.01%
  cpu          : usr=1.28%, sys=4.55%, ctx=88506112, majf=0, minf=166
  IO depths    : 1=0.1%, 2=108.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=114546628/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=902007MB, aggrb=128285KB/s, minb=128285KB/s, maxb=128285KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=893351MB, aggrb=127054KB/s, minb=127054KB/s, maxb=127054KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=894896MB, aggrb=127274KB/s, minb=127274KB/s, maxb=127274KB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sdb: ios=0/125152246, merge=0/0, ticks=0/75818440, in_queue=75755819, util=100.00%
  sdc: ios=0/123947319, merge=0/0, ticks=0/76059300, in_queue=75996235, util=100.00%
  sdd: ios=0/124191900, merge=0/0, ticks=0/76142839, in_queue=76082658, util=100.00%


###################################################################################
# 
# Sun 29 Nov 2015 02:34:54 AM EST
# 64kB randwrite with 1 jobs and 2 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  write: io=3078.3GB, bw=448303KB/s, iops=7004, runt=7200001msec
     lat (usec): min=58, max=694151, avg=1426.25, stdev=2758.16
xfs
  write: io=3126.9GB, bw=455378KB/s, iops=7115, runt=7200001msec
     lat (usec): min=61, max=1644.5K, avg=1404.05, stdev=2337.28
xfs
  write: io=3272.1GB, bw=476660KB/s, iops=7447, runt=7200001msec
     lat (usec): min=55, max=887524, avg=1341.15, stdev=3291.70

Totals
IOPS: 21566
MBPS: 1346
Average Latency (μsec): 1390

xfs-01: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-02: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-03: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-04: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-05: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-11: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-12: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-13: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-14: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-15: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-21: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-22: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-23: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-24: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
xfs-25: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=32163: Sun Nov 29 04:44:55 2015
  write: io=3078.3GB, bw=448303KB/s, iops=7004, runt=7200001msec
    slat (usec): min=5, max=749, avg=13.02, stdev= 6.12
    clat (usec): min=18, max=694144, avg=1413.29, stdev=2758.19
     lat (usec): min=58, max=694151, avg=1426.25, stdev=2758.16
    clat percentiles (usec):
     |  1.00th=[  302],  5.00th=[  466], 10.00th=[  596], 20.00th=[  796],
     | 30.00th=[  964], 40.00th=[ 1112], 50.00th=[ 1272], 60.00th=[ 1448],
     | 70.00th=[ 1640], 80.00th=[ 1928], 90.00th=[ 2384], 95.00th=[ 2800],
     | 99.00th=[ 3696], 99.50th=[ 4080], 99.90th=[ 5088], 99.95th=[ 5664],
     | 99.99th=[ 7840]
    bw (KB  /s): min=    0, max=121344, per=20.08%, avg=90015.34, stdev=12327.59
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.43%, 500=5.73%
    lat (usec) : 750=11.46%, 1000=14.91%
    lat (msec) : 2=49.62%, 4=17.27%, 10=0.57%, 20=0.01%, 50=0.01%
    lat (msec) : 750=0.01%
  cpu          : usr=1.20%, sys=1.81%, ctx=53820407, majf=0, minf=168
  IO depths    : 1=0.1%, 2=110.2%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=50434125/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=5): err= 0: pid=32168: Sun Nov 29 04:44:55 2015
  write: io=3126.9GB, bw=455378KB/s, iops=7115, runt=7200001msec
    slat (usec): min=5, max=682, avg=12.89, stdev= 5.97
    clat (usec): min=33, max=1644.4K, avg=1390.98, stdev=2337.23
     lat (usec): min=61, max=1644.5K, avg=1404.05, stdev=2337.28
    clat percentiles (usec):
     |  1.00th=[  306],  5.00th=[  474], 10.00th=[  604], 20.00th=[  796],
     | 30.00th=[  956], 40.00th=[ 1096], 50.00th=[ 1256], 60.00th=[ 1416],
     | 70.00th=[ 1624], 80.00th=[ 1896], 90.00th=[ 2352], 95.00th=[ 2768],
     | 99.00th=[ 3632], 99.50th=[ 3984], 99.90th=[ 4960], 99.95th=[ 5536],
     | 99.99th=[ 7520]
    bw (KB  /s): min=    0, max=123776, per=20.06%, avg=91348.29, stdev=12303.98
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.39%, 500=5.53%, 750=11.54%
    lat (usec) : 1000=15.57%
    lat (msec) : 2=50.08%, 4=16.38%, 10=0.50%, 20=0.01%, 750=0.01%
    lat (msec) : 2000=0.01%
  cpu          : usr=1.23%, sys=1.81%, ctx=54761320, majf=0, minf=170
  IO depths    : 1=0.1%, 2=110.2%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=51230016/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=5): err= 0: pid=32173: Sun Nov 29 04:44:55 2015
  write: io=3272.1GB, bw=476660KB/s, iops=7447, runt=7200001msec
    slat (usec): min=5, max=693, avg=15.04, stdev= 6.91
    clat (usec): min=26, max=887515, avg=1325.93, stdev=3291.75
     lat (usec): min=55, max=887524, avg=1341.15, stdev=3291.70
    clat percentiles (usec):
     |  1.00th=[  231],  5.00th=[  354], 10.00th=[  450], 20.00th=[  612],
     | 30.00th=[  764], 40.00th=[  924], 50.00th=[ 1096], 60.00th=[ 1320],
     | 70.00th=[ 1576], 80.00th=[ 1880], 90.00th=[ 2384], 95.00th=[ 2960],
     | 99.00th=[ 4256], 99.50th=[ 4768], 99.90th=[ 6048], 99.95th=[ 6752],
     | 99.99th=[ 9920]
    bw (KB  /s): min=    0, max=191488, per=20.11%, avg=95844.26, stdev=16545.00
    lat (usec) : 50=0.01%, 100=0.01%, 250=1.42%, 500=11.51%, 750=16.18%
    lat (usec) : 1000=15.41%
    lat (msec) : 2=38.84%, 4=15.29%, 10=1.35%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
  cpu          : usr=1.45%, sys=2.29%, ctx=50503829, majf=0, minf=173
  IO depths    : 1=0.1%, 2=110.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=53624294/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
  WRITE: io=3078.3GB, aggrb=448303KB/s, minb=448303KB/s, maxb=448303KB/s, mint=7200001msec, maxt=7200001msec

Run status group 1 (all jobs):
  WRITE: io=3126.9GB, aggrb=455377KB/s, minb=455377KB/s, maxb=455377KB/s, mint=7200001msec, maxt=7200001msec

Run status group 2 (all jobs):
  WRITE: io=3272.1GB, aggrb=476660KB/s, minb=476660KB/s, maxb=476660KB/s, mint=7200001msec, maxt=7200001msec

Disk stats (read/write):
  sdb: ios=0/55567798, merge=0/0, ticks=0/77323963, in_queue=77306596, util=100.00%
  sdc: ios=0/56442118, merge=0/0, ticks=0/77412719, in_queue=77395875, util=100.00%
  sdd: ios=0/59193766, merge=0/0, ticks=0/77008188, in_queue=76979016, util=100.00%


###################################################################################
# 
# Sun 29 Nov 2015 04:45:56 AM EST
# 64kB randread with 2 jobs and 10 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=23028GB, bw=3275.2MB/s, iops=52401, runt=7200003msec
     lat (usec): min=52, max=25614, avg=1907.11, stdev=763.10
xfs
  read : io=22915GB, bw=3259.4MB/s, iops=52144, runt=7200003msec
     lat (usec): min=55, max=24792, avg=1916.55, stdev=780.33
xfs
  read : io=23050GB, bw=3278.2MB/s, iops=52450, runt=7200003msec
     lat (usec): min=55, max=26959, avg=1905.37, stdev=849.82

Totals
IOPS: 156995
MBPS: 9812
Average Latency (μsec): 1909

xfs-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-02: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-03: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-04: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-05: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-12: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-13: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-14: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-15: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-21: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-22: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-23: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-24: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
xfs-25: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=10
...
fio-2.1.10
Starting 30 processes

xfs-01: (groupid=0, jobs=10): err= 0: pid=34151: Sun Nov 29 06:55:57 2015
  read : io=23028GB, bw=3275.2MB/s, iops=52401, runt=7200003msec
    slat (usec): min=2, max=732, avg= 9.63, stdev= 4.77
    clat (usec): min=45, max=25608, avg=1897.30, stdev=763.10
     lat (usec): min=52, max=25614, avg=1907.11, stdev=763.10
    clat percentiles (usec):
     |  1.00th=[  620],  5.00th=[  852], 10.00th=[ 1020], 20.00th=[ 1272],
     | 30.00th=[ 1464], 40.00th=[ 1640], 50.00th=[ 1816], 60.00th=[ 1992],
     | 70.00th=[ 2192], 80.00th=[ 2448], 90.00th=[ 2832], 95.00th=[ 3216],
     | 99.00th=[ 4256], 99.50th=[ 4832], 99.90th=[ 6176], 99.95th=[ 6752],
     | 99.99th=[ 8640]
    bw (KB  /s): min=    0, max=354816, per=10.00%, avg=335458.67, stdev=5353.32
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.20%, 750=2.60%
    lat (usec) : 1000=6.40%
    lat (msec) : 2=51.19%, 4=38.16%, 10=1.43%, 20=0.01%, 50=0.01%
  cpu          : usr=1.80%, sys=7.38%, ctx=380222592, majf=0, minf=1954
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=377291817/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
xfs-11: (groupid=1, jobs=10): err= 0: pid=34161: Sun Nov 29 06:55:57 2015
  read : io=22915GB, bw=3259.4MB/s, iops=52144, runt=7200003msec
    slat (usec): min=3, max=701, avg= 9.00, stdev= 4.39
    clat (usec): min=47, max=24785, avg=1907.47, stdev=780.32
     lat (usec): min=55, max=24792, avg=1916.55, stdev=780.33
    clat percentiles (usec):
     |  1.00th=[  612],  5.00th=[  844], 10.00th=[ 1012], 20.00th=[ 1256],
     | 30.00th=[ 1464], 40.00th=[ 1640], 50.00th=[ 1816], 60.00th=[ 2008],
     | 70.00th=[ 2192], 80.00th=[ 2448], 90.00th=[ 2864], 95.00th=[ 3248],
     | 99.00th=[ 4320], 99.50th=[ 4896], 99.90th=[ 6304], 99.95th=[ 6944],
     | 99.99th=[ 8896]
    bw (KB  /s): min=    0, max=360832, per=10.00%, avg=333814.42, stdev=6783.41
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.23%, 750=2.77%
    lat (usec) : 1000=6.50%
    lat (msec) : 2=50.30%, 4=38.61%, 10=1.59%, 20=0.01%, 50=0.01%
  cpu          : usr=1.82%, sys=6.82%, ctx=380550849, majf=0, minf=1958
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=375441464/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10
xfs-21: (groupid=2, jobs=10): err= 0: pid=34171: Sun Nov 29 06:55:57 2015
  read : io=23050GB, bw=3278.2MB/s, iops=52450, runt=7200003msec
    slat (usec): min=2, max=704, avg= 8.94, stdev= 4.41
    clat (usec): min=48, max=26947, avg=1896.30, stdev=849.82
     lat (usec): min=55, max=26959, avg=1905.37, stdev=849.82
    clat percentiles (usec):
     |  1.00th=[  588],  5.00th=[  812], 10.00th=[  980], 20.00th=[ 1224],
     | 30.00th=[ 1416], 40.00th=[ 1608], 50.00th=[ 1784], 60.00th=[ 1976],
     | 70.00th=[ 2192], 80.00th=[ 2448], 90.00th=[ 2864], 95.00th=[ 3312],
     | 99.00th=[ 4768], 99.50th=[ 5536], 99.90th=[ 7584], 99.95th=[ 8512],
     | 99.99th=[11200]
    bw (KB  /s): min=    0, max=367744, per=10.00%, avg=335772.73, stdev=10145.60
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.30%, 750=3.25%
    lat (usec) : 1000=7.20%
    lat (msec) : 2=50.63%, 4=36.50%, 10=2.11%, 20=0.02%, 50=0.01%
  cpu          : usr=1.75%, sys=6.87%, ctx=382673996, majf=0, minf=1960
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=377643934/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=10

Run status group 0 (all jobs):
   READ: io=23028GB, aggrb=3275.2MB/s, minb=3275.2MB/s, maxb=3275.2MB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=22915GB, aggrb=3259.4MB/s, minb=3259.4MB/s, maxb=3259.4MB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=23050GB, aggrb=3278.2MB/s, minb=3278.2MB/s, maxb=3278.2MB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sdb: ios=408511670/25, merge=7/0, ticks=775592602/0, in_queue=775333397, util=100.00%
  sdc: ios=406531400/25, merge=1/0, ticks=775765890/0, in_queue=775543530, util=100.00%
  sdd: ios=408981953/24, merge=3/0, ticks=775804539/0, in_queue=775579147, util=100.00%


###################################################################################
# 
# Sun 29 Nov 2015 06:56:57 AM EST
# 32kB randread with 4 jobs and 6 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=22283GB, bw=3169.2MB/s, iops=101411, runt=7200003msec
     lat (usec): min=46, max=20694, avg=1182.15, stdev=527.94
xfs
  read : io=22284GB, bw=3169.3MB/s, iops=101415, runt=7200004msec
     lat (usec): min=49, max=22340, avg=1182.10, stdev=525.66
xfs
  read : io=22632GB, bw=3218.8MB/s, iops=103000, runt=7200003msec
     lat (usec): min=48, max=22344, avg=1163.92, stdev=522.33

Totals
IOPS: 305826
MBPS: 9556
Average Latency (μsec): 1175

xfs-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-02: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-03: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-04: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-05: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-12: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-13: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-14: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-15: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-21: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-22: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-23: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-24: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
xfs-25: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=6
...
fio-2.1.10
Starting 60 processes

xfs-01: (groupid=0, jobs=20): err= 0: pid=35806: Sun Nov 29 09:06:59 2015
  read : io=22283GB, bw=3169.2MB/s, iops=101411, runt=7200003msec
    slat (usec): min=2, max=2511, avg= 9.76, stdev=11.31
    clat (usec): min=0, max=20689, avg=1172.24, stdev=527.84
     lat (usec): min=46, max=20694, avg=1182.15, stdev=527.94
    clat percentiles (usec):
     |  1.00th=[  410],  5.00th=[  524], 10.00th=[  604], 20.00th=[  732],
     | 30.00th=[  844], 40.00th=[  956], 50.00th=[ 1064], 60.00th=[ 1192],
     | 70.00th=[ 1352], 80.00th=[ 1560], 90.00th=[ 1864], 95.00th=[ 2160],
     | 99.00th=[ 2832], 99.50th=[ 3120], 99.90th=[ 3792], 99.95th=[ 4128],
     | 99.99th=[ 5280]
    bw (KB  /s): min=    0, max=237440, per=5.00%, avg=162301.22, stdev=2569.28
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=3.90%, 750=17.57%, 1000=22.56%
    lat (msec) : 2=48.54%, 4=7.36%, 10=0.06%, 20=0.01%, 50=0.01%
  cpu          : usr=1.34%, sys=7.47%, ctx=726128163, majf=0, minf=1634
  IO depths    : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=730160876/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
xfs-11: (groupid=1, jobs=20): err= 0: pid=35826: Sun Nov 29 09:06:59 2015
  read : io=22284GB, bw=3169.3MB/s, iops=101415, runt=7200004msec
    slat (usec): min=2, max=2549, avg= 9.38, stdev=10.89
    clat (usec): min=0, max=22331, avg=1172.63, stdev=525.57
     lat (usec): min=49, max=22340, avg=1182.10, stdev=525.66
    clat percentiles (usec):
     |  1.00th=[  410],  5.00th=[  524], 10.00th=[  604], 20.00th=[  732],
     | 30.00th=[  844], 40.00th=[  956], 50.00th=[ 1080], 60.00th=[ 1208],
     | 70.00th=[ 1352], 80.00th=[ 1560], 90.00th=[ 1864], 95.00th=[ 2160],
     | 99.00th=[ 2800], 99.50th=[ 3120], 99.90th=[ 3760], 99.95th=[ 4128],
     | 99.99th=[ 5216]
    bw (KB  /s): min=    0, max=196480, per=5.00%, avg=162308.10, stdev=2565.22
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=3.95%, 750=17.40%, 1000=22.44%
    lat (msec) : 2=48.88%, 4=7.27%, 10=0.06%, 20=0.01%, 50=0.01%
  cpu          : usr=1.41%, sys=7.18%, ctx=728530020, majf=0, minf=1685
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=730194872/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
xfs-21: (groupid=2, jobs=20): err= 0: pid=35846: Sun Nov 29 09:06:59 2015
  read : io=22632GB, bw=3218.8MB/s, iops=103000, runt=7200003msec
    slat (usec): min=2, max=2450, avg= 9.25, stdev=10.86
    clat (usec): min=0, max=22338, avg=1154.58, stdev=522.24
     lat (usec): min=48, max=22344, avg=1163.92, stdev=522.33
    clat percentiles (usec):
     |  1.00th=[  410],  5.00th=[  524], 10.00th=[  604], 20.00th=[  732],
     | 30.00th=[  844], 40.00th=[  948], 50.00th=[ 1064], 60.00th=[ 1176],
     | 70.00th=[ 1320], 80.00th=[ 1512], 90.00th=[ 1800], 95.00th=[ 2096],
     | 99.00th=[ 2800], 99.50th=[ 3184], 99.90th=[ 4512], 99.95th=[ 5344],
     | 99.99th=[ 6752]
    bw (KB  /s): min=    0, max=195264, per=5.00%, avg=164842.63, stdev=2700.53
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=3.98%, 750=17.43%, 1000=23.15%
    lat (msec) : 2=49.25%, 4=6.04%, 10=0.15%, 20=0.01%, 50=0.01%
  cpu          : usr=1.43%, sys=7.18%, ctx=740865335, majf=0, minf=1671
  IO depths    : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=741600419/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6

Run status group 0 (all jobs):
   READ: io=22283GB, aggrb=3169.2MB/s, minb=3169.2MB/s, maxb=3169.2MB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=22284GB, aggrb=3169.3MB/s, minb=3169.3MB/s, maxb=3169.3MB/s, mint=7200004msec, maxt=7200004msec

Run status group 2 (all jobs):
   READ: io=22632GB, aggrb=3218.8MB/s, minb=3218.8MB/s, maxb=3218.8MB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sdb: ios=791138804/25, merge=1/0, ticks=924295372/0, in_queue=923580849, util=100.00%
  sdc: ios=791108098/25, merge=2/0, ticks=924899279/0, in_queue=924203049, util=100.00%
  sdd: ios=803458747/25, merge=1/0, ticks=924871490/0, in_queue=924187957, util=100.00%


###################################################################################
# 
# Sun 29 Nov 2015 09:07:59 AM EST
# 16kB randread with 2 jobs and 14 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=16977GB, bw=2414.6MB/s, iops=154528, runt=7200003msec
     lat (usec): min=45, max=40817, avg=904.90, stdev=239.44
xfs
  read : io=16938GB, bw=2409.3MB/s, iops=154177, runt=7200003msec
     lat (usec): min=69, max=38054, avg=906.96, stdev=233.24
xfs
  read : io=17059GB, bw=2426.2MB/s, iops=155272, runt=7200006msec
     lat (usec): min=109, max=49357, avg=900.56, stdev=276.24

Totals
IOPS: 463977
MBPS: 7249
Average Latency (μsec): 903

xfs-01: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-02: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-03: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-04: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-05: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-11: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-12: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-13: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-14: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-15: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-21: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-22: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-23: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-24: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
xfs-25: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=14
...
fio-2.1.10
Starting 30 processes

xfs-01: (groupid=0, jobs=10): err= 0: pid=37523: Sun Nov 29 11:18:01 2015
  read : io=16977GB, bw=2414.6MB/s, iops=154528, runt=7200003msec
    slat (usec): min=1, max=30531, avg=15.66, stdev=36.14
    clat (usec): min=15, max=39875, avg=889.09, stdev=237.40
     lat (usec): min=45, max=40817, avg=904.90, stdev=239.44
    clat percentiles (usec):
     |  1.00th=[  596],  5.00th=[  676], 10.00th=[  716], 20.00th=[  756],
     | 30.00th=[  796], 40.00th=[  820], 50.00th=[  852], 60.00th=[  892],
     | 70.00th=[  932], 80.00th=[  980], 90.00th=[ 1064], 95.00th=[ 1160],
     | 99.00th=[ 1608], 99.50th=[ 2384], 99.90th=[ 3472], 99.95th=[ 3664],
     | 99.99th=[ 4832]
    bw (KB  /s): min=    0, max=437568, per=10.00%, avg=247264.06, stdev=4584.08
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.23%
    lat (usec) : 750=17.70%, 1000=64.68%
    lat (msec) : 2=16.73%, 4=0.64%, 10=0.02%, 20=0.01%, 50=0.01%
  cpu          : usr=2.88%, sys=26.40%, ctx=646855595, majf=0, minf=915
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1112607331/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=14
xfs-11: (groupid=1, jobs=10): err= 0: pid=37533: Sun Nov 29 11:18:01 2015
  read : io=16938GB, bw=2409.3MB/s, iops=154177, runt=7200003msec
    slat (usec): min=1, max=28045, avg=14.75, stdev=35.35
    clat (usec): min=22, max=37262, avg=892.13, stdev=231.35
     lat (usec): min=69, max=38054, avg=906.96, stdev=233.24
    clat percentiles (usec):
     |  1.00th=[  612],  5.00th=[  684], 10.00th=[  716], 20.00th=[  764],
     | 30.00th=[  796], 40.00th=[  828], 50.00th=[  860], 60.00th=[  892],
     | 70.00th=[  932], 80.00th=[  988], 90.00th=[ 1064], 95.00th=[ 1160],
     | 99.00th=[ 1576], 99.50th=[ 2320], 99.90th=[ 3440], 99.95th=[ 3632],
     | 99.99th=[ 4512]
    bw (KB  /s): min=    0, max=357440, per=10.00%, avg=246701.67, stdev=4100.76
    lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.14%, 750=16.51%
    lat (usec) : 1000=65.69%
    lat (msec) : 2=17.02%, 4=0.62%, 10=0.02%, 20=0.01%, 50=0.01%
  cpu          : usr=2.93%, sys=24.68%, ctx=656234259, majf=0, minf=920
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1110077316/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=14
xfs-21: (groupid=2, jobs=10): err= 0: pid=37543: Sun Nov 29 11:18:01 2015
  read : io=17059GB, bw=2426.2MB/s, iops=155272, runt=7200006msec
    slat (usec): min=1, max=33160, avg=14.99, stdev=35.41
    clat (usec): min=0, max=49329, avg=885.45, stdev=274.61
     lat (usec): min=109, max=49357, avg=900.56, stdev=276.24
    clat percentiles (usec):
     |  1.00th=[  596],  5.00th=[  676], 10.00th=[  708], 20.00th=[  748],
     | 30.00th=[  788], 40.00th=[  812], 50.00th=[  844], 60.00th=[  884],
     | 70.00th=[  924], 80.00th=[  972], 90.00th=[ 1064], 95.00th=[ 1160],
     | 99.00th=[ 1640], 99.50th=[ 2608], 99.90th=[ 4576], 99.95th=[ 5280],
     | 99.99th=[ 6048]
    bw (KB  /s): min=    0, max=385728, per=10.00%, avg=248453.84, stdev=4222.53
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=0.18%, 750=19.37%, 1000=64.21%
    lat (msec) : 2=15.53%, 4=0.57%, 10=0.14%, 20=0.01%, 50=0.01%
  cpu          : usr=2.92%, sys=25.50%, ctx=656856168, majf=0, minf=916
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1117959894/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=14

Run status group 0 (all jobs):
   READ: io=16977GB, aggrb=2414.6MB/s, minb=2414.6MB/s, maxb=2414.6MB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=16938GB, aggrb=2409.3MB/s, minb=2409.3MB/s, maxb=2409.3MB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=17059GB, aggrb=2426.2MB/s, minb=2426.2MB/s, maxb=2426.2MB/s, mint=7200006msec, maxt=7200006msec

Disk stats (read/write):
  sdb: ios=1204668092/25, merge=1449/0, ticks=1034530149/10, in_queue=1032928058, util=100.00%
  sdc: ios=1201961148/25, merge=1204/0, ticks=1038216891/11, in_queue=1036853291, util=100.00%
  sdd: ios=1210398963/25, merge=1162/0, ticks=1036745596/9, in_queue=1035302120, util=100.00%


###################################################################################
# 
# Sun 29 Nov 2015 11:19:01 AM EST
# 8kB randread with 3 jobs and 6 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=5607.3GB, bw=816613KB/s, iops=102076, runt=7200002msec
     lat (usec): min=39, max=17021, avg=880.44, stdev=911.12
xfs
  read : io=4524.1GB, bw=658993KB/s, iops=82374, runt=7200002msec
     lat (usec): min=38, max=16266, avg=1091.45, stdev=973.02
xfs
  read : io=5642.2GB, bw=821699KB/s, iops=102712, runt=7200005msec
     lat (usec): min=43, max=20080, avg=875.00, stdev=920.69

Totals
IOPS: 287162
MBPS: 2242
Average Latency (μsec): 948

xfs-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-02: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-03: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-04: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-05: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-12: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-13: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-14: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-15: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-21: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-22: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-23: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-24: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
xfs-25: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=6
...
fio-2.1.10
Starting 45 processes

xfs-01: (groupid=0, jobs=15): err= 0: pid=39204: Sun Nov 29 13:29:03 2015
  read : io=5607.3GB, bw=816613KB/s, iops=102076, runt=7200002msec
    slat (usec): min=1, max=2943, avg=15.69, stdev=17.55
    clat (usec): min=0, max=17013, avg=863.92, stdev=913.84
     lat (usec): min=39, max=17021, avg=880.44, stdev=911.12
    clat percentiles (usec):
     |  1.00th=[  120],  5.00th=[  139], 10.00th=[  149], 20.00th=[  169],
     | 30.00th=[  189], 40.00th=[  213], 50.00th=[  245], 60.00th=[  330],
     | 70.00th=[ 1496], 80.00th=[ 1848], 90.00th=[ 2320], 95.00th=[ 2576],
     | 99.00th=[ 2960], 99.50th=[ 3088], 99.90th=[ 4016], 99.95th=[ 4448],
     | 99.99th=[ 5152]
    bw (KB  /s): min=    0, max=286880, per=6.67%, avg=54462.79, stdev=67479.40
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.16%, 250=50.90%, 500=11.58%, 750=0.37%, 1000=0.12%
    lat (msec) : 2=20.89%, 4=15.86%, 10=0.10%, 20=0.01%
  cpu          : usr=1.70%, sys=14.86%, ctx=495361846, majf=0, minf=687
  IO depths    : 1=0.1%, 2=0.1%, 4=112.1%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=734951453/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
xfs-11: (groupid=1, jobs=15): err= 0: pid=39219: Sun Nov 29 13:29:03 2015
  read : io=4524.1GB, bw=658993KB/s, iops=82374, runt=7200002msec
    slat (usec): min=1, max=6249, avg=11.23, stdev=11.01
    clat (usec): min=0, max=16263, avg=1079.96, stdev=975.92
     lat (usec): min=38, max=16266, avg=1091.45, stdev=973.02
    clat percentiles (usec):
     |  1.00th=[  120],  5.00th=[  137], 10.00th=[  149], 20.00th=[  173],
     | 30.00th=[  197], 40.00th=[  229], 50.00th=[  322], 60.00th=[ 1544],
     | 70.00th=[ 1784], 80.00th=[ 2064], 90.00th=[ 2480], 95.00th=[ 2672],
     | 99.00th=[ 3024], 99.50th=[ 3184], 99.90th=[ 4320], 99.95th=[ 4704],
     | 99.99th=[ 5280]
    bw (KB  /s): min=    0, max=284032, per=6.67%, avg=43952.17, stdev=59029.23
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.10%, 250=43.99%, 500=7.32%, 750=0.17%, 1000=0.09%
    lat (msec) : 2=26.53%, 4=21.65%, 10=0.16%, 20=0.01%
  cpu          : usr=1.41%, sys=9.22%, ctx=470622667, majf=0, minf=701
  IO depths    : 1=0.1%, 2=0.1%, 4=115.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=593093841/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6
xfs-21: (groupid=2, jobs=15): err= 0: pid=39234: Sun Nov 29 13:29:03 2015
  read : io=5642.2GB, bw=821699KB/s, iops=102712, runt=7200005msec
    slat (usec): min=1, max=2369, avg=12.95, stdev=14.09
    clat (usec): min=0, max=20075, avg=861.40, stdev=922.90
     lat (usec): min=43, max=20080, avg=875.00, stdev=920.69
    clat percentiles (usec):
     |  1.00th=[  123],  5.00th=[  143], 10.00th=[  159], 20.00th=[  189],
     | 30.00th=[  227], 40.00th=[  290], 50.00th=[  378], 60.00th=[  442],
     | 70.00th=[ 1480], 80.00th=[ 1880], 90.00th=[ 2352], 95.00th=[ 2608],
     | 99.00th=[ 3024], 99.50th=[ 3248], 99.90th=[ 5408], 99.95th=[ 6240],
     | 99.99th=[ 7328]
    bw (KB  /s): min=    0, max=280832, per=6.67%, avg=54802.85, stdev=60155.32
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.10%, 250=34.38%, 500=32.05%, 750=2.85%, 1000=0.11%
    lat (msec) : 2=13.61%, 4=16.59%, 10=0.31%, 20=0.01%, 50=0.01%
  cpu          : usr=1.86%, sys=13.43%, ctx=536909310, majf=0, minf=712
  IO depths    : 1=0.1%, 2=0.1%, 4=113.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=739529147/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=6

Run status group 0 (all jobs):
   READ: io=5607.3GB, aggrb=816612KB/s, minb=816612KB/s, maxb=816612KB/s, mint=7200002msec, maxt=7200002msec

Run status group 1 (all jobs):
   READ: io=4524.1GB, aggrb=658993KB/s, minb=658993KB/s, maxb=658993KB/s, mint=7200002msec, maxt=7200002msec

Run status group 2 (all jobs):
   READ: io=5642.2GB, aggrb=821698KB/s, minb=821698KB/s, maxb=821698KB/s, mint=7200005msec, maxt=7200005msec

Disk stats (read/write):
  sdb: ios=823589357/25, merge=0/0, ticks=679733835/18, in_queue=678190929, util=100.00%
  sdc: ios=684085662/25, merge=0/0, ticks=690091252/16, in_queue=689271536, util=100.00%
  sdd: ios=837904106/25, merge=1/0, ticks=682726156/25, in_queue=681492752, util=100.00%


###################################################################################
# 
# Sun 29 Nov 2015 01:30:04 PM EST
# 4kB randread with 3 jobs and 8 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=3022.1GB, bw=440240KB/s, iops=110059, runt=7200004msec
     lat (usec): min=45, max=17590, avg=1089.08, stdev=1257.91
xfs
  read : io=2687.4GB, bw=391373KB/s, iops=97843, runt=7200003msec
     lat (usec): min=42, max=24336, avg=1225.28, stdev=1303.64
xfs
  read : io=3160.5GB, bw=460273KB/s, iops=115068, runt=7200003msec
     lat (usec): min=40, max=19264, avg=1041.64, stdev=1253.73

Totals
IOPS: 322970
MBPS: 1260
Average Latency (μsec): 1118

xfs-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-02: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-03: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-04: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-05: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-12: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-13: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-14: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-15: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-21: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-22: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-23: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-24: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
xfs-25: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=8
...
fio-2.1.10
Starting 45 processes

xfs-01: (groupid=0, jobs=15): err= 0: pid=40878: Sun Nov 29 15:40:07 2015
  read : io=3022.1GB, bw=440240KB/s, iops=110059, runt=7200004msec
    slat (usec): min=1, max=3043, avg=15.45, stdev=16.98
    clat (usec): min=0, max=17585, avg=1073.89, stdev=1260.45
     lat (usec): min=45, max=17590, avg=1089.08, stdev=1257.91
    clat percentiles (usec):
     |  1.00th=[  112],  5.00th=[  127], 10.00th=[  137], 20.00th=[  153],
     | 30.00th=[  169], 40.00th=[  189], 50.00th=[  215], 60.00th=[  270],
     | 70.00th=[ 2480], 80.00th=[ 2704], 90.00th=[ 2928], 95.00th=[ 3120],
     | 99.00th=[ 3632], 99.50th=[ 3824], 99.90th=[ 4256], 99.95th=[ 4640],
     | 99.99th=[ 5408]
    bw (KB  /s): min=    0, max=226944, per=6.67%, avg=29362.14, stdev=47198.98
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.15%
    lat (usec) : 250=57.39%, 500=8.49%, 750=0.35%, 1000=0.09%
    lat (msec) : 2=0.14%, 4=33.13%, 10=0.25%, 20=0.01%
  cpu          : usr=1.65%, sys=14.55%, ctx=394337096, majf=0, minf=624
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=109.3%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=792431931/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-11: (groupid=1, jobs=15): err= 0: pid=40893: Sun Nov 29 15:40:07 2015
  read : io=2687.4GB, bw=391373KB/s, iops=97843, runt=7200003msec
    slat (usec): min=1, max=3877, avg=13.19, stdev=14.78
    clat (usec): min=0, max=24330, avg=1211.93, stdev=1306.55
     lat (usec): min=42, max=24336, avg=1225.28, stdev=1303.64
    clat percentiles (usec):
     |  1.00th=[  110],  5.00th=[  125], 10.00th=[  135], 20.00th=[  153],
     | 30.00th=[  173], 40.00th=[  195], 50.00th=[  231], 60.00th=[  374],
     | 70.00th=[ 2608], 80.00th=[ 2768], 90.00th=[ 2960], 95.00th=[ 3184],
     | 99.00th=[ 3664], 99.50th=[ 3856], 99.90th=[ 4384], 99.95th=[ 4832],
     | 99.99th=[ 5536]
    bw (KB  /s): min=    0, max=218088, per=6.67%, avg=26103.59, stdev=43859.21
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.16%
    lat (usec) : 250=52.97%, 500=7.81%, 750=0.28%, 1000=0.07%
    lat (msec) : 2=0.14%, 4=38.26%, 10=0.32%, 20=0.01%, 50=0.01%
  cpu          : usr=1.42%, sys=11.07%, ctx=391048499, majf=0, minf=647
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=106.9%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=704471393/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-21: (groupid=2, jobs=15): err= 0: pid=40908: Sun Nov 29 15:40:07 2015
  read : io=3160.5GB, bw=460273KB/s, iops=115068, runt=7200003msec
    slat (usec): min=1, max=3513, avg=14.06, stdev=15.16
    clat (usec): min=0, max=19260, avg=1027.11, stdev=1256.00
     lat (usec): min=40, max=19264, avg=1041.64, stdev=1253.73
    clat percentiles (usec):
     |  1.00th=[  112],  5.00th=[  129], 10.00th=[  139], 20.00th=[  157],
     | 30.00th=[  173], 40.00th=[  193], 50.00th=[  219], 60.00th=[  262],
     | 70.00th=[ 2384], 80.00th=[ 2704], 90.00th=[ 2928], 95.00th=[ 3152],
     | 99.00th=[ 3664], 99.50th=[ 3920], 99.90th=[ 5216], 99.95th=[ 6304],
     | 99.99th=[ 7648]
    bw (KB  /s): min=    0, max=214728, per=6.67%, avg=30697.60, stdev=47063.40
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.16%
    lat (usec) : 250=57.75%, 500=10.49%, 750=0.34%, 1000=0.09%
    lat (msec) : 2=0.14%, 4=30.64%, 10=0.40%, 20=0.01%
  cpu          : usr=1.69%, sys=13.75%, ctx=421044408, majf=0, minf=652
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=105.2%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=828492238/w=0/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=3022.1GB, aggrb=440239KB/s, minb=440239KB/s, maxb=440239KB/s, mint=7200004msec, maxt=7200004msec

Run status group 1 (all jobs):
   READ: io=2687.4GB, aggrb=391372KB/s, minb=391372KB/s, maxb=391372KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=3160.5GB, aggrb=460273KB/s, minb=460273KB/s, maxb=460273KB/s, mint=7200003msec, maxt=7200003msec

Disk stats (read/write):
  sdb: ios=866447051/25, merge=0/0, ticks=903794733/64, in_queue=902298567, util=100.00%
  sdc: ios=753298519/25, merge=0/0, ticks=913181472/32, in_queue=912164096, util=100.00%
  sdd: ios=871731739/25, merge=0/0, ticks=907709989/66, in_queue=906433880, util=100.00%


###################################################################################
# 
# Sun 29 Nov 2015 03:41:07 PM EST
# 8kB randrw with 4 jobs and 2 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=1387.9GB, bw=202119KB/s, iops=25264, runt=7200003msec
     lat (usec): min=53, max=17188, avg=1093.52, stdev=673.01
  write: io=700042MB, bw=99561KB/s, iops=12445, runt=7200003msec
     lat (usec): min=29, max=6634, avg=990.17, stdev=440.01
xfs
  read : io=1375.1GB, bw=200391KB/s, iops=25048, runt=7200003msec
     lat (usec): min=82, max=17084, avg=1105.21, stdev=681.99
  write: io=694054MB, bw=98710KB/s, iops=12338, runt=7200003msec
     lat (usec): min=27, max=11345, avg=994.12, stdev=436.41
xfs
  read : io=1381.5GB, bw=201190KB/s, iops=25148, runt=7200002msec
     lat (usec): min=43, max=23391, avg=1144.26, stdev=811.34
  write: io=696795MB, bw=99100KB/s, iops=12387, runt=7200002msec
     lat (usec): min=27, max=10232, avg=902.06, stdev=481.04

Totals
IOPS: 112630
MBPS: 877
Average Latency (μsec): 2076

xfs-01: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-02: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-03: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-04: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-05: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-11: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-12: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-13: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-14: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-15: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-21: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-22: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-23: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-24: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
xfs-25: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2
...
fio-2.1.10
Starting 60 processes

xfs-01: (groupid=0, jobs=20): err= 0: pid=42569: Sun Nov 29 17:51:10 2015
  read : io=1387.9GB, bw=202119KB/s, iops=25264, runt=7200003msec
    slat (usec): min=1, max=937, avg=10.22, stdev=10.70
    clat (usec): min=0, max=17171, avg=1083.23, stdev=672.53
     lat (usec): min=53, max=17188, avg=1093.52, stdev=673.01
    clat percentiles (usec):
     |  1.00th=[  266],  5.00th=[  322], 10.00th=[  370], 20.00th=[  524],
     | 30.00th=[  644], 40.00th=[  756], 50.00th=[  924], 60.00th=[ 1176],
     | 70.00th=[ 1352], 80.00th=[ 1608], 90.00th=[ 1816], 95.00th=[ 2024],
     | 99.00th=[ 3632], 99.50th=[ 3984], 99.90th=[ 4448], 99.95th=[ 4768],
     | 99.99th=[ 7648]
    bw (KB  /s): min=    0, max=14928, per=5.00%, avg=10112.22, stdev=343.62
  write: io=700042MB, bw=99561KB/s, iops=12445, runt=7200003msec
    slat (usec): min=2, max=6627, avg=12.51, stdev=12.03
    clat (usec): min=0, max=5059, avg=977.58, stdev=439.25
     lat (usec): min=29, max=6634, avg=990.17, stdev=440.01
    clat percentiles (usec):
     |  1.00th=[  229],  5.00th=[  286], 10.00th=[  334], 20.00th=[  498],
     | 30.00th=[  708], 40.00th=[  836], 50.00th=[ 1020], 60.00th=[ 1160],
     | 70.00th=[ 1272], 80.00th=[ 1416], 90.00th=[ 1512], 95.00th=[ 1640],
     | 99.00th=[ 1816], 99.50th=[ 1848], 99.90th=[ 1960], 99.95th=[ 2008],
     | 99.99th=[ 2128]
    bw (KB  /s): min=    0, max= 7744, per=5.00%, avg=4981.27, stdev=251.23
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.98%, 500=17.98%, 750=18.51%, 1000=13.99%
    lat (msec) : 2=45.06%, 4=3.17%, 10=0.31%, 20=0.01%
  cpu          : usr=0.66%, sys=3.15%, ctx=273220834, majf=0, minf=680
  IO depths    : 1=0.1%, 2=108.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=181907282/w=89605357/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-11: (groupid=1, jobs=20): err= 0: pid=42589: Sun Nov 29 17:51:10 2015
  read : io=1375.1GB, bw=200391KB/s, iops=25048, runt=7200003msec
    slat (usec): min=1, max=787, avg=10.11, stdev=10.80
    clat (usec): min=0, max=17074, avg=1095.03, stdev=681.55
     lat (usec): min=82, max=17084, avg=1105.21, stdev=681.99
    clat percentiles (usec):
     |  1.00th=[  266],  5.00th=[  326], 10.00th=[  378], 20.00th=[  532],
     | 30.00th=[  652], 40.00th=[  772], 50.00th=[  940], 60.00th=[ 1192],
     | 70.00th=[ 1352], 80.00th=[ 1608], 90.00th=[ 1832], 95.00th=[ 2064],
     | 99.00th=[ 3664], 99.50th=[ 4016], 99.90th=[ 4512], 99.95th=[ 4896],
     | 99.99th=[ 7584]
    bw (KB  /s): min=    0, max=16752, per=5.00%, avg=10025.58, stdev=336.84
  write: io=694054MB, bw=98710KB/s, iops=12338, runt=7200003msec
    slat (usec): min=1, max=11335, avg=12.45, stdev=12.35
    clat (usec): min=0, max=9396, avg=981.59, stdev=435.69
     lat (usec): min=27, max=11345, avg=994.12, stdev=436.41
    clat percentiles (usec):
     |  1.00th=[  225],  5.00th=[  290], 10.00th=[  338], 20.00th=[  516],
     | 30.00th=[  716], 40.00th=[  844], 50.00th=[ 1032], 60.00th=[ 1160],
     | 70.00th=[ 1272], 80.00th=[ 1416], 90.00th=[ 1512], 95.00th=[ 1624],
     | 99.00th=[ 1800], 99.50th=[ 1848], 99.90th=[ 1960], 99.95th=[ 2024],
     | 99.99th=[ 2128]
    bw (KB  /s): min=    0, max= 8224, per=5.00%, avg=4938.76, stdev=250.94
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=1.03%, 500=17.25%, 750=18.31%, 1000=14.35%
    lat (msec) : 2=45.28%, 4=3.42%, 10=0.34%, 20=0.01%
  cpu          : usr=0.64%, sys=3.12%, ctx=271270388, majf=0, minf=685
  IO depths    : 1=0.1%, 2=108.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=180352039/w=88838941/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2
xfs-21: (groupid=2, jobs=20): err= 0: pid=42609: Sun Nov 29 17:51:10 2015
  read : io=1381.5GB, bw=201190KB/s, iops=25148, runt=7200002msec
    slat (usec): min=1, max=703, avg= 9.95, stdev=10.50
    clat (usec): min=0, max=23374, avg=1134.24, stdev=811.30
     lat (usec): min=43, max=23391, avg=1144.26, stdev=811.34
    clat percentiles (usec):
     |  1.00th=[  249],  5.00th=[  298], 10.00th=[  326], 20.00th=[  394],
     | 30.00th=[  502], 40.00th=[  652], 50.00th=[ 1064], 60.00th=[ 1336],
     | 70.00th=[ 1432], 80.00th=[ 1576], 90.00th=[ 2160], 95.00th=[ 2480],
     | 99.00th=[ 3920], 99.50th=[ 4256], 99.90th=[ 5152], 99.95th=[ 6240],
     | 99.99th=[11840]
    bw (KB  /s): min=    0, max=25488, per=5.00%, avg=10065.74, stdev=331.66
  write: io=696795MB, bw=99100KB/s, iops=12387, runt=7200002msec
    slat (usec): min=2, max=9123, avg=12.22, stdev=12.02
    clat (usec): min=0, max=10225, avg=889.76, stdev=480.94
     lat (usec): min=27, max=10232, avg=902.06, stdev=481.04
    clat percentiles (usec):
     |  1.00th=[  207],  5.00th=[  258], 10.00th=[  290], 20.00th=[  350],
     | 30.00th=[  454], 40.00th=[  660], 50.00th=[  932], 60.00th=[ 1128],
     | 70.00th=[ 1240], 80.00th=[ 1384], 90.00th=[ 1512], 95.00th=[ 1656],
     | 99.00th=[ 1752], 99.50th=[ 1784], 99.90th=[ 1864], 99.95th=[ 1960],
     | 99.99th=[ 2256]
    bw (KB  /s): min=    0, max=12240, per=5.00%, avg=4958.30, stdev=261.28
    lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.02%
    lat (usec) : 250=1.92%, 500=29.08%, 750=12.72%, 1000=6.47%
    lat (msec) : 2=39.33%, 4=9.86%, 10=0.57%, 20=0.01%, 50=0.01%
  cpu          : usr=0.62%, sys=3.09%, ctx=271669362, majf=0, minf=711
  IO depths    : 1=0.1%, 2=108.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=181070953/w=89189705/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):
   READ: io=1387.9GB, aggrb=202119KB/s, minb=202119KB/s, maxb=202119KB/s, mint=7200003msec, maxt=7200003msec
  WRITE: io=700042MB, aggrb=99561KB/s, minb=99561KB/s, maxb=99561KB/s, mint=7200003msec, maxt=7200003msec

Run status group 1 (all jobs):
   READ: io=1375.1GB, aggrb=200391KB/s, minb=200391KB/s, maxb=200391KB/s, mint=7200003msec, maxt=7200003msec
  WRITE: io=694054MB, aggrb=98709KB/s, minb=98709KB/s, maxb=98709KB/s, mint=7200003msec, maxt=7200003msec

Run status group 2 (all jobs):
   READ: io=1381.5GB, aggrb=201189KB/s, minb=201189KB/s, maxb=201189KB/s, mint=7200002msec, maxt=7200002msec
  WRITE: io=696795MB, aggrb=99099KB/s, minb=99099KB/s, maxb=99099KB/s, mint=7200002msec, maxt=7200002msec

Disk stats (read/write):
  sdb: ios=197251609/97152029, merge=0/0, ticks=212702978/94938219, in_queue=307341342, util=100.00%
  sdc: ios=195570988/96324907, merge=0/0, ticks=213154140/94482254, in_queue=307353729, util=100.00%
  sdd: ios=196376580/96717751, merge=0/0, ticks=221793439/85883091, in_queue=307399834, util=100.00%


###################################################################################
# 
# Sun 29 Nov 2015 05:52:10 PM EST
# 64kB randrw with 1 jobs and 8 I/Os outstanding
# CentOS release 6.7 Linux 2.6.32-573.7.1.el6.x86_64
# 
###################################################################################

Summary
xfs
  read : io=4479.4GB, bw=652349KB/s, iops=10192, runt=7200006msec
     lat (usec): min=47, max=722715, avg=3751.12, stdev=16108.71
  write: io=2207.3GB, bw=321453KB/s, iops=5022, runt=7200006msec
     lat (usec): min=40, max=691204, avg=346.77, stdev=1237.62
xfs
  read : io=4529.3GB, bw=659617KB/s, iops=10306, runt=7200005msec
     lat (usec): min=44, max=728565, avg=3719.14, stdev=15898.01
  write: io=2231.8GB, bw=325015KB/s, iops=5078, runt=7200005msec
     lat (usec): min=40, max=734268, avg=324.88, stdev=1446.99
xfs
  read : io=4216.3GB, bw=614037KB/s, iops=9594, runt=7200006msec
     lat (usec): min=48, max=697214, avg=4048.34, stdev=17583.29
  write: io=2077.5GB, bw=302544KB/s, iops=4727, runt=7200006msec
     lat (usec): min=37, max=617366, avg=240.86, stdev=336.20

Totals
IOPS: 44919
MBPS: 2805
Average Latency (μsec): 4142

xfs-01: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-02: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-03: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-04: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-05: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-11: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-12: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-13: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-14: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-15: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-21: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-22: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-23: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-24: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
xfs-25: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8
fio-2.1.10
Starting 15 processes

xfs-01: (groupid=0, jobs=5): err= 0: pid=44361: Sun Nov 29 20:02:11 2015
  read : io=4479.4GB, bw=652349KB/s, iops=10192, runt=7200006msec
    slat (usec): min=3, max=690, avg=10.87, stdev= 4.50
    clat (usec): min=41, max=722709, avg=3740.08, stdev=16108.72
     lat (usec): min=47, max=722715, avg=3751.12, stdev=16108.71
    clat percentiles (usec):
     |  1.00th=[  556],  5.00th=[  796], 10.00th=[ 1032], 20.00th=[ 1528],
     | 30.00th=[ 2064], 40.00th=[ 2608], 50.00th=[ 3088], 60.00th=[ 3600],
     | 70.00th=[ 4128], 80.00th=[ 4896], 90.00th=[ 5984], 95.00th=[ 7008],
     | 99.00th=[ 9280], 99.50th=[10432], 99.90th=[14784], 99.95th=[651264],
     | 99.99th=[684032]
    bw (KB  /s): min=    0, max=169984, per=20.85%, avg=136001.52, stdev=30661.88
  write: io=2207.3GB, bw=321453KB/s, iops=5022, runt=7200006msec
    slat (usec): min=5, max=693, avg=16.86, stdev= 6.12
    clat (usec): min=0, max=691187, avg=329.71, stdev=1237.60
     lat (usec): min=40, max=691204, avg=346.77, stdev=1237.62
    clat percentiles (usec):
     |  1.00th=[   45],  5.00th=[   55], 10.00th=[   67], 20.00th=[   95],
     | 30.00th=[  125], 40.00th=[  159], 50.00th=[  199], 60.00th=[  251],
     | 70.00th=[  322], 80.00th=[  434], 90.00th=[  692], 95.00th=[ 1128],
     | 99.00th=[ 2064], 99.50th=[ 2480], 99.90th=[ 3440], 99.95th=[ 3856],
     | 99.99th=[ 5280]
    bw (KB  /s): min=   93, max=89856, per=20.86%, avg=67047.07, stdev=15328.62
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.81%, 100=6.32%
    lat (usec) : 250=12.62%, 500=8.28%, 750=4.76%, 1000=4.44%
    lat (msec) : 2=14.66%, 4=26.34%, 10=21.33%, 20=0.40%, 50=0.01%
    lat (msec) : 750=0.04%
  cpu          : usr=1.91%, sys=4.71%, ctx=107387263, majf=0, minf=164
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=107.6%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=73389243/w=36163463/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-11: (groupid=1, jobs=5): err= 0: pid=44366: Sun Nov 29 20:02:11 2015
  read : io=4529.3GB, bw=659617KB/s, iops=10306, runt=7200005msec
    slat (usec): min=3, max=708, avg=10.35, stdev= 4.46
    clat (usec): min=38, max=728557, avg=3708.61, stdev=15898.02
     lat (usec): min=44, max=728565, avg=3719.14, stdev=15898.01
    clat percentiles (usec):
     |  1.00th=[  548],  5.00th=[  780], 10.00th=[ 1012], 20.00th=[ 1480],
     | 30.00th=[ 2024], 40.00th=[ 2576], 50.00th=[ 3088], 60.00th=[ 3568],
     | 70.00th=[ 4128], 80.00th=[ 4896], 90.00th=[ 5984], 95.00th=[ 7008],
     | 99.00th=[ 9280], 99.50th=[10432], 99.90th=[14400], 99.95th=[651264],
     | 99.99th=[684032]
    bw (KB  /s): min=    0, max=173184, per=20.84%, avg=137431.82, stdev=31046.92
  write: io=2231.8GB, bw=325015KB/s, iops=5078, runt=7200005msec
    slat (usec): min=5, max=712, avg=16.23, stdev= 6.14
    clat (usec): min=0, max=734256, avg=308.43, stdev=1446.95
     lat (usec): min=40, max=734268, avg=324.88, stdev=1446.99
    clat percentiles (usec):
     |  1.00th=[   43],  5.00th=[   52], 10.00th=[   63], 20.00th=[   88],
     | 30.00th=[  115], 40.00th=[  147], 50.00th=[  183], 60.00th=[  229],
     | 70.00th=[  294], 80.00th=[  398], 90.00th=[  636], 95.00th=[ 1064],
     | 99.00th=[ 2008], 99.50th=[ 2416], 99.90th=[ 3344], 99.95th=[ 3792],
     | 99.99th=[ 5216]
    bw (KB  /s): min=   94, max=94720, per=20.85%, avg=67749.45, stdev=15502.80
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=1.24%
    lat (usec) : 100=6.78%, 250=12.94%, 500=7.76%, 750=4.54%, 1000=4.51%
    lat (msec) : 2=14.65%, 4=25.97%, 10=21.17%, 20=0.39%, 50=0.01%
    lat (msec) : 750=0.04%
  cpu          : usr=1.80%, sys=4.61%, ctx=108973853, majf=0, minf=169
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=107.7%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=74206892/w=36564226/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8
xfs-21: (groupid=2, jobs=5): err= 0: pid=44371: Sun Nov 29 20:02:11 2015
  read : io=4216.3GB, bw=614037KB/s, iops=9594, runt=7200006msec
    slat (usec): min=2, max=687, avg=10.42, stdev= 5.20
    clat (usec): min=42, max=697195, avg=4037.78, stdev=17583.29
     lat (usec): min=48, max=697214, avg=4048.34, stdev=17583.29
    clat percentiles (usec):
     |  1.00th=[  454],  5.00th=[  660], 10.00th=[  964], 20.00th=[ 1512],
     | 30.00th=[ 2040], 40.00th=[ 2576], 50.00th=[ 3152], 60.00th=[ 3696],
     | 70.00th=[ 4384], 80.00th=[ 5216], 90.00th=[ 6624], 95.00th=[ 7904],
     | 99.00th=[11072], 99.50th=[12864], 99.90th=[24192], 99.95th=[618496],
     | 99.99th=[651264]
    bw (KB  /s): min=    0, max=175360, per=20.93%, avg=128538.51, stdev=33048.03
  write: io=2077.5GB, bw=302544KB/s, iops=4727, runt=7200006msec
    slat (usec): min=5, max=661, avg=17.03, stdev= 7.20
    clat (usec): min=0, max=617345, avg=223.63, stdev=335.46
     lat (usec): min=37, max=617366, avg=240.86, stdev=336.20
    clat percentiles (usec):
     |  1.00th=[   39],  5.00th=[   45], 10.00th=[   50], 20.00th=[   60],
     | 30.00th=[   72], 40.00th=[   87], 50.00th=[  106], 60.00th=[  137],
     | 70.00th=[  191], 80.00th=[  330], 90.00th=[  564], 95.00th=[  772],
     | 99.00th=[ 1432], 99.50th=[ 1544], 99.90th=[ 1720], 99.95th=[ 1768],
     | 99.99th=[ 1880]
    bw (KB  /s): min=    0, max=93952, per=20.94%, avg=63347.99, stdev=16497.77
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=3.08%
    lat (usec) : 100=12.39%, 250=9.42%, 500=5.19%, 750=5.65%, 1000=3.05%
    lat (msec) : 2=13.87%, 4=23.73%, 10=22.49%, 20=1.02%, 50=0.05%
    lat (msec) : 750=0.05%
  cpu          : usr=1.66%, sys=4.35%, ctx=97311089, majf=0, minf=164
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=107.8%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=69079237/w=34036205/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=8

Run status group 0 (all jobs):
   READ: io=4479.4GB, aggrb=652348KB/s, minb=652348KB/s, maxb=652348KB/s, mint=7200006msec, maxt=7200006msec
  WRITE: io=2207.3GB, aggrb=321452KB/s, minb=321452KB/s, maxb=321452KB/s, mint=7200006msec, maxt=7200006msec

Run status group 1 (all jobs):
   READ: io=4529.3GB, aggrb=659616KB/s, minb=659616KB/s, maxb=659616KB/s, mint=7200005msec, maxt=7200005msec
  WRITE: io=2231.8GB, aggrb=325015KB/s, minb=325015KB/s, maxb=325015KB/s, mint=7200005msec, maxt=7200005msec

Run status group 2 (all jobs):
   READ: io=4216.3GB, aggrb=614037KB/s, minb=614037KB/s, maxb=614037KB/s, mint=7200006msec, maxt=7200006msec
  WRITE: io=2077.5GB, aggrb=302543KB/s, minb=302543KB/s, maxb=302543KB/s, mint=7200006msec, maxt=7200006msec

Disk stats (read/write):
  sdb: ios=78975662/38915257, merge=0/0, ticks=297373677/13147624, in_queue=310463908, util=100.00%
  sdc: ios=79928887/39383264, merge=0/0, ticks=297928822/12422669, in_queue=310296994, util=100.00%
  sdd: ios=74483936/36698942, merge=0/0, ticks=302246681/8246876, in_queue=310435315, util=100.00%