################################################################################### # ION SR-71mach5 # with (24) Intel DC S3610 1600GB SSDs in (3) RAID 5 arrays of 8.75TiB each # each with (5) 1.75TiB partitions # All Partitions on each array are xfs # format and mount options listed below ################################################################################### 2015 December 04, 12:44 CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 600 s ramp + 7200 s run, per test
Block Size |
Test |
Num Jobs |
## I/Os Queued |
IOPS |
MBps |
Latency (μsec) | |
---|---|---|---|---|---|---|---|
8kB | randwrite | 1 | 1 | 45248 | 352 | 329 | |
64kB | randwrite | 1 | 1 | 23038 | 1439 | 647 | |
8kB | randrw | 2 | 2 | 104713 | 816 | 1074 | |
64kB | randrw | 2 | 2 | 39651 | 2475 | 2339 | |
1mB | write | 1 | 2 | 5335 | 5335 | 5627 | |
1mB | read | 1 | 4 | 10851 | 10851 | 5526 | |
256kB | write | 1 | 2 | 33062 | 8264 | 904 | |
256kB | read | 1 | 4 | 35237 | 8808 | 1700 | |
256kB | read | 1 | 14 | 43297 | 10823 | 4848 | |
64kB | write | 1 | 4 | 118831 | 7425 | 503 | |
64kB | read | 1 | 4 | 92764 | 5796 | 645 | |
64kB | read | 1 | 16 | 154264 | 9640 | 1554 | |
64kB | randread | 1 | 8 | 128025 | 8000 | 935 | |
64kB | randread | 4 | 4 | 153082 | 9566 | 1566 | |
32kB | randread | 2 | 8 | 286734 | 8959 | 835 | |
32kB | randread | 4 | 8 | 309483 | 9670 | 1549 | |
16kB | randread | 2 | 8 | 448881 | 7013 | 533 | |
8kB | randread | 4 | 2 | 455924 | 3560 | 261 | |
8kB | randread | 4 | 4 | 474122 | 3703 | 504 | |
4kB | randread | 2 | 4 | 474274 | 1851 | 251 | |
4kB | randread | 4 | 6 | 487307 | 1902 | 737 |
################################################################################### # # SR-71mach5 Configuration # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### ################################################################################### # # mkfs commands # ################################################################################### #!/bin/bash # # format 5 partitions on each of 3 RAID 5 volumes with XFS # mkfs.xfs -L C0P1 -d su=65536,sw=7 -l su=64k /dev/sdb1 mkfs.xfs -L C1P1 -d su=65536,sw=7 -l su=64k /dev/sdc1 mkfs.xfs -L C2P1 -d su=65536,sw=7 -l su=64k /dev/sdd1 mkfs.xfs -L C0P2 -d su=65536,sw=7 -l su=64k /dev/sdb2 mkfs.xfs -L C1P2 -d su=65536,sw=7 -l su=64k /dev/sdc2 mkfs.xfs -L C2P2 -d su=65536,sw=7 -l su=64k /dev/sdd2 mkfs.xfs -L C0P3 -d su=65536,sw=7 -l su=64k /dev/sdb3 mkfs.xfs -L C1P3 -d su=65536,sw=7 -l su=64k /dev/sdc3 mkfs.xfs -L C2P3 -d su=65536,sw=7 -l su=64k /dev/sdd3 mkfs.xfs -L C0P4 -d su=65536,sw=7 -l su=64k /dev/sdb4 mkfs.xfs -L C1P4 -d su=65536,sw=7 -l su=64k /dev/sdc4 mkfs.xfs -L C2P4 -d su=65536,sw=7 -l su=64k /dev/sdd4 mkfs.xfs -L C0P5 -d su=65536,sw=7 -l su=64k /dev/sdb5 mkfs.xfs -L C1P5 -d su=65536,sw=7 -l su=64k /dev/sdc5 mkfs.xfs -L C2P5 -d su=65536,sw=7 -l su=64k /dev/sdd5 ################################################################################### # # /etc/fstab # kernel parameter: elevator=deadline # ################################################################################### LABEL=C0P1 /sr71/C0P1 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C0P2 /sr71/C0P2 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C0P3 /sr71/C0P3 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C0P4 /sr71/C0P4 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C0P5 /sr71/C0P5 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C1P1 /sr71/C1P1 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C1P2 /sr71/C1P2 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C1P3 /sr71/C1P3 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C1P4 /sr71/C1P4 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C1P5 /sr71/C1P5 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C2P1 /sr71/C2P1 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C2P2 /sr71/C2P2 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C2P3 /sr71/C2P3 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C2P4 /sr71/C2P4 xfs sync,noatime,nodiratime,nobarrier 0 0 LABEL=C2P5 /sr71/C2P5 xfs sync,noatime,nodiratime,nobarrier 0 0 ################################################################################### # # actual mount paramters # ################################################################################### /dev/sdb2 on /sr71/C1P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sdb5 on /sr71/C1P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sda5 on /sr71/C0P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sdb1 on /sr71/C1P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sda1 on /sr71/C0P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sda4 on /sr71/C0P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sda2 on /sr71/C0P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sda3 on /sr71/C0P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sdb3 on /sr71/C1P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sdb4 on /sr71/C1P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sdc2 on /sr71/C2P2 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sdc5 on /sr71/C2P5 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sdc3 on /sr71/C2P3 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sdc4 on /sr71/C2P4 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) /dev/sdc1 on /sr71/C2P1 type xfs (rw,noatime,nodiratime,sync,seclabel,wsync,attr2,nobarrier,inode64,logbsize=64k,sunit=128,swidth=896,noquota) ################################################################################### # # fio job file fio-2.1.10 # ################################################################################### 60 seconds between fio tests [global] ioengine=libaio size=1536g direct=1 time_based ramp_time=${RAMPTIME} runtime=${RUNTIME} iodepth=${IODEPTH} bs=${BS} rw=${RW} rwmixread=${RWMIXREAD} numjobs=${NUMJOBS} group_reporting # disable some measurements to reduce system overhead # cannot: breaks ALL latency reporting # disable_clat=1 # disable_slat=1 # disable_lat=1 # HOWTO lists disable_bw but fio-2.1.10 exits with global error # disable_bw=1 # write log options seem to keep fio from completing # write_bw_log=mach5 # write_iops_log=mach5 # write_lat_log=mach5 [xfs-01] new_group filename=/sr71/C0P1/fio.tst [xfs-02] filename=/sr71/C0P2/fio.tst [xfs-03] filename=/sr71/C0P3/fio.tst [xfs-04] filename=/sr71/C0P4/fio.tst [xfs-05] filename=/sr71/C0P5/fio.tst [xfs-11] new_group filename=/sr71/C1P1/fio.tst [xfs-12] filename=/sr71/C1P2/fio.tst [xfs-13] filename=/sr71/C1P3/fio.tst [xfs-14] filename=/sr71/C1P4/fio.tst [xfs-15] filename=/sr71/C1P5/fio.tst [xfs-21] new_group filename=/sr71/C2P1/fio.tst [xfs-22] filename=/sr71/C2P2/fio.tst [xfs-23] filename=/sr71/C2P3/fio.tst [xfs-24] filename=/sr71/C2P4/fio.tst [xfs-25] filename=/sr71/C2P5/fio.tst ################################################################################### # # 201512041244 test details # ################################################################################### ################################################################################### # # Fri 04 Dec 2015 12:45:05 PM EST # 8kB randwrite with 1 jobs and 1 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs write: io=854340MB, bw=121506KB/s, iops=15188, runt=7200002msec lat (usec): min=32, max=13076, avg=327.11, stdev=392.08 xfs write: io=845663MB, bw=120272KB/s, iops=15034, runt=7200001msec lat (usec): min=34, max=4047, avg=330.63, stdev=392.45 xfs write: io=845258MB, bw=120214KB/s, iops=15026, runt=7200002msec lat (usec): min=33, max=11412, avg=330.84, stdev=431.24 Totals IOPS: 45248 MBPS: 352 Average Latency (μsec): 329 xfs-01: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-02: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-03: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-04: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-05: (g=0): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-11: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-12: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-13: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-14: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-15: (g=1): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-21: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-22: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-23: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-24: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 xfs-25: (g=2): rw=randwrite, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=1 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=63516: Fri Dec 4 14:55:07 2015 write: io=854340MB, bw=121506KB/s, iops=15188, runt=7200002msec slat (usec): min=3, max=10186, avg=13.56, stdev= 5.65 clat (usec): min=0, max=13063, avg=313.57, stdev=391.98 lat (usec): min=32, max=13076, avg=327.11, stdev=392.08 clat percentiles (usec): | 1.00th=[ 43], 5.00th=[ 57], 10.00th=[ 63], 20.00th=[ 68], | 30.00th=[ 72], 40.00th=[ 75], 50.00th=[ 82], 60.00th=[ 97], | 70.00th=[ 302], 80.00th=[ 828], 90.00th=[ 988], 95.00th=[ 1176], | 99.00th=[ 1352], 99.50th=[ 1384], 99.90th=[ 1432], 99.95th=[ 1448], | 99.99th=[ 1496] bw (KB /s): min= 0, max=34208, per=20.01%, avg=24315.28, stdev=385.79 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=2.26% lat (usec) : 100=58.34%, 250=7.99%, 500=8.70%, 750=1.94%, 1000=11.24% lat (msec) : 2=9.53%, 4=0.01%, 10=0.01%, 20=0.01% cpu : usr=1.53%, sys=6.85%, ctx=118514539, majf=0, minf=8609 IO depths : 1=108.4%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=109355464/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=1 xfs-11: (groupid=1, jobs=5): err= 0: pid=63521: Fri Dec 4 14:55:07 2015 write: io=845663MB, bw=120272KB/s, iops=15034, runt=7200001msec slat (usec): min=3, max=2163, avg=13.06, stdev= 5.33 clat (usec): min=0, max=4036, avg=317.47, stdev=392.26 lat (usec): min=34, max=4047, avg=330.63, stdev=392.45 clat percentiles (usec): | 1.00th=[ 45], 5.00th=[ 58], 10.00th=[ 64], 20.00th=[ 69], | 30.00th=[ 73], 40.00th=[ 77], 50.00th=[ 84], 60.00th=[ 103], | 70.00th=[ 310], 80.00th=[ 828], 90.00th=[ 996], 95.00th=[ 1176], | 99.00th=[ 1352], 99.50th=[ 1384], 99.90th=[ 1432], 99.95th=[ 1464], | 99.99th=[ 1496] bw (KB /s): min= 0, max=26112, per=20.01%, avg=24068.63, stdev=363.98 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=1.82% lat (usec) : 100=57.29%, 250=9.12%, 500=8.69%, 750=2.13%, 1000=11.23% lat (msec) : 2=9.72%, 4=0.01%, 10=0.01% cpu : usr=1.46%, sys=6.61%, ctx=117348025, majf=0, minf=9634 IO depths : 1=108.4%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=108244857/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=1 xfs-21: (groupid=2, jobs=5): err= 0: pid=63526: Fri Dec 4 14:55:07 2015 write: io=845258MB, bw=120214KB/s, iops=15026, runt=7200002msec slat (usec): min=3, max=2181, avg=13.20, stdev= 5.46 clat (usec): min=0, max=11399, avg=317.61, stdev=431.12 lat (usec): min=33, max=11412, avg=330.84, stdev=431.24 clat percentiles (usec): | 1.00th=[ 43], 5.00th=[ 56], 10.00th=[ 62], 20.00th=[ 68], | 30.00th=[ 71], 40.00th=[ 74], 50.00th=[ 79], 60.00th=[ 87], | 70.00th=[ 135], 80.00th=[ 756], 90.00th=[ 1128], 95.00th=[ 1224], | 99.00th=[ 1512], 99.50th=[ 1528], 99.90th=[ 1576], 99.95th=[ 1592], | 99.99th=[ 1688] bw (KB /s): min= 0, max=40144, per=20.01%, avg=24057.30, stdev=439.07 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=2.39% lat (usec) : 100=64.01%, 250=5.93%, 500=3.71%, 750=3.75%, 1000=5.40% lat (msec) : 2=14.79%, 4=0.01%, 10=0.01%, 20=0.01% cpu : usr=1.42%, sys=6.66%, ctx=117267690, majf=0, minf=8989 IO depths : 1=108.4%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=108192965/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=1 Run status group 0 (all jobs): WRITE: io=854340MB, aggrb=121506KB/s, minb=121506KB/s, maxb=121506KB/s, mint=7200002msec, maxt=7200002msec Run status group 1 (all jobs): WRITE: io=845663MB, aggrb=120272KB/s, minb=120272KB/s, maxb=120272KB/s, mint=7200001msec, maxt=7200001msec Run status group 2 (all jobs): WRITE: io=845258MB, aggrb=120214KB/s, minb=120214KB/s, maxb=120214KB/s, mint=7200002msec, maxt=7200002msec Disk stats (read/write): sda: ios=0/118489725, merge=0/0, ticks=0/37045995, in_queue=36997313, util=100.00% sdb: ios=0/117324147, merge=0/0, ticks=0/37084981, in_queue=37042708, util=100.00% sdc: ios=0/117243486, merge=0/0, ticks=0/37070433, in_queue=37027547, util=100.00% ################################################################################### # # Fri 04 Dec 2015 02:56:07 PM EST # 64kB randwrite with 1 jobs and 1 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs write: io=3357.2GB, bw=488918KB/s, iops=7639, runt=7200001msec lat (usec): min=61, max=694853, avg=650.94, stdev=3159.15 xfs write: io=3362.1GB, bw=489762KB/s, iops=7652, runt=7200001msec lat (usec): min=59, max=921558, avg=649.89, stdev=3359.52 xfs write: io=3404.6GB, bw=495816KB/s, iops=7747, runt=7200001msec lat (usec): min=56, max=1131.7K, avg=642.06, stdev=4402.89 Totals IOPS: 23038 MBPS: 1439 Average Latency (μsec): 647 xfs-01: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-02: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-03: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-04: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-05: (g=0): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-11: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-12: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-13: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-14: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-15: (g=1): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-21: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-22: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-23: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-24: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 xfs-25: (g=2): rw=randwrite, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=69156: Fri Dec 4 17:06:08 2015 write: io=3357.2GB, bw=488918KB/s, iops=7639, runt=7200001msec slat (usec): min=7, max=1158, avg=25.65, stdev= 9.19 clat (usec): min=1, max=694800, avg=624.91, stdev=3159.10 lat (usec): min=61, max=694853, avg=650.94, stdev=3159.15 clat percentiles (usec): | 1.00th=[ 103], 5.00th=[ 147], 10.00th=[ 185], 20.00th=[ 270], | 30.00th=[ 342], 40.00th=[ 422], 50.00th=[ 510], 60.00th=[ 612], | 70.00th=[ 732], 80.00th=[ 892], 90.00th=[ 1160], 95.00th=[ 1416], | 99.00th=[ 2040], 99.50th=[ 2320], 99.90th=[ 3024], 99.95th=[ 3344], | 99.99th=[ 4576] bw (KB /s): min= 0, max=120576, per=20.21%, avg=98827.62, stdev=12100.67 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (usec) : 100=0.84%, 250=16.86%, 500=31.37%, 750=22.21%, 1000=13.62% lat (msec) : 2=14.00%, 4=1.09%, 10=0.01%, 20=0.01%, 750=0.01% cpu : usr=1.70%, sys=5.65%, ctx=59723779, majf=0, minf=22546 IO depths : 1=108.5%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=55003240/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=1 xfs-11: (groupid=1, jobs=5): err= 0: pid=69161: Fri Dec 4 17:06:08 2015 write: io=3362.1GB, bw=489762KB/s, iops=7652, runt=7200001msec slat (usec): min=7, max=1158, avg=25.19, stdev= 8.93 clat (usec): min=1, max=921537, avg=624.34, stdev=3359.47 lat (usec): min=59, max=921558, avg=649.89, stdev=3359.52 clat percentiles (usec): | 1.00th=[ 101], 5.00th=[ 145], 10.00th=[ 183], 20.00th=[ 270], | 30.00th=[ 342], 40.00th=[ 422], 50.00th=[ 510], 60.00th=[ 612], | 70.00th=[ 732], 80.00th=[ 892], 90.00th=[ 1144], 95.00th=[ 1400], | 99.00th=[ 2024], 99.50th=[ 2288], 99.90th=[ 2928], 99.95th=[ 3248], | 99.99th=[ 4448] bw (KB /s): min= 0, max=120192, per=20.24%, avg=99120.29, stdev=12077.31 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (usec) : 100=0.91%, 250=16.84%, 500=31.22%, 750=22.42%, 1000=13.75% lat (msec) : 2=13.82%, 4=1.03%, 10=0.01%, 20=0.01%, 750=0.01% lat (msec) : 1000=0.01% cpu : usr=1.67%, sys=5.53%, ctx=59708804, majf=0, minf=23357 IO depths : 1=108.3%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=55098228/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=1 xfs-21: (groupid=2, jobs=5): err= 0: pid=69166: Fri Dec 4 17:06:08 2015 write: io=3404.6GB, bw=495816KB/s, iops=7747, runt=7200001msec slat (usec): min=7, max=1175, avg=25.47, stdev= 9.32 clat (usec): min=1, max=1131.7K, avg=616.23, stdev=4402.68 lat (usec): min=56, max=1131.7K, avg=642.06, stdev=4402.89 clat percentiles (usec): | 1.00th=[ 92], 5.00th=[ 124], 10.00th=[ 159], 20.00th=[ 223], | 30.00th=[ 302], 40.00th=[ 370], 50.00th=[ 454], 60.00th=[ 556], | 70.00th=[ 692], 80.00th=[ 868], 90.00th=[ 1176], 95.00th=[ 1448], | 99.00th=[ 2064], 99.50th=[ 2416], 99.90th=[ 3408], 99.95th=[ 4080], | 99.99th=[12224] bw (KB /s): min= 0, max=189696, per=20.28%, avg=100543.08, stdev=17925.36 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (usec) : 100=1.92%, 250=21.11%, 500=31.67%, 750=19.14%, 1000=11.35% lat (msec) : 2=13.61%, 4=1.15%, 10=0.04%, 20=0.01%, 50=0.01% lat (msec) : 100=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%, 2000=0.01% cpu : usr=1.64%, sys=5.66%, ctx=60550031, majf=0, minf=26261 IO depths : 1=108.5%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=55779323/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=1 Run status group 0 (all jobs): WRITE: io=3357.2GB, aggrb=488917KB/s, minb=488917KB/s, maxb=488917KB/s, mint=7200001msec, maxt=7200001msec Run status group 1 (all jobs): WRITE: io=3362.1GB, aggrb=489761KB/s, minb=489761KB/s, maxb=489761KB/s, mint=7200001msec, maxt=7200001msec Run status group 2 (all jobs): WRITE: io=3404.6GB, aggrb=495816KB/s, minb=495816KB/s, maxb=495816KB/s, mint=7200001msec, maxt=7200001msec Disk stats (read/write): sda: ios=0/59691755, merge=0/0, ticks=0/37145570, in_queue=37095400, util=100.00% sdb: ios=0/59677509, merge=0/0, ticks=0/37191957, in_queue=37144806, util=100.00% sdc: ios=0/60516706, merge=0/0, ticks=0/37236819, in_queue=37192550, util=100.00% ################################################################################### # # Fri 04 Dec 2015 05:07:09 PM EST # 8kB randrw with 2 jobs and 2 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=1287.9GB, bw=187560KB/s, iops=23444, runt=7200002msec lat (usec): min=38, max=18029, avg=634.13, stdev=647.99 write: io=649827MB, bw=92420KB/s, iops=11552, runt=7200002msec lat (usec): min=28, max=10605, avg=438.90, stdev=433.80 xfs read : io=1289.9GB, bw=187848KB/s, iops=23480, runt=7200003msec lat (usec): min=36, max=16767, avg=624.64, stdev=621.12 write: io=650824MB, bw=92562KB/s, iops=11570, runt=7200003msec lat (usec): min=29, max=12197, avg=455.54, stdev=440.62 xfs read : io=1275.8GB, bw=185794KB/s, iops=23224, runt=7200003msec lat (usec): min=40, max=21893, avg=649.26, stdev=707.86 write: io=643686MB, bw=91546KB/s, iops=11443, runt=7200003msec lat (usec): min=28, max=12979, avg=424.72, stdev=448.09 Totals IOPS: 104713 MBPS: 816 Average Latency (μsec): 1074 xfs-01: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-02: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-03: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-04: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-05: (g=0): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-11: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-12: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-13: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-14: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-15: (g=1): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-21: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-22: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-23: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-24: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-25: (g=2): rw=randrw, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... fio-2.1.10 Starting 30 processes xfs-01: (groupid=0, jobs=10): err= 0: pid=74995: Fri Dec 4 19:17:11 2015 read : io=1287.9GB, bw=187560KB/s, iops=23444, runt=7200002msec slat (usec): min=2, max=6557, avg=16.44, stdev=11.01 clat (usec): min=0, max=18021, avg=617.56, stdev=648.04 lat (usec): min=38, max=18029, avg=634.13, stdev=647.99 clat percentiles (usec): | 1.00th=[ 135], 5.00th=[ 155], 10.00th=[ 169], 20.00th=[ 191], | 30.00th=[ 215], 40.00th=[ 247], 50.00th=[ 314], 60.00th=[ 450], | 70.00th=[ 676], 80.00th=[ 1064], 90.00th=[ 1336], 95.00th=[ 1752], | 99.00th=[ 3248], 99.50th=[ 3504], 99.90th=[ 4128], 99.95th=[ 4576], | 99.99th=[ 6624] bw (KB /s): min= 0, max=27760, per=10.01%, avg=18765.68, stdev=570.66 write: io=649827MB, bw=92420KB/s, iops=11552, runt=7200002msec slat (usec): min=2, max=8909, avg=18.11, stdev=11.75 clat (usec): min=0, max=10590, avg=420.65, stdev=433.17 lat (usec): min=28, max=10605, avg=438.90, stdev=433.80 clat percentiles (usec): | 1.00th=[ 48], 5.00th=[ 69], 10.00th=[ 83], 20.00th=[ 103], | 30.00th=[ 120], 40.00th=[ 139], 50.00th=[ 163], 60.00th=[ 258], | 70.00th=[ 532], 80.00th=[ 932], 90.00th=[ 1208], 95.00th=[ 1336], | 99.00th=[ 1448], 99.50th=[ 1496], 99.90th=[ 1560], 99.95th=[ 1576], | 99.99th=[ 1640] bw (KB /s): min= 0, max=12672, per=10.01%, avg=9246.75, stdev=432.04 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.36% lat (usec) : 100=5.64%, 250=40.91%, 500=18.12%, 750=9.25%, 1000=5.17% lat (msec) : 2=17.79%, 4=2.67%, 10=0.09%, 20=0.01% cpu : usr=1.40%, sys=9.22%, ctx=237898032, majf=0, minf=39880 IO depths : 1=0.1%, 2=108.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=168803785/w=83177855/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-11: (groupid=1, jobs=10): err= 0: pid=75005: Fri Dec 4 19:17:11 2015 read : io=1289.9GB, bw=187848KB/s, iops=23480, runt=7200003msec slat (usec): min=2, max=8260, avg=17.12, stdev=11.85 clat (usec): min=0, max=16752, avg=607.44, stdev=621.17 lat (usec): min=36, max=16767, avg=624.64, stdev=621.12 clat percentiles (usec): | 1.00th=[ 137], 5.00th=[ 159], 10.00th=[ 173], 20.00th=[ 195], | 30.00th=[ 217], 40.00th=[ 247], 50.00th=[ 310], 60.00th=[ 446], | 70.00th=[ 684], 80.00th=[ 1064], 90.00th=[ 1336], 95.00th=[ 1688], | 99.00th=[ 3152], 99.50th=[ 3440], 99.90th=[ 4016], 99.95th=[ 4384], | 99.99th=[ 6240] bw (KB /s): min= 0, max=32240, per=10.01%, avg=18794.58, stdev=552.32 write: io=650824MB, bw=92562KB/s, iops=11570, runt=7200003msec slat (usec): min=2, max=11077, avg=18.75, stdev=12.56 clat (usec): min=0, max=10735, avg=436.69, stdev=440.11 lat (usec): min=29, max=12197, avg=455.54, stdev=440.62 clat percentiles (usec): | 1.00th=[ 53], 5.00th=[ 75], 10.00th=[ 89], 20.00th=[ 109], | 30.00th=[ 126], 40.00th=[ 145], 50.00th=[ 171], 60.00th=[ 290], | 70.00th=[ 548], 80.00th=[ 964], 90.00th=[ 1240], 95.00th=[ 1352], | 99.00th=[ 1464], 99.50th=[ 1496], 99.90th=[ 1560], 99.95th=[ 1592], | 99.99th=[ 1656] bw (KB /s): min= 0, max=15168, per=10.01%, avg=9261.11, stdev=418.51 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.24% lat (usec) : 100=4.68%, 250=41.56%, 500=18.47%, 750=8.85%, 1000=5.08% lat (msec) : 2=18.80%, 4=2.24%, 10=0.07%, 20=0.01% cpu : usr=1.38%, sys=9.51%, ctx=236356371, majf=0, minf=49767 IO depths : 1=0.1%, 2=108.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=169062974/w=83305494/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-21: (groupid=2, jobs=10): err= 0: pid=75015: Fri Dec 4 19:17:11 2015 read : io=1275.8GB, bw=185794KB/s, iops=23224, runt=7200003msec slat (usec): min=2, max=7545, avg=16.65, stdev=11.50 clat (usec): min=0, max=21870, avg=632.49, stdev=707.85 lat (usec): min=40, max=21893, avg=649.26, stdev=707.86 clat percentiles (usec): | 1.00th=[ 135], 5.00th=[ 153], 10.00th=[ 167], 20.00th=[ 185], | 30.00th=[ 205], 40.00th=[ 227], 50.00th=[ 266], 60.00th=[ 398], | 70.00th=[ 740], 80.00th=[ 1096], 90.00th=[ 1432], 95.00th=[ 2024], | 99.00th=[ 3344], 99.50th=[ 3696], 99.90th=[ 4640], 99.95th=[ 5216], | 99.99th=[10688] bw (KB /s): min= 0, max=25104, per=10.01%, avg=18588.83, stdev=592.25 write: io=643686MB, bw=91546KB/s, iops=11443, runt=7200003msec slat (usec): min=2, max=7477, avg=18.25, stdev=12.22 clat (usec): min=0, max=12970, avg=406.34, stdev=447.49 lat (usec): min=28, max=12979, avg=424.72, stdev=448.09 clat percentiles (usec): | 1.00th=[ 50], 5.00th=[ 70], 10.00th=[ 82], 20.00th=[ 100], | 30.00th=[ 115], 40.00th=[ 131], 50.00th=[ 149], 60.00th=[ 181], | 70.00th=[ 434], 80.00th=[ 964], 90.00th=[ 1176], 95.00th=[ 1288], | 99.00th=[ 1592], 99.50th=[ 1624], 99.90th=[ 1688], 99.95th=[ 1704], | 99.99th=[ 1736] bw (KB /s): min= 0, max=12752, per=10.01%, avg=9159.77, stdev=433.27 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.30% lat (usec) : 100=6.22%, 250=46.44%, 500=13.67%, 750=5.61%, 1000=5.75% lat (msec) : 2=18.23%, 4=3.56%, 10=0.20%, 20=0.01%, 50=0.01% cpu : usr=1.35%, sys=9.16%, ctx=235131167, majf=0, minf=47171 IO depths : 1=0.1%, 2=108.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=167215087/w=82391864/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 Run status group 0 (all jobs): READ: io=1287.9GB, aggrb=187559KB/s, minb=187559KB/s, maxb=187559KB/s, mint=7200002msec, maxt=7200002msec WRITE: io=649827MB, aggrb=92419KB/s, minb=92419KB/s, maxb=92419KB/s, mint=7200002msec, maxt=7200002msec Run status group 1 (all jobs): READ: io=1289.9GB, aggrb=187847KB/s, minb=187847KB/s, maxb=187847KB/s, mint=7200003msec, maxt=7200003msec WRITE: io=650824MB, aggrb=92561KB/s, minb=92561KB/s, maxb=92561KB/s, mint=7200003msec, maxt=7200003msec Run status group 2 (all jobs): READ: io=1275.8GB, aggrb=185794KB/s, minb=185794KB/s, maxb=185794KB/s, mint=7200003msec, maxt=7200003msec WRITE: io=643686MB, aggrb=91546KB/s, minb=91546KB/s, maxb=91546KB/s, mint=7200003msec, maxt=7200003msec Disk stats (read/write): sda: ios=183088677/90211370, merge=0/0, ticks=112675266/37995568, in_queue=150765789, util=100.00% sdb: ios=183392830/90361511, merge=0/0, ticks=110949968/39542142, in_queue=150616787, util=100.00% sdc: ios=181415930/89383765, merge=0/0, ticks=114329148/36355294, in_queue=150894772, util=100.00% ################################################################################### # # Fri 04 Dec 2015 07:18:11 PM EST # 64kB randrw with 2 jobs and 2 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=3910.5GB, bw=569495KB/s, iops=8898, runt=7200003msec lat (usec): min=55, max=713591, avg=2159.30, stdev=7649.18 write: io=1926.2GB, bw=280516KB/s, iops=4383, runt=7200003msec lat (usec): min=42, max=9870, avg=171.19, stdev=208.13 xfs read : io=3921.3GB, bw=571076KB/s, iops=8923, runt=7200012msec lat (usec): min=53, max=713479, avg=2154.64, stdev=8015.81 write: io=1931.6GB, bw=281294KB/s, iops=4395, runt=7200012msec lat (usec): min=44, max=12299, avg=167.95, stdev=206.82 xfs read : io=3843.2GB, bw=559695KB/s, iops=8745, runt=7200003msec lat (usec): min=53, max=676938, avg=2202.07, stdev=7465.60 write: io=1892.9GB, bw=275662KB/s, iops=4307, runt=7200003msec lat (usec): min=41, max=20762, avg=164.23, stdev=208.56 Totals IOPS: 39651 MBPS: 2475 Average Latency (μsec): 2339 xfs-01: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-02: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-03: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-04: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-05: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-11: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-12: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-13: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-14: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-15: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-21: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-22: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-23: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-24: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... xfs-25: (g=2): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=2 ... fio-2.1.10 Starting 30 processes xfs-01: (groupid=0, jobs=10): err= 0: pid=80368: Fri Dec 4 21:28:13 2015 read : io=3910.5GB, bw=569495KB/s, iops=8898, runt=7200003msec slat (usec): min=4, max=1230, avg=20.35, stdev= 8.61 clat (usec): min=1, max=713573, avg=2138.75, stdev=7649.07 lat (usec): min=55, max=713591, avg=2159.30, stdev=7649.18 clat percentiles (usec): | 1.00th=[ 386], 5.00th=[ 474], 10.00th=[ 556], 20.00th=[ 748], | 30.00th=[ 1080], 40.00th=[ 1368], 50.00th=[ 1720], 60.00th=[ 2160], | 70.00th=[ 2672], 80.00th=[ 3216], 90.00th=[ 3920], 95.00th=[ 4640], | 99.00th=[ 6240], 99.50th=[ 7008], 99.90th=[ 9792], 99.95th=[12224], | 99.99th=[659456] bw (KB /s): min= 0, max=70898, per=10.17%, avg=57933.71, stdev=7938.91 write: io=1926.2GB, bw=280516KB/s, iops=4383, runt=7200003msec slat (usec): min=5, max=1682, avg=24.23, stdev=10.55 clat (usec): min=0, max=9856, avg=146.72, stdev=206.97 lat (usec): min=42, max=9870, avg=171.19, stdev=208.13 clat percentiles (usec): | 1.00th=[ 45], 5.00th=[ 50], 10.00th=[ 52], 20.00th=[ 58], | 30.00th=[ 64], 40.00th=[ 71], 50.00th=[ 80], 60.00th=[ 92], | 70.00th=[ 112], 80.00th=[ 155], 90.00th=[ 278], 95.00th=[ 402], | 99.00th=[ 1176], 99.50th=[ 1240], 99.90th=[ 1352], 99.95th=[ 1384], | 99.99th=[ 1448] bw (KB /s): min= 0, max=41088, per=10.18%, avg=28542.59, stdev=4410.13 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=1.64% lat (usec) : 100=19.59%, 250=7.91%, 500=6.99%, 750=9.05%, 1000=5.00% lat (msec) : 2=20.76%, 4=22.79%, 10=6.20%, 20=0.06%, 50=0.01% lat (msec) : 750=0.01% cpu : usr=0.95%, sys=4.22%, ctx=97455723, majf=0, minf=336171 IO depths : 1=0.1%, 2=108.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=64068179/w=31558061/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-11: (groupid=1, jobs=10): err= 0: pid=80378: Fri Dec 4 21:28:13 2015 read : io=3921.3GB, bw=571076KB/s, iops=8923, runt=7200012msec slat (usec): min=4, max=1143, avg=20.46, stdev= 8.70 clat (usec): min=1, max=713460, avg=2133.98, stdev=8015.71 lat (usec): min=53, max=713479, avg=2154.64, stdev=8015.81 clat percentiles (usec): | 1.00th=[ 386], 5.00th=[ 474], 10.00th=[ 548], 20.00th=[ 740], | 30.00th=[ 1080], 40.00th=[ 1352], 50.00th=[ 1704], 60.00th=[ 2128], | 70.00th=[ 2640], 80.00th=[ 3184], 90.00th=[ 3920], 95.00th=[ 4640], | 99.00th=[ 6240], 99.50th=[ 7008], 99.90th=[ 9664], 99.95th=[11840], | 99.99th=[659456] bw (KB /s): min= 0, max=71394, per=10.19%, avg=58194.58, stdev=8417.04 write: io=1931.6GB, bw=281294KB/s, iops=4395, runt=7200012msec slat (usec): min=5, max=1296, avg=24.30, stdev=10.59 clat (usec): min=0, max=12285, avg=143.41, stdev=205.59 lat (usec): min=44, max=12299, avg=167.95, stdev=206.82 clat percentiles (usec): | 1.00th=[ 44], 5.00th=[ 49], 10.00th=[ 52], 20.00th=[ 57], | 30.00th=[ 63], 40.00th=[ 69], 50.00th=[ 78], 60.00th=[ 90], | 70.00th=[ 108], 80.00th=[ 149], 90.00th=[ 274], 95.00th=[ 390], | 99.00th=[ 1192], 99.50th=[ 1240], 99.90th=[ 1352], 99.95th=[ 1384], | 99.99th=[ 1448] bw (KB /s): min= 0, max=41644, per=10.19%, avg=28668.90, stdev=4627.90 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=1.95% lat (usec) : 100=19.85%, 250=7.45%, 500=6.98%, 750=9.40%, 1000=4.97% lat (msec) : 2=20.58%, 4=22.66%, 10=6.11%, 20=0.05%, 50=0.01% lat (msec) : 750=0.01% cpu : usr=0.96%, sys=4.25%, ctx=97792369, majf=0, minf=363307 IO depths : 1=0.1%, 2=108.1%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=64246201/w=31645624/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-21: (groupid=2, jobs=10): err= 0: pid=80388: Fri Dec 4 21:28:13 2015 read : io=3843.2GB, bw=559695KB/s, iops=8745, runt=7200003msec slat (usec): min=4, max=1353, avg=20.24, stdev= 8.46 clat (usec): min=29, max=676924, avg=2181.63, stdev=7465.50 lat (usec): min=53, max=676938, avg=2202.07, stdev=7465.60 clat percentiles (usec): | 1.00th=[ 386], 5.00th=[ 470], 10.00th=[ 548], 20.00th=[ 708], | 30.00th=[ 1032], 40.00th=[ 1336], 50.00th=[ 1672], 60.00th=[ 2128], | 70.00th=[ 2672], 80.00th=[ 3280], 90.00th=[ 4128], 95.00th=[ 4960], | 99.00th=[ 6944], 99.50th=[ 7840], 99.90th=[13376], 99.95th=[18048], | 99.99th=[610304] bw (KB /s): min= 0, max=72960, per=10.16%, avg=56851.97, stdev=8280.85 write: io=1892.9GB, bw=275662KB/s, iops=4307, runt=7200003msec slat (usec): min=4, max=1292, avg=24.19, stdev=10.46 clat (usec): min=0, max=20735, avg=139.79, stdev=207.38 lat (usec): min=41, max=20762, avg=164.23, stdev=208.56 clat percentiles (usec): | 1.00th=[ 44], 5.00th=[ 48], 10.00th=[ 51], 20.00th=[ 55], | 30.00th=[ 60], 40.00th=[ 67], 50.00th=[ 75], 60.00th=[ 85], | 70.00th=[ 103], 80.00th=[ 141], 90.00th=[ 274], 95.00th=[ 386], | 99.00th=[ 1224], 99.50th=[ 1288], 99.90th=[ 1400], 99.95th=[ 1448], | 99.99th=[ 1528] bw (KB /s): min= 0, max=40448, per=10.16%, avg=28005.47, stdev=4558.37 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=2.56% lat (usec) : 100=20.06%, 250=6.68%, 500=7.17%, 750=10.08%, 1000=5.01% lat (msec) : 2=19.84%, 4=21.15%, 10=7.34%, 20=0.09%, 50=0.01% lat (msec) : 750=0.01% cpu : usr=0.94%, sys=4.13%, ctx=95970987, majf=0, minf=292201 IO depths : 1=0.1%, 2=108.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=62965689/w=31011963/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 Run status group 0 (all jobs): READ: io=3910.5GB, aggrb=569494KB/s, minb=569494KB/s, maxb=569494KB/s, mint=7200003msec, maxt=7200003msec WRITE: io=1926.2GB, aggrb=280515KB/s, minb=280515KB/s, maxb=280515KB/s, mint=7200003msec, maxt=7200003msec Run status group 1 (all jobs): READ: io=3921.3GB, aggrb=571076KB/s, minb=571076KB/s, maxb=571076KB/s, mint=7200012msec, maxt=7200012msec WRITE: io=1931.6GB, aggrb=281293KB/s, minb=281293KB/s, maxb=281293KB/s, mint=7200012msec, maxt=7200012msec Run status group 2 (all jobs): READ: io=3843.2GB, aggrb=559694KB/s, minb=559694KB/s, maxb=559694KB/s, mint=7200003msec, maxt=7200003msec WRITE: io=1892.9GB, aggrb=275661KB/s, minb=275661KB/s, maxb=275661KB/s, mint=7200003msec, maxt=7200003msec Disk stats (read/write): sda: ios=69227566/34096523, merge=0/0, ticks=148196321/4998934, in_queue=153195776, util=100.00% sdb: ios=69462141/34212385, merge=0/0, ticks=148271385/4897222, in_queue=153174257, util=100.00% sdc: ios=68008284/33493261, merge=0/0, ticks=148557138/4678283, in_queue=153261444, util=100.00% ################################################################################### # # Fri 04 Dec 2015 09:29:13 PM EST # 1mB write with 1 jobs and 2 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs write: io=12898GB, bw=1834.4MB/s, iops=1834, runt=7200004msec lat (msec): min=1, max=57, avg= 5.45, stdev= 2.84 xfs write: io=12772GB, bw=1816.5MB/s, iops=1816, runt=7200002msec lat (usec): min=476, max=32149, avg=5502.10, stdev=2933.37 xfs write: io=11848GB, bw=1685.9MB/s, iops=1685, runt=7200002msec lat (msec): min=1, max=46, avg= 5.93, stdev= 3.03 Totals IOPS: 5335 MBPS: 5335 Average Latency (μsec): 5627 xfs-01: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-02: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-03: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-04: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-05: (g=0): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-11: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-12: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-13: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-14: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-15: (g=1): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-21: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-22: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-23: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-24: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 xfs-25: (g=2): rw=write, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=2 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=86628: Fri Dec 4 23:39:14 2015 write: io=12898GB, bw=1834.4MB/s, iops=1834, runt=7200004msec slat (usec): min=41, max=6546, avg=128.72, stdev=19.12 clat (msec): min=1, max=57, avg= 5.32, stdev= 2.83 lat (msec): min=1, max=57, avg= 5.45, stdev= 2.84 clat percentiles (usec): | 1.00th=[ 2928], 5.00th=[ 3152], 10.00th=[ 3216], 20.00th=[ 3344], | 30.00th=[ 3440], 40.00th=[ 3536], 50.00th=[ 3696], 60.00th=[ 4384], | 70.00th=[ 5984], 80.00th=[ 7712], 90.00th=[ 9792], 95.00th=[11328], | 99.00th=[14016], 99.50th=[15040], 99.90th=[18304], 99.95th=[19840], | 99.99th=[23168] bw (KB /s): min= 1, max=587776, per=20.03%, avg=376203.82, stdev=137817.52 lat (msec) : 2=0.01%, 4=55.35%, 10=35.81%, 20=8.79%, 50=0.05% lat (msec) : 100=0.01% cpu : usr=1.78%, sys=3.89%, ctx=13871510, majf=0, minf=488950 IO depths : 1=0.1%, 2=105.6%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=13207356/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-11: (groupid=1, jobs=5): err= 0: pid=86633: Fri Dec 4 23:39:14 2015 write: io=12772GB, bw=1816.5MB/s, iops=1816, runt=7200002msec slat (usec): min=39, max=4570, avg=127.81, stdev=20.22 clat (usec): min=379, max=32019, avg=5373.87, stdev=2929.20 lat (usec): min=476, max=32149, avg=5502.10, stdev=2933.37 clat percentiles (usec): | 1.00th=[ 2960], 5.00th=[ 3152], 10.00th=[ 3216], 20.00th=[ 3344], | 30.00th=[ 3408], 40.00th=[ 3504], 50.00th=[ 3664], 60.00th=[ 4384], | 70.00th=[ 6048], 80.00th=[ 7840], 90.00th=[ 9920], 95.00th=[11840], | 99.00th=[14400], 99.50th=[15424], 99.90th=[18048], 99.95th=[19584], | 99.99th=[23168] bw (KB /s): min= 1, max=587776, per=20.03%, avg=372545.09, stdev=142384.55 lat (usec) : 500=0.01% lat (msec) : 2=0.01%, 4=55.69%, 10=34.67%, 20=9.60%, 50=0.04% cpu : usr=1.73%, sys=3.84%, ctx=13732695, majf=0, minf=498068 IO depths : 1=0.1%, 2=105.5%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=13078105/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-21: (groupid=2, jobs=5): err= 0: pid=86638: Fri Dec 4 23:39:14 2015 write: io=11848GB, bw=1685.9MB/s, iops=1685, runt=7200002msec slat (usec): min=48, max=2826, avg=139.77, stdev=26.11 clat (msec): min=1, max=46, avg= 5.79, stdev= 3.04 lat (msec): min=1, max=46, avg= 5.93, stdev= 3.03 clat percentiles (usec): | 1.00th=[ 2672], 5.00th=[ 3024], 10.00th=[ 3216], 20.00th=[ 3440], | 30.00th=[ 3632], 40.00th=[ 4128], 50.00th=[ 4640], 60.00th=[ 5536], | 70.00th=[ 6432], 80.00th=[ 7968], 90.00th=[10176], 95.00th=[11968], | 99.00th=[15680], 99.50th=[17280], 99.90th=[21376], 99.95th=[27264], | 99.99th=[31872] bw (KB /s): min= 1, max=585728, per=20.03%, avg=345576.51, stdev=112438.00 lat (msec) : 2=0.08%, 4=37.47%, 10=51.68%, 20=10.63%, 50=0.15% cpu : usr=1.69%, sys=3.99%, ctx=12403319, majf=0, minf=1009342 IO depths : 1=0.1%, 2=106.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=12132575/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 Run status group 0 (all jobs): WRITE: io=12898GB, aggrb=1834.4MB/s, minb=1834.4MB/s, maxb=1834.4MB/s, mint=7200004msec, maxt=7200004msec Run status group 1 (all jobs): WRITE: io=12772GB, aggrb=1816.5MB/s, minb=1816.5MB/s, maxb=1816.5MB/s, mint=7200002msec, maxt=7200002msec Run status group 2 (all jobs): WRITE: io=11848GB, aggrb=1685.9MB/s, minb=1685.9MB/s, maxb=1685.9MB/s, mint=7200002msec, maxt=7200002msec Disk stats (read/write): sda: ios=0/55769369, merge=0/0, ticks=0/291770089, in_queue=291734637, util=100.00% sdb: ios=0/55187184, merge=0/0, ticks=0/292238435, in_queue=292203265, util=100.00% sdc: ios=0/51426189, merge=0/0, ticks=0/290113425, in_queue=290088876, util=100.00% ################################################################################### # # Fri 04 Dec 2015 11:40:14 PM EST # 1mB read with 1 jobs and 4 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=25315GB, bw=3600.5MB/s, iops=3600, runt=7200006msec lat (usec): min=429, max=27870, avg=5553.09, stdev=665.86 xfs read : io=25403GB, bw=3612.9MB/s, iops=3612, runt=7200006msec lat (usec): min=446, max=25573, avg=5534.01, stdev=683.13 xfs read : io=25590GB, bw=3639.5MB/s, iops=3639, runt=7200008msec lat (usec): min=496, max=27456, avg=5493.46, stdev=764.88 Totals IOPS: 10851 MBPS: 10851 Average Latency (μsec): 5526 xfs-01: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-02: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-03: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-04: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-05: (g=0): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-11: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-12: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-13: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-14: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-15: (g=1): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-21: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-22: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-23: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-24: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 xfs-25: (g=2): rw=read, bs=1M-1M/1M-1M/1M-1M, ioengine=libaio, iodepth=4 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=91904: Sat Dec 5 01:50:15 2015 read : io=25315GB, bw=3600.5MB/s, iops=3600, runt=7200006msec slat (usec): min=37, max=4234, avg=137.38, stdev=44.16 clat (usec): min=359, max=27733, avg=5415.48, stdev=666.48 lat (usec): min=429, max=27870, avg=5553.09, stdev=665.86 clat percentiles (usec): | 1.00th=[ 4384], 5.00th=[ 4640], 10.00th=[ 4768], 20.00th=[ 4896], | 30.00th=[ 5024], 40.00th=[ 5152], 50.00th=[ 5280], 60.00th=[ 5472], | 70.00th=[ 5664], 80.00th=[ 5856], 90.00th=[ 6176], 95.00th=[ 6496], | 99.00th=[ 7776], 99.50th=[ 8384], 99.90th=[ 9664], 99.95th=[10304], | 99.99th=[13504] bw (KB /s): min= 1, max=915456, per=20.02%, avg=737960.57, stdev=14442.96 lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.07%, 10=99.86%, 20=0.07%, 50=0.01% cpu : usr=0.29%, sys=11.32%, ctx=26160329, majf=0, minf=1074372 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=25922977/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-11: (groupid=1, jobs=5): err= 0: pid=91909: Sat Dec 5 01:50:15 2015 read : io=25403GB, bw=3612.9MB/s, iops=3612, runt=7200006msec slat (usec): min=37, max=3669, avg=137.29, stdev=43.88 clat (usec): min=326, max=25406, avg=5396.16, stdev=683.95 lat (usec): min=446, max=25573, avg=5534.01, stdev=683.13 clat percentiles (usec): | 1.00th=[ 4320], 5.00th=[ 4640], 10.00th=[ 4768], 20.00th=[ 4896], | 30.00th=[ 5024], 40.00th=[ 5088], 50.00th=[ 5216], 60.00th=[ 5408], | 70.00th=[ 5600], 80.00th=[ 5856], 90.00th=[ 6176], 95.00th=[ 6560], | 99.00th=[ 7776], 99.50th=[ 8384], 99.90th=[ 9664], 99.95th=[10304], | 99.99th=[13248] bw (KB /s): min= 1, max=909540, per=20.02%, avg=740505.70, stdev=12952.57 lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.08%, 10=99.85%, 20=0.07%, 50=0.01% cpu : usr=0.29%, sys=11.34%, ctx=26278784, majf=0, minf=1071255 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=26012366/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-21: (groupid=2, jobs=5): err= 0: pid=91914: Sat Dec 5 01:50:15 2015 read : io=25590GB, bw=3639.5MB/s, iops=3639, runt=7200008msec slat (usec): min=38, max=3179, avg=134.28, stdev=43.28 clat (usec): min=398, max=27322, avg=5359.06, stdev=765.62 lat (usec): min=496, max=27456, avg=5493.46, stdev=764.88 clat percentiles (usec): | 1.00th=[ 4320], 5.00th=[ 4576], 10.00th=[ 4704], 20.00th=[ 4832], | 30.00th=[ 4960], 40.00th=[ 5024], 50.00th=[ 5152], 60.00th=[ 5280], | 70.00th=[ 5536], 80.00th=[ 5792], 90.00th=[ 6176], 95.00th=[ 6624], | 99.00th=[ 8384], 99.50th=[ 9024], 99.90th=[10944], 99.95th=[11840], | 99.99th=[15808] bw (KB /s): min= 1, max=915672, per=20.02%, avg=745966.10, stdev=14214.33 lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.12%, 10=99.66%, 20=0.21%, 50=0.01% cpu : usr=0.29%, sys=11.18%, ctx=26688458, majf=0, minf=693092 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=26204275/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: io=25315GB, aggrb=3600.5MB/s, minb=3600.5MB/s, maxb=3600.5MB/s, mint=7200006msec, maxt=7200006msec Run status group 1 (all jobs): READ: io=25403GB, aggrb=3612.9MB/s, minb=3612.9MB/s, maxb=3612.9MB/s, mint=7200006msec, maxt=7200006msec Run status group 2 (all jobs): READ: io=25590GB, aggrb=3639.5MB/s, minb=3639.5MB/s, maxb=3639.5MB/s, mint=7200008msec, maxt=7200008msec Disk stats (read/write): sda: ios=112336427/10, merge=0/0, ticks=541986477/0, in_queue=542105457, util=100.00% sdb: ios=112721466/10, merge=0/0, ticks=538006282/0, in_queue=538106524, util=100.00% sdc: ios=113563500/10, merge=0/0, ticks=532693019/0, in_queue=532839116, util=100.00% ################################################################################### # # Sat 05 Dec 2015 01:51:16 AM EST # 256kB write with 1 jobs and 2 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs write: io=19384GB, bw=2756.9MB/s, iops=11027, runt=7200001msec lat (usec): min=182, max=21450, avg=904.86, stdev=470.39 xfs write: io=19367GB, bw=2754.5MB/s, iops=11017, runt=7200001msec lat (usec): min=176, max=23138, avg=905.65, stdev=475.23 xfs write: io=19368GB, bw=2754.7MB/s, iops=11018, runt=7200001msec lat (usec): min=238, max=11955, avg=905.52, stdev=165.48 Totals IOPS: 33062 MBPS: 8264 Average Latency (μsec): 904 xfs-01: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-02: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-03: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-04: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-05: (g=0): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-11: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-12: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-13: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-14: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-15: (g=1): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-21: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-22: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-23: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-24: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 xfs-25: (g=2): rw=write, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=2 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=98037: Sat Dec 5 04:01:17 2015 write: io=19384GB, bw=2756.9MB/s, iops=11027, runt=7200001msec slat (usec): min=11, max=1861, avg=36.71, stdev= 8.59 clat (usec): min=131, max=21422, avg=867.87, stdev=470.46 lat (usec): min=182, max=21450, avg=904.86, stdev=470.39 clat percentiles (usec): | 1.00th=[ 516], 5.00th=[ 596], 10.00th=[ 644], 20.00th=[ 692], | 30.00th=[ 732], 40.00th=[ 772], 50.00th=[ 812], 60.00th=[ 860], | 70.00th=[ 908], 80.00th=[ 980], 90.00th=[ 1096], 95.00th=[ 1256], | 99.00th=[ 1528], 99.50th=[ 1640], 99.90th=[10944], 99.95th=[11456], | 99.99th=[11968] bw (KB /s): min= 0, max=694784, per=20.01%, avg=564909.75, stdev=17982.50 lat (usec) : 250=0.01%, 500=0.76%, 750=33.94%, 1000=47.92% lat (msec) : 2=17.03%, 4=0.04%, 10=0.17%, 20=0.12%, 50=0.01% cpu : usr=3.02%, sys=8.91%, ctx=73338528, majf=0, minf=204388 IO depths : 1=0.1%, 2=108.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=79395693/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-11: (groupid=1, jobs=5): err= 0: pid=98042: Sat Dec 5 04:01:17 2015 write: io=19367GB, bw=2754.5MB/s, iops=11017, runt=7200001msec slat (usec): min=11, max=1427, avg=36.65, stdev= 8.48 clat (usec): min=43, max=23078, avg=868.69, stdev=475.32 lat (usec): min=176, max=23138, avg=905.65, stdev=475.23 clat percentiles (usec): | 1.00th=[ 524], 5.00th=[ 596], 10.00th=[ 636], 20.00th=[ 684], | 30.00th=[ 724], 40.00th=[ 764], 50.00th=[ 812], 60.00th=[ 852], | 70.00th=[ 908], 80.00th=[ 980], 90.00th=[ 1112], 95.00th=[ 1272], | 99.00th=[ 1528], 99.50th=[ 1656], 99.90th=[10688], 99.95th=[11328], | 99.99th=[11712] bw (KB /s): min= 0, max=680622, per=20.01%, avg=564425.62, stdev=16624.18 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.52%, 750=35.29% lat (usec) : 1000=46.21% lat (msec) : 2=17.62%, 4=0.05%, 10=0.19%, 20=0.12%, 50=0.01% cpu : usr=3.01%, sys=8.87%, ctx=73658813, majf=0, minf=198905 IO depths : 1=0.1%, 2=108.4%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=79327664/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-21: (groupid=2, jobs=5): err= 0: pid=98047: Sat Dec 5 04:01:17 2015 write: io=19368GB, bw=2754.7MB/s, iops=11018, runt=7200001msec slat (usec): min=11, max=1202, avg=38.01, stdev= 9.05 clat (usec): min=63, max=11904, avg=867.20, stdev=166.03 lat (usec): min=238, max=11955, avg=905.52, stdev=165.48 clat percentiles (usec): | 1.00th=[ 516], 5.00th=[ 620], 10.00th=[ 668], 20.00th=[ 732], | 30.00th=[ 780], 40.00th=[ 812], 50.00th=[ 852], 60.00th=[ 900], | 70.00th=[ 940], 80.00th=[ 996], 90.00th=[ 1080], 95.00th=[ 1176], | 99.00th=[ 1304], 99.50th=[ 1352], 99.90th=[ 1448], 99.95th=[ 1480], | 99.99th=[ 1576] bw (KB /s): min= 0, max=678400, per=20.01%, avg=564469.34, stdev=16099.59 lat (usec) : 100=0.01%, 250=0.01%, 500=0.84%, 750=23.07%, 1000=56.58% lat (msec) : 2=19.51%, 4=0.01%, 10=0.01%, 20=0.01% cpu : usr=3.04%, sys=9.28%, ctx=72008670, majf=0, minf=234313 IO depths : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=79333053/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 Run status group 0 (all jobs): WRITE: io=19384GB, aggrb=2756.9MB/s, minb=2756.9MB/s, maxb=2756.9MB/s, mint=7200001msec, maxt=7200001msec Run status group 1 (all jobs): WRITE: io=19367GB, aggrb=2754.5MB/s, minb=2754.5MB/s, maxb=2754.5MB/s, mint=7200001msec, maxt=7200001msec Run status group 2 (all jobs): WRITE: io=19368GB, aggrb=2754.7MB/s, minb=2754.7MB/s, maxb=2754.7MB/s, mint=7200001msec, maxt=7200001msec Disk stats (read/write): sda: ios=0/86027599, merge=0/0, ticks=0/74362562, in_queue=74274463, util=100.00% sdb: ios=0/85975680, merge=0/0, ticks=0/74401843, in_queue=74317527, util=100.00% sdc: ios=0/85952495, merge=0/0, ticks=0/74245750, in_queue=74169219, util=100.00% ################################################################################### # # Sat 05 Dec 2015 04:02:17 AM EST # 256kB read with 1 jobs and 4 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=20649GB, bw=2936.8MB/s, iops=11746, runt=7200003msec lat (usec): min=115, max=22142, avg=1701.05, stdev=471.42 xfs read : io=20655GB, bw=2937.7MB/s, iops=11750, runt=7200002msec lat (usec): min=109, max=20650, avg=1700.55, stdev=487.26 xfs read : io=20640GB, bw=2935.6MB/s, iops=11741, runt=7200002msec lat (usec): min=111, max=28130, avg=1701.78, stdev=557.87 Totals IOPS: 35237 MBPS: 8808 Average Latency (μsec): 1700 xfs-01: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-02: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-03: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-04: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-05: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-11: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-12: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-13: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-14: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-15: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-21: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-22: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-23: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-24: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 xfs-25: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=4 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=104284: Sat Dec 5 06:12:18 2015 read : io=20649GB, bw=2936.8MB/s, iops=11746, runt=7200003msec slat (usec): min=10, max=1027, avg=31.44, stdev= 6.86 clat (usec): min=88, max=22102, avg=1669.38, stdev=471.34 lat (usec): min=115, max=22142, avg=1701.05, stdev=471.42 clat percentiles (usec): | 1.00th=[ 956], 5.00th=[ 1144], 10.00th=[ 1224], 20.00th=[ 1352], | 30.00th=[ 1432], 40.00th=[ 1512], 50.00th=[ 1592], 60.00th=[ 1672], | 70.00th=[ 1784], 80.00th=[ 1912], 90.00th=[ 2128], 95.00th=[ 2384], | 99.00th=[ 3568], 99.50th=[ 3984], 99.90th=[ 5152], 99.95th=[ 5728], | 99.99th=[ 7072] bw (KB /s): min= 0, max=820736, per=20.01%, avg=601785.06, stdev=14045.83 lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.09%, 1000=1.38% lat (msec) : 2=83.63%, 4=14.41%, 10=0.49%, 20=0.01%, 50=0.01% cpu : usr=0.87%, sys=9.82%, ctx=82972598, majf=0, minf=307002 IO depths : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=84578115/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-11: (groupid=1, jobs=5): err= 0: pid=104289: Sat Dec 5 06:12:18 2015 read : io=20655GB, bw=2937.7MB/s, iops=11750, runt=7200002msec slat (usec): min=10, max=1174, avg=31.71, stdev= 6.89 clat (usec): min=86, max=20617, avg=1668.64, stdev=487.19 lat (usec): min=109, max=20650, avg=1700.55, stdev=487.26 clat percentiles (usec): | 1.00th=[ 932], 5.00th=[ 1128], 10.00th=[ 1224], 20.00th=[ 1336], | 30.00th=[ 1432], 40.00th=[ 1512], 50.00th=[ 1592], 60.00th=[ 1672], | 70.00th=[ 1768], 80.00th=[ 1912], 90.00th=[ 2128], 95.00th=[ 2448], | 99.00th=[ 3632], 99.50th=[ 4080], 99.90th=[ 5280], 99.95th=[ 5856], | 99.99th=[ 7136] bw (KB /s): min= 0, max=798208, per=20.01%, avg=601961.91, stdev=14916.16 lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.12%, 1000=1.63% lat (msec) : 2=83.29%, 4=14.39%, 10=0.56%, 20=0.01%, 50=0.01% cpu : usr=0.87%, sys=9.88%, ctx=82407848, majf=0, minf=307397 IO depths : 1=0.1%, 2=0.1%, 4=108.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=84602835/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-21: (groupid=2, jobs=5): err= 0: pid=104294: Sat Dec 5 06:12:18 2015 read : io=20640GB, bw=2935.6MB/s, iops=11741, runt=7200002msec slat (usec): min=10, max=1241, avg=31.54, stdev= 6.86 clat (usec): min=83, max=28095, avg=1670.04, stdev=557.81 lat (usec): min=111, max=28130, avg=1701.78, stdev=557.87 clat percentiles (usec): | 1.00th=[ 916], 5.00th=[ 1112], 10.00th=[ 1208], 20.00th=[ 1336], | 30.00th=[ 1416], 40.00th=[ 1496], 50.00th=[ 1576], 60.00th=[ 1656], | 70.00th=[ 1752], 80.00th=[ 1880], 90.00th=[ 2128], 95.00th=[ 2448], | 99.00th=[ 4016], 99.50th=[ 4896], 99.90th=[ 6432], 99.95th=[ 7072], | 99.99th=[ 9024] bw (KB /s): min= 0, max=714240, per=20.01%, avg=601533.55, stdev=17788.54 lat (usec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.15%, 1000=1.83% lat (msec) : 2=84.09%, 4=12.90%, 10=1.02%, 20=0.01%, 50=0.01% cpu : usr=0.87%, sys=9.82%, ctx=82353801, majf=0, minf=292899 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=84542359/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: io=20649GB, aggrb=2936.8MB/s, minb=2936.8MB/s, maxb=2936.8MB/s, mint=7200003msec, maxt=7200003msec Run status group 1 (all jobs): READ: io=20655GB, aggrb=2937.7MB/s, minb=2937.7MB/s, maxb=2937.7MB/s, mint=7200002msec, maxt=7200002msec Run status group 2 (all jobs): READ: io=20640GB, aggrb=2935.6MB/s, minb=2935.6MB/s, maxb=2935.6MB/s, mint=7200002msec, maxt=7200002msec Disk stats (read/write): sda: ios=91717644/10, merge=0/0, ticks=152886672/0, in_queue=152840387, util=100.00% sdb: ios=91574306/10, merge=0/0, ticks=152835763/0, in_queue=152788927, util=100.00% sdc: ios=91531358/10, merge=0/0, ticks=152862762/0, in_queue=152824206, util=100.00% ################################################################################### # # Sat 05 Dec 2015 06:13:18 AM EST # 256kB read with 1 jobs and 14 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=25267GB, bw=3593.6MB/s, iops=14374, runt=7200006msec lat (usec): min=154, max=27007, avg=4868.35, stdev=891.17 xfs read : io=25328GB, bw=3602.3MB/s, iops=14409, runt=7200006msec lat (usec): min=159, max=25889, avg=4856.60, stdev=1015.66 xfs read : io=25514GB, bw=3628.7MB/s, iops=14514, runt=7200006msec lat (usec): min=146, max=30376, avg=4821.19, stdev=1049.62 Totals IOPS: 43297 MBPS: 10823 Average Latency (μsec): 4848 xfs-01: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-02: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-03: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-04: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-05: (g=0): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-11: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-12: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-13: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-14: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-15: (g=1): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-21: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-22: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-23: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-24: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 xfs-25: (g=2): rw=read, bs=256K-256K/256K-256K/256K-256K, ioengine=libaio, iodepth=14 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=109403: Sat Dec 5 08:23:19 2015 read : io=25267GB, bw=3593.6MB/s, iops=14374, runt=7200006msec slat (usec): min=10, max=2159, avg=30.07, stdev= 7.17 clat (usec): min=127, max=26979, avg=4838.10, stdev=891.08 lat (usec): min=154, max=27007, avg=4868.35, stdev=891.17 clat percentiles (usec): | 1.00th=[ 2704], 5.00th=[ 3216], 10.00th=[ 3568], 20.00th=[ 4192], | 30.00th=[ 4576], 40.00th=[ 4768], 50.00th=[ 4896], 60.00th=[ 5088], | 70.00th=[ 5216], 80.00th=[ 5472], 90.00th=[ 5792], 95.00th=[ 6176], | 99.00th=[ 7008], 99.50th=[ 7520], 99.90th=[ 8896], 99.95th=[ 9536], | 99.99th=[11328] bw (KB /s): min= 0, max=962048, per=20.01%, avg=736315.97, stdev=48130.76 lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2=0.09%, 4=16.94%, 10=82.95%, 20=0.03%, 50=0.01% cpu : usr=1.04%, sys=11.50%, ctx=101223845, majf=0, minf=880074 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=103493859/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=14 xfs-11: (groupid=1, jobs=5): err= 0: pid=109408: Sat Dec 5 08:23:19 2015 read : io=25328GB, bw=3602.3MB/s, iops=14409, runt=7200006msec slat (usec): min=10, max=2357, avg=29.83, stdev= 7.06 clat (usec): min=133, max=25843, avg=4826.61, stdev=1015.57 lat (usec): min=159, max=25889, avg=4856.60, stdev=1015.66 clat percentiles (usec): | 1.00th=[ 2576], 5.00th=[ 3120], 10.00th=[ 3440], 20.00th=[ 3952], | 30.00th=[ 4384], 40.00th=[ 4704], 50.00th=[ 4896], 60.00th=[ 5088], | 70.00th=[ 5280], 80.00th=[ 5536], 90.00th=[ 5984], 95.00th=[ 6368], | 99.00th=[ 7584], 99.50th=[ 8160], 99.90th=[ 9536], 99.95th=[10176], | 99.99th=[12480] bw (KB /s): min= 0, max=914432, per=20.01%, avg=738100.26, stdev=43264.81 lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2=0.16%, 4=20.85%, 10=78.92%, 20=0.06%, 50=0.01% cpu : usr=1.02%, sys=11.43%, ctx=101591216, majf=0, minf=785621 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.4%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=103744977/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=14 xfs-21: (groupid=2, jobs=5): err= 0: pid=109413: Sat Dec 5 08:23:19 2015 read : io=25514GB, bw=3628.7MB/s, iops=14514, runt=7200006msec slat (usec): min=10, max=2842, avg=30.09, stdev= 7.07 clat (usec): min=123, max=30347, avg=4790.88, stdev=1049.53 lat (usec): min=146, max=30376, avg=4821.19, stdev=1049.62 clat percentiles (usec): | 1.00th=[ 2480], 5.00th=[ 3088], 10.00th=[ 3408], 20.00th=[ 3888], | 30.00th=[ 4320], 40.00th=[ 4640], 50.00th=[ 4896], 60.00th=[ 5088], | 70.00th=[ 5280], 80.00th=[ 5536], 90.00th=[ 5920], 95.00th=[ 6368], | 99.00th=[ 7712], 99.50th=[ 8384], 99.90th=[10176], 99.95th=[10944], | 99.99th=[13888] bw (KB /s): min= 0, max=897024, per=20.01%, avg=743515.38, stdev=44147.22 lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2=0.27%, 4=22.16%, 10=77.44%, 20=0.11%, 50=0.01% cpu : usr=1.05%, sys=11.59%, ctx=101750101, majf=0, minf=881410 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=104506094/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=14 Run status group 0 (all jobs): READ: io=25267GB, aggrb=3593.6MB/s, minb=3593.6MB/s, maxb=3593.6MB/s, mint=7200006msec, maxt=7200006msec Run status group 1 (all jobs): READ: io=25328GB, aggrb=3602.3MB/s, minb=3602.3MB/s, maxb=3602.3MB/s, mint=7200006msec, maxt=7200006msec Run status group 2 (all jobs): READ: io=25514GB, aggrb=3628.7MB/s, minb=3628.7MB/s, maxb=3628.7MB/s, mint=7200006msec, maxt=7200006msec Disk stats (read/write): sda: ios=112107031/0, merge=0/0, ticks=542431494/0, in_queue=542569402, util=100.00% sdb: ios=112422879/0, merge=0/0, ticks=542457206/0, in_queue=542586233, util=100.00% sdc: ios=113219903/0, merge=0/0, ticks=542411187/0, in_queue=542579483, util=100.00% ################################################################################### # # Sat 05 Dec 2015 08:24:20 AM EST # 64kB write with 1 jobs and 4 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs write: io=17338GB, bw=2465.9MB/s, iops=39453, runt=7200001msec lat (usec): min=80, max=15197, avg=506.16, stdev=140.38 xfs write: io=17491GB, bw=2487.6MB/s, iops=39801, runt=7200001msec lat (usec): min=75, max=15048, avg=501.73, stdev=139.81 xfs write: io=17393GB, bw=2473.7MB/s, iops=39577, runt=7200001msec lat (usec): min=116, max=14836, avg=504.54, stdev=133.76 Totals IOPS: 118831 MBPS: 7425 Average Latency (μsec): 503 xfs-01: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-02: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-03: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-04: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-05: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-11: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-12: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-13: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-14: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-15: (g=1): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-21: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-22: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-23: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-24: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-25: (g=2): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=114517: Sat Dec 5 10:34:21 2015 write: io=17338GB, bw=2465.9MB/s, iops=39453, runt=7200001msec slat (usec): min=3, max=3527, avg=12.64, stdev= 6.04 clat (usec): min=28, max=15186, avg=493.41, stdev=140.58 lat (usec): min=80, max=15197, avg=506.16, stdev=140.38 clat percentiles (usec): | 1.00th=[ 258], 5.00th=[ 302], 10.00th=[ 330], 20.00th=[ 374], | 30.00th=[ 410], 40.00th=[ 438], 50.00th=[ 466], 60.00th=[ 498], | 70.00th=[ 556], 80.00th=[ 628], 90.00th=[ 692], 95.00th=[ 748], | 99.00th=[ 844], 99.50th=[ 916], 99.90th=[ 1020], 99.95th=[ 1048], | 99.99th=[ 1096] bw (KB /s): min= 0, max=585472, per=20.00%, avg=505118.06, stdev=20984.49 lat (usec) : 50=0.01%, 100=0.01%, 250=0.68%, 500=59.85%, 750=34.51% lat (usec) : 1000=4.81% lat (msec) : 2=0.15%, 4=0.01%, 10=0.01%, 20=0.01% cpu : usr=2.62%, sys=12.23%, ctx=159039477, majf=0, minf=73175 IO depths : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=284061880/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-11: (groupid=1, jobs=5): err= 0: pid=114522: Sat Dec 5 10:34:21 2015 write: io=17491GB, bw=2487.6MB/s, iops=39801, runt=7200001msec slat (usec): min=3, max=14770, avg=12.89, stdev= 6.29 clat (usec): min=28, max=15024, avg=488.83, stdev=140.02 lat (usec): min=75, max=15048, avg=501.73, stdev=139.81 clat percentiles (usec): | 1.00th=[ 253], 5.00th=[ 298], 10.00th=[ 326], 20.00th=[ 370], | 30.00th=[ 406], 40.00th=[ 434], 50.00th=[ 458], 60.00th=[ 490], | 70.00th=[ 556], 80.00th=[ 628], 90.00th=[ 684], 95.00th=[ 748], | 99.00th=[ 844], 99.50th=[ 908], 99.90th=[ 988], 99.95th=[ 1012], | 99.99th=[ 1064] bw (KB /s): min= 0, max=580480, per=20.00%, avg=509575.59, stdev=20127.36 lat (usec) : 50=0.01%, 100=0.01%, 250=0.83%, 500=61.31%, 750=33.15% lat (usec) : 1000=4.63% lat (msec) : 2=0.08%, 4=0.01%, 10=0.01%, 20=0.01% cpu : usr=2.67%, sys=12.56%, ctx=161002374, majf=0, minf=71320 IO depths : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=286569590/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-21: (groupid=2, jobs=5): err= 0: pid=114527: Sat Dec 5 10:34:21 2015 write: io=17393GB, bw=2473.7MB/s, iops=39577, runt=7200001msec slat (usec): min=3, max=7859, avg=13.78, stdev= 6.71 clat (usec): min=26, max=14825, avg=490.72, stdev=134.01 lat (usec): min=116, max=14836, avg=504.54, stdev=133.76 clat percentiles (usec): | 1.00th=[ 266], 5.00th=[ 306], 10.00th=[ 334], 20.00th=[ 378], | 30.00th=[ 410], 40.00th=[ 434], 50.00th=[ 462], 60.00th=[ 494], | 70.00th=[ 556], 80.00th=[ 620], 90.00th=[ 692], 95.00th=[ 740], | 99.00th=[ 812], 99.50th=[ 836], 99.90th=[ 876], 99.95th=[ 900], | 99.99th=[ 948] bw (KB /s): min= 0, max=574080, per=20.00%, avg=506713.12, stdev=23846.71 lat (usec) : 50=0.01%, 100=0.01%, 250=0.42%, 500=60.92%, 750=34.19% lat (usec) : 1000=4.47% lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01% cpu : usr=2.73%, sys=13.44%, ctx=157987006, majf=0, minf=73279 IO depths : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=0/w=284958752/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): WRITE: io=17338GB, aggrb=2465.9MB/s, minb=2465.9MB/s, maxb=2465.9MB/s, mint=7200001msec, maxt=7200001msec Run status group 1 (all jobs): WRITE: io=17491GB, aggrb=2487.6MB/s, minb=2487.6MB/s, maxb=2487.6MB/s, mint=7200001msec, maxt=7200001msec Run status group 2 (all jobs): WRITE: io=17393GB, aggrb=2473.7MB/s, minb=2473.7MB/s, maxb=2473.7MB/s, mint=7200001msec, maxt=7200001msec Disk stats (read/write): sda: ios=0/307967225, merge=0/0, ticks=0/148747512, in_queue=148720028, util=100.00% sdb: ios=0/310568776, merge=0/0, ticks=0/148538126, in_queue=148523944, util=100.00% sdc: ios=0/308764826, merge=0/0, ticks=0/147774741, in_queue=147807990, util=100.00% ################################################################################### # # Sat 05 Dec 2015 10:35:21 AM EST # 64kB read with 1 jobs and 4 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=13955GB, bw=1984.8MB/s, iops=31755, runt=7200001msec lat (usec): min=42, max=15674, avg=628.94, stdev=140.47 xfs read : io=13311GB, bw=1893.2MB/s, iops=30290, runt=7200001msec lat (usec): min=38, max=14631, avg=659.41, stdev=145.91 xfs read : io=13500GB, bw=1919.1MB/s, iops=30719, runt=7200001msec lat (usec): min=44, max=17190, avg=650.17, stdev=161.06 Totals IOPS: 92764 MBPS: 5796 Average Latency (μsec): 645 xfs-01: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-02: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-03: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-04: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-05: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-11: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-12: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-13: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-14: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-15: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-21: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-22: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-23: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-24: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 xfs-25: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=120706: Sat Dec 5 12:45:22 2015 read : io=13955GB, bw=1984.8MB/s, iops=31755, runt=7200001msec slat (usec): min=3, max=843, avg=10.32, stdev= 3.79 clat (usec): min=2, max=15667, avg=618.53, stdev=140.44 lat (usec): min=42, max=15674, avg=628.94, stdev=140.47 clat percentiles (usec): | 1.00th=[ 386], 5.00th=[ 438], 10.00th=[ 466], 20.00th=[ 510], | 30.00th=[ 540], 40.00th=[ 572], 50.00th=[ 604], 60.00th=[ 636], | 70.00th=[ 676], 80.00th=[ 716], 90.00th=[ 780], 95.00th=[ 844], | 99.00th=[ 988], 99.50th=[ 1064], 99.90th=[ 1384], 99.95th=[ 2288], | 99.99th=[ 3248] bw (KB /s): min= 0, max=531584, per=20.00%, avg=406554.33, stdev=18993.37 lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=17.34% lat (usec) : 750=68.33%, 1000=13.44% lat (msec) : 2=0.83%, 4=0.06%, 10=0.01%, 20=0.01% cpu : usr=1.43%, sys=9.79%, ctx=223355172, majf=0, minf=53631 IO depths : 1=0.1%, 2=0.1%, 4=108.1%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=228640616/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-11: (groupid=1, jobs=5): err= 0: pid=120711: Sat Dec 5 12:45:22 2015 read : io=13311GB, bw=1893.2MB/s, iops=30290, runt=7200001msec slat (usec): min=3, max=1248, avg=10.35, stdev= 3.86 clat (usec): min=28, max=14624, avg=648.92, stdev=145.84 lat (usec): min=38, max=14631, avg=659.41, stdev=145.91 clat percentiles (usec): | 1.00th=[ 394], 5.00th=[ 450], 10.00th=[ 486], 20.00th=[ 532], | 30.00th=[ 572], 40.00th=[ 604], 50.00th=[ 636], 60.00th=[ 676], | 70.00th=[ 708], 80.00th=[ 756], 90.00th=[ 820], 95.00th=[ 876], | 99.00th=[ 1032], 99.50th=[ 1096], 99.90th=[ 1416], 99.95th=[ 2288], | 99.99th=[ 3280] bw (KB /s): min= 0, max=472960, per=20.00%, avg=387798.47, stdev=16770.30 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=12.64%, 750=66.58% lat (usec) : 1000=19.46% lat (msec) : 2=1.26%, 4=0.06%, 10=0.01%, 20=0.01% cpu : usr=1.37%, sys=9.39%, ctx=213329175, majf=0, minf=58365 IO depths : 1=0.1%, 2=0.1%, 4=108.2%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=218089251/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-21: (groupid=2, jobs=5): err= 0: pid=120716: Sat Dec 5 12:45:22 2015 read : io=13500GB, bw=1919.1MB/s, iops=30719, runt=7200001msec slat (usec): min=3, max=1280, avg=10.50, stdev= 3.87 clat (usec): min=23, max=17180, avg=639.59, stdev=160.97 lat (usec): min=44, max=17190, avg=650.17, stdev=161.06 clat percentiles (usec): | 1.00th=[ 398], 5.00th=[ 450], 10.00th=[ 482], 20.00th=[ 524], | 30.00th=[ 564], 40.00th=[ 596], 50.00th=[ 620], 60.00th=[ 660], | 70.00th=[ 700], 80.00th=[ 740], 90.00th=[ 804], 95.00th=[ 868], | 99.00th=[ 1012], 99.50th=[ 1096], 99.90th=[ 1896], 99.95th=[ 2992], | 99.99th=[ 5152] bw (KB /s): min= 0, max=480256, per=20.00%, avg=393294.65, stdev=21035.12 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=13.93%, 750=67.41% lat (usec) : 1000=17.53% lat (msec) : 2=1.04%, 4=0.07%, 10=0.03%, 20=0.01% cpu : usr=1.40%, sys=9.67%, ctx=214858681, majf=0, minf=55109 IO depths : 1=0.1%, 2=0.1%, 4=108.5%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=221179893/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: io=13955GB, aggrb=1984.8MB/s, minb=1984.8MB/s, maxb=1984.8MB/s, mint=7200001msec, maxt=7200001msec Run status group 1 (all jobs): READ: io=13311GB, aggrb=1893.2MB/s, minb=1893.2MB/s, maxb=1893.2MB/s, mint=7200001msec, maxt=7200001msec Run status group 2 (all jobs): READ: io=13500GB, aggrb=1919.1MB/s, minb=1919.1MB/s, maxb=1919.1MB/s, mint=7200001msec, maxt=7200001msec Disk stats (read/write): sda: ios=247133874/10, merge=0/0, ticks=152881473/0, in_queue=152874571, util=100.00% sdb: ios=236014034/10, merge=0/0, ticks=152994259/0, in_queue=152984442, util=100.00% sdc: ios=240052216/10, merge=0/0, ticks=152879602/0, in_queue=152877382, util=100.00% ################################################################################### # # Sat 05 Dec 2015 12:46:23 PM EST # 64kB read with 1 jobs and 16 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=22589GB, bw=3212.8MB/s, iops=51403, runt=7200002msec lat (usec): min=43, max=20613, avg=1555.57, stdev=398.21 xfs read : io=22342GB, bw=3177.6MB/s, iops=50840, runt=7200002msec lat (usec): min=51, max=20199, avg=1572.78, stdev=413.40 xfs read : io=22861GB, bw=3251.4MB/s, iops=52021, runt=7200002msec lat (usec): min=53, max=23660, avg=1537.09, stdev=398.30 Totals IOPS: 154264 MBPS: 9640 Average Latency (μsec): 1554 xfs-01: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-02: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-03: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-04: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-05: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-11: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-12: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-13: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-14: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-15: (g=1): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-21: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-22: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-23: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-24: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 xfs-25: (g=2): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=16 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=125833: Sat Dec 5 14:56:24 2015 read : io=22589GB, bw=3212.8MB/s, iops=51403, runt=7200002msec slat (usec): min=3, max=7232, avg=11.42, stdev= 4.78 clat (usec): min=36, max=20593, avg=1543.95, stdev=398.15 lat (usec): min=43, max=20613, avg=1555.57, stdev=398.21 clat percentiles (usec): | 1.00th=[ 796], 5.00th=[ 972], 10.00th=[ 1080], 20.00th=[ 1208], | 30.00th=[ 1304], 40.00th=[ 1400], 50.00th=[ 1496], 60.00th=[ 1608], | 70.00th=[ 1720], 80.00th=[ 1864], 90.00th=[ 2064], 95.00th=[ 2256], | 99.00th=[ 2640], 99.50th=[ 2800], 99.90th=[ 3216], 99.95th=[ 3504], | 99.99th=[ 4512] bw (KB /s): min= 0, max=755328, per=20.00%, avg=658044.94, stdev=21662.44 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.55% lat (usec) : 1000=5.48% lat (msec) : 2=81.27%, 4=12.67%, 10=0.02%, 20=0.01%, 50=0.01% cpu : usr=2.16%, sys=16.73%, ctx=325458800, majf=0, minf=201665 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=108.3%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=370103487/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=16 xfs-11: (groupid=1, jobs=5): err= 0: pid=125838: Sat Dec 5 14:56:24 2015 read : io=22342GB, bw=3177.6MB/s, iops=50840, runt=7200002msec slat (usec): min=3, max=4595, avg=10.96, stdev= 4.51 clat (usec): min=42, max=20192, avg=1561.62, stdev=413.34 lat (usec): min=51, max=20199, avg=1572.78, stdev=413.40 clat percentiles (usec): | 1.00th=[ 780], 5.00th=[ 964], 10.00th=[ 1064], 20.00th=[ 1208], | 30.00th=[ 1320], 40.00th=[ 1416], 50.00th=[ 1512], 60.00th=[ 1624], | 70.00th=[ 1752], 80.00th=[ 1896], 90.00th=[ 2128], 95.00th=[ 2320], | 99.00th=[ 2672], 99.50th=[ 2832], 99.90th=[ 3184], 99.95th=[ 3376], | 99.99th=[ 4192] bw (KB /s): min= 0, max=758016, per=20.00%, avg=650839.39, stdev=24440.60 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.68% lat (usec) : 1000=5.71% lat (msec) : 2=79.11%, 4=14.47%, 10=0.01%, 20=0.01%, 50=0.01% cpu : usr=2.09%, sys=16.03%, ctx=326942072, majf=0, minf=194705 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=108.3%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=366051519/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=16 xfs-21: (groupid=2, jobs=5): err= 0: pid=125843: Sat Dec 5 14:56:24 2015 read : io=22861GB, bw=3251.4MB/s, iops=52021, runt=7200002msec slat (usec): min=3, max=1389, avg=11.72, stdev= 4.80 clat (usec): min=46, max=23653, avg=1525.31, stdev=398.22 lat (usec): min=53, max=23660, avg=1537.09, stdev=398.30 clat percentiles (usec): | 1.00th=[ 780], 5.00th=[ 964], 10.00th=[ 1064], 20.00th=[ 1192], | 30.00th=[ 1304], 40.00th=[ 1384], 50.00th=[ 1480], 60.00th=[ 1576], | 70.00th=[ 1688], 80.00th=[ 1832], 90.00th=[ 2040], 95.00th=[ 2224], | 99.00th=[ 2608], 99.50th=[ 2768], 99.90th=[ 3408], 99.95th=[ 3984], | 99.99th=[ 5792] bw (KB /s): min= 0, max=812544, per=20.00%, avg=665949.97, stdev=25269.16 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.69% lat (usec) : 1000=5.84% lat (msec) : 2=81.93%, 4=11.47%, 10=0.05%, 20=0.01%, 50=0.01% cpu : usr=2.19%, sys=17.39%, ctx=325225711, majf=0, minf=194181 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=108.4%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=374552488/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=16 Run status group 0 (all jobs): READ: io=22589GB, aggrb=3212.8MB/s, minb=3212.8MB/s, maxb=3212.8MB/s, mint=7200002msec, maxt=7200002msec Run status group 1 (all jobs): READ: io=22342GB, aggrb=3177.6MB/s, minb=3177.6MB/s, maxb=3177.6MB/s, mint=7200002msec, maxt=7200002msec Run status group 2 (all jobs): READ: io=22861GB, aggrb=3251.4MB/s, minb=3251.4MB/s, maxb=3251.4MB/s, mint=7200002msec, maxt=7200002msec Disk stats (read/write): sda: ios=400850907/0, merge=0/0, ticks=618705538/0, in_queue=619747050, util=100.00% sdb: ios=396507984/0, merge=0/0, ticks=618943630/0, in_queue=619868954, util=100.00% sdc: ios=405819749/0, merge=0/0, ticks=618381965/0, in_queue=619797095, util=100.00% ################################################################################### # # Sat 05 Dec 2015 02:57:24 PM EST # 64kB randread with 1 jobs and 8 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=18918GB, bw=2690.6MB/s, iops=43048, runt=7200001msec lat (usec): min=55, max=16889, avg=928.05, stdev=318.21 xfs read : io=18560GB, bw=2639.7MB/s, iops=42234, runt=7200002msec lat (usec): min=53, max=17563, avg=945.97, stdev=318.39 xfs read : io=18784GB, bw=2671.5MB/s, iops=42743, runt=7200002msec lat (usec): min=56, max=19439, avg=934.67, stdev=324.62 Totals IOPS: 128025 MBPS: 8000 Average Latency (μsec): 935 xfs-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-02: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-03: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-04: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-05: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-12: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-13: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-14: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-15: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-21: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-22: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-23: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-24: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 xfs-25: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=8 fio-2.1.10 Starting 15 processes xfs-01: (groupid=0, jobs=5): err= 0: pid=131304: Sat Dec 5 17:07:25 2015 read : io=18918GB, bw=2690.6MB/s, iops=43048, runt=7200001msec slat (usec): min=3, max=1817, avg=12.19, stdev= 4.58 clat (usec): min=47, max=16871, avg=915.74, stdev=318.15 lat (usec): min=55, max=16889, avg=928.05, stdev=318.21 clat percentiles (usec): | 1.00th=[ 446], 5.00th=[ 524], 10.00th=[ 580], 20.00th=[ 652], | 30.00th=[ 724], 40.00th=[ 788], 50.00th=[ 860], 60.00th=[ 932], | 70.00th=[ 1032], 80.00th=[ 1144], 90.00th=[ 1336], 95.00th=[ 1496], | 99.00th=[ 1896], 99.50th=[ 2064], 99.90th=[ 2768], 99.95th=[ 3184], | 99.99th=[ 4128] bw (KB /s): min= 0, max=571904, per=20.00%, avg=551101.50, stdev=8752.00 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=3.37%, 750=30.96% lat (usec) : 1000=33.01% lat (msec) : 2=32.00%, 4=0.64%, 10=0.01%, 20=0.01% cpu : usr=2.33%, sys=15.20%, ctx=282021092, majf=0, minf=107845 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=309952629/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 xfs-11: (groupid=1, jobs=5): err= 0: pid=131309: Sat Dec 5 17:07:25 2015 read : io=18560GB, bw=2639.7MB/s, iops=42234, runt=7200002msec slat (usec): min=3, max=1411, avg=11.10, stdev= 3.88 clat (usec): min=44, max=17549, avg=934.76, stdev=318.37 lat (usec): min=53, max=17563, avg=945.97, stdev=318.39 clat percentiles (usec): | 1.00th=[ 454], 5.00th=[ 540], 10.00th=[ 596], 20.00th=[ 676], | 30.00th=[ 740], 40.00th=[ 804], 50.00th=[ 876], 60.00th=[ 948], | 70.00th=[ 1048], 80.00th=[ 1160], 90.00th=[ 1352], 95.00th=[ 1528], | 99.00th=[ 1912], 99.50th=[ 2096], 99.90th=[ 2640], 99.95th=[ 3024], | 99.99th=[ 3920] bw (KB /s): min= 0, max=571264, per=20.00%, avg=540679.71, stdev=10657.28 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=2.71%, 750=28.54% lat (usec) : 1000=34.30% lat (msec) : 2=33.73%, 4=0.71%, 10=0.01%, 20=0.01% cpu : usr=2.31%, sys=13.71%, ctx=279241666, majf=0, minf=108698 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=304089879/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 xfs-21: (groupid=2, jobs=5): err= 0: pid=131314: Sat Dec 5 17:07:25 2015 read : io=18784GB, bw=2671.5MB/s, iops=42743, runt=7200002msec slat (usec): min=3, max=2799, avg=12.47, stdev= 4.73 clat (usec): min=32, max=19426, avg=922.09, stdev=324.58 lat (usec): min=56, max=19439, avg=934.67, stdev=324.62 clat percentiles (usec): | 1.00th=[ 450], 5.00th=[ 524], 10.00th=[ 580], 20.00th=[ 660], | 30.00th=[ 724], 40.00th=[ 788], 50.00th=[ 860], 60.00th=[ 940], | 70.00th=[ 1032], 80.00th=[ 1144], 90.00th=[ 1336], 95.00th=[ 1512], | 99.00th=[ 1912], 99.50th=[ 2096], 99.90th=[ 2800], 99.95th=[ 3408], | 99.99th=[ 5344] bw (KB /s): min= 0, max=573824, per=20.00%, avg=547185.28, stdev=10714.09 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=3.13%, 750=30.39% lat (usec) : 1000=33.37% lat (msec) : 2=32.41%, 4=0.67%, 10=0.03%, 20=0.01% cpu : usr=2.26%, sys=15.52%, ctx=282144899, majf=0, minf=106837 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=307750857/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 Run status group 0 (all jobs): READ: io=18918GB, aggrb=2690.6MB/s, minb=2690.6MB/s, maxb=2690.6MB/s, mint=7200001msec, maxt=7200001msec Run status group 1 (all jobs): READ: io=18560GB, aggrb=2639.7MB/s, minb=2639.7MB/s, maxb=2639.7MB/s, mint=7200002msec, maxt=7200002msec Run status group 2 (all jobs): READ: io=18784GB, aggrb=2671.5MB/s, minb=2671.5MB/s, maxb=2671.5MB/s, mint=7200002msec, maxt=7200002msec Disk stats (read/write): sda: ios=335781722/0, merge=0/0, ticks=307161541/0, in_queue=307530180, util=100.00% sdb: ios=329450646/0, merge=0/0, ticks=307397699/0, in_queue=307615986, util=100.00% sdc: ios=333370262/0, merge=0/0, ticks=307082670/0, in_queue=307621500, util=100.00% ################################################################################### # # Sat 05 Dec 2015 05:08:25 PM EST # 64kB randread with 4 jobs and 4 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=22449GB, bw=3192.8MB/s, iops=51083, runt=7200003msec lat (usec): min=59, max=21974, avg=1564.54, stdev=557.47 xfs read : io=22252GB, bw=3164.8MB/s, iops=50635, runt=7200003msec lat (usec): min=56, max=20757, avg=1578.51, stdev=573.45 xfs read : io=22572GB, bw=3210.3MB/s, iops=51364, runt=7200002msec lat (usec): min=56, max=23413, avg=1556.03, stdev=573.67 Totals IOPS: 153082 MBPS: 9566 Average Latency (μsec): 1566 xfs-01: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-02: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-03: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-04: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-05: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-11: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-12: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-13: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-14: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-15: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-21: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-22: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-23: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-24: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... xfs-25: (g=2): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4 ... fio-2.1.10 Starting 60 processes xfs-01: (groupid=0, jobs=20): err= 0: pid=136970: Sat Dec 5 19:18:27 2015 read : io=22449GB, bw=3192.8MB/s, iops=51083, runt=7200003msec slat (usec): min=3, max=4695, avg=14.22, stdev= 5.59 clat (usec): min=27, max=21967, avg=1550.17, stdev=557.42 lat (usec): min=59, max=21974, avg=1564.54, stdev=557.47 clat percentiles (usec): | 1.00th=[ 580], 5.00th=[ 748], 10.00th=[ 876], 20.00th=[ 1048], | 30.00th=[ 1208], 40.00th=[ 1352], 50.00th=[ 1496], 60.00th=[ 1640], | 70.00th=[ 1816], 80.00th=[ 2008], 90.00th=[ 2288], 95.00th=[ 2544], | 99.00th=[ 3056], 99.50th=[ 3248], 99.90th=[ 3760], 99.95th=[ 4016], | 99.99th=[ 4960] bw (KB /s): min= 0, max=172032, per=5.00%, avg=163549.87, stdev=2445.15 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=0.24%, 750=4.78% lat (usec) : 1000=11.67% lat (msec) : 2=62.94%, 4=20.31%, 10=0.05%, 20=0.01%, 50=0.01% cpu : usr=0.84%, sys=5.37%, ctx=380462846, majf=0, minf=283402 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=367802203/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-11: (groupid=1, jobs=20): err= 0: pid=136990: Sat Dec 5 19:18:27 2015 read : io=22252GB, bw=3164.8MB/s, iops=50635, runt=7200003msec slat (usec): min=3, max=6719, avg=13.17, stdev= 5.68 clat (usec): min=0, max=20747, avg=1565.23, stdev=573.40 lat (usec): min=56, max=20757, avg=1578.51, stdev=573.45 clat percentiles (usec): | 1.00th=[ 588], 5.00th=[ 748], 10.00th=[ 868], 20.00th=[ 1048], | 30.00th=[ 1208], 40.00th=[ 1352], 50.00th=[ 1496], 60.00th=[ 1656], | 70.00th=[ 1832], 80.00th=[ 2040], 90.00th=[ 2320], 95.00th=[ 2608], | 99.00th=[ 3120], 99.50th=[ 3344], 99.90th=[ 3856], 99.95th=[ 4128], | 99.99th=[ 5024] bw (KB /s): min= 0, max=171008, per=5.00%, avg=162115.65, stdev=2628.87 lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.23% lat (usec) : 750=4.72%, 1000=11.84% lat (msec) : 2=61.64%, 4=21.50%, 10=0.07%, 20=0.01%, 50=0.01% cpu : usr=0.77%, sys=4.91%, ctx=379181628, majf=0, minf=263657 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=364575306/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-21: (groupid=2, jobs=20): err= 0: pid=137010: Sat Dec 5 19:18:27 2015 read : io=22572GB, bw=3210.3MB/s, iops=51364, runt=7200002msec slat (usec): min=3, max=8547, avg=13.70, stdev= 5.74 clat (usec): min=2, max=23405, avg=1542.19, stdev=573.62 lat (usec): min=56, max=23413, avg=1556.03, stdev=573.67 clat percentiles (usec): | 1.00th=[ 572], 5.00th=[ 740], 10.00th=[ 852], 20.00th=[ 1032], | 30.00th=[ 1192], 40.00th=[ 1336], 50.00th=[ 1480], 60.00th=[ 1624], | 70.00th=[ 1800], 80.00th=[ 2008], 90.00th=[ 2320], 95.00th=[ 2576], | 99.00th=[ 3120], 99.50th=[ 3344], 99.90th=[ 3984], 99.95th=[ 4448], | 99.99th=[ 6176] bw (KB /s): min= 0, max=241792, per=5.00%, avg=164449.98, stdev=3227.01 lat (usec) : 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01% lat (usec) : 500=0.26%, 750=5.23%, 1000=12.32% lat (msec) : 2=61.92%, 4=20.17%, 10=0.09%, 20=0.01%, 50=0.01% cpu : usr=0.81%, sys=5.18%, ctx=383351656, majf=0, minf=276915 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=369824535/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: io=22449GB, aggrb=3192.8MB/s, minb=3192.8MB/s, maxb=3192.8MB/s, mint=7200003msec, maxt=7200003msec Run status group 1 (all jobs): READ: io=22252GB, aggrb=3164.8MB/s, minb=3164.8MB/s, maxb=3164.8MB/s, mint=7200003msec, maxt=7200003msec Run status group 2 (all jobs): READ: io=22572GB, aggrb=3210.3MB/s, minb=3210.3MB/s, maxb=3210.3MB/s, mint=7200002msec, maxt=7200002msec Disk stats (read/write): sda: ios=398463529/0, merge=0/0, ticks=618189134/0, in_queue=619463660, util=100.00% sdb: ios=394975502/0, merge=0/0, ticks=618589768/0, in_queue=619863110, util=100.00% sdc: ios=400625314/0, merge=0/0, ticks=618317711/0, in_queue=619989711, util=100.00% ################################################################################### # # Sat 05 Dec 2015 07:19:27 PM EST # 32kB randread with 2 jobs and 8 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=20978GB, bw=2983.6MB/s, iops=95472, runt=7200002msec lat (usec): min=48, max=22253, avg=836.99, stdev=326.11 xfs read : io=20878GB, bw=2969.3MB/s, iops=95017, runt=7200002msec lat (usec): min=46, max=18472, avg=841.01, stdev=329.03 xfs read : io=21148GB, bw=3007.7MB/s, iops=96245, runt=7200002msec lat (usec): min=37, max=19087, avg=830.24, stdev=320.65 Totals IOPS: 286734 MBPS: 8959 Average Latency (μsec): 835 xfs-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-02: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-03: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-04: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-05: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-12: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-13: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-14: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-15: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-21: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-22: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-23: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-24: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-25: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... fio-2.1.10 Starting 30 processes xfs-01: (groupid=0, jobs=10): err= 0: pid=142168: Sat Dec 5 21:29:29 2015 read : io=20978GB, bw=2983.6MB/s, iops=95472, runt=7200002msec slat (usec): min=2, max=9945, avg= 8.63, stdev= 4.59 clat (usec): min=12, max=22240, avg=828.26, stdev=326.07 lat (usec): min=48, max=22253, avg=836.99, stdev=326.11 clat percentiles (usec): | 1.00th=[ 346], 5.00th=[ 418], 10.00th=[ 470], 20.00th=[ 556], | 30.00th=[ 628], 40.00th=[ 692], 50.00th=[ 772], 60.00th=[ 852], | 70.00th=[ 948], 80.00th=[ 1064], 90.00th=[ 1256], 95.00th=[ 1448], | 99.00th=[ 1848], 99.50th=[ 2008], 99.90th=[ 2416], 99.95th=[ 2640], | 99.99th=[ 3312] bw (KB /s): min= 0, max=329152, per=10.00%, avg=305544.51, stdev=3184.57 lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.02%, 500=13.23% lat (usec) : 750=34.17%, 1000=27.46% lat (msec) : 2=24.60%, 4=0.51%, 10=0.01%, 20=0.01%, 50=0.01% cpu : usr=2.14%, sys=12.18%, ctx=650991552, majf=0, minf=111481 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=687399976/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 xfs-11: (groupid=1, jobs=10): err= 0: pid=142178: Sat Dec 5 21:29:29 2015 read : io=20878GB, bw=2969.3MB/s, iops=95017, runt=7200002msec slat (usec): min=2, max=7937, avg= 9.20, stdev= 5.05 clat (usec): min=11, max=18465, avg=831.72, stdev=328.97 lat (usec): min=46, max=18472, avg=841.01, stdev=329.03 clat percentiles (usec): | 1.00th=[ 342], 5.00th=[ 422], 10.00th=[ 474], 20.00th=[ 556], | 30.00th=[ 628], 40.00th=[ 700], 50.00th=[ 772], 60.00th=[ 852], | 70.00th=[ 948], 80.00th=[ 1064], 90.00th=[ 1272], 95.00th=[ 1448], | 99.00th=[ 1848], 99.50th=[ 2024], 99.90th=[ 2512], 99.95th=[ 2864], | 99.99th=[ 3504] bw (KB /s): min= 0, max=313472, per=10.00%, avg=304087.29, stdev=3467.33 lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.03%, 500=12.69% lat (usec) : 750=34.47%, 1000=27.48% lat (msec) : 2=24.78%, 4=0.55%, 10=0.01%, 20=0.01% cpu : usr=2.12%, sys=12.84%, ctx=646443755, majf=0, minf=111121 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=684122614/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 xfs-21: (groupid=2, jobs=10): err= 0: pid=142188: Sat Dec 5 21:29:29 2015 read : io=21148GB, bw=3007.7MB/s, iops=96245, runt=7200002msec slat (usec): min=2, max=10319, avg= 9.16, stdev= 5.03 clat (usec): min=1, max=19078, avg=820.98, stdev=320.59 lat (usec): min=37, max=19087, avg=830.24, stdev=320.65 clat percentiles (usec): | 1.00th=[ 342], 5.00th=[ 418], 10.00th=[ 470], 20.00th=[ 556], | 30.00th=[ 620], 40.00th=[ 692], 50.00th=[ 764], 60.00th=[ 844], | 70.00th=[ 940], 80.00th=[ 1064], 90.00th=[ 1240], 95.00th=[ 1416], | 99.00th=[ 1800], 99.50th=[ 1960], 99.90th=[ 2384], 99.95th=[ 2704], | 99.99th=[ 3920] bw (KB /s): min= 0, max=326912, per=10.00%, avg=308019.20, stdev=3228.89 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=0.02%, 500=13.37%, 750=34.58%, 1000=27.71% lat (msec) : 2=23.88%, 4=0.42%, 10=0.01%, 20=0.01% cpu : usr=2.15%, sys=12.97%, ctx=654044003, majf=0, minf=109891 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=692967654/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 Run status group 0 (all jobs): READ: io=20978GB, aggrb=2983.6MB/s, minb=2983.6MB/s, maxb=2983.6MB/s, mint=7200002msec, maxt=7200002msec Run status group 1 (all jobs): READ: io=20878GB, aggrb=2969.3MB/s, minb=2969.3MB/s, maxb=2969.3MB/s, mint=7200002msec, maxt=7200002msec Run status group 2 (all jobs): READ: io=21148GB, aggrb=3007.7MB/s, minb=3007.7MB/s, maxb=3007.7MB/s, mint=7200002msec, maxt=7200002msec Disk stats (read/write): sda: ios=744711869/0, merge=0/0, ticks=616581194/0, in_queue=617904697, util=100.00% sdb: ios=741201820/0, merge=0/0, ticks=616343610/0, in_queue=618052537, util=100.00% sdc: ios=750763322/0, merge=0/0, ticks=616207757/0, in_queue=618621595, util=100.00% ################################################################################### # # Sat 05 Dec 2015 09:30:29 PM EST # 32kB randread with 4 jobs and 8 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=22510GB, bw=3201.5MB/s, iops=102445, runt=7200004msec lat (usec): min=46, max=30707, avg=1560.75, stdev=764.75 xfs read : io=22533GB, bw=3204.8MB/s, iops=102550, runt=7200003msec lat (usec): min=50, max=23074, avg=1559.27, stdev=762.17 xfs read : io=22959GB, bw=3265.3MB/s, iops=104488, runt=7200003msec lat (usec): min=49, max=24971, avg=1530.35, stdev=733.18 Totals IOPS: 309483 MBPS: 9670 Average Latency (μsec): 1549 xfs-01: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-02: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-03: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-04: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-05: (g=0): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-11: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-12: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-13: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-14: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-15: (g=1): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-21: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-22: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-23: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-24: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... xfs-25: (g=2): rw=randread, bs=32K-32K/32K-32K/32K-32K, ioengine=libaio, iodepth=8 ... fio-2.1.10 Starting 60 processes xfs-01: (groupid=0, jobs=20): err= 0: pid=1239: Sat Dec 5 23:40:31 2015 read : io=22510GB, bw=3201.5MB/s, iops=102445, runt=7200004msec slat (usec): min=2, max=15103, avg=53.02, stdev=260.70 clat (usec): min=1, max=21933, avg=1507.64, stdev=732.05 lat (usec): min=46, max=30707, avg=1560.75, stdev=764.75 clat percentiles (usec): | 1.00th=[ 486], 5.00th=[ 620], 10.00th=[ 724], 20.00th=[ 884], | 30.00th=[ 1032], 40.00th=[ 1176], 50.00th=[ 1336], 60.00th=[ 1544], | 70.00th=[ 1784], 80.00th=[ 2096], 90.00th=[ 2544], 95.00th=[ 2928], | 99.00th=[ 3728], 99.50th=[ 4048], 99.90th=[ 4704], 99.95th=[ 4960], | 99.99th=[ 5600] bw (KB /s): min= 0, max=227328, per=5.00%, avg=163975.38, stdev=2188.22 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=0.01%, 500=1.27%, 750=10.38%, 1000=16.60% lat (msec) : 2=49.36%, 4=21.85%, 10=0.54%, 20=0.01%, 50=0.01% cpu : usr=1.07%, sys=6.95%, ctx=599608608, majf=0, minf=273144 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.4%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=737610048/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 xfs-11: (groupid=1, jobs=20): err= 0: pid=1259: Sat Dec 5 23:40:31 2015 read : io=22533GB, bw=3204.8MB/s, iops=102550, runt=7200003msec slat (usec): min=2, max=12487, avg=52.55, stdev=263.80 clat (usec): min=9, max=22215, avg=1506.67, stdev=728.33 lat (usec): min=50, max=23074, avg=1559.27, stdev=762.17 clat percentiles (usec): | 1.00th=[ 482], 5.00th=[ 620], 10.00th=[ 724], 20.00th=[ 884], | 30.00th=[ 1032], 40.00th=[ 1176], 50.00th=[ 1352], 60.00th=[ 1544], | 70.00th=[ 1768], 80.00th=[ 2064], 90.00th=[ 2512], 95.00th=[ 2928], | 99.00th=[ 3696], 99.50th=[ 4016], 99.90th=[ 4704], 99.95th=[ 4960], | 99.99th=[ 5600] bw (KB /s): min= 0, max=254080, per=5.00%, avg=164142.91, stdev=2184.54 lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01% lat (usec) : 500=1.30%, 750=10.28%, 1000=16.52% lat (msec) : 2=49.64%, 4=21.74%, 10=0.52%, 20=0.01%, 50=0.01% cpu : usr=0.98%, sys=6.01%, ctx=603692430, majf=0, minf=249669 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=738366057/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 xfs-21: (groupid=2, jobs=20): err= 0: pid=1282: Sat Dec 5 23:40:31 2015 read : io=22959GB, bw=3265.3MB/s, iops=104488, runt=7200003msec slat (usec): min=2, max=18575, avg=51.62, stdev=259.48 clat (usec): min=22, max=24962, avg=1478.63, stdev=699.27 lat (usec): min=49, max=24971, avg=1530.35, stdev=733.18 clat percentiles (usec): | 1.00th=[ 482], 5.00th=[ 620], 10.00th=[ 724], 20.00th=[ 884], | 30.00th=[ 1032], 40.00th=[ 1176], 50.00th=[ 1320], 60.00th=[ 1512], | 70.00th=[ 1736], 80.00th=[ 2024], 90.00th=[ 2448], 95.00th=[ 2832], | 99.00th=[ 3600], 99.50th=[ 3888], 99.90th=[ 4576], 99.95th=[ 4832], | 99.99th=[ 5600] bw (KB /s): min= 0, max=244800, per=5.00%, avg=167245.06, stdev=2220.63 lat (usec) : 50=0.01%, 100=0.01%, 250=0.01%, 500=1.36%, 750=10.15% lat (usec) : 1000=16.65% lat (msec) : 2=51.26%, 4=20.18%, 10=0.39%, 20=0.01%, 50=0.01% cpu : usr=0.98%, sys=5.91%, ctx=616648050, majf=0, minf=263588 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=752316632/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 Run status group 0 (all jobs): READ: io=22510GB, aggrb=3201.5MB/s, minb=3201.5MB/s, maxb=3201.5MB/s, mint=7200004msec, maxt=7200004msec Run status group 1 (all jobs): READ: io=22533GB, aggrb=3204.8MB/s, minb=3204.8MB/s, maxb=3204.8MB/s, mint=7200003msec, maxt=7200003msec Run status group 2 (all jobs): READ: io=22959GB, aggrb=3265.3MB/s, minb=3265.3MB/s, maxb=3265.3MB/s, mint=7200003msec, maxt=7200003msec Disk stats (read/write): sda: ios=799206083/0, merge=0/0, ticks=1074999612/0, in_queue=1076680252, util=100.00% sdb: ios=800001829/0, merge=0/0, ticks=1072401462/0, in_queue=1073717368, util=100.00% sdc: ios=815088337/0, merge=0/0, ticks=1071027161/0, in_queue=1072785766, util=100.00% ################################################################################### # # Sat 05 Dec 2015 11:41:31 PM EST # 16kB randread with 2 jobs and 8 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=16420GB, bw=2335.4MB/s, iops=149462, runt=7200001msec lat (usec): min=72, max=15692, avg=534.37, stdev=119.41 xfs read : io=16377GB, bw=2329.2MB/s, iops=149067, runt=7200001msec lat (usec): min=119, max=13972, avg=535.72, stdev=120.43 xfs read : io=16518GB, bw=2349.3MB/s, iops=150352, runt=7200003msec lat (usec): min=83, max=16383, avg=531.36, stdev=122.17 Totals IOPS: 448881 MBPS: 7013 Average Latency (μsec): 533 xfs-01: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-02: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-03: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-04: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-05: (g=0): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-11: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-12: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-13: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-14: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-15: (g=1): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-21: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-22: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-23: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-24: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... xfs-25: (g=2): rw=randread, bs=16K-16K/16K-16K/16K-16K, ioengine=libaio, iodepth=8 ... fio-2.1.10 Starting 30 processes xfs-01: (groupid=0, jobs=10): err= 0: pid=6572: Sun Dec 6 01:51:33 2015 read : io=16420GB, bw=2335.4MB/s, iops=149462, runt=7200001msec slat (usec): min=2, max=10218, avg=14.25, stdev=10.46 clat (usec): min=10, max=15685, avg=520.07, stdev=119.95 lat (usec): min=72, max=15692, avg=534.37, stdev=119.41 clat percentiles (usec): | 1.00th=[ 342], 5.00th=[ 378], 10.00th=[ 398], 20.00th=[ 426], | 30.00th=[ 454], 40.00th=[ 482], 50.00th=[ 502], 60.00th=[ 524], | 70.00th=[ 556], 80.00th=[ 604], 90.00th=[ 660], 95.00th=[ 716], | 99.00th=[ 844], 99.50th=[ 908], 99.90th=[ 1144], 99.95th=[ 2024], | 99.99th=[ 3088] bw (KB /s): min= 0, max=274112, per=10.00%, avg=239147.56, stdev=2215.59 lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=48.56% lat (usec) : 750=48.07%, 1000=3.16% lat (msec) : 2=0.15%, 4=0.05%, 10=0.01%, 20=0.01% cpu : usr=2.22%, sys=27.99%, ctx=676556949, majf=0, minf=63859 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1076129027/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 xfs-11: (groupid=1, jobs=10): err= 0: pid=6582: Sun Dec 6 01:51:33 2015 read : io=16377GB, bw=2329.2MB/s, iops=149067, runt=7200001msec slat (usec): min=2, max=12084, avg= 8.36, stdev= 7.12 clat (usec): min=1, max=13964, avg=527.23, stdev=120.46 lat (usec): min=119, max=13972, avg=535.72, stdev=120.43 clat percentiles (usec): | 1.00th=[ 342], 5.00th=[ 382], 10.00th=[ 402], 20.00th=[ 430], | 30.00th=[ 458], 40.00th=[ 490], 50.00th=[ 510], 60.00th=[ 532], | 70.00th=[ 572], 80.00th=[ 612], 90.00th=[ 676], 95.00th=[ 732], | 99.00th=[ 860], 99.50th=[ 924], 99.90th=[ 1112], 99.95th=[ 1544], | 99.99th=[ 2992] bw (KB /s): min= 0, max=378560, per=10.00%, avg=238519.46, stdev=2316.59 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=0.01%, 500=45.06%, 750=50.96%, 1000=3.76% lat (msec) : 2=0.18%, 4=0.04%, 10=0.01%, 20=0.01% cpu : usr=2.25%, sys=17.78%, ctx=747390396, majf=0, minf=61430 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1073288239/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 xfs-21: (groupid=2, jobs=10): err= 0: pid=6592: Sun Dec 6 01:51:33 2015 read : io=16518GB, bw=2349.3MB/s, iops=150352, runt=7200003msec slat (usec): min=2, max=9384, avg=10.41, stdev= 7.97 clat (usec): min=6, max=16379, avg=520.90, stdev=122.52 lat (usec): min=83, max=16383, avg=531.36, stdev=122.17 clat percentiles (usec): | 1.00th=[ 338], 5.00th=[ 378], 10.00th=[ 398], 20.00th=[ 426], | 30.00th=[ 454], 40.00th=[ 482], 50.00th=[ 506], 60.00th=[ 532], | 70.00th=[ 564], 80.00th=[ 604], 90.00th=[ 668], 95.00th=[ 724], | 99.00th=[ 852], 99.50th=[ 908], 99.90th=[ 1080], 99.95th=[ 1352], | 99.99th=[ 3408] bw (KB /s): min= 0, max=333984, per=10.00%, avg=240575.18, stdev=2486.16 lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01% lat (usec) : 500=48.10%, 750=48.33%, 1000=3.37% lat (msec) : 2=0.15%, 4=0.03%, 10=0.01%, 20=0.01% cpu : usr=2.28%, sys=21.19%, ctx=711576031, majf=0, minf=61426 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=108.3%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1082540444/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=8 Run status group 0 (all jobs): READ: io=16420GB, aggrb=2335.4MB/s, minb=2335.4MB/s, maxb=2335.4MB/s, mint=7200001msec, maxt=7200001msec Run status group 1 (all jobs): READ: io=16377GB, aggrb=2329.2MB/s, minb=2329.2MB/s, maxb=2329.2MB/s, mint=7200001msec, maxt=7200001msec Run status group 2 (all jobs): READ: io=16518GB, aggrb=2349.3MB/s, minb=2349.3MB/s, maxb=2349.3MB/s, mint=7200003msec, maxt=7200003msec Disk stats (read/write): sda: ios=1165803932/0, merge=0/0, ticks=599638672/0, in_queue=601844367, util=100.00% sdb: ios=1162722583/0, merge=0/0, ticks=608706711/0, in_queue=610038394, util=100.00% sdc: ios=1172687119/0, merge=0/0, ticks=605566261/0, in_queue=607515969, util=100.00% ################################################################################### # # Sun 06 Dec 2015 01:52:33 AM EST # 8kB randread with 4 jobs and 2 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=8400.7GB, bw=1194.8MB/s, iops=152928, runt=7200003msec lat (usec): min=55, max=18120, avg=260.53, stdev=62.74 xfs read : io=8265.9GB, bw=1175.6MB/s, iops=150474, runt=7200001msec lat (usec): min=97, max=18233, avg=264.75, stdev=60.77 xfs read : io=8378.4GB, bw=1191.6MB/s, iops=152522, runt=7200002msec lat (usec): min=82, max=15482, avg=261.20, stdev=71.33 Totals IOPS: 455924 MBPS: 3560 Average Latency (μsec): 261 xfs-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-02: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-03: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-04: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-05: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-12: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-13: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-14: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-15: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-21: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-22: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-23: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-24: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... xfs-25: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=2 ... fio-2.1.10 Starting 60 processes xfs-01: (groupid=0, jobs=20): err= 0: pid=12731: Sun Dec 6 04:02:36 2015 read : io=8400.7GB, bw=1194.8MB/s, iops=152928, runt=7200003msec slat (usec): min=2, max=8649, avg=14.86, stdev=10.58 clat (usec): min=0, max=18110, avg=245.58, stdev=62.75 lat (usec): min=55, max=18120, avg=260.53, stdev=62.74 clat percentiles (usec): | 1.00th=[ 173], 5.00th=[ 191], 10.00th=[ 201], 20.00th=[ 213], | 30.00th=[ 221], 40.00th=[ 231], 50.00th=[ 239], 60.00th=[ 249], | 70.00th=[ 262], 80.00th=[ 274], 90.00th=[ 298], 95.00th=[ 318], | 99.00th=[ 358], 99.50th=[ 382], 99.90th=[ 628], 99.95th=[ 1384], | 99.99th=[ 2704] bw (KB /s): min= 0, max=99360, per=5.00%, avg=61176.92, stdev=696.75 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (usec) : 100=0.01%, 250=61.28%, 500=38.60%, 750=0.04%, 1000=0.02% lat (msec) : 2=0.03%, 4=0.03%, 10=0.01%, 20=0.01% cpu : usr=1.51%, sys=15.98%, ctx=1053655116, majf=0, minf=62406 IO depths : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1101082588/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-11: (groupid=1, jobs=20): err= 0: pid=12751: Sun Dec 6 04:02:36 2015 read : io=8265.9GB, bw=1175.6MB/s, iops=150474, runt=7200001msec slat (usec): min=1, max=12613, avg=13.79, stdev=10.53 clat (usec): min=0, max=18194, avg=251.01, stdev=60.75 lat (usec): min=97, max=18233, avg=264.75, stdev=60.77 clat percentiles (usec): | 1.00th=[ 175], 5.00th=[ 193], 10.00th=[ 203], 20.00th=[ 215], | 30.00th=[ 225], 40.00th=[ 235], 50.00th=[ 245], 60.00th=[ 255], | 70.00th=[ 270], 80.00th=[ 282], 90.00th=[ 306], 95.00th=[ 326], | 99.00th=[ 370], 99.50th=[ 390], 99.90th=[ 494], 99.95th=[ 1004], | 99.99th=[ 2640] bw (KB /s): min= 0, max=93296, per=5.00%, avg=60195.86, stdev=626.36 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (usec) : 100=0.01%, 250=55.22%, 500=44.68%, 750=0.03%, 1000=0.02% lat (msec) : 2=0.02%, 4=0.03%, 10=0.01%, 20=0.01% cpu : usr=1.46%, sys=14.59%, ctx=1040422347, majf=0, minf=51149 IO depths : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1083420079/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 xfs-21: (groupid=2, jobs=20): err= 0: pid=12771: Sun Dec 6 04:02:36 2015 read : io=8378.4GB, bw=1191.6MB/s, iops=152522, runt=7200002msec slat (usec): min=1, max=9509, avg=14.56, stdev=10.37 clat (usec): min=0, max=15462, avg=246.52, stdev=71.33 lat (usec): min=82, max=15482, avg=261.20, stdev=71.33 clat percentiles (usec): | 1.00th=[ 173], 5.00th=[ 191], 10.00th=[ 201], 20.00th=[ 213], | 30.00th=[ 221], 40.00th=[ 231], 50.00th=[ 239], 60.00th=[ 251], | 70.00th=[ 262], 80.00th=[ 278], 90.00th=[ 302], 95.00th=[ 322], | 99.00th=[ 362], 99.50th=[ 382], 99.90th=[ 458], 99.95th=[ 828], | 99.99th=[ 3792] bw (KB /s): min= 0, max=79216, per=5.00%, avg=61014.75, stdev=669.27 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (usec) : 100=0.01%, 250=59.80%, 500=40.12%, 750=0.03%, 1000=0.01% lat (msec) : 2=0.02%, 4=0.02%, 10=0.01%, 20=0.01% cpu : usr=1.51%, sys=15.59%, ctx=1052933241, majf=0, minf=60930 IO depths : 1=0.1%, 2=108.3%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1098160327/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=2 Run status group 0 (all jobs): READ: io=8400.7GB, aggrb=1194.8MB/s, minb=1194.8MB/s, maxb=1194.8MB/s, mint=7200003msec, maxt=7200003msec Run status group 1 (all jobs): READ: io=8265.9GB, aggrb=1175.6MB/s, minb=1175.6MB/s, maxb=1175.6MB/s, mint=7200001msec, maxt=7200001msec Run status group 2 (all jobs): READ: io=8378.4GB, aggrb=1191.6MB/s, minb=1191.6MB/s, maxb=1191.6MB/s, mint=7200002msec, maxt=7200002msec Disk stats (read/write): sda: ios=1192841716/0, merge=0/0, ticks=295150901/0, in_queue=296284610, util=100.00% sdb: ios=1173803604/0, merge=0/0, ticks=295957657/0, in_queue=297241601, util=100.00% sdc: ios=1189706915/0, merge=0/0, ticks=295344166/0, in_queue=297018649, util=100.00% ################################################################################### # # Sun 06 Dec 2015 04:03:36 AM EST # 8kB randread with 4 jobs and 4 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=8708.3GB, bw=1238.5MB/s, iops=158524, runt=7200002msec lat (usec): min=72, max=25028, avg=503.49, stdev=77.03 xfs read : io=8635.3GB, bw=1228.2MB/s, iops=157199, runt=7200003msec lat (usec): min=110, max=22678, avg=507.74, stdev=92.75 xfs read : io=8701.2GB, bw=1237.6MB/s, iops=158399, runt=7200005msec lat (usec): min=106, max=21424, avg=503.89, stdev=196.33 Totals IOPS: 474122 MBPS: 3703 Average Latency (μsec): 504 xfs-01: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-02: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-03: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-04: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-05: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-11: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-12: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-13: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-14: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-15: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-21: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-22: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-23: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-24: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... xfs-25: (g=2): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=4 ... fio-2.1.10 Starting 60 processes xfs-01: (groupid=0, jobs=20): err= 0: pid=18896: Sun Dec 6 06:13:40 2015 read : io=8708.3GB, bw=1238.5MB/s, iops=158524, runt=7200002msec slat (usec): min=1, max=15258, avg=18.94, stdev=16.90 clat (usec): min=0, max=21201, avg=484.47, stdev=77.17 lat (usec): min=72, max=25028, avg=503.49, stdev=77.03 clat percentiles (usec): | 1.00th=[ 374], 5.00th=[ 398], 10.00th=[ 418], 20.00th=[ 442], | 30.00th=[ 458], 40.00th=[ 470], 50.00th=[ 478], 60.00th=[ 490], | 70.00th=[ 506], 80.00th=[ 524], 90.00th=[ 556], 95.00th=[ 580], | 99.00th=[ 628], 99.50th=[ 652], 99.90th=[ 1020], 99.95th=[ 1912], | 99.99th=[ 2992] bw (KB /s): min= 0, max=168624, per=5.00%, avg=63416.19, stdev=724.26 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=0.03%, 500=66.60%, 750=33.21%, 1000=0.06% lat (msec) : 2=0.06%, 4=0.05%, 10=0.01%, 20=0.01%, 50=0.01% cpu : usr=1.33%, sys=19.79%, ctx=929147375, majf=0, minf=88672 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1141378045/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-11: (groupid=1, jobs=20): err= 0: pid=18916: Sun Dec 6 06:13:40 2015 read : io=8635.3GB, bw=1228.2MB/s, iops=157199, runt=7200003msec slat (usec): min=1, max=14677, avg=19.70, stdev=17.39 clat (usec): min=0, max=18853, avg=488.03, stdev=92.98 lat (usec): min=110, max=22678, avg=507.74, stdev=92.75 clat percentiles (usec): | 1.00th=[ 370], 5.00th=[ 398], 10.00th=[ 418], 20.00th=[ 442], | 30.00th=[ 458], 40.00th=[ 470], 50.00th=[ 482], 60.00th=[ 494], | 70.00th=[ 510], 80.00th=[ 532], 90.00th=[ 556], 95.00th=[ 580], | 99.00th=[ 636], 99.50th=[ 676], 99.90th=[ 1784], 99.95th=[ 2576], | 99.99th=[ 3120] bw (KB /s): min= 0, max=101984, per=5.00%, avg=62886.24, stdev=693.50 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=0.03%, 500=64.05%, 750=35.66%, 1000=0.09% lat (msec) : 2=0.09%, 4=0.09%, 10=0.01%, 20=0.01% cpu : usr=1.33%, sys=20.37%, ctx=924791451, majf=0, minf=99194 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1131835638/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-21: (groupid=2, jobs=20): err= 0: pid=18936: Sun Dec 6 06:13:40 2015 read : io=8701.2GB, bw=1237.6MB/s, iops=158399, runt=7200005msec slat (usec): min=1, max=16978, avg=18.50, stdev=16.54 clat (usec): min=0, max=21406, avg=485.28, stdev=196.40 lat (usec): min=106, max=21424, avg=503.89, stdev=196.33 clat percentiles (usec): | 1.00th=[ 342], 5.00th=[ 382], 10.00th=[ 398], 20.00th=[ 426], | 30.00th=[ 446], 40.00th=[ 462], 50.00th=[ 474], 60.00th=[ 486], | 70.00th=[ 502], 80.00th=[ 524], 90.00th=[ 548], 95.00th=[ 580], | 99.00th=[ 652], 99.50th=[ 1012], 99.90th=[ 4048], 99.95th=[ 4832], | 99.99th=[ 5536] bw (KB /s): min= 0, max=134416, per=5.00%, avg=63366.63, stdev=782.20 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=0.05%, 500=69.70%, 750=29.61%, 1000=0.13% lat (msec) : 2=0.20%, 4=0.20%, 10=0.10%, 20=0.01%, 50=0.01% cpu : usr=1.34%, sys=19.39%, ctx=934831188, majf=0, minf=95101 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1140477810/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: io=8708.3GB, aggrb=1238.5MB/s, minb=1238.5MB/s, maxb=1238.5MB/s, mint=7200002msec, maxt=7200002msec Run status group 1 (all jobs): READ: io=8635.3GB, aggrb=1228.2MB/s, minb=1228.2MB/s, maxb=1228.2MB/s, mint=7200003msec, maxt=7200003msec Run status group 2 (all jobs): READ: io=8701.2GB, aggrb=1237.6MB/s, minb=1237.6MB/s, maxb=1237.6MB/s, mint=7200005msec, maxt=7200005msec Disk stats (read/write): sda: ios=1236636585/0, merge=0/0, ticks=596780002/0, in_queue=599497089, util=100.00% sdb: ios=1226218027/0, merge=0/0, ticks=596550341/0, in_queue=599459546, util=100.00% sdc: ios=1235663496/0, merge=0/0, ticks=597239403/0, in_queue=600815571, util=100.00% ################################################################################### # # Sun 06 Dec 2015 06:14:40 AM EST # 4kB randread with 2 jobs and 4 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=4359.9GB, bw=634941KB/s, iops=158735, runt=7200001msec lat (usec): min=50, max=14828, avg=251.22, stdev=94.94 xfs read : io=4335.6GB, bw=631407KB/s, iops=157851, runt=7200001msec lat (usec): min=44, max=15200, avg=252.53, stdev=113.85 xfs read : io=4331.5GB, bw=630754KB/s, iops=157688, runt=7200001msec lat (usec): min=43, max=17838, avg=252.76, stdev=202.34 Totals IOPS: 474274 MBPS: 1851 Average Latency (μsec): 251 xfs-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-02: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-03: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-04: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-05: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-12: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-13: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-14: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-15: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-21: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-22: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-23: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-24: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... xfs-25: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4 ... fio-2.1.10 Starting 30 processes xfs-01: (groupid=0, jobs=10): err= 0: pid=24063: Sun Dec 6 08:24:43 2015 read : io=4359.9GB, bw=634941KB/s, iops=158735, runt=7200001msec slat (usec): min=1, max=12739, avg=11.77, stdev= 8.90 clat (usec): min=0, max=14821, avg=239.31, stdev=95.19 lat (usec): min=50, max=14828, avg=251.22, stdev=94.94 clat percentiles (usec): | 1.00th=[ 169], 5.00th=[ 187], 10.00th=[ 197], 20.00th=[ 209], | 30.00th=[ 217], 40.00th=[ 225], 50.00th=[ 233], 60.00th=[ 241], | 70.00th=[ 249], 80.00th=[ 262], 90.00th=[ 278], 95.00th=[ 294], | 99.00th=[ 334], 99.50th=[ 374], 99.90th=[ 2064], 99.95th=[ 2512], | 99.99th=[ 2896] bw (KB /s): min= 0, max=94728, per=10.00%, avg=63495.53, stdev=967.08 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (usec) : 100=0.01%, 250=70.14%, 500=29.48%, 750=0.09%, 1000=0.06% lat (msec) : 2=0.12%, 4=0.11%, 10=0.01%, 20=0.01% cpu : usr=2.48%, sys=25.59%, ctx=801738863, majf=0, minf=20014 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1142894261/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-11: (groupid=1, jobs=10): err= 0: pid=24073: Sun Dec 6 08:24:43 2015 read : io=4335.6GB, bw=631407KB/s, iops=157851, runt=7200001msec slat (usec): min=1, max=7979, avg=12.20, stdev= 9.68 clat (usec): min=0, max=15196, avg=240.21, stdev=114.07 lat (usec): min=44, max=15200, avg=252.53, stdev=113.85 clat percentiles (usec): | 1.00th=[ 165], 5.00th=[ 185], 10.00th=[ 195], 20.00th=[ 207], | 30.00th=[ 215], 40.00th=[ 223], 50.00th=[ 231], 60.00th=[ 239], | 70.00th=[ 249], 80.00th=[ 262], 90.00th=[ 278], 95.00th=[ 294], | 99.00th=[ 342], 99.50th=[ 564], 99.90th=[ 2352], 99.95th=[ 2640], | 99.99th=[ 2960] bw (KB /s): min= 0, max=103328, per=10.00%, avg=63142.32, stdev=755.16 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=70.97%, 500=28.49%, 750=0.12%, 1000=0.08% lat (msec) : 2=0.17%, 4=0.16%, 10=0.01%, 20=0.01% cpu : usr=2.39%, sys=26.06%, ctx=799663717, majf=0, minf=21301 IO depths : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1136533497/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 xfs-21: (groupid=2, jobs=10): err= 0: pid=24083: Sun Dec 6 08:24:43 2015 read : io=4331.5GB, bw=630754KB/s, iops=157688, runt=7200001msec slat (usec): min=1, max=12610, avg=11.10, stdev= 9.03 clat (usec): min=0, max=17831, avg=241.64, stdev=202.42 lat (usec): min=43, max=17838, avg=252.76, stdev=202.34 clat percentiles (usec): | 1.00th=[ 151], 5.00th=[ 175], 10.00th=[ 187], 20.00th=[ 201], | 30.00th=[ 211], 40.00th=[ 219], 50.00th=[ 227], 60.00th=[ 235], | 70.00th=[ 245], 80.00th=[ 258], 90.00th=[ 274], 95.00th=[ 294], | 99.00th=[ 346], 99.50th=[ 908], 99.90th=[ 4016], 99.95th=[ 4640], | 99.99th=[ 5280] bw (KB /s): min= 0, max=98728, per=10.00%, avg=63077.00, stdev=946.73 lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (usec) : 100=0.01%, 250=73.94%, 500=25.41%, 750=0.09%, 1000=0.07% lat (msec) : 2=0.16%, 4=0.22%, 10=0.10%, 20=0.01% cpu : usr=2.43%, sys=24.04%, ctx=820290138, majf=0, minf=21481 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1135357230/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: io=4359.9GB, aggrb=634941KB/s, minb=634941KB/s, maxb=634941KB/s, mint=7200001msec, maxt=7200001msec Run status group 1 (all jobs): READ: io=4335.6GB, aggrb=631407KB/s, minb=631407KB/s, maxb=631407KB/s, mint=7200001msec, maxt=7200001msec Run status group 2 (all jobs): READ: io=4331.5GB, aggrb=630753KB/s, minb=630753KB/s, maxb=630753KB/s, mint=7200001msec, maxt=7200001msec Disk stats (read/write): sda: ios=1238206338/0, merge=0/0, ticks=291747883/0, in_queue=292809466, util=100.00% sdb: ios=1231420594/0, merge=0/0, ticks=291347602/0, in_queue=292621697, util=100.00% sdc: ios=1229587889/0, merge=0/0, ticks=292919564/0, in_queue=294408732, util=100.00% ################################################################################### # # Sun 06 Dec 2015 08:25:43 AM EST # 4kB randread with 4 jobs and 6 I/Os outstanding # CentOS Linux release 7.1.1503 Linux 3.10.0-229.20.1.el7.x86_64 # ################################################################################### Summary xfs read : io=4460.5GB, bw=649596KB/s, iops=162398, runt=7200002msec lat (usec): min=59, max=20090, avg=737.81, stdev=114.41 xfs read : io=4454.3GB, bw=648695KB/s, iops=162173, runt=7200002msec lat (usec): min=101, max=19830, avg=738.77, stdev=106.55 xfs read : io=4469.8GB, bw=650948KB/s, iops=162736, runt=7200001msec lat (usec): min=84, max=19163, avg=736.28, stdev=153.74 Totals IOPS: 487307 MBPS: 1902 Average Latency (μsec): 737 xfs-01: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-02: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-03: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-04: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-05: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-11: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-12: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-13: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-14: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-15: (g=1): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-21: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-22: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-23: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-24: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... xfs-25: (g=2): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=6 ... fio-2.1.10 Starting 60 processes xfs-01: (groupid=0, jobs=20): err= 0: pid=29737: Sun Dec 6 10:35:47 2015 read : io=4460.5GB, bw=649596KB/s, iops=162398, runt=7200002msec slat (usec): min=1, max=18269, avg=18.03, stdev=14.95 clat (usec): min=0, max=18725, avg=719.64, stdev=114.96 lat (usec): min=59, max=20090, avg=737.81, stdev=114.41 clat percentiles (usec): | 1.00th=[ 628], 5.00th=[ 652], 10.00th=[ 660], 20.00th=[ 676], | 30.00th=[ 684], 40.00th=[ 700], 50.00th=[ 716], 60.00th=[ 724], | 70.00th=[ 740], 80.00th=[ 756], 90.00th=[ 772], 95.00th=[ 788], | 99.00th=[ 836], 99.50th=[ 996], 99.90th=[ 2736], 99.95th=[ 3088], | 99.99th=[ 3440] bw (KB /s): min= 0, max=71400, per=5.00%, avg=32482.91, stdev=427.29 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=0.01%, 500=0.11%, 750=78.42%, 1000=20.96% lat (msec) : 2=0.29%, 4=0.21%, 10=0.01%, 20=0.01% cpu : usr=1.37%, sys=19.56%, ctx=933590115, majf=0, minf=75015 IO depths : 1=0.1%, 2=0.1%, 4=108.4%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1169272162/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=6 xfs-11: (groupid=1, jobs=20): err= 0: pid=29757: Sun Dec 6 10:35:47 2015 read : io=4454.3GB, bw=648695KB/s, iops=162173, runt=7200002msec slat (usec): min=1, max=15681, avg=14.69, stdev=14.27 clat (usec): min=0, max=19822, avg=724.43, stdev=106.83 lat (usec): min=101, max=19830, avg=738.77, stdev=106.55 clat percentiles (usec): | 1.00th=[ 636], 5.00th=[ 660], 10.00th=[ 668], 20.00th=[ 684], | 30.00th=[ 692], 40.00th=[ 708], 50.00th=[ 716], 60.00th=[ 732], | 70.00th=[ 740], 80.00th=[ 756], 90.00th=[ 780], 95.00th=[ 796], | 99.00th=[ 836], 99.50th=[ 900], 99.90th=[ 2640], 99.95th=[ 3024], | 99.99th=[ 3408] bw (KB /s): min= 0, max=47928, per=5.00%, avg=32438.44, stdev=328.11 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=0.01%, 500=0.05%, 750=74.95%, 1000=24.57% lat (msec) : 2=0.23%, 4=0.18%, 10=0.01%, 20=0.01% cpu : usr=1.35%, sys=16.20%, ctx=940028653, majf=0, minf=65737 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1167651532/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=6 xfs-21: (groupid=2, jobs=20): err= 0: pid=29777: Sun Dec 6 10:35:47 2015 read : io=4469.8GB, bw=650948KB/s, iops=162736, runt=7200001msec slat (usec): min=1, max=17704, avg=18.33, stdev=14.91 clat (usec): min=0, max=19158, avg=717.87, stdev=154.12 lat (usec): min=84, max=19163, avg=736.28, stdev=153.74 clat percentiles (usec): | 1.00th=[ 612], 5.00th=[ 644], 10.00th=[ 660], 20.00th=[ 668], | 30.00th=[ 684], 40.00th=[ 700], 50.00th=[ 708], 60.00th=[ 724], | 70.00th=[ 732], 80.00th=[ 748], 90.00th=[ 772], 95.00th=[ 788], | 99.00th=[ 828], 99.50th=[ 876], 99.90th=[ 3440], 99.95th=[ 4576], | 99.99th=[ 5600] bw (KB /s): min= 0, max=52912, per=5.00%, avg=32550.51, stdev=361.27 lat (usec) : 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (usec) : 250=0.01%, 500=0.09%, 750=79.65%, 1000=19.88% lat (msec) : 2=0.18%, 4=0.13%, 10=0.07%, 20=0.01% cpu : usr=1.36%, sys=19.84%, ctx=931888168, majf=0, minf=71240 IO depths : 1=0.1%, 2=0.1%, 4=108.3%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued : total=r=1171705652/w=0/d=0, short=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=6 Run status group 0 (all jobs): READ: io=4460.5GB, aggrb=649595KB/s, minb=649595KB/s, maxb=649595KB/s, mint=7200002msec, maxt=7200002msec Run status group 1 (all jobs): READ: io=4454.3GB, aggrb=648695KB/s, minb=648695KB/s, maxb=648695KB/s, mint=7200002msec, maxt=7200002msec Run status group 2 (all jobs): READ: io=4469.8GB, aggrb=650947KB/s, minb=650947KB/s, maxb=650947KB/s, mint=7200001msec, maxt=7200001msec Disk stats (read/write): sda: ios=1266941472/0, merge=0/0, ticks=909758076/0, in_queue=914856859, util=100.00% sdb: ios=1265058418/0, merge=0/0, ticks=912564127/0, in_queue=917307956, util=100.00% sdc: ios=1269508963/0, merge=0/0, ticks=909313832/0, in_queue=915774464, util=100.00%