###########################################################################
#
# 64kB Random R(67%)/W(33%) on SR-71mach4 running CentOS 6.4
# 3 simultaneous Clients
#
###########################################################################
 Total IOPS: 3558
 Total MBps:  222
Ave Latency: 24.54ms



###########################################################################
#
# 64kB Random R(67%)/W(33%) on SR-71mach4 running CentOS 6.4 Wed Dec  4 19:10:56 2013
# Client 1 running CentOS 6.4. 2x10GbE to 2 shares
#
###########################################################################

job0: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job0: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
job1: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job1: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.0.10
Starting 6 processes

job0: (groupid=0, jobs=3): err= 0: pid=39314: Wed Dec  4 19:10:56 2013
  read : io=89552MB, bw=25472KB/s, iops=398 , runt=3600017msec
    slat (usec): min=1 , max=297 , avg=10.71, stdev= 8.69
    clat (usec): min=198 , max=87490 , avg=18060.65, stdev=6645.39
     lat (usec): min=274 , max=87501 , avg=18071.79, stdev=6645.47
    clat percentiles (usec):
     |  1.00th=[ 6560],  5.00th=[ 8896], 10.00th=[10304], 20.00th=[12352],
     | 30.00th=[14016], 40.00th=[15552], 50.00th=[17024], 60.00th=[18816],
     | 70.00th=[20864], 80.00th=[23424], 90.00th=[27008], 95.00th=[30336],
     | 99.00th=[37120], 99.50th=[39680], 99.90th=[45824], 99.95th=[48896],
     | 99.99th=[57088]
    bw (KB/s)  : min= 4039, max=22139, per=33.37%, avg=8499.41, stdev=1190.63
  write: io=44107MB, bw=12546KB/s, iops=196 , runt=3600017msec
    slat (usec): min=4 , max=551 , avg=17.85, stdev=12.86
    clat (usec): min=920 , max=128800 , avg=24493.51, stdev=8836.65
     lat (usec): min=990 , max=128814 , avg=24511.81, stdev=8836.74
    clat percentiles (usec):
     |  1.00th=[ 9152],  5.00th=[12352], 10.00th=[14272], 20.00th=[17024],
     | 30.00th=[19072], 40.00th=[21120], 50.00th=[23168], 60.00th=[25472],
     | 70.00th=[28288], 80.00th=[31360], 90.00th=[36608], 95.00th=[40704],
     | 99.00th=[49408], 99.50th=[52992], 99.90th=[61184], 99.95th=[65280],
     | 99.99th=[78336]
    bw (KB/s)  : min= 1644, max=11324, per=33.37%, avg=4185.82, stdev=698.50
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.04%, 10=6.31%, 20=49.11%, 50=44.21%
    lat (msec) : 100=0.32%, 250=0.01%
  cpu          : usr=0.18%, sys=0.26%, ctx=2088500, majf=0, minf=82
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1432828/w=705704/d=0, short=r=0/w=0/d=0
job1: (groupid=1, jobs=3): err= 0: pid=39317: Wed Dec  4 19:10:56 2013
  read : io=89648MB, bw=25500KB/s, iops=398 , runt=3600015msec
    slat (usec): min=2 , max=253 , avg=11.32, stdev=10.16
    clat (usec): min=295 , max=91409 , avg=18063.52, stdev=6678.52
     lat (usec): min=326 , max=91418 , avg=18075.28, stdev=6678.59
    clat percentiles (usec):
     |  1.00th=[ 6496],  5.00th=[ 8896], 10.00th=[10304], 20.00th=[12352],
     | 30.00th=[14016], 40.00th=[15552], 50.00th=[17024], 60.00th=[18816],
     | 70.00th=[20864], 80.00th=[23424], 90.00th=[27008], 95.00th=[30336],
     | 99.00th=[37120], 99.50th=[39680], 99.90th=[45824], 99.95th=[48896],
     | 99.99th=[58624]
    bw (KB/s)  : min= 4652, max=18103, per=33.37%, avg=8508.05, stdev=1177.64
  write: io=44038MB, bw=12526KB/s, iops=195 , runt=3600015msec
    slat (usec): min=4 , max=517 , avg=19.05, stdev=16.66
    clat (usec): min=954 , max=111059 , avg=24483.25, stdev=8776.28
     lat (usec): min=979 , max=111102 , avg=24502.75, stdev=8776.31
    clat percentiles (usec):
     |  1.00th=[ 9152],  5.00th=[12352], 10.00th=[14272], 20.00th=[17024],
     | 30.00th=[19072], 40.00th=[21120], 50.00th=[23168], 60.00th=[25472],
     | 70.00th=[28288], 80.00th=[31360], 90.00th=[36096], 95.00th=[40704],
     | 99.00th=[49408], 99.50th=[52480], 99.90th=[61184], 99.95th=[65280],
     | 99.99th=[79360]
    bw (KB/s)  : min= 1889, max=10687, per=33.37%, avg=4179.37, stdev=707.70
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.05%, 10=6.37%, 20=49.00%, 50=44.27%
    lat (msec) : 100=0.31%, 250=0.01%
  cpu          : usr=0.18%, sys=0.26%, ctx=2121641, majf=0, minf=84
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1434367/w=704615/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=89552MB, aggrb=25472KB/s, minb=25472KB/s, maxb=25472KB/s, mint=3600017msec, maxt=3600017msec
  WRITE: io=44107MB, aggrb=12545KB/s, minb=12545KB/s, maxb=12545KB/s, mint=3600017msec, maxt=3600017msec

Run status group 1 (all jobs):
   READ: io=89648MB, aggrb=25499KB/s, minb=25499KB/s, maxb=25499KB/s, mint=3600015msec, maxt=3600015msec
  WRITE: io=44038MB, aggrb=12526KB/s, minb=12526KB/s, maxb=12526KB/s, mint=3600015msec, maxt=3600015msec
2013-12-04 19:10:56

###########################################################################
#
# 64kB Random R(67%)/W(33%) on SR-71mach4 running CentOS 6.4 Wed Dec  4 19:10:56 2013
# Client 2 running CentOS 6.4. 2x10GbE to 2 shares
#
###########################################################################

job1: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job1: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
job2: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job2: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.0.10
Starting 6 processes

job1: (groupid=0, jobs=3): err= 0: pid=38226: Wed Dec  4 19:10:58 2013
  read : io=89502MB, bw=25458KB/s, iops=397 , runt=3600016msec
    slat (usec): min=1 , max=482 , avg=18.44, stdev=16.48
    clat (usec): min=249 , max=91044 , avg=18024.67, stdev=6680.32
     lat (usec): min=373 , max=91072 , avg=18043.58, stdev=6680.42
    clat percentiles (usec):
     |  1.00th=[ 6368],  5.00th=[ 8768], 10.00th=[10304], 20.00th=[12224],
     | 30.00th=[13888], 40.00th=[15552], 50.00th=[17024], 60.00th=[18816],
     | 70.00th=[20864], 80.00th=[23424], 90.00th=[27008], 95.00th=[30336],
     | 99.00th=[37120], 99.50th=[39680], 99.90th=[45824], 99.95th=[48896],
     | 99.99th=[57088]
    bw (KB/s)  : min= 4562, max=19456, per=33.37%, avg=8494.84, stdev=1187.42
  write: io=44114MB, bw=12548KB/s, iops=196 , runt=3600016msec
    slat (usec): min=4 , max=578 , avg=29.69, stdev=2624502.75.07
    clat (msec): min=1 , max=108 , avg=24.54, stdev= 8.76
     lat (msec): min=1 , max=108 , avg=24.57, stdev= 8.76
    clat percentiles (usec):
     |  1.00th=[ 9536],  5.00th=[12480], 10.00th=[14400], 20.00th=[17024],
     | 30.00th=[19072], 40.00th=[21120], 50.00th=[23424], 60.00th=[25472],
     | 70.00th=[28288], 80.00th=[31616], 90.00th=[36608], 95.00th=[40704],
     | 99.00th=[49408], 99.50th=[52992], 99.90th=[60672], 99.95th=[64768],
     | 99.99th=[77312]
    bw (KB/s)  : min= 2011, max= 9472, per=33.37%, avg=4186.60, stdev=696.77
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.05%, 10=6.44%, 20=48.95%, 50=44.23%
    lat (msec) : 100=0.31%, 250=0.01%
  cpu          : usr=0.27%, sys=0.34%, ctx=2545371, majf=0, minf=82
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1432032/w=705818/d=0, short=r=0/w=0/d=0
job2: (groupid=1, jobs=3): err= 0: pid=38229: Wed Dec  4 19:10:58 2013
  read : io=89528MB, bw=25466KB/s, iops=397 , runt=3600018msec
    slat (usec): min=2 , max=392 , avg=18.89, stdev=16.94
    clat (usec): min=394 , max=91571 , avg=18025.84, stdev=6667.00
     lat (usec): min=427 , max=91599 , avg=18045.20, stdev=6667.04
    clat percentiles (usec):
     |  1.00th=[ 6432],  5.00th=[ 8768], 10.00th=[10304], 20.00th=[12224],
     | 30.00th=[14016], 40.00th=[15552], 50.00th=[17024], 60.00th=[18816],
     | 70.00th=[20864], 80.00th=[23424], 90.00th=[27008], 95.00th=[30336],
     | 99.00th=[36608], 99.50th=[39680], 99.90th=[45824], 99.95th=[48896],
     | 99.99th=[58624]
    bw (KB/s)  : min= 4224, max=20031, per=33.37%, avg=8497.18, stdev=1191.50
  write: io=44117MB, bw=12549KB/s, iops=196 , runt=3600018msec
    slat (usec): min=4 , max=620 , avg=30.84, stdev=27.80
    clat (msec): min=2 , max=115 , avg=24.53, stdev= 8.78
     lat (msec): min=2 , max=115 , avg=24.57, stdev= 8.78
    clat percentiles (usec):
     |  1.00th=[ 9408],  5.00th=[12480], 10.00th=[14400], 20.00th=[17024],
     | 30.00th=[19072], 40.00th=[21120], 50.00th=[23168], 60.00th=[25472],
     | 70.00th=[28288], 80.00th=[31616], 90.00th=[36608], 95.00th=[40704],
     | 99.00th=[49408], 99.50th=[52992], 99.90th=[61184], 99.95th=[66048],
     | 99.99th=[78336]
    bw (KB/s)  : min= 1753, max= 9561, per=33.37%, avg=4186.95, stdev=702.14
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.05%, 10=6.39%, 20=48.99%, 50=44.24%
    lat (msec) : 100=0.32%, 250=0.01%
  cpu          : usr=0.22%, sys=0.35%, ctx=2517249, majf=0, minf=84
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1432443/w=705872/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=89502MB, aggrb=25458KB/s, minb=25458KB/s, maxb=25458KB/s, mint=3600016msec, maxt=3600016msec
  WRITE: io=44114MB, aggrb=12547KB/s, minb=12547KB/s, maxb=12547KB/s, mint=3600016msec, maxt=3600016msec

Run status group 1 (all jobs):
   READ: io=89528MB, aggrb=25465KB/s, minb=25465KB/s, maxb=25465KB/s, mint=3600018msec, maxt=3600018msec
  WRITE: io=44117MB, aggrb=12548KB/s, minb=12548KB/s, maxb=12548KB/s, mint=3600018msec, maxt=3600018msec
2013-12-04 19:10:58


###########################################################################
#
# 64kB Random R(67%)/W(33%) on SR-71mach4 running CentOS 6.4 Wed Dec  4 19:10:56 2013
# Client 3 running CentOS 6.4. 2x10GbE to 2 shares
#
###########################################################################

job2: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job2: (g=0): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
job0: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job0: (g=1): rw=randrw, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.0.10
Starting 6 processes

job2: (groupid=0, jobs=3): err= 0: pid=37974: Wed Dec  4 19:10:59 2013
  read : io=89465MB, bw=25448KB/s, iops=397 , runt=3600008msec
    slat (usec): min=2 , max=184 , avg= 6.78, stdev= 4.89
    clat (usec): min=199 , max=89188 , avg=18060.04, stdev=6663.61
     lat (usec): min=204 , max=89194 , avg=18067.08, stdev=6663.65
    clat percentiles (usec):
     |  1.00th=[ 6496],  5.00th=[ 8896], 10.00th=[10304], 20.00th=[12352],
     | 30.00th=[14016], 40.00th=[15552], 50.00th=[17024], 60.00th=[18816],
     | 70.00th=[20864], 80.00th=[23424], 90.00th=[27008], 95.00th=[30336],
     | 99.00th=[37120], 99.50th=[39168], 99.90th=[45312], 99.95th=[48896],
     | 99.99th=[57600]
    bw (KB/s)  : min= 4291, max=16286, per=33.36%, avg=8489.98, stdev=1180.35
  write: io=44124MB, bw=12551KB/s, iops=196 , runt=3600008msec
    slat (usec): min=4 , max=179 , avg=10.97, stdev= 7.26
    clat (msec): min=1 , max=123 , avg=24.54, stdev= 8.79
     lat (msec): min=1 , max=123 , avg=24.55, stdev= 8.79
    clat percentiles (usec):
     |  1.00th=[ 9408],  5.00th=[12480], 10.00th=[14400], 20.00th=[17024],
     | 30.00th=[19072], 40.00th=[21120], 50.00th=[23424], 60.00th=[25472],
     | 70.00th=[28288], 80.00th=[31616], 90.00th=[36608], 95.00th=[40704],
     | 99.00th=[49408], 99.50th=[52992], 99.90th=[61184], 99.95th=[65280],
     | 99.99th=[78336]
    bw (KB/s)  : min= 1893, max= 8600, per=33.36%, avg=4187.04, stdev=697.53
    lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.05%, 10=6.31%, 20=48.98%, 50=44.34%
    lat (msec) : 100=0.32%, 250=0.01%
  cpu          : usr=0.14%, sys=0.15%, ctx=2082391, majf=0, minf=82
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1431438/w=705977/d=0, short=r=0/w=0/d=0
job0: (groupid=1, jobs=3): err= 0: pid=37977: Wed Dec  4 19:10:59 2013
  read : io=89504MB, bw=25459KB/s, iops=397 , runt=3600010msec
    slat (usec): min=1 , max=165 , avg=10.56, stdev=11.04
    clat (usec): min=284 , max=91684 , avg=18055.74, stdev=6670.34
     lat (usec): min=289 , max=91690 , avg=18066.59, stdev=6670.49
    clat percentiles (usec):
     |  1.00th=[ 6496],  5.00th=[ 8768], 10.00th=[10304], 20.00th=[12352],
     | 30.00th=[14016], 40.00th=[15552], 50.00th=[17024], 60.00th=[18816],
     | 70.00th=[20864], 80.00th=[23424], 90.00th=[27008], 95.00th=[30336],
     | 99.00th=[37120], 99.50th=[39680], 99.90th=[45824], 99.95th=[48896],
     | 99.99th=[57600]
    bw (KB/s)  : min= 4317, max=16286, per=33.36%, avg=8493.88, stdev=1181.92
  write: io=44092MB, bw=12542KB/s, iops=195 , runt=3600010msec
    slat (usec): min=4 , max=212 , avg=17.30, stdev=17.83
    clat (msec): min=1 , max=108 , avg=24.53, stdev= 8.81
     lat (msec): min=1 , max=108 , avg=24.55, stdev= 8.81
    clat percentiles (usec):
     |  1.00th=[ 9280],  5.00th=[12480], 10.00th=[14400], 20.00th=[17024],
     | 30.00th=[19072], 40.00th=[21120], 50.00th=[23168], 60.00th=[25472],
     | 70.00th=[28288], 80.00th=[31616], 90.00th=[36608], 95.00th=[40704],
     | 99.00th=[49408], 99.50th=[52992], 99.90th=[61184], 99.95th=[65280],
     | 99.99th=[79360]
    bw (KB/s)  : min= 1897, max= 9088, per=33.36%, avg=4184.03, stdev=693.03
    lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
    lat (msec) : 2=0.01%, 4=0.05%, 10=6.33%, 20=48.96%, 50=44.32%
    lat (msec) : 100=0.33%, 250=0.01%
  cpu          : usr=0.14%, sys=0.19%, ctx=2364362, majf=0, minf=84
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=1432068/w=705474/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=89465MB, aggrb=25447KB/s, minb=25447KB/s, maxb=25447KB/s, mint=3600008msec, maxt=3600008msec
  WRITE: io=44124MB, aggrb=12550KB/s, minb=12550KB/s, maxb=12550KB/s, mint=3600008msec, maxt=3600008msec

Run status group 1 (all jobs):
   READ: io=89504MB, aggrb=25458KB/s, minb=25458KB/s, maxb=25458KB/s, mint=3600010msec, maxt=3600010msec
  WRITE: io=44092MB, aggrb=12541KB/s, minb=12541KB/s, maxb=12541KB/s, mint=3600010msec, maxt=3600010msec
2013-12-04 19:10:59

###########################################################################
#
# 64kB Random Read on SR-71mach4 running CentOS 6.4
# 3 simultaneous Clients
#
###########################################################################
 Total IOPS: 81,673
 Total MBps:  5,104
Ave Latency: 0.88ms



###########################################################################
#
# 64kB Random Read on SR-71mach4 running CentOS 6.4 Wed Dec  4 20:11:57 2013
# Client 1 running CentOS 6.4. 2x10GbE to 2 shares
#
###########################################################################

job0: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job0: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
job1: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job1: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.0.10
Starting 6 processes

job0: (groupid=0, jobs=3): err= 0: pid=39746: Wed Dec  4 20:11:57 2013
  read : io=2962.8GB, bw=862953KB/s, iops=13483 , runt=3600001msec
    slat (usec): min=1 , max=1216 , avg= 6.07, stdev= 7.53
    clat (usec): min=2 , max=41889 , avg=881.27, stdev=300.39
     lat (usec): min=191 , max=41894 , avg=887.59, stdev=300.74
    clat percentiles (usec):
     |  1.00th=[  486],  5.00th=[  572], 10.00th=[  620], 20.00th=[  692],
     | 30.00th=[  740], 40.00th=[  772], 50.00th=[  812], 60.00th=[  868],
     | 70.00th=[  932], 80.00th=[  996], 90.00th=[ 1160], 95.00th=[ 1480],
     | 99.00th=[ 2096], 99.50th=[ 2288], 99.90th=[ 2736], 99.95th=[ 2992],
     | 99.99th=[ 4080]
    bw (KB/s)  : min=94080, max=406528, per=33.35%, avg=287760.09, stdev=49073.37
    lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=1.35%
    lat (usec) : 750=33.29%, 1000=45.88%
    lat (msec) : 2=18.07%, 4=1.40%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.85%, sys=2.96%, ctx=38368318, majf=0, minf=274
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=48541092/w=0/d=0, short=r=0/w=0/d=0
job1: (groupid=1, jobs=3): err= 0: pid=39749: Wed Dec  4 20:11:57 2013
  read : io=2981.4GB, bw=868390KB/s, iops=13568 , runt=3600001msec
    slat (usec): min=1 , max=1357 , avg= 6.26, stdev= 7.75
    clat (usec): min=2 , max=21318 , avg=875.96, stdev=295.78
     lat (usec): min=196 , max=21323 , avg=882.46, stdev=296.16
    clat percentiles (usec):
     |  1.00th=[  486],  5.00th=[  572], 10.00th=[  620], 20.00th=[  692],
     | 30.00th=[  732], 40.00th=[  764], 50.00th=[  812], 60.00th=[  860],
     | 70.00th=[  924], 80.00th=[  988], 90.00th=[ 1128], 95.00th=[ 1464],
     | 99.00th=[ 2096], 99.50th=[ 2256], 99.90th=[ 2704], 99.95th=[ 2960],
     | 99.99th=[ 4128]
    bw (KB/s)  : min=93952, max=392704, per=33.35%, avg=289574.72, stdev=49158.43
    lat (usec) : 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=1.37%, 750=33.76%, 1000=46.25%
    lat (msec) : 2=17.26%, 4=1.36%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.67%, sys=3.01%, ctx=38878438, majf=0, minf=276
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=48846936/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=2962.8GB, aggrb=862952KB/s, minb=862952KB/s, maxb=862952KB/s, mint=3600001msec, maxt=3600001msec

Run status group 1 (all jobs):
   READ: io=2981.4GB, aggrb=868389KB/s, minb=868389KB/s, maxb=868389KB/s, mint=3600001msec, maxt=3600001msec
2013-12-04 20:11:57

###########################################################################
#
# 64kB Random Read on SR-71mach4 running CentOS 6.4 Wed Dec  4 20:11:58 2013
# Client 2 running CentOS 6.4. 2x10GbE to 2 shares
#
###########################################################################

job1: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job1: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
job2: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job2: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.0.10
Starting 6 processes

job1: (groupid=0, jobs=3): err= 0: pid=38656: Wed Dec  4 20:11:58 2013
  read : io=2989.6GB, bw=870624KB/s, iops=13603 , runt=3600001msec
    slat (usec): min=1 , max=1199 , avg= 5.89, stdev= 7.51
    clat (usec): min=2 , max=41767 , avg=873.70, stdev=294.61
     lat (usec): min=199 , max=41773 , avg=879.81, stdev=294.97
    clat percentiles (usec):
     |  1.00th=[  486],  5.00th=[  572], 10.00th=[  620], 20.00th=[  692],
     | 30.00th=[  732], 40.00th=[  764], 50.00th=[  812], 60.00th=[  860],
     | 70.00th=[  916], 80.00th=[  988], 90.00th=[ 1128], 95.00th=[ 1432],
     | 99.00th=[ 2096], 99.50th=[ 2256], 99.90th=[ 2704], 99.95th=[ 2992],
     | 99.99th=[ 4128]
    bw (KB/s)  : min=94208, max=349824, per=33.35%, avg=290321.02, stdev=49091.69
    lat (usec) : 4=0.01%, 10=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=1.43%, 750=33.59%, 1000=46.30%
    lat (msec) : 2=17.33%, 4=1.34%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.73%, sys=2.86%, ctx=39015456, majf=0, minf=274
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=48972592/w=0/d=0, short=r=0/w=0/d=0
job2: (groupid=1, jobs=3): err= 0: pid=38659: Wed Dec  4 20:11:58 2013
  read : io=2993.2GB, bw=872061KB/s, iops=13625 , runt=3600001msec
    slat (usec): min=1 , max=1207 , avg= 5.82, stdev= 7.29
    clat (usec): min=2 , max=42938 , avg=872.74, stdev=294.49
     lat (usec): min=202 , max=42946 , avg=878.78, stdev=294.80
    clat percentiles (usec):
     |  1.00th=[  486],  5.00th=[  572], 10.00th=[  620], 20.00th=[  692],
     | 30.00th=[  732], 40.00th=[  764], 50.00th=[  812], 60.00th=[  860],
     | 70.00th=[  916], 80.00th=[  988], 90.00th=[ 1128], 95.00th=[ 1432],
     | 99.00th=[ 2096], 99.50th=[ 2256], 99.90th=[ 2704], 99.95th=[ 2960],
     | 99.99th=[ 4048]
    bw (KB/s)  : min=87760, max=356096, per=33.35%, avg=290798.40, stdev=49231.82
    lat (usec) : 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
    lat (usec) : 500=1.44%, 750=33.67%, 1000=46.29%
    lat (msec) : 2=17.25%, 4=1.34%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.57%, sys=2.85%, ctx=39144575, majf=0, minf=276
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=49053444/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=2989.6GB, aggrb=870623KB/s, minb=870623KB/s, maxb=870623KB/s, mint=3600001msec, maxt=3600001msec

Run status group 1 (all jobs):
   READ: io=2993.2GB, aggrb=872060KB/s, minb=872060KB/s, maxb=872060KB/s, mint=3600001msec, maxt=3600001msec
2013-12-04 20:11:58

###########################################################################
#
# 64kB Random Read on SR-71mach4 running CentOS 6.4 Wed Dec  4 20:11:59 2013
# Client 3 running CentOS 6.4. 2x10GbE to 2 shares
#
###########################################################################

job2: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job2: (g=0): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
job0: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
...
job0: (g=1): rw=randread, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=4
fio-2.0.10
Starting 6 processes

job2: (groupid=0, jobs=3): err= 0: pid=38394: Wed Dec  4 20:11:59 2013
  read : io=2991.5GB, bw=871318KB/s, iops=13614 , runt=3600001msec
    slat (usec): min=1 , max=687 , avg= 5.56, stdev= 6.39
    clat (usec): min=177 , max=82432 , avg=873.75, stdev=295.65
     lat (usec): min=183 , max=82440 , avg=879.53, stdev=295.78
    clat percentiles (usec):
     |  1.00th=[  482],  5.00th=[  564], 10.00th=[  620], 20.00th=[  692],
     | 30.00th=[  732], 40.00th=[  764], 50.00th=[  812], 60.00th=[  868],
     | 70.00th=[  924], 80.00th=[  988], 90.00th=[ 1128], 95.00th=[ 1416],
     | 99.00th=[ 2096], 99.50th=[ 2288], 99.90th=[ 2768], 99.95th=[ 2992],
     | 99.99th=[ 3856]
    bw (KB/s)  : min=94208, max=393216, per=33.35%, avg=290550.76, stdev=49662.44
    lat (usec) : 250=0.01%, 500=1.58%, 750=33.57%, 1000=46.29%
    lat (msec) : 2=17.16%, 4=1.40%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=1.67%, sys=2.72%, ctx=38439876, majf=0, minf=274
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=49011635/w=0/d=0, short=r=0/w=0/d=0
job0: (groupid=1, jobs=3): err= 0: pid=38397: Wed Dec  4 20:11:59 2013
  read : io=3027.9GB, bw=881925KB/s, iops=13780 , runt=3600001msec
    slat (usec): min=1 , max=664 , avg= 6.03, stdev= 7.09
    clat (usec): min=2 , max=42594 , avg=862.82, stdev=289.59
     lat (usec): min=175 , max=42599 , avg=869.07, stdev=289.80
    clat percentiles (usec):
     |  1.00th=[  490],  5.00th=[  564], 10.00th=[  620], 20.00th=[  692],
     | 30.00th=[  732], 40.00th=[  756], 50.00th=[  804], 60.00th=[  852],
     | 70.00th=[  908], 80.00th=[  972], 90.00th=[ 1096], 95.00th=[ 1384],
     | 99.00th=[ 2096], 99.50th=[ 2256], 99.90th=[ 2736], 99.95th=[ 2992],
     | 99.99th=[ 3920]
    bw (KB/s)  : min=94147, max=390784, per=33.35%, avg=294080.73, stdev=50545.29
    lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=1.40%
    lat (usec) : 750=35.68%, 1000=46.40%
    lat (msec) : 2=15.14%, 4=1.36%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=1.61%, sys=2.75%, ctx=40594319, majf=0, minf=276
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=49608286/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=2991.5GB, aggrb=871317KB/s, minb=871317KB/s, maxb=871317KB/s, mint=3600001msec, maxt=3600001msec

Run status group 1 (all jobs):
   READ: io=3027.9GB, aggrb=881924KB/s, minb=881924KB/s, maxb=881924KB/s, mint=3600001msec, maxt=3600001msec
2013-12-04 20:11:59

###########################################################################
#
# 8kB Random Read on SR-71mach4 running CentOS 6.4
# 3 simultaneous Clients
#
###########################################################################
 Total IOPS: 220,863
 Total MBps:  1,725
Ave Latency: 651ms



###########################################################################
#
# 8kB Random Read on SR-71mach4 running CentOS 6.4 Wed Dec  4 21:12:57 2013
# Client 1 running CentOS 6.4. 2x10GbE to 2 shares
#
###########################################################################

job0: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
job0: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
job1: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
job1: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
fio-2.0.10
Starting 4 processes

job0: (groupid=0, jobs=2): err= 0: pid=40183: Wed Dec  4 21:12:57 2013
  read : io=1014.6GB, bw=295517KB/s, iops=36939 , runt=3600001msec
    slat (usec): min=1 , max=2903 , avg= 3.01, stdev= 4.92
    clat (usec): min=11 , max=54378 , avg=645.23, stdev=196.25
     lat (usec): min=98 , max=54379 , avg=648.40, stdev=196.04
    clat percentiles (usec):
     |  1.00th=[  274],  5.00th=[  378], 10.00th=[  458], 20.00th=[  506],
     | 30.00th=[  556], 40.00th=[  604], 50.00th=[  636], 60.00th=[  676],
     | 70.00th=[  716], 80.00th=[  756], 90.00th=[  852], 95.00th=[  932],
     | 99.00th=[ 1144], 99.50th=[ 1240], 99.90th=[ 1592], 99.95th=[ 1912],
     | 99.99th=[ 3600]
    bw (KB/s)  : min=76000, max=221008, per=50.01%, avg=147787.12, stdev=6666.22
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.46%, 500=17.57%
    lat (usec) : 750=60.06%, 1000=18.98%
    lat (msec) : 2=2.89%, 4=0.04%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=3.95%, sys=5.53%, ctx=43816835, majf=0, minf=102
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=132982686/w=0/d=0, short=r=0/w=0/d=0
job1: (groupid=1, jobs=2): err= 0: pid=40185: Wed Dec  4 21:12:57 2013
  read : io=1012.3GB, bw=294849KB/s, iops=36856 , runt=3600001msec
    slat (usec): min=1 , max=4154 , avg= 3.55, stdev= 7.94
    clat (usec): min=1 , max=40716 , avg=646.32, stdev=200.74
     lat (usec): min=92 , max=40718 , avg=650.03, stdev=200.30
    clat percentiles (usec):
     |  1.00th=[  270],  5.00th=[  370], 10.00th=[  454], 20.00th=[  506],
     | 30.00th=[  556], 40.00th=[  604], 50.00th=[  636], 60.00th=[  676],
     | 70.00th=[  724], 80.00th=[  764], 90.00th=[  860], 95.00th=[  940],
     | 99.00th=[ 1160], 99.50th=[ 1272], 99.90th=[ 1656], 99.95th=[ 1976],
     | 99.99th=[ 3760]
    bw (KB/s)  : min=75008, max=220992, per=50.01%, avg=147453.26, stdev=6858.72
    lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.52%
    lat (usec) : 500=17.92%, 750=59.11%, 1000=19.29%
    lat (msec) : 2=3.10%, 4=0.04%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=3.69%, sys=5.90%, ctx=49094050, majf=0, minf=104
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=132682081/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=1014.6GB, aggrb=295516KB/s, minb=295516KB/s, maxb=295516KB/s, mint=3600001msec, maxt=3600001msec

Run status group 1 (all jobs):
   READ: io=1012.3GB, aggrb=294848KB/s, minb=294848KB/s, maxb=294848KB/s, mint=3600001msec, maxt=3600001msec
   

###########################################################################
#
# 8kB Random Read on SR-71mach4 running CentOS 6.4 Wed Dec  4 21:12:59 2013
# Client 2 running CentOS 6.4. 2x10GbE to 2 shares
#
###########################################################################

job1: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
job1: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
job2: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
job2: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
fio-2.0.10
Starting 4 processes

job1: (groupid=0, jobs=2): err= 0: pid=39091: Wed Dec  4 21:12:59 2013
  read : io=993.89GB, bw=289258KB/s, iops=36157 , runt=3600001msec
    slat (usec): min=1 , max=3360 , avg= 9.01, stdev=14.60
    clat (usec): min=10 , max=54080 , avg=653.00, stdev=259.46
     lat (usec): min=90 , max=54094 , avg=662.21, stdev=256.43
    clat percentiles (usec):
     |  1.00th=[  215],  5.00th=[  294], 10.00th=[  358], 20.00th=[  462],
     | 30.00th=[  524], 40.00th=[  588], 50.00th=[  644], 60.00th=[  692],
     | 70.00th=[  748], 80.00th=[  820], 90.00th=[  940], 95.00th=[ 1048],
     | 99.00th=[ 1368], 99.50th=[ 1528], 99.90th=[ 2096], 99.95th=[ 2512],
     | 99.99th=[ 4256]
    bw (KB/s)  : min=74816, max=181136, per=50.01%, avg=144651.67, stdev=7556.02
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.05%, 250=2.14%, 500=23.60%
    lat (usec) : 750=44.96%, 1000=22.38%
    lat (msec) : 2=6.74%, 4=0.11%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=4.29%, sys=8.69%, ctx=170354820, majf=0, minf=104
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=130165920/w=0/d=0, short=r=0/w=0/d=0
job2: (groupid=1, jobs=2): err= 0: pid=39093: Wed Dec  4 21:12:59 2013
  read : io=991.97GB, bw=288925KB/s, iops=36115 , runt=3600001msec
    slat (usec): min=1 , max=3642 , avg= 9.30, stdev=15.92
    clat (usec): min=7 , max=41770 , avg=653.54, stdev=259.58
     lat (usec): min=86 , max=41776 , avg=663.05, stdev=256.59
    clat percentiles (usec):
     |  1.00th=[  217],  5.00th=[  298], 10.00th=[  362], 20.00th=[  462],
     | 30.00th=[  524], 40.00th=[  588], 50.00th=[  644], 60.00th=[  692],
     | 70.00th=[  748], 80.00th=[  820], 90.00th=[  940], 95.00th=[ 1048],
     | 99.00th=[ 1368], 99.50th=[ 1544], 99.90th=[ 2160], 99.95th=[ 2544],
     | 99.99th=[ 4256]
    bw (KB/s)  : min=76048, max=172800, per=50.01%, avg=144486.91, stdev=7974.08
    lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.06%, 250=1.99%
    lat (usec) : 500=23.66%, 750=45.15%, 1000=22.29%
    lat (msec) : 2=6.71%, 4=0.12%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=4.24%, sys=9.38%, ctx=170133421, majf=0, minf=102
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=130016390/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=993.89GB, aggrb=289257KB/s, minb=289257KB/s, maxb=289257KB/s, mint=3600001msec, maxt=3600001msec

Run status group 1 (all jobs):
   READ: io=991.97GB, aggrb=288925KB/s, minb=288925KB/s, maxb=288925KB/s, mint=3600001msec, maxt=3600001msec
   

###########################################################################
#
# 8kB Random Read on SR-71mach4 running CentOS 6.4 Wed Dec  4 21:13:00 2013
# Client 3 running CentOS 6.4. 2x10GbE to 2 shares
#
###########################################################################

job2: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
job2: (g=0): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
job0: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
job0: (g=1): rw=randread, bs=8K-8K/8K-8K/8K-8K, ioengine=libaio, iodepth=12
fio-2.0.10
Starting 4 processes

job2: (groupid=0, jobs=2): err= 0: pid=38832: Wed Dec  4 21:13:00 2013
  read : io=1026.5GB, bw=298979KB/s, iops=37372 , runt=3600001msec
    slat (usec): min=1 , max=378 , avg= 2.38, stdev= 2.09
    clat (usec): min=19 , max=53384 , avg=638.74, stdev=184.27
     lat (usec): min=97 , max=53386 , avg=641.25, stdev=184.09
    clat percentiles (usec):
     |  1.00th=[  290],  5.00th=[  390], 10.00th=[  462], 20.00th=[  510],
     | 30.00th=[  564], 40.00th=[  604], 50.00th=[  628], 60.00th=[  660],
     | 70.00th=[  708], 80.00th=[  748], 90.00th=[  828], 95.00th=[  900],
     | 99.00th=[ 1096], 99.50th=[ 1176], 99.90th=[ 1464], 99.95th=[ 1704],
     | 99.99th=[ 3408]
    bw (KB/s)  : min=77808, max=209712, per=50.01%, avg=149518.99, stdev=6694.67
    lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.33%, 500=16.98%
    lat (usec) : 750=63.13%, 1000=17.42%
    lat (msec) : 2=2.11%, 4=0.03%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=3.23%, sys=4.44%, ctx=44202475, majf=0, minf=104
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=134540779/w=0/d=0, short=r=0/w=0/d=0
job0: (groupid=1, jobs=2): err= 0: pid=38834: Wed Dec  4 21:13:00 2013
  read : io=1027.1GB, bw=299400KB/s, iops=37424 , runt=3600001msec
    slat (usec): min=1 , max=1520 , avg= 4.80, stdev= 5.16
    clat (usec): min=20 , max=44399 , avg=635.41, stdev=213.83
     lat (usec): min=88 , max=44404 , avg=640.35, stdev=212.76
    clat percentiles (usec):
     |  1.00th=[  247],  5.00th=[  334], 10.00th=[  382], 20.00th=[  486],
     | 30.00th=[  540], 40.00th=[  596], 50.00th=[  628], 60.00th=[  668],
     | 70.00th=[  724], 80.00th=[  764], 90.00th=[  868], 95.00th=[  964],
     | 99.00th=[ 1160], 99.50th=[ 1256], 99.90th=[ 1544], 99.95th=[ 1768],
     | 99.99th=[ 3440]
    bw (KB/s)  : min=76848, max=175584, per=50.01%, avg=149717.63, stdev=7250.68
    lat (usec) : 50=0.01%, 100=0.01%, 250=1.17%, 500=21.92%, 750=53.69%
    lat (usec) : 1000=19.66%
    lat (msec) : 2=3.53%, 4=0.02%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=3.48%, sys=6.10%, ctx=158965262, majf=0, minf=102
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=134729934/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
   READ: io=1026.5GB, aggrb=298979KB/s, minb=298979KB/s, maxb=298979KB/s, mint=3600001msec, maxt=3600001msec

Run status group 1 (all jobs):
   READ: io=1027.1GB, aggrb=299399KB/s, minb=299399KB/s, maxb=299399KB/s, mint=3600001msec, maxt=3600001msec
   


###########################################################################
#
# All tests:
# mkfs.ext4 -L a0 -b 4096 -E stride=16,stripe-width=112 /ssd/a0/fio.tst1
# mkfs.ext4 -L a1 -b 4096 -E stride=16,stripe-width=112 /ssd/a1/fio.tst1
# mkfs.ext4 -L a2 -b 4096 -E stride=16,stripe-width=112 /ssd/a2/fio.tst1
# LABEL=a0		/ssd/a0			ext4	noatime,data=writeback,barrier=0,nobh,errors=remount-ro 0 2
# LABEL=a1		/ssd/a1			ext4	noatime,data=writeback,barrier=0,nobh,errors=remount-ro 0 2
# LABEL=a2		/ssd/a2			ext4	noatime,data=writeback,barrier=0,nobh,errors=remount-ro 0 2
# ifconfig eth0 10.10.1.1 netmask 255.255.255.0 mtu 9014 up
# ifconfig eth1 10.10.2.1 netmask 255.255.255.0 mtu 9014 up
# mount -t nfs4 -o async,rsize=65536,wsize=65536,noatime 10.10.1.10:/ssd/a0/share1 /root/a0
# mount -t nfs4 -o async,rsize=65536,wsize=65536,noatime 10.10.2.10:/ssd/a1/share0 /root/a1
#
###########################################################################