diff --git a/Makefile b/Makefile
index 6f9561b..f5a588f 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ vpath %.h $(SEARCHPATH)
 DEPS += io500-util.h io500-debug.h io500-opt.h
 OBJSO += util.o
 OBJSO += ini-parse.o phase_dbg.o phase_opt.o phase_timestamp.o
-OBJSO += phase_find.o phase_find_easy.o phase_find_hard.o phase_ior_easy.o phase_ior_easy_read.o phase_mdtest.o phase_ior.o phase_ior_easy_write.o phase_ior_hard.o phase_ior_hard_read.o phase_ior_hard_write.o phase_mdtest_easy.o phase_mdtest_easy_delete.o phase_mdtest_easy_stat.o phase_mdtest_easy_write.o phase_mdtest_hard.o phase_mdtest_hard_delete.o phase_mdtest_hard_read.o phase_mdtest_hard_stat.o phase_mdtest_hard_write.o phase_ior_rnd1MB.o phase_ior_rnd4K.o phase_ior_rnd_write4K.o phase_ior_rnd_read4K.o phase_ior_rnd_write1MB.o phase_ior_rnd_read1MB.o phase_mdworkbench.o phase_mdworkbench_create.o phase_mdworkbench_delete.o phase_mdworkbench_bench.o phase_ior_rnd_read4k-easywrite.o
+OBJSO += phase_find.o phase_find_easy.o phase_find_hard.o phase_ior_easy.o phase_ior_easy_read.o phase_mdtest.o phase_ior.o phase_ior_easy_write.o phase_ior_hard.o phase_ior_hard_read.o phase_ior_hard_write.o phase_mdtest_easy.o phase_mdtest_easy_delete.o phase_mdtest_easy_stat.o phase_mdtest_easy_write.o phase_mdtest_hard.o phase_mdtest_hard_delete.o phase_mdtest_hard_read.o phase_mdtest_hard_stat.o phase_mdtest_hard_write.o phase_ior_rnd1MB.o phase_ior_rnd4K.o phase_ior_rnd_write4K.o phase_ior_rnd_read4K.o phase_ior_rnd_write1MB.o phase_ior_rnd_read1MB.o phase_mdworkbench.o phase_mdworkbench_create.o phase_mdworkbench_delete.o phase_mdworkbench_bench.o phase_ior_rnd_read4k-easywrite.o  phase_concurrent.o
 
 OBJS = $(patsubst %,./build/%,$(OBJSO))
 
diff --git a/find/sfind.sh b/find/sfind.sh
index b76e8fd..e7bf349 100755
--- a/find/sfind.sh
+++ b/find/sfind.sh
@@ -9,6 +9,10 @@
 # When you are done pring 'x/y' where x is matched files and y is total files searched.
 # There is a parallel version also install in bin that might be better
 
+# The script should support the following extra options
+# -e for deletion of found files 
+# -E for deletion of directories
+
 function parse_rates {
   #find -D rates gives a weird thing like this:
   #Predicate success rates after completion
@@ -18,7 +22,20 @@ function parse_rates {
   echo $rates | tr " " "\n" | grep '/' | $1 -1 | cut -d \/ -f 2 | cut -d = -f 1
 }
 
-rates=`find -D rates $* 2>&1 | grep -A1 Predicate | tail -1` 
+IN=""
+for var in "$@"
+do
+    if [[ $var == "-e" ]] ; then 
+      continue
+    fi
+    if [[ $var == "-E" ]] ; then 
+      IN="$IN -exec rm -f {} ;"
+      continue
+    fi
+    IN="$IN $var"
+done
+
+rates=$(find -D rates $IN 2>&1 | grep -A1 Predicate | tail -1)
 total_files=$(parse_rates 'head')
 match_files=$(parse_rates 'tail')
 echo "MATCHED $match_files/$total_files"
diff --git a/include/io500-opt.h b/include/io500-opt.h
index 3e1ec16..faf0433 100644
--- a/include/io500-opt.h
+++ b/include/io500-opt.h
@@ -27,7 +27,8 @@ typedef struct{
 
   int rank;
   int mpi_size;
-  int io_buffers_on_gpu; /* are the I/O buffers to be allocated on a GPU */
+  int allocateBufferDevice; /* where are the I/O buffers allocated and processed */
+  int gpuDirect; /* useGPUDirect */
 
   char * api;
   char * apiArgs; // for IOR and mdtest
diff --git a/include/io500-phase.h b/include/io500-phase.h
index c558455..45ff35a 100644
--- a/include/io500-phase.h
+++ b/include/io500-phase.h
@@ -8,9 +8,10 @@
 #define DEBUG_OPTION "INVALIDATES RUN; FOR DEBUGGING."
 
 typedef enum {
-  IO500_NO_SCORE,
+  IO500_NO_SCORE = 0,
   IO500_SCORE_MD,
   IO500_SCORE_BW,
+  IO500_SCORE_CONCURRENT,
   IO500_SCORE_LAST
 } io500_phase_score_group;
 
@@ -37,7 +38,7 @@ typedef struct{
   io500_phase_score_group group;
 } u_phase_t;
 
-#define IO500_PHASES (2 + 1 + 2*3 + 3 + 3 + 4 + 5 + 3 + 4 + 1)
+#define IO500_PHASES (2 + 1 + 2*3 + 3 + 3 + 4 + 5 + 3 + 4 + 1 + 1)
 
 extern u_phase_t p_opt;
 extern u_phase_t p_debug;
@@ -65,6 +66,8 @@ extern u_phase_t p_mdworkbench_create;
 extern u_phase_t p_mdworkbench_bench;
 extern u_phase_t p_mdworkbench_delete;
 
+extern u_phase_t p_concurrent;
+
 extern u_phase_t p_ior_easy;
 extern u_phase_t p_ior_easy_write;
 extern u_phase_t p_ior_easy_read;
diff --git a/include/io500-util.h b/include/io500-util.h
index 997cd14..23b786d 100644
--- a/include/io500-util.h
+++ b/include/io500-util.h
@@ -82,7 +82,7 @@ void u_print_timestamp(FILE * out);
 void * u_malloc(int size);
 
 
-FILE * u_res_file_prep(char const * name);
+FILE * u_res_file_prep(char const * name, int rank);
 void u_res_file_close(FILE * out);
 
 uint32_t u_phase_unique_random_number(char const * phase_name);
diff --git a/src/main.c b/src/main.c
index 389f507..17fc99e 100644
--- a/src/main.c
+++ b/src/main.c
@@ -22,7 +22,16 @@ FILE* file_out = NULL;
 static char const * io500_phase_str[IO500_SCORE_LAST] = {
   "NO SCORE",
   "MD",
-  "BW"};
+  "BW", 
+  "CONCURRENT"
+  };
+
+static char const * io500_unit_str[IO500_SCORE_LAST] = {
+  "     ",
+  "kIOPS",
+  "GiB/s",
+  "score"
+};
 
 extern int rank;
 
@@ -127,14 +136,19 @@ static void print_cfg_hash(FILE * out, ini_section_t ** cfg){
 
 static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t * hash){
   double overall_score = 1;
-  for(io500_phase_score_group g=1; g < IO500_SCORE_LAST; g++){
+  int groups = IO500_SCORE_BW;
+  if(extended){
+    groups = IO500_SCORE_CONCURRENT;
+  }
+  memset(scores, 0, sizeof(double) * IO500_SCORE_LAST);
+  for(io500_phase_score_group g=1; g <= groups; g++){
     char score_string[2048];
     char *p = score_string;
     double score = 1;
     int numbers = 0;
-    p += sprintf(p, " %s = (", io500_phase_str[g]);
+    p += sprintf(p, " %s = (1.0", io500_phase_str[g]);
     for(int i=0; i < IO500_PHASES; i++){
-      if(phases[i]->group == g && (extended || ! (phases[i]->type & IO500_PHASE_FLAG_OPTIONAL)) ){
+      if( phases[i]->group == g && (extended || ! (phases[i]->type & IO500_PHASE_FLAG_OPTIONAL)) ){
         double t = phases[i]->score;
         if(t <= 0){
           continue;
@@ -157,7 +171,7 @@ static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t
 
     overall_score *= score;
   }
-  return sqrt(overall_score);
+  return pow(overall_score, 1.0 / groups);
 }
 
 static const char * io500_mode_str(io500_mode mode){
@@ -169,6 +183,92 @@ static const char * io500_mode_str(io500_mode mode){
   }
 }
 
+typedef struct {
+  int run;
+  int valid;
+  double score;
+  double time;
+} io500_phase_result_t;
+
+typedef struct{
+  double score[IO500_SCORE_LAST];
+  double scoreX[IO500_SCORE_LAST];
+  double runtime;
+} io500_overall_results_t;
+
+static io500_phase_result_t io500_results[IO500_PHASES];
+static io500_overall_results_t io500_overall_results;
+
+static void dump_io500_phase(FILE * o, int i){
+  io500_phase_result_t * r = & io500_results[i];
+  if(! r->run) return;
+  u_phase_t * phase = phases[i];
+  char score_str[40];
+  sprintf(score_str, "%f", r->score);
+  fprintf(o, "%25s\t%15s\t%s\t%.3fs\t%s\n", phase->name, score_str, io500_unit_str[phase->group], r->time, r->valid ? "" : "INVALID");
+}
+
+static void dump_io500_results(void){
+    FILE * res;
+    char file[PATH_MAX];
+    sprintf(file, "%s/result_pretty.txt", opt.resdir);
+    res = fopen(file, "w");
+    if(! res){
+      FATAL("Could not open \"%s\" for writing (%s)\n", file, strerror(errno));
+    } 
+    
+    fprintf(res, "%25s\t%fs\n", "RUNTIME", io500_overall_results.runtime);
+    fprintf(res, "%25s", "SCORE");
+    for(int i=0; i < IO500_SCORE_LAST; i++){
+      fprintf(res, "\t%f\t%s\t", io500_overall_results.score[i], io500_unit_str[i]);
+    }
+    fprintf(res, "\n");
+    
+    if(opt.mode == IO500_MODE_EXTENDED){
+      fprintf(res, "%25s", "SCOREX");
+      for(int i=0; i < IO500_SCORE_LAST; i++){
+        fprintf(res, "\t%f\t%s\t", io500_overall_results.scoreX[i], io500_unit_str[i]);
+      }
+      fprintf(res, "\n");
+    }
+    
+    // ior easy
+    dump_io500_phase(res, 3);
+    dump_io500_phase(res, 24);
+    // ior hard
+    dump_io500_phase(res, 15);
+    dump_io500_phase(res, 26);
+    // rand4k
+    dump_io500_phase(res, 5);
+    dump_io500_phase(res, 19);
+    // rand1M
+    dump_io500_phase(res, 8);
+    dump_io500_phase(res, 20);
+    // concurrent
+    dump_io500_phase(res, 23);
+    
+    // MD easy
+    dump_io500_phase(res, 7);
+    dump_io500_phase(res, 25);
+    dump_io500_phase(res, 29);
+    // MD hard
+    dump_io500_phase(res, 17);
+    dump_io500_phase(res, 27);
+    dump_io500_phase(res, 30);
+    dump_io500_phase(res, 31);
+    // MDW
+    dump_io500_phase(res, 22);
+    dump_io500_phase(res, 28);
+    // find
+    dump_io500_phase(res, 18);
+    // find easy
+    dump_io500_phase(res, 13);    
+    // find hard
+    dump_io500_phase(res, 21);
+    
+    fclose(res);
+}
+
 int main(int argc, char ** argv){
   int mpi_init = 0;
   file_out = stdout;
@@ -201,6 +301,9 @@ int main(int argc, char ** argv){
     }
     goto out;
   }
+  
+  memset(io500_results, 0, sizeof(io500_results));
+  double t_io500_start = GetTimeStamp();
 
   mpi_init = 1;
   MPI_Init(& argc, & argv);
@@ -372,11 +475,12 @@ int main(int argc, char ** argv){
 
   for(int i=0; i < IO500_PHASES; i++){
     u_phase_t * phase = phases[i];
-    if(! phase->run) continue;
+    if(! phase->run) continue;        
     if( cleanup_only && ! (phase->type & IO500_PHASE_REMOVE) ) continue;
     if(! RUN_PHASE(phase)){
       continue;
     }
+    io500_results[i].run = 1;
 
     if(opt.drop_caches && ! (phase->type & IO500_PHASE_DUMMY) ){
       DEBUG_INFO("Dropping cache\n");
@@ -448,6 +552,11 @@ int main(int argc, char ** argv){
           opt.is_valid_run = 0;
         }
       }
+    
+      io500_results[i].score = score;
+      io500_results[i].time = runtime;
+      io500_results[i].valid = opt.is_valid_phase;
+      
       if(! (phase->type & IO500_PHASE_DUMMY)){
         PRINT_PAIR("score", "%f\n", score);
       }
@@ -459,7 +568,12 @@ int main(int argc, char ** argv){
       }else{
         dupprintf("[      ]");
       }
-      dupprintf(" %20s %15s %s : time %.3f seconds%s\n", phase->name, score_str, phase->name[0] == 'i' ? "GiB/s" : "kIOPS", runtime, valid_str);
+      dupprintf(" %25s %15s %s : time %.3f seconds%s\n", phase->name, score_str, io500_unit_str[phase->group], runtime, valid_str);
+      
+      PRINT_PAIR("t_delta", "%.4f\n", runtime);
+      PRINT_PAIR_HEADER("t_end");
+      u_print_timestamp(file_out);
+      fprintf(file_out, "\n");
     }
     if(phase->group > IO500_NO_SCORE){
       if(phase->type & IO500_PHASE_FLAG_OPTIONAL){
@@ -469,14 +583,6 @@ int main(int argc, char ** argv){
       }
     }
     phases[i]->score = score;
-
-
-    if(opt.verbosity > 0 && opt.rank == 0){
-      PRINT_PAIR("t_delta", "%.4f\n", runtime);
-      PRINT_PAIR_HEADER("t_end");
-      u_print_timestamp(file_out);
-      fprintf(file_out, "\n");
-    }
   }
 
   MPI_Barrier(MPI_COMM_WORLD);
@@ -494,7 +600,10 @@ int main(int argc, char ** argv){
     PRINT_PAIR("hash", "%X\n", (int) score_hash);
     dupprintf("[SCORE ] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n",
     scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_score, valid_str);
-
+    
+    scores[IO500_NO_SCORE] = overall_score;
+    memcpy(io500_overall_results.score, scores, sizeof(scores));
+    
     // extended run
     if(opt.mode == IO500_MODE_EXTENDED){
       valid_str = opt.is_valid_extended_run ? "" : " [INVALID]";
@@ -507,8 +616,13 @@ int main(int argc, char ** argv){
       PRINT_PAIR("SCORE", "%f%s\n", overall_extended_score, valid_str);
       PRINT_PAIR("hash", "%X\n", (int) score_extended_hash);
       dupprintf("[SCOREX] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n", scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_extended_score, valid_str);
+      
+      scores[IO500_NO_SCORE] = overall_score;
+      memcpy(io500_overall_results.scoreX, scores, sizeof(scores));
     }
-
+    
+    io500_overall_results.runtime = GetTimeStamp() - t_io500_start;
+    dump_io500_results();
     printf("\nThe result files are stored in the directory: %s\n", opt.resdir);
   }
 
diff --git a/src/phase-definitions.h b/src/phase-definitions.h
index 2f97541..36aca26 100644
--- a/src/phase-definitions.h
+++ b/src/phase-definitions.h
@@ -36,6 +36,8 @@ static u_phase_t * phases[IO500_PHASES] = {
 
   & p_mdworkbench_bench,
 
+  & p_concurrent,
+
   & p_ior_easy_read,
   & p_mdtest_easy_stat,
 
diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c
new file mode 100644
index 0000000..391818a
--- /dev/null
+++ b/src/phase_concurrent.c
@@ -0,0 +1,234 @@
+#include <sys/stat.h>
+#include <unistd.h>
+#include <mpi.h>
+#include <math.h>
+
+#include <io500-phase.h>
+#include <phase_concurrent.h>
+
+#define CBENCHS 3
+
+typedef struct{
+  int run;
+  char * command[CBENCHS];
+  mdworkbench_results_t * mdw;
+  IOR_point_t * iorr;
+  IOR_point_t * iorw;
+  int color; // benchmark that is run
+  float ratio_write;
+  float ratio_read;
+  int procs[3];
+} opt_concurrent_bench;
+
+static opt_concurrent_bench o;
+
+static ini_option_t option[] = {
+  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},
+  {"ratio-write", "The ratio of processes that perform write - this ratio has the highest priority", 0, INI_FLOAT, "0.2", & o.ratio_write},
+  {"ratio-read", "The ratio of processes that perform random read - remaining processes will do metadata", 0, INI_FLOAT, "0.4", & o.ratio_read},  
+  {NULL} };
+
+static double run(void){
+  opt_concurrent_bench d = o;
+    
+  MPI_Comm ccom;
+  /*
+    Split processes into 3 groups and benchmarks:
+    benchmark 0 - 20% parallel write - ior easy
+    benchmark 1 - 40% parallel read 1 MB - rnd1MB read
+    benchmark 2 - 40% md-workbench for concurrent usage
+  */
+  /* Sanity check of parameters */
+  if(d.ratio_write > 1){
+    d.ratio_write = 0;
+  }
+  if(d.ratio_read > 1){
+    d.ratio_read = 0;
+  }
+
+  int workloads[] = {opt.mpi_size * d.ratio_write, opt.mpi_size * (d.ratio_write+d.ratio_read), opt.mpi_size};
+  int procs[] = {workloads[0], workloads[1] - workloads[0], workloads[2] - workloads[1]};
+  memcpy(o.procs, procs, sizeof(int)*3);
+  
+  if(procs[0] + procs[1] == 0 || procs[0] + procs[2] == 0 || procs[1] + procs[2] == 0){
+    INVALID("The concurrent phase doesn't make sense with two benchmarks using 0 processes\n");
+    //return 0;
+  }
+  
+  int color = (opt.rank < workloads[0]) ? 0 : (opt.rank < workloads[1] ? 1 : 2);
+  o.color = color;
+  UMPI_CHECK(MPI_Comm_split(MPI_COMM_WORLD, color, opt.rank, & ccom));
+  
+  int crank;
+  MPI_Comm_rank(ccom, & crank);
+  /* printf("%d - %d - rank %d\n", opt.rank, color, crank); */
+  if(opt.rank == 0){
+    PRINT_PAIR("ranks-write", "%d\n", procs[0]);
+    PRINT_PAIR("ranks-read", "%d\n", procs[1]);
+    PRINT_PAIR("ranks-md", "%d\n", procs[2]);
+  }
+    
+  u_argv_t * argv[CBENCHS];
+  for(int i=0; i < CBENCHS; i++){
+    argv[i] = u_argv_create();
+  }
+    
+  /* prepare benchmark settings */ 
+  {
+  opt_ior_easy d = ior_easy_o;
+  ior_easy_add_params(argv[0], 0, 1);
+  u_argv_push(argv[0], "-o");	/* filename for output file */
+  u_argv_push_printf(argv[0], "%s/ior-easy-concurrent/ior_file_easy", opt.datadir);
+  u_argv_push(argv[0], "-w");	/* write file */
+  u_argv_push_printf(argv[0], "-D=%d", opt.stonewall); /* deadline for stonewall in seconds */
+  u_argv_push_printf(argv[0], "-O=minTimeDuration=%d", opt.stonewall); /* minimum runtime */
+  //u_argv_push_default_if_set_api_options(argv[0], "-a", d.api, o.api);
+  u_argv_push_default_if_set_api_options(argv[0], "-a", d.api, d.api); // TODO USE RND WRITE OPTION
+  u_argv_push(argv[0], "-O");
+  u_argv_push_printf(argv[0], "saveRankPerformanceDetailsCSV=%s/concurrent-ior-easy-write.csv", opt.resdir);
+  o.command[0] = u_flatten_argv(argv[0]);
+  PRINT_PAIR("exe-easy-write", "%s\n", o.command[0]);
+  }
+  
+  {
+  opt_ior_rnd d = ior_rnd1MB_o;
+  ior_rnd1MB_add_params(argv[1]);
+  u_argv_push(argv[1], "-r");
+  u_argv_push_default_if_set_api_options(argv[1], "-a", d.api, d.api); // TODO USE RND WRITE OPTION
+  u_argv_push(argv[1], "-O");
+  u_argv_push_printf(argv[1], "saveRankPerformanceDetailsCSV=%s/concurrent-ior-rnd1MB-read.csv", opt.resdir);
+  u_argv_push_printf(argv[1], "-D=%d", opt.stonewall); /* deadline for stonewall in seconds */
+  u_argv_push_printf(argv[1], "-O=minTimeDuration=%d", opt.stonewall); /* minimum runtime */
+  o.command[1] = u_flatten_argv(argv[1]);
+  PRINT_PAIR("exe-rnd1MB-read", "%s\n", o.command[1]);
+  }
+  
+  {
+  mdworkbench_add_params(argv[2], 0);
+  u_argv_push(argv[2], "-R=1"); /* 1 repetition */
+  u_argv_push(argv[2], "-X"); /* turn read verification on */
+  u_argv_push_printf(argv[2], "-w=%d", opt.stonewall);
+  u_argv_push_printf(argv[2], "-o=%s/mdworkbench", opt.datadir);
+  u_argv_push_printf(argv[2], "--run-info-file=%s/mdworkbench.status", opt.resdir );  
+  u_argv_push(argv[2], "-2"); /* run pre-create or benchmarking phase */
+  o.command[2] = u_flatten_argv(argv[2]);  
+  PRINT_PAIR("exe-md-workbench", "%s\n", o.command[2]);
+  }
+  
+  if(opt.dry_run || d.run == 0){
+    for(int i=0; i < CBENCHS; i++){
+      u_argv_free(argv[i]);
+    }
+    return 0;
+  }    
+  double score = 0;
+  double time = 0;
+  if(color == 0){    
+    // run ior easy write
+    FILE * out = u_res_file_prep("concurrent-ior-easy-write", crank);
+    score = ior_process_write(argv[0], out, & o.iorw, ccom);
+    if(o.iorw){
+      time = o.iorw->time;
+    }
+  }else if(color == 1){
+    // run ior rnd 1MB read
+    FILE * out = u_res_file_prep("concurrent-rnd1MB-read", crank);
+    score = ior_process_read(argv[1], out, & o.iorr, ccom);
+    if(o.iorr){
+      time = o.iorr->time;
+    }
+  }else if(color == 2){
+    // run md_workbench
+    FILE * out = u_res_file_prep("concurrent-md-workbench", crank);
+    mdworkbench_process(argv[2], out, & o.mdw, ccom);
+    /* calculate score */
+    double rate = o.mdw->result[0].rate;
+    PRINT_PAIR("maxOpTime", "%f\n", o.mdw->result[0].max_op_time);
+    score = rate / 1000.0;
+    if(o.mdw){
+      time = o.mdw->result[0].runtime;
+    }
+  }
+  
+  if(crank == 0){    
+    // exchange scores, calculate geometric mean score
+    double scores[CBENCHS] = {0};
+    double times[CBENCHS] = {0};
+    if(opt.rank == 0){
+      MPI_Request req_score, req_time;
+      UMPI_CHECK(MPI_Isend(& score, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD, & req_score));
+      UMPI_CHECK(MPI_Isend(& time, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD, & req_time));
+      
+      double aggregated_score = 1;
+      int benchmarks = 0;
+      if (procs[0] != 0){
+        scores[0] = score;
+        times[0] = time;
+        benchmarks++;
+      }
+      for(int i=1; i < CBENCHS; i++){
+        if (procs[i] == 0){
+          scores[i] = 0;
+          times[i] = 0;
+          continue;
+        }
+        benchmarks++;
+        UMPI_CHECK(MPI_Recv(& scores[i], 1, MPI_DOUBLE, workloads[i-1], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE));
+        UMPI_CHECK(MPI_Recv(& times[i], 1, MPI_DOUBLE, workloads[i-1], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE));
+        aggregated_score = aggregated_score * (scores[i] / procs[i]);
+      }
+      PRINT_PAIR("score-ior-easy-write", "%f\n", scores[0]);
+      PRINT_PAIR("score-ior-rnd1MB-read", "%f\n", scores[1]);
+      PRINT_PAIR("score-ior-md-workbench", "%f\n", scores[2]);
+      PRINT_PAIR("time-ior-easy-write", "%f\n", times[0]);
+      PRINT_PAIR("time-ior-rnd1MB-read", "%f\n", times[1]);
+      PRINT_PAIR("time-ior-md-workbench", "%f\n", times[2]);
+      score = pow(aggregated_score, 1.0/benchmarks) * opt.mpi_size;
+    }else{
+      UMPI_CHECK(MPI_Send(& score, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD));
+      UMPI_CHECK(MPI_Send(& time, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD));
+    }
+  }else{
+    score = 0;
+  }
+  return score;
+}
+
+
+static void validate(void){
+  u_create_datadir("ior-easy-concurrent");
+}
+
+static void cleanup(void){
+  opt_concurrent_bench d = o;
+    
+  if (opt.dry_run || ! d.run) return;
+  
+  if(opt.rank == 0){
+    if(! ior_easy_o.filePerProc){
+      u_purge_file("ior-easy-concurrent/ior_file_easy");
+    }
+  }
+  if(ior_easy_o.filePerProc){
+      char filename[PATH_MAX];
+      if(opt.rank < d.procs[0]){
+        sprintf(filename, "ior-easy-concurrent/ior_file_easy.%08d", opt.rank);
+        u_purge_file(filename);
+      }
+  }
+  if(opt.rank == 0){
+    u_purge_datadir("ior-easy-concurrent");
+  }
+}
+
+
+u_phase_t p_concurrent = {
+  "concurrent",
+  IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL,
+  option,
+  validate,
+  run,
+  0,
+  .cleanup = cleanup,
+  .group = IO500_SCORE_CONCURRENT,
+};
diff --git a/src/phase_concurrent.h b/src/phase_concurrent.h
new file mode 100644
index 0000000..6f253b7
--- /dev/null
+++ b/src/phase_concurrent.h
@@ -0,0 +1,7 @@
+#ifndef IO500_PHASE_CONCURRENT_H
+#define IO500_PHASE_CONCURRENT_H
+
+#include <phase_mdworkbench.h>
+#include <phase_ior.h>
+
+#endif
diff --git a/src/phase_find.c b/src/phase_find.c
index 932ee99..c7e31af 100644
--- a/src/phase_find.c
+++ b/src/phase_find.c
@@ -60,7 +60,7 @@ double run_find(const char * phase_name, opt_find * of){
         fprintf(fd, "runtime: %f rate: %f\n", of->pfind_res->runtime, of->pfind_res->rate);
         fprintf(fd, "rank, errors, unknown, found, total, checked, job steal msgs received, work items send, job steal msgs send, work items stolen, time spend in job stealing in s, number of completion tokens send\n");
         fprintf(fd, "0, %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64, res->errors, res->unknown_file, res->found_files, res->total_files, res->checked_dirents);
-        fprintf(fd, ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %.3fs, %"PRIu64"\n", res->monitor.job_steal_inbound, res->monitor.work_send, res->monitor.job_steal_tries, res->monitor.work_stolen, res->monitor.job_steal_mpitime_us / 1000000.0, res->monitor.completion_tokens_send);
+        fprintf(fd, ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %.3f, %"PRIu64"\n", res->monitor.job_steal_inbound, res->monitor.work_send, res->monitor.job_steal_tries, res->monitor.work_stolen, res->monitor.job_steal_mpitime_us / 1000000.0, res->monitor.completion_tokens_send);
         for(int i=1; i < of->nproc; i++){
           MPI_Recv(& res->errors, 5, MPI_LONG_LONG_INT, i, 4712, of->pfind_com, MPI_STATUS_IGNORE);
           fprintf(fd, "%d, %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64, i, res->errors, res->unknown_file, res->found_files, res->total_files, res->checked_dirents);
@@ -98,7 +98,7 @@ double run_find(const char * phase_name, opt_find * of){
       ERROR("Failed to run find command: \"%s\" error: %s\n", of->command, strerror(errno));
       return -1;
     }
-    char line[1024];
+    char line[PATH_MAX];
     uint64_t hits = 0;
     *line = '\0';
     while (fgets(line, sizeof(line), fp) != NULL) {
@@ -175,16 +175,7 @@ static void validate(void){
     u_argv_push(argv, "-name");
     u_argv_push(argv, "*01*");
     u_argv_push(argv, "-C");
-    if(of.pfind_steal_from_next){
-      u_argv_push(argv, "-N");
-    }
-    if(of.pfind_par_single_dir_access_hash){
-      u_argv_push(argv, "-H");
-      u_argv_push(argv, "1");
-    }
-    u_argv_push(argv, "-q");
-    u_argv_push_printf(argv, "%d", of.pfind_queue_length);
-
+    
     pfind_prepare_arguments(argv, & of);
   }
 }
@@ -226,6 +217,16 @@ void pfind_prepare_arguments(u_argv_t * argv, opt_find * of){
         com = MPI_COMM_NULL;
       }
     }
+    
+    if(of->pfind_steal_from_next){
+      u_argv_push(argv, "-N");
+    }
+    if(of->pfind_par_single_dir_access_hash){
+      u_argv_push(argv, "-H");
+      u_argv_push(argv, "1");
+    }
+    u_argv_push(argv, "-q");
+    u_argv_push_printf(argv, "%d", of->pfind_queue_length);
 
     of->command = u_flatten_argv(argv);
     of->pfind_com = com;
diff --git a/src/phase_find_easy.c b/src/phase_find_easy.c
index b2f3236..52f49c2 100644
--- a/src/phase_find_easy.c
+++ b/src/phase_find_easy.c
@@ -29,7 +29,7 @@ static ini_option_t option[] = {
 static void validate(void){
   if(of.run == 0) return;
   if(of.ext_find){
-    char args[1024];
+    char args[PATH_MAX];
     sprintf(args, "%s/mdtest-easy/ -name \"*01*\"", opt.datadir);
     external_find_prepare_arguments(args, & of);
   }else{
@@ -39,15 +39,6 @@ static void validate(void){
     u_argv_push(argv, "-name");
     u_argv_push(argv, "*01*");
     u_argv_push(argv, "-C");
-    if(of.pfind_steal_from_next){
-      u_argv_push(argv, "-N");
-    }
-    if(of.pfind_par_single_dir_access_hash){
-      u_argv_push(argv, "-H");
-      u_argv_push(argv, "1");
-    }
-    u_argv_push(argv, "-q");
-    u_argv_push_printf(argv, "%d", of.pfind_queue_length);
 
     pfind_prepare_arguments(argv, & of);
   }
diff --git a/src/phase_find_hard.c b/src/phase_find_hard.c
index 2481a32..5eaf119 100644
--- a/src/phase_find_hard.c
+++ b/src/phase_find_hard.c
@@ -12,7 +12,7 @@ static opt_find of;
 
 static double run(void){
   if(of.run == 0) return 0.0;
-  return run_find("find-hard", & of);;
+  return run_find("find-hard", & of);
 }
 
 static ini_option_t option[] = {
@@ -29,7 +29,7 @@ static ini_option_t option[] = {
 static void validate(void){
   if(of.run == 0) return;
   if(of.ext_find){
-    char args[1024];
+    char args[PATH_MAX];
     sprintf(args, "%s/mdtest-hard/ -newer %s/timestampfile -size 3901c -name \"*01*\"", opt.datadir, opt.resdir);
     external_find_prepare_arguments(args, & of);
   }else{
@@ -43,15 +43,6 @@ static void validate(void){
     u_argv_push(argv, "-name");
     u_argv_push(argv, "*01*");
     u_argv_push(argv, "-C");
-    if(of.pfind_steal_from_next){
-      u_argv_push(argv, "-N");
-    }
-    if(of.pfind_par_single_dir_access_hash){
-      u_argv_push(argv, "-H");
-      u_argv_push(argv, "1");
-    }
-    u_argv_push(argv, "-q");
-    u_argv_push_printf(argv, "%d", of.pfind_queue_length);
 
     pfind_prepare_arguments(argv, & of);
   }
diff --git a/src/phase_ior.c b/src/phase_ior.c
index 2a0ed18..ed30afd 100644
--- a/src/phase_ior.c
+++ b/src/phase_ior.c
@@ -2,8 +2,8 @@
 #include <assert.h>
 
 
-double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){
-  IOR_test_t * test = ior_run(argv->size, argv->vector, MPI_COMM_WORLD, out);
+double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com){
+  IOR_test_t * test = ior_run(argv->size, argv->vector, com, out);
   assert(test);
   IOR_results_t * res = test->results;
   assert(res);
@@ -31,8 +31,8 @@ double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){
   return tp;
 }
 
-double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){
-  IOR_results_t * res = ior_run(argv->size, argv->vector, MPI_COMM_WORLD, out)->results;
+double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com){
+  IOR_results_t * res = ior_run(argv->size, argv->vector, com, out)->results;
   u_res_file_close(out);
   u_argv_free(argv);
 
@@ -40,7 +40,7 @@ double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){
   *res_out = p;
 
   if(res->errors){
-    INVALID("Errors (%d) occured during phase in IOR. This invalidates your run.\n", res->errors);
+    INVALID("Errors (%d) occurred during phase in IOR. This invalidates your run.\n", res->errors);
   }
   double tp = p->aggFileSizeForBW / p->time / GIBIBYTE;
   return tp;
diff --git a/src/phase_ior.h b/src/phase_ior.h
index 3df6888..135899f 100644
--- a/src/phase_ior.h
+++ b/src/phase_ior.h
@@ -34,6 +34,7 @@ typedef struct{
 
   int random_prefill_bytes;
   uint64_t block_size;
+  int segments;
   int verbosity;
 } opt_ior_rnd;
 
@@ -41,13 +42,13 @@ extern opt_ior_rnd ior_rnd4K_o;
 extern opt_ior_rnd ior_rnd1MB_o;
 
 
-void ior_easy_add_params(u_argv_t * argv, int addStdFlags);
+void ior_easy_add_params(u_argv_t * argv, int useStatusFile, int addStdFlags);
 void ior_hard_add_params(u_argv_t * argv);
 void ior_rnd4K_add_params(u_argv_t * argv);
 void ior_rnd1MB_add_params(u_argv_t * argv);
 
 // Generic helpers
-double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out);
-double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out);
+double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com);
+double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com);
 
 #endif
diff --git a/src/phase_ior_easy.c b/src/phase_ior_easy.c
index 3b224a2..5c6e146 100644
--- a/src/phase_ior_easy.c
+++ b/src/phase_ior_easy.c
@@ -21,7 +21,7 @@ static void validate(void){
 }
 
 static void cleanup(void){
-  if (opt.dry_run) return;
+  if (opt.dry_run || ! ior_easy_o.run) return;
   
   if(opt.rank == 0){
     char filename[PATH_MAX];
@@ -42,7 +42,7 @@ static void cleanup(void){
   }
 }
 
-void ior_easy_add_params(u_argv_t * argv, int addStdFlags){
+void ior_easy_add_params(u_argv_t * argv, int useStatusFile, int addStdFlags){
   opt_ior_easy d = ior_easy_o;
 
   u_argv_push(argv, "./ior");
@@ -50,9 +50,12 @@ void ior_easy_add_params(u_argv_t * argv, int addStdFlags){
   for(int i=0; i < ior_easy_o.verbosity; i++){
     u_argv_push(argv, "-v");	/* verbose */
   }
-  if(opt.io_buffers_on_gpu){
+  if(opt.allocateBufferDevice){
     u_argv_push(argv, "-O");
-    u_argv_push(argv, "allocateBufferOnGPU=1");
+    u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
   u_argv_push(argv, "-C");	/* reorder tasks in constant order for read */
   u_argv_push(argv, "-Q");	/* task per node offset */
@@ -66,9 +69,13 @@ void ior_easy_add_params(u_argv_t * argv, int addStdFlags){
   u_argv_push(argv, "-e");	/* fsync upon write close */
   u_argv_push(argv, "-o");	/* filename for output file */
   u_argv_push_printf(argv, "%s/ior-easy/ior_file_easy", opt.datadir);
-  if(addStdFlags){
+
+  if(useStatusFile){
     u_argv_push(argv, "-O");	/* additional IOR options */
     u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-easy.stonewall", opt.resdir);
+  }
+
+  if(addStdFlags){
     u_argv_push(argv, "-t");	/* transfer size */
     u_argv_push(argv, d.transferSize);
     u_argv_push(argv, "-b");	/* blocksize in bytes */
diff --git a/src/phase_ior_easy_read.c b/src/phase_ior_easy_read.c
index 543885d..babe03f 100644
--- a/src/phase_ior_easy_read.c
+++ b/src/phase_ior_easy_read.c
@@ -30,7 +30,7 @@ static double run(void){
   opt_ior_easy d = ior_easy_o;
 
   u_argv_t * argv = u_argv_create();
-  ior_easy_add_params(argv, 1);
+  ior_easy_add_params(argv, 1, 1);
   u_argv_push(argv, "-r");	/* read existing file */
   u_argv_push(argv, "-R");	/* verify data read */
   u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api);
@@ -49,8 +49,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_easy_read.name);
-  return ior_process_read(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_easy_read.name, opt.rank);
+  return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 
diff --git a/src/phase_ior_easy_write.c b/src/phase_ior_easy_write.c
index 87e97b5..2f9a3f6 100644
--- a/src/phase_ior_easy_write.c
+++ b/src/phase_ior_easy_write.c
@@ -29,7 +29,7 @@ static double run(void){
   opt_ior_easy d = ior_easy_o;
 
   u_argv_t * argv = u_argv_create();
-  ior_easy_add_params(argv, 1);
+  ior_easy_add_params(argv, 1, 1);
   u_argv_push(argv, "-w");	/* write file */
   u_argv_push(argv, "-D");	/* deadline for stonewall in seconds */
   u_argv_push_printf(argv, "%d", opt.stonewall);
@@ -51,8 +51,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_easy_write.name);
-  return ior_process_write(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_easy_write.name, opt.rank);
+  return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 u_phase_t p_ior_easy_write = {
diff --git a/src/phase_ior_hard.c b/src/phase_ior_hard.c
index 9162943..74425de 100644
--- a/src/phase_ior_hard.c
+++ b/src/phase_ior_hard.c
@@ -21,7 +21,7 @@ static void validate(void){
 }
 
 static void cleanup(void){
-  if (opt.dry_run) return;
+  if (opt.dry_run || ! ior_hard_o.run) return;
   
   if( opt.rank == 0){
     char filename[PATH_MAX];
@@ -40,9 +40,12 @@ void ior_hard_add_params(u_argv_t * argv){
   for(int i=0; i < ior_hard_o.verbosity; i++){
     u_argv_push(argv, "-v");	/* verbose */
   }
-  if(opt.io_buffers_on_gpu){
+  if(opt.allocateBufferDevice){
     u_argv_push(argv, "-O");
-    u_argv_push(argv, "allocateBufferOnGPU=1");
+    u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
   u_argv_push(argv, "-C");	/* reorder tasks in constant order for read */
   u_argv_push(argv, "-Q");	/* task per node offset */
diff --git a/src/phase_ior_hard_read.c b/src/phase_ior_hard_read.c
index 5b35ee7..8c59650 100644
--- a/src/phase_ior_hard_read.c
+++ b/src/phase_ior_hard_read.c
@@ -51,8 +51,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_hard_read.name);
-  return ior_process_read(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_hard_read.name, opt.rank);
+  return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 
diff --git a/src/phase_ior_hard_write.c b/src/phase_ior_hard_write.c
index 91ea568..660c2d4 100644
--- a/src/phase_ior_hard_write.c
+++ b/src/phase_ior_hard_write.c
@@ -52,8 +52,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_hard_write.name);
-  return ior_process_write(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_hard_write.name, opt.rank);
+  return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 u_phase_t p_ior_hard_write = {
diff --git a/src/phase_ior_rnd1MB.c b/src/phase_ior_rnd1MB.c
index 956c803..4963eb3 100644
--- a/src/phase_ior_rnd1MB.c
+++ b/src/phase_ior_rnd1MB.c
@@ -12,6 +12,7 @@ static ini_option_t option[] = {
   {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_rnd1MB_o.run},
   {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_rnd1MB_o.verbosity},
   {"randomPrefill", "Prefill the file with this blocksize in bytes, e.g., 2097152", 0, INI_INT, "0", & ior_rnd1MB_o.random_prefill_bytes},
+  {"segmentCount", "Number of segments", 0, INI_INT, "10000000", & ior_rnd1MB_o.segments},
   {NULL} };
 
 
@@ -37,9 +38,12 @@ void ior_rnd1MB_add_params(u_argv_t * argv){
   for(int i=0; i < ior_rnd1MB_o.verbosity; i++){
     u_argv_push(argv, "-v");
   }
-  if(opt.io_buffers_on_gpu){
+  if(opt.allocateBufferDevice){
     u_argv_push(argv, "-O");
-    u_argv_push(argv, "allocateBufferOnGPU=1");
+    u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
   u_argv_push(argv, "-Q=1");
   //u_argv_push(argv, "-F");
@@ -55,7 +59,7 @@ void ior_rnd1MB_add_params(u_argv_t * argv){
   u_argv_push(argv, "-k");
   u_argv_push(argv, "-t=1048576");
   u_argv_push_printf(argv, "-b=%ld", d.block_size);
-  u_argv_push_printf(argv, "-s=%d", 10000000);
+  u_argv_push_printf(argv, "-s=%d", d.segments); /* number of segments */
 }
 
 u_phase_t p_ior_rnd1MB = {
diff --git a/src/phase_ior_rnd4K.c b/src/phase_ior_rnd4K.c
index 6e6b773..fc4f0b8 100644
--- a/src/phase_ior_rnd4K.c
+++ b/src/phase_ior_rnd4K.c
@@ -12,6 +12,7 @@ static ini_option_t option[] = {
   {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_rnd4K_o.run},
   {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_rnd4K_o.verbosity},
   {"randomPrefill", "Prefill the file with this blocksize in bytes, e.g., 2097152", 0, INI_INT, "0", & ior_rnd4K_o.random_prefill_bytes},
+  {"segmentCount", "Number of segments", 0, INI_INT, "10000000", & ior_rnd4K_o.segments},
   {NULL} };
 
 
@@ -37,9 +38,12 @@ void ior_rnd4K_add_params(u_argv_t * argv){
   for(int i=0; i < ior_rnd4K_o.verbosity; i++){
     u_argv_push(argv, "-v");
   }
-  if(opt.io_buffers_on_gpu){
+  if(opt.allocateBufferDevice){
     u_argv_push(argv, "-O");
-    u_argv_push(argv, "allocateBufferOnGPU=1");
+    u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
   u_argv_push(argv, "-Q=1");
   //u_argv_push(argv, "-F");
@@ -55,7 +59,7 @@ void ior_rnd4K_add_params(u_argv_t * argv){
   u_argv_push(argv, "-k");
   u_argv_push(argv, "-t=4096");
   u_argv_push_printf(argv, "-b=%ld", d.block_size);
-  u_argv_push_printf(argv, "-s=%d", 10000000);
+  u_argv_push_printf(argv, "-s=%d", d.segments); /* number of segments */
 }
 
 u_phase_t p_ior_rnd4K = {
diff --git a/src/phase_ior_rnd_read1MB.c b/src/phase_ior_rnd_read1MB.c
index 0693d41..708da4b 100644
--- a/src/phase_ior_rnd_read1MB.c
+++ b/src/phase_ior_rnd_read1MB.c
@@ -44,8 +44,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_rnd1MB_read.name);
-  return ior_process_read(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_rnd1MB_read.name, opt.rank);
+  return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 
diff --git a/src/phase_ior_rnd_read4K.c b/src/phase_ior_rnd_read4K.c
index 275b88c..8bd77ec 100644
--- a/src/phase_ior_rnd_read4K.c
+++ b/src/phase_ior_rnd_read4K.c
@@ -44,8 +44,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_rnd4K_read.name);
-  return ior_process_read(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_rnd4K_read.name, opt.rank);
+  return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 
diff --git a/src/phase_ior_rnd_read4k-easywrite.c b/src/phase_ior_rnd_read4k-easywrite.c
index f8a71cd..f6ba36e 100644
--- a/src/phase_ior_rnd_read4k-easywrite.c
+++ b/src/phase_ior_rnd_read4k-easywrite.c
@@ -30,7 +30,7 @@ static double run(void){
   opt_ior_rnd_read d = o;
 
   u_argv_t * argv = u_argv_create();
-  ior_easy_add_params(argv, 0);
+  ior_easy_add_params(argv, 0, 0);
   u_argv_push(argv, "-r");
   u_argv_push(argv, "-D");
   u_argv_push_printf(argv, "%d", opt.stonewall);
@@ -50,8 +50,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_rnd4K_read.name);
-  return ior_process_read(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_rnd4K_read.name, opt.rank);
+  return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 
diff --git a/src/phase_ior_rnd_write1MB.c b/src/phase_ior_rnd_write1MB.c
index e256f2a..3272dd7 100644
--- a/src/phase_ior_rnd_write1MB.c
+++ b/src/phase_ior_rnd_write1MB.c
@@ -28,7 +28,7 @@ static void validate(void){
 }
 
 static void cleanup(void){
-  if (opt.dry_run) return;
+  if (opt.dry_run || ! ior_rnd1MB_o.run) return;
 
   if(opt.rank == 0){
     char filename[PATH_MAX];
@@ -62,8 +62,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_rnd1MB_write.name);
-  return ior_process_write(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_rnd1MB_write.name, opt.rank);
+  return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 u_phase_t p_ior_rnd1MB_write = {
diff --git a/src/phase_ior_rnd_write4K.c b/src/phase_ior_rnd_write4K.c
index d21d6b8..627dc49 100644
--- a/src/phase_ior_rnd_write4K.c
+++ b/src/phase_ior_rnd_write4K.c
@@ -28,7 +28,7 @@ static void validate(void){
 }
 
 static void cleanup(void){
-  if (opt.dry_run) return;
+  if (opt.dry_run || ! ior_rnd4K_o.run) return;
 
   if(opt.rank == 0){
     char filename[PATH_MAX];
@@ -62,8 +62,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_rnd4K_write.name);
-  return ior_process_write(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_rnd4K_write.name, opt.rank);
+  return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 u_phase_t p_ior_rnd4K_write = {
diff --git a/src/phase_mdtest.c b/src/phase_mdtest.c
index 2b8b2e3..8428f1f 100644
--- a/src/phase_mdtest.c
+++ b/src/phase_mdtest.c
@@ -7,6 +7,10 @@ void p_mdtest_run(u_argv_t * argv, FILE * out, mdtest_generic_res * d, mdtest_te
   d->items = res->items[test];
   d->rate = res->rate[test] / 1000;
   d->rate_stonewall = res->stonewall_item_sum[test] / res->stonewall_time[test] / 1000;
+  INFO_PAIR("errors", "%.0f\n", (float) res->total_errors);
+  if(res->total_errors){
+    INVALID("Verification errors %.0f \n", (float) res->total_errors);
+  }
   u_res_file_close(out);
   u_argv_free(argv);
   free(res);
diff --git a/src/phase_mdtest_easy.c b/src/phase_mdtest_easy.c
index 4b9a424..6862a35 100644
--- a/src/phase_mdtest_easy.c
+++ b/src/phase_mdtest_easy.c
@@ -32,8 +32,11 @@ void mdtest_easy_add_params(u_argv_t * argv){
 
   u_argv_push(argv, "./mdtest");
   u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType);
-  if(opt.io_buffers_on_gpu){
-    u_argv_push(argv, "--allocateBufferOnGPU");
+  if(opt.allocateBufferDevice){
+    u_argv_push_printf(argv, "--allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
   u_argv_push(argv, "-n");	/* number of files per process */
   u_argv_push_printf(argv, "%"PRIu64, d.g.files_per_proc);
diff --git a/src/phase_mdtest_easy_delete.c b/src/phase_mdtest_easy_delete.c
index 7504750..92b85bf 100644
--- a/src/phase_mdtest_easy_delete.c
+++ b/src/phase_mdtest_easy_delete.c
@@ -35,7 +35,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_easy_delete.name);
+  FILE * out = u_res_file_prep(p_mdtest_easy_delete.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_REMOVE_NUM);
 
   return o.res.rate;
diff --git a/src/phase_mdtest_easy_stat.c b/src/phase_mdtest_easy_stat.c
index 5fa2c81..db39181 100644
--- a/src/phase_mdtest_easy_stat.c
+++ b/src/phase_mdtest_easy_stat.c
@@ -35,7 +35,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_easy_stat.name);
+  FILE * out = u_res_file_prep(p_mdtest_easy_stat.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_STAT_NUM);
   return o.res.rate;
 }
diff --git a/src/phase_mdtest_easy_write.c b/src/phase_mdtest_easy_write.c
index f0f2b65..f1db4a3 100644
--- a/src/phase_mdtest_easy_write.c
+++ b/src/phase_mdtest_easy_write.c
@@ -42,7 +42,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_easy_write.name);
+  FILE * out = u_res_file_prep(p_mdtest_easy_write.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_CREATE_NUM);
 
   PRINT_PAIR("rate-stonewall", "%f\n", o.res.rate_stonewall);
diff --git a/src/phase_mdtest_hard.c b/src/phase_mdtest_hard.c
index 5c5244c..a1376dc 100644
--- a/src/phase_mdtest_hard.c
+++ b/src/phase_mdtest_hard.c
@@ -35,9 +35,13 @@ void mdtest_hard_add_params(u_argv_t * argv){
 
   u_argv_push(argv, "./mdtest");
   u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType);
-  if(opt.io_buffers_on_gpu){
-    u_argv_push(argv, "--allocateBufferOnGPU");
+  if(opt.allocateBufferDevice){
+    u_argv_push_printf(argv, "--allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
+
   u_argv_push(argv, "-n");	/* number of files per process */
   u_argv_push_printf(argv, "%"PRIu64, d.g.files_per_proc);
 
diff --git a/src/phase_mdtest_hard_delete.c b/src/phase_mdtest_hard_delete.c
index 2a53ca7..9699569 100644
--- a/src/phase_mdtest_hard_delete.c
+++ b/src/phase_mdtest_hard_delete.c
@@ -35,7 +35,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_hard_delete.name);
+  FILE * out = u_res_file_prep(p_mdtest_hard_delete.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_REMOVE_NUM);
 
   return o.res.rate;
diff --git a/src/phase_mdtest_hard_read.c b/src/phase_mdtest_hard_read.c
index 6152ca5..86238af 100644
--- a/src/phase_mdtest_hard_read.c
+++ b/src/phase_mdtest_hard_read.c
@@ -36,7 +36,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_hard_read.name);
+  FILE * out = u_res_file_prep(p_mdtest_hard_read.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_READ_NUM);
 
   return o.res.rate;
diff --git a/src/phase_mdtest_hard_stat.c b/src/phase_mdtest_hard_stat.c
index cb3f7ec..2b48944 100644
--- a/src/phase_mdtest_hard_stat.c
+++ b/src/phase_mdtest_hard_stat.c
@@ -35,7 +35,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_hard_stat.name);
+  FILE * out = u_res_file_prep(p_mdtest_hard_stat.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_STAT_NUM);
   return o.res.rate;
 }
diff --git a/src/phase_mdtest_hard_write.c b/src/phase_mdtest_hard_write.c
index fce389b..590ea6e 100644
--- a/src/phase_mdtest_hard_write.c
+++ b/src/phase_mdtest_hard_write.c
@@ -45,7 +45,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_hard_write.name);
+  FILE * out = u_res_file_prep(p_mdtest_hard_write.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_CREATE_NUM);
 
   PRINT_PAIR("rate-stonewall", "%f\n", o.res.rate_stonewall);
diff --git a/src/phase_mdworkbench.c b/src/phase_mdworkbench.c
index 1b14970..7fc7f9c 100644
--- a/src/phase_mdworkbench.c
+++ b/src/phase_mdworkbench.c
@@ -32,8 +32,8 @@ static void validate(void){
 }
 
 
-void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out){
-  mdworkbench_results_t * res = md_workbench_run(argv->size, argv->vector, MPI_COMM_WORLD, out);
+void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out, MPI_Comm com){
+  mdworkbench_results_t * res = md_workbench_run(argv->size, argv->vector, com, out);
   u_res_file_close(out);
   u_argv_free(argv);
 
@@ -53,9 +53,12 @@ void mdworkbench_add_params(u_argv_t * argv, int is_create){
   for(int i=0; i < mdworkbench_o.verbosity; i++){
     u_argv_push(argv, "-v");
   }
-  if(opt.io_buffers_on_gpu){
-    u_argv_push(argv, "--allocateBufferOnGPU");
-  }  
+  if(opt.allocateBufferDevice){
+    u_argv_push_printf(argv, "--allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
+  }
   u_argv_push(argv, "--process-reports");
   u_argv_push_default_if_set_api_options(argv, "-a", d->api, d->api);
   u_argv_push_printf(argv, "-o=%s/mdworkbench", opt.datadir);
diff --git a/src/phase_mdworkbench.h b/src/phase_mdworkbench.h
index 5ff0550..867b03c 100644
--- a/src/phase_mdworkbench.h
+++ b/src/phase_mdworkbench.h
@@ -16,6 +16,6 @@ typedef struct{
 extern opt_mdworkbench mdworkbench_o;
 
 void mdworkbench_add_params(u_argv_t * argv, int is_create);
-void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out);
+void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out, MPI_Comm com);
 
 #endif
diff --git a/src/phase_mdworkbench_bench.c b/src/phase_mdworkbench_bench.c
index 9e7010a..c9881c7 100644
--- a/src/phase_mdworkbench_bench.c
+++ b/src/phase_mdworkbench_bench.c
@@ -33,12 +33,15 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_mdworkbench_bench.name);
-  mdworkbench_process(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_mdworkbench_bench.name, opt.rank);
+  mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD);
   if(o.res->count != 2){
-    INVALID("During the md-workbench phase not two iterations are performed but %d This invalidates your run.\n", o.res->count);
+    INVALID("During the md-workbench phase not two iterations are performed but %d. This invalidates your run.\n", o.res->count);
     return 0.0;
   }
+  if(o.res->errors){
+    INVALID("md-workbench reported %lld errors\n", (long long int) o.res->errors);
+  }
 
   double rate = o.res->result[1].rate;
   if(   o.res->result[0].rate       < rate * 0.5
diff --git a/src/phase_mdworkbench_create.c b/src/phase_mdworkbench_create.c
index eff9041..0805154 100644
--- a/src/phase_mdworkbench_create.c
+++ b/src/phase_mdworkbench_create.c
@@ -31,8 +31,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_mdworkbench_create.name);
-  mdworkbench_process(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_mdworkbench_create.name, opt.rank);
+  mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD);
   PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time);
 
   double rate = o.res->result[0].rate;
diff --git a/src/phase_mdworkbench_delete.c b/src/phase_mdworkbench_delete.c
index af79b4f..e543a9d 100644
--- a/src/phase_mdworkbench_delete.c
+++ b/src/phase_mdworkbench_delete.c
@@ -3,48 +3,51 @@
 #include <mpi.h>
 
 #include <io500-phase.h>
-#include <phase_mdworkbench.h>
+#include <pfind-options.h>
+#include <phase_find.h>
 
-typedef struct{
-  int run;
-  char * command;
-  mdworkbench_results_t * res;
-} opt_mdworkbench_delete;
-
-static opt_mdworkbench_delete o;
+static opt_find of;
 
 static ini_option_t option[] = {
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},
+  {"external-script", "Set to an external script to perform the find phase", 0, INI_STRING, NULL, & of.ext_find},
+  {"external-mpi-args", "Startup arguments for external scripts, some MPI's may not support this!", 0, INI_STRING, "", & of.ext_mpi},
+  {"external-extra-args", "Extra arguments for the external scripts", 0, INI_STRING, "", & of.ext_args},
+  {"nproc", "Set the number of processes for pfind/the external script", 0, INI_UINT, NULL, & of.nproc},
+  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & of.run},
+  {"pfind-queue-length", "Pfind queue length", 0, INI_INT, "10000", & of.pfind_queue_length},
+  {"pfind-steal-next", "Pfind Steal from next", 0, INI_BOOL, "FALSE", & of.pfind_steal_from_next},
+  {"pfind-parallelize-single-dir-access-using-hashing", "Parallelize the readdir by using hashing. Your system must support this!", 0, INI_BOOL, "FALSE", &  of.pfind_par_single_dir_access_hash},
   {NULL} };
 
-static double run(void){
-  opt_mdworkbench d = mdworkbench_o;
-
-  u_argv_t * argv = u_argv_create();
-  mdworkbench_add_params(argv, 0);
-  u_argv_push(argv, "-3");
-
-  o.command = u_flatten_argv(argv);
-  PRINT_PAIR("exe", "%s\n", o.command);
-  if(opt.dry_run || d.run == 0){
-    u_argv_free(argv);
-    return 0;
+static void validate(void){
+  if(of.ext_find){
+    char args[PATH_MAX];
+    sprintf(args, "%s/mdworkbench/ -E -e", opt.datadir);
+    external_find_prepare_arguments(args, & of);
+  }else{
+    u_argv_t * argv = u_argv_create();
+    u_argv_push(argv, "./pfind");
+    u_argv_push_printf(argv, "%s/mdworkbench", opt.datadir);  
+    u_argv_push(argv, "-C");
+    u_argv_push(argv, "-E");
+    u_argv_push(argv, "-e");
+    
+    pfind_prepare_arguments(argv, & of);
   }
-  FILE * out = u_res_file_prep(p_mdworkbench_delete.name);
-  mdworkbench_process(argv, out, & o.res);
-  PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time);
+}
 
-  double rate = o.res->result[0].rate;
-  return rate / 1000.0;
+static double run(void){
+  if(opt.dry_run || of.run == 0) return 0.0;
+  return run_find(p_mdworkbench_delete.name, & of);
 }
 
 
 u_phase_t p_mdworkbench_delete = {
-  "mdworkbench-delete",
+  "mdworkbench-find-delete",
   IO500_PHASE_REMOVE | IO500_PHASE_FLAG_OPTIONAL,
   option,
-  NULL,
+  validate,
   run,
   0,
-  .group = IO500_NO_SCORE,
+  .group = IO500_SCORE_MD,
 };
diff --git a/src/phase_mdworkbench_delete_mdw.c b/src/phase_mdworkbench_delete_mdw.c
new file mode 100644
index 0000000..b7df011
--- /dev/null
+++ b/src/phase_mdworkbench_delete_mdw.c
@@ -0,0 +1,50 @@
+#include <sys/stat.h>
+#include <unistd.h>
+#include <mpi.h>
+
+#include <io500-phase.h>
+#include <phase_mdworkbench.h>
+
+typedef struct{
+  int run;
+  char * command;
+  mdworkbench_results_t * res;
+} opt_mdworkbench_delete;
+
+static opt_mdworkbench_delete o;
+
+static ini_option_t option[] = {
+  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},
+  {NULL} };
+
+static void validate(void){
+}
+
+static double run(void){
+  opt_mdworkbench d = mdworkbench_o;
+  u_argv_t * argv = u_argv_create();
+  mdworkbench_add_params(argv, 0);
+  u_argv_push(argv, "-3");
+  o.command = u_flatten_argv(argv);
+  
+  if(opt.dry_run || d.run == 0){
+    return 0;
+  }  
+  FILE * out = u_res_file_prep(p_mdworkbench_delete.name, opt.rank);
+  mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD);
+  PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time);
+  
+  double rate = o.res->result[0].rate;
+  return rate / 1000.0;    
+}
+
+
+u_phase_t p_mdworkbench_delete = {
+  "mdworkbench-delete",
+  IO500_PHASE_REMOVE | IO500_PHASE_FLAG_OPTIONAL,
+  option,
+  validate,
+  run,
+  0,
+  .group = IO500_NO_SCORE,
+};
diff --git a/src/phase_opt.c b/src/phase_opt.c
index ea8f176..fc018e4 100644
--- a/src/phase_opt.c
+++ b/src/phase_opt.c
@@ -10,7 +10,8 @@ static ini_option_t option[] = {
   {"api", "The general API for the tests (to create/delete the datadir, extra options will be passed to IOR/mdtest)", 0, INI_STRING, "POSIX", & opt.api},
   {"drop-caches", "Purge the caches, this is useful for testing and needed for single node runs", 0, INI_BOOL, "FALSE", & opt.drop_caches},
   {"drop-caches-cmd", "Cache purging command, invoked before each I/O phase", 0, INI_STRING, "sudo -n bash -c \"echo 3 > /proc/sys/vm/drop_caches\"", & opt.drop_caches_cmd},
-  {"io-buffers-on-gpu", "Allocate the I/O buffers on the GPU", 0, INI_BOOL, "FALSE", & opt.io_buffers_on_gpu},
+  {"allocateBufferDevice", "Allocate the I/O buffers on 0=CPU, 1=Managed Check CPU, 2=Managed Check GPU, 3=GPU", 0, INI_INT, "0", & opt.allocateBufferDevice},
+  {"gpuDirect", "Use GPUDirect only useful with Buffers > 0", 0, INI_BOOL, "FALSE", & opt.gpuDirect},
   {"verbosity", "The verbosity level between 1 and 10", 0, INI_UINT, "1", & opt.verbosity},
   {"scc", "Use the rules for the Student Cluster Competition", 0, INI_BOOL, "FALSE", & opt.scc},
   {"dataPacketType", "Type of packet that will be created [timestamp|offset|incompressible|random]", 0, INI_STRING, "timestamp", & opt.dataPacketType},
diff --git a/src/util.c b/src/util.c
index b104f7b..caebfa3 100644
--- a/src/util.c
+++ b/src/util.c
@@ -235,9 +235,9 @@ void u_print_timestamp(FILE * out){
   fprintf(out, "%s", buffer);
 }
 
-FILE * u_res_file_prep(char const * name){
+FILE * u_res_file_prep(char const * name, int rank){
   FILE * out = stdout;
-  if(opt.rank == 0){
+  if(rank == 0){
     char fname[PATH_MAX];
     sprintf(fname, "%s/%s.txt", opt.resdir, name);
     INFO_PAIR("result-file", "%s\n", fname);