diff --git a/Makefile b/Makefile
index 6f9561b..e7dd3c4 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ vpath %.h $(SEARCHPATH)
 DEPS += io500-util.h io500-debug.h io500-opt.h
 OBJSO += util.o
 OBJSO += ini-parse.o phase_dbg.o phase_opt.o phase_timestamp.o
-OBJSO += phase_find.o phase_find_easy.o phase_find_hard.o phase_ior_easy.o phase_ior_easy_read.o phase_mdtest.o phase_ior.o phase_ior_easy_write.o phase_ior_hard.o phase_ior_hard_read.o phase_ior_hard_write.o phase_mdtest_easy.o phase_mdtest_easy_delete.o phase_mdtest_easy_stat.o phase_mdtest_easy_write.o phase_mdtest_hard.o phase_mdtest_hard_delete.o phase_mdtest_hard_read.o phase_mdtest_hard_stat.o phase_mdtest_hard_write.o phase_ior_rnd1MB.o phase_ior_rnd4K.o phase_ior_rnd_write4K.o phase_ior_rnd_read4K.o phase_ior_rnd_write1MB.o phase_ior_rnd_read1MB.o phase_mdworkbench.o phase_mdworkbench_create.o phase_mdworkbench_delete.o phase_mdworkbench_bench.o phase_ior_rnd_read4k-easywrite.o
+OBJSO += phase_find.o phase_ior_easy.o phase_ior_easy_read.o phase_mdtest.o phase_ior.o phase_ior_easy_write.o phase_ior_hard.o phase_ior_hard_read.o phase_ior_hard_write.o phase_mdtest_easy.o phase_mdtest_easy_delete.o phase_mdtest_easy_stat.o phase_mdtest_easy_write.o phase_mdtest_hard.o phase_mdtest_hard_delete.o phase_mdtest_hard_read.o phase_mdtest_hard_stat.o phase_mdtest_hard_write.o phase_ior_rnd_read4k-easywrite.o
 
 OBJS = $(patsubst %,./build/%,$(OBJSO))
 
diff --git a/config-debug-run.ini b/config-debug-run.ini
index 2750988..481698c 100644
--- a/config-debug-run.ini
+++ b/config-debug-run.ini
@@ -5,5 +5,4 @@ timestamp-resultdir = FALSE
 
 [debug]
 stonewall-time = 1
-pause-dir = ./pause/
 
diff --git a/find/sfind.sh b/find/sfind.sh
index b76e8fd..e7bf349 100755
--- a/find/sfind.sh
+++ b/find/sfind.sh
@@ -9,6 +9,10 @@
 # When you are done pring 'x/y' where x is matched files and y is total files searched.
 # There is a parallel version also install in bin that might be better
 
+# The script should support the following extra options
+# -e for deletion of found files 
+# -E for deletion of directories
+
 function parse_rates {
   #find -D rates gives a weird thing like this:
   #Predicate success rates after completion
@@ -18,7 +22,20 @@ function parse_rates {
   echo $rates | tr " " "\n" | grep '/' | $1 -1 | cut -d \/ -f 2 | cut -d = -f 1
 }
 
-rates=`find -D rates $* 2>&1 | grep -A1 Predicate | tail -1` 
+IN=""
+for var in "$@"
+do
+    if [[ $var == "-e" ]] ; then 
+      continue
+    fi
+    if [[ $var == "-E" ]] ; then 
+      IN="$IN -exec rm -f {} ;"
+      continue
+    fi
+    IN="$IN $var"
+done
+
+rates=$(find -D rates $IN 2>&1 | grep -A1 Predicate | tail -1)
 total_files=$(parse_rates 'head')
 match_files=$(parse_rates 'tail')
 echo "MATCHED $match_files/$total_files"
diff --git a/include/io500-opt.h b/include/io500-opt.h
index 3e1ec16..faf0433 100644
--- a/include/io500-opt.h
+++ b/include/io500-opt.h
@@ -27,7 +27,8 @@ typedef struct{
 
   int rank;
   int mpi_size;
-  int io_buffers_on_gpu; /* are the I/O buffers to be allocated on a GPU */
+  int allocateBufferDevice; /* where are the I/O buffers allocated and processed */
+  int gpuDirect; /* useGPUDirect */
 
   char * api;
   char * apiArgs; // for IOR and mdtest
diff --git a/include/io500-phase.h b/include/io500-phase.h
index c558455..e180205 100644
--- a/include/io500-phase.h
+++ b/include/io500-phase.h
@@ -8,7 +8,7 @@
 #define DEBUG_OPTION "INVALIDATES RUN; FOR DEBUGGING."
 
 typedef enum {
-  IO500_NO_SCORE,
+  IO500_NO_SCORE = 0,
   IO500_SCORE_MD,
   IO500_SCORE_BW,
   IO500_SCORE_LAST
@@ -20,8 +20,7 @@ typedef enum {
   IO500_PHASE_WRITE = 2,
   IO500_PHASE_READ = 4,
   IO500_PHASE_UPDATE = 8,
-  IO500_PHASE_REMOVE = 16,
-  IO500_PHASE_FLAG_OPTIONAL = 32
+  IO500_PHASE_REMOVE = 16
 } io500_phase_type;
 
 typedef struct{
@@ -37,7 +36,7 @@ typedef struct{
   io500_phase_score_group group;
 } u_phase_t;
 
-#define IO500_PHASES (2 + 1 + 2*3 + 3 + 3 + 4 + 5 + 3 + 4 + 1)
+#define IO500_PHASES (2 + 1 + 1 + 3 + 4 + 4 + 5)
 
 extern u_phase_t p_opt;
 extern u_phase_t p_debug;
@@ -45,26 +44,11 @@ extern u_phase_t p_debug;
 extern u_phase_t p_timestamp;
 
 extern u_phase_t p_find;
-extern u_phase_t p_find_easy;
-extern u_phase_t p_find_hard;
 
 extern u_phase_t p_ior_hard;
 extern u_phase_t p_ior_hard_write;
 extern u_phase_t p_ior_hard_read;
 
-extern u_phase_t p_ior_rnd4K;
-extern u_phase_t p_ior_rnd4K_write;
-extern u_phase_t p_ior_rnd4K_read;
-
-extern u_phase_t p_ior_rnd1MB;
-extern u_phase_t p_ior_rnd1MB_write;
-extern u_phase_t p_ior_rnd1MB_read;
-
-extern u_phase_t p_mdworkbench;
-extern u_phase_t p_mdworkbench_create;
-extern u_phase_t p_mdworkbench_bench;
-extern u_phase_t p_mdworkbench_delete;
-
 extern u_phase_t p_ior_easy;
 extern u_phase_t p_ior_easy_write;
 extern u_phase_t p_ior_easy_read;
diff --git a/include/io500-util.h b/include/io500-util.h
index 997cd14..23b786d 100644
--- a/include/io500-util.h
+++ b/include/io500-util.h
@@ -82,7 +82,7 @@ void u_print_timestamp(FILE * out);
 void * u_malloc(int size);
 
 
-FILE * u_res_file_prep(char const * name);
+FILE * u_res_file_prep(char const * name, int rank);
 void u_res_file_close(FILE * out);
 
 uint32_t u_phase_unique_random_number(char const * phase_name);
diff --git a/prepare.sh b/prepare.sh
index cc3f20d..8826a4e 100755
--- a/prepare.sh
+++ b/prepare.sh
@@ -7,7 +7,7 @@ echo It will also attempt to build the benchmarks
 echo It will output OK at the end if builds succeed
 echo
 
-IOR_HASH=348572ccfe8963
+IOR_HASH=de7dc94f468d8cb8f
 PFIND_HASH=aaba722a178
 
 INSTALL_DIR=$PWD
diff --git a/src/main.c b/src/main.c
index 389f507..aba1331 100644
--- a/src/main.c
+++ b/src/main.c
@@ -15,14 +15,20 @@
 
 #define INVALID_RUN(...) do{ if (opt.rank == 0){fprintf(file_out, "; ERROR INVALID "__VA_ARGS__); printf("ERROR INVALID (%s:%d) ", __FILE__, __LINE__); printf(__VA_ARGS__); fflush(file_out); opt.is_valid_run = 0; } }while(0);
 
-#define RUN_PHASE(phase) ( ! (phase->type & IO500_PHASE_FLAG_OPTIONAL && opt.mode == IO500_MODE_STANDARD) )
 
 FILE* file_out = NULL;
 
 static char const * io500_phase_str[IO500_SCORE_LAST] = {
   "NO SCORE",
   "MD",
-  "BW"};
+  "BW"
+  };
+
+static char const * io500_unit_str[IO500_SCORE_LAST] = {
+  "     ",
+  "kIOPS",
+  "GiB/s"
+};
 
 extern int rank;
 
@@ -125,16 +131,18 @@ static void print_cfg_hash(FILE * out, ini_section_t ** cfg){
 
 #define dupprintf(...) do{ if(opt.rank == 0) { fprintf(res_summary, __VA_ARGS__); printf(__VA_ARGS__); } }while(0);
 
-static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t * hash){
+static double calc_score(double scores[IO500_SCORE_LAST], uint32_t * hash){
   double overall_score = 1;
-  for(io500_phase_score_group g=1; g < IO500_SCORE_LAST; g++){
+  int groups = IO500_SCORE_BW;
+  memset(scores, 0, sizeof(double) * IO500_SCORE_LAST);
+  for(io500_phase_score_group g=1; g <= groups; g++){
     char score_string[2048];
     char *p = score_string;
     double score = 1;
     int numbers = 0;
-    p += sprintf(p, " %s = (", io500_phase_str[g]);
+    p += sprintf(p, " %s = (1.0", io500_phase_str[g]);
     for(int i=0; i < IO500_PHASES; i++){
-      if(phases[i]->group == g && (extended || ! (phases[i]->type & IO500_PHASE_FLAG_OPTIONAL)) ){
+      if( phases[i]->group == g ){
         double t = phases[i]->score;
         if(t <= 0){
           continue;
@@ -157,16 +165,86 @@ static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t
 
     overall_score *= score;
   }
-  return sqrt(overall_score);
+  return pow(overall_score, 1.0 / groups);
 }
 
-static const char * io500_mode_str(io500_mode mode){
-  switch(mode){
-    case (IO500_MODE_STANDARD): return "standard";
-    case (IO500_MODE_EXTENDED): return "extended";
-    default:
-      return "unknown";
+typedef struct {
+  int run;
+  int valid;
+  double score;
+  double time;
+} io500_phase_result_t;
+
+typedef struct{
+  double score[IO500_SCORE_LAST];
+  double scoreX[IO500_SCORE_LAST];
+  double runtime;
+} io500_overall_results_t;
+
+static io500_phase_result_t io500_results[IO500_PHASES];
+static io500_overall_results_t io500_overall_results;
+
+static void dump_io500_phase(FILE * o, char * name){
+  int i;
+  u_phase_t * phase = NULL;
+  io500_phase_result_t * r;
+  for(i=0; i < IO500_PHASES; i++){
+    if (strcmp(phases[i]->name, name) == 0) {
+      phase = phases[i];
+      r = & io500_results[i];
+      if(! r->run) return;
+      break;
+    }
   }
+  
+  char score_str[40];
+  sprintf(score_str, "%f", r->score);
+  fprintf(o, "%25s\t%15s\t%s\t%.3fs\t%s\n", phase->name, score_str, io500_unit_str[phase->group], r->time, r->valid ? "" : "INVALID");
+}
+
+static void dump_io500_results(void){
+    FILE * res;
+    char file[PATH_MAX];
+    sprintf(file, "%s/result_pretty.txt", opt.resdir);
+    res = fopen(file, "w");
+    if(! res){
+      FATAL("Could not open \"%s\" for writing (%s)\n", file, strerror(errno));
+    } 
+    
+    fprintf(res, "%25s\t%fs\n", "RUNTIME", io500_overall_results.runtime);
+    fprintf(res, "%25s", "SCORE");
+    for(int i=0; i < IO500_SCORE_LAST; i++){
+      fprintf(res, "\t%f\t%s\t", io500_overall_results.score[i], io500_unit_str[i]);
+    }
+    fprintf(res, "\n");
+    
+    if(opt.mode == IO500_MODE_EXTENDED){
+      fprintf(res, "%25s", "SCOREX");
+      for(int i=0; i < IO500_SCORE_LAST; i++){
+        fprintf(res, "\t%f\t%s\t", io500_overall_results.scoreX[i], io500_unit_str[i]);
+      }
+      fprintf(res, "\n");
+    }
+    
+    // ior easy
+    dump_io500_phase(res, "ior-easy-write");
+    dump_io500_phase(res, "ior-easy-read");
+    // ior hard
+    dump_io500_phase(res, "ior-hard-write");
+    dump_io500_phase(res, "ior-hard-read");  
+    // MD easy
+    dump_io500_phase(res, "mdtest-easy-write");
+    dump_io500_phase(res, "mdtest-easy-stat");
+    dump_io500_phase(res, "mdtest-easy-delete");
+    // MD hard
+    dump_io500_phase(res, "mdtest-hard-write");
+    dump_io500_phase(res, "mdtest-hard-stat");
+    dump_io500_phase(res, "mdtest-hard-read");
+    dump_io500_phase(res, "mdtest-hard-delete");
+    // find
+    dump_io500_phase(res, "find");
+    
+    fclose(res);
 }
 
 int main(int argc, char ** argv){
@@ -201,6 +279,9 @@ int main(int argc, char ** argv){
     }
     goto out;
   }
+  
+  memset(io500_results, 0, sizeof(io500_results));
+  double t_io500_start = GetTimeStamp();
 
   mpi_init = 1;
   MPI_Init(& argc, & argv);
@@ -326,7 +407,6 @@ int main(int argc, char ** argv){
   PRINT_PAIR("version", "%s\n", VERSION);
   print_cfg_hash(file_out, cfg);
   PRINT_PAIR("result-dir", "%s\n", opt.resdir);
-  PRINT_PAIR("mode", "%s\n", io500_mode_str(opt.mode));
 
   if(opt.rank == 0){
     // create configuration in result directory to ensure it is preserved
@@ -356,13 +436,12 @@ int main(int argc, char ** argv){
 
   // manage a hash for the scores
   uint32_t score_hash = 0;
-  uint32_t score_extended_hash = 0;
   u_hash_update_key_val(& score_hash, "version", VERSION);
 
-  dupprintf("IO500 version %s (%s)\n", VERSION, io500_mode_str(opt.mode));
+  dupprintf("IO500 version %s\n", VERSION);
 
   for(int i=0; i < IO500_PHASES; i++){
-    if(RUN_PHASE(phases[i]) && phases[i]->validate){
+    if((phases[i]) && phases[i]->validate){
       phases[i]->validate();
     }
   }
@@ -372,11 +451,12 @@ int main(int argc, char ** argv){
 
   for(int i=0; i < IO500_PHASES; i++){
     u_phase_t * phase = phases[i];
-    if(! phase->run) continue;
+    if(! phase->run) continue;        
     if( cleanup_only && ! (phase->type & IO500_PHASE_REMOVE) ) continue;
-    if(! RUN_PHASE(phase)){
+    if(! (phase)){
       continue;
     }
+    io500_results[i].run = 1;
 
     if(opt.drop_caches && ! (phase->type & IO500_PHASE_DUMMY) ){
       DEBUG_INFO("Dropping cache\n");
@@ -444,10 +524,12 @@ int main(int argc, char ** argv){
       }
       if(! opt.is_valid_phase){
         opt.is_valid_extended_run = 0;
-        if(! (phase->type & IO500_PHASE_FLAG_OPTIONAL)){
-          opt.is_valid_run = 0;
-        }
       }
+    
+      io500_results[i].score = score;
+      io500_results[i].time = runtime;
+      io500_results[i].valid = opt.is_valid_phase;
+      
       if(! (phase->type & IO500_PHASE_DUMMY)){
         PRINT_PAIR("score", "%f\n", score);
       }
@@ -459,24 +541,17 @@ int main(int argc, char ** argv){
       }else{
         dupprintf("[      ]");
       }
-      dupprintf(" %20s %15s %s : time %.3f seconds%s\n", phase->name, score_str, phase->name[0] == 'i' ? "GiB/s" : "kIOPS", runtime, valid_str);
-    }
-    if(phase->group > IO500_NO_SCORE){
-      if(phase->type & IO500_PHASE_FLAG_OPTIONAL){
-        u_hash_update_key_val_dbl(& score_extended_hash, phase->name, score);
-      }else{
-        u_hash_update_key_val_dbl(& score_hash, phase->name, score);
-      }
-    }
-    phases[i]->score = score;
-
-
-    if(opt.verbosity > 0 && opt.rank == 0){
+      dupprintf(" %25s %15s %s : time %.3f seconds%s\n", phase->name, score_str, io500_unit_str[phase->group], runtime, valid_str);
+      
       PRINT_PAIR("t_delta", "%.4f\n", runtime);
       PRINT_PAIR_HEADER("t_end");
       u_print_timestamp(file_out);
       fprintf(file_out, "\n");
     }
+    if(phase->group > IO500_NO_SCORE){
+      u_hash_update_key_val_dbl(& score_hash, phase->name, score);
+    }
+    phases[i]->score = score;
   }
 
   MPI_Barrier(MPI_COMM_WORLD);
@@ -485,7 +560,7 @@ int main(int argc, char ** argv){
     // compute the overall score
     fprintf(file_out, "\n[SCORE]\n");
     double scores[IO500_SCORE_LAST];
-    double overall_score = calc_score(scores, 0, & score_hash);
+    double overall_score = calc_score(scores, & score_hash);
     PRINT_PAIR("SCORE", "%f%s\n", overall_score, valid_str);
     u_hash_update_key_val_dbl(& score_hash, "SCORE", overall_score);
     if( ! opt.is_valid_run ){
@@ -494,26 +569,17 @@ int main(int argc, char ** argv){
     PRINT_PAIR("hash", "%X\n", (int) score_hash);
     dupprintf("[SCORE ] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n",
     scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_score, valid_str);
-
-    // extended run
-    if(opt.mode == IO500_MODE_EXTENDED){
-      valid_str = opt.is_valid_extended_run ? "" : " [INVALID]";
-      fprintf(file_out, "\n[SCOREX]\n");
-      double overall_extended_score = calc_score(scores, 1, & score_extended_hash);
-      u_hash_update_key_val_dbl(& score_extended_hash, "SCORE", overall_extended_score);
-      if( ! opt.is_valid_extended_run ){
-        u_hash_update_key_val(& score_extended_hash, "valid", "NO");
-      }
-      PRINT_PAIR("SCORE", "%f%s\n", overall_extended_score, valid_str);
-      PRINT_PAIR("hash", "%X\n", (int) score_extended_hash);
-      dupprintf("[SCOREX] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n", scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_extended_score, valid_str);
-    }
-
+    
+    scores[IO500_NO_SCORE] = overall_score;
+    memcpy(io500_overall_results.score, scores, sizeof(scores));
+    
+    io500_overall_results.runtime = GetTimeStamp() - t_io500_start;
+    dump_io500_results();
     printf("\nThe result files are stored in the directory: %s\n", opt.resdir);
   }
 
   for(int i=0; i < IO500_PHASES; i++){
-    if(RUN_PHASE(phases[i]) && phases[i]->cleanup){
+    if((phases[i]) && phases[i]->cleanup){
       phases[i]->cleanup();
     }
   }
diff --git a/src/phase-definitions.h b/src/phase-definitions.h
index 2f97541..22aeea3 100644
--- a/src/phase-definitions.h
+++ b/src/phase-definitions.h
@@ -5,22 +5,11 @@ static u_phase_t * phases[IO500_PHASES] = {
   & p_ior_easy,
   & p_ior_easy_write,
 
-  & p_ior_rnd4K,
-  & p_ior_rnd4K_write,
-
   & p_mdtest_easy,
   & p_mdtest_easy_write,
   
-  & p_ior_rnd1MB,
-  & p_ior_rnd1MB_write,
-
-  & p_mdworkbench,
-  & p_mdworkbench_create,
-
   & p_timestamp,
 
-  & p_find_easy,
-
   & p_ior_hard,
   & p_ior_hard_write,
 
@@ -29,21 +18,12 @@ static u_phase_t * phases[IO500_PHASES] = {
 
   & p_find,
 
-  & p_ior_rnd4K_read,
-  & p_ior_rnd1MB_read,
-
-  & p_find_hard,
-
-  & p_mdworkbench_bench,
-
   & p_ior_easy_read,
   & p_mdtest_easy_stat,
 
   & p_ior_hard_read,
   & p_mdtest_hard_stat,
 
-  & p_mdworkbench_delete,
-
   & p_mdtest_easy_delete,
   & p_mdtest_hard_read,
   & p_mdtest_hard_delete,
diff --git a/src/phase_find.c b/src/phase_find.c
index 932ee99..c7e31af 100644
--- a/src/phase_find.c
+++ b/src/phase_find.c
@@ -60,7 +60,7 @@ double run_find(const char * phase_name, opt_find * of){
         fprintf(fd, "runtime: %f rate: %f\n", of->pfind_res->runtime, of->pfind_res->rate);
         fprintf(fd, "rank, errors, unknown, found, total, checked, job steal msgs received, work items send, job steal msgs send, work items stolen, time spend in job stealing in s, number of completion tokens send\n");
         fprintf(fd, "0, %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64, res->errors, res->unknown_file, res->found_files, res->total_files, res->checked_dirents);
-        fprintf(fd, ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %.3fs, %"PRIu64"\n", res->monitor.job_steal_inbound, res->monitor.work_send, res->monitor.job_steal_tries, res->monitor.work_stolen, res->monitor.job_steal_mpitime_us / 1000000.0, res->monitor.completion_tokens_send);
+        fprintf(fd, ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %.3f, %"PRIu64"\n", res->monitor.job_steal_inbound, res->monitor.work_send, res->monitor.job_steal_tries, res->monitor.work_stolen, res->monitor.job_steal_mpitime_us / 1000000.0, res->monitor.completion_tokens_send);
         for(int i=1; i < of->nproc; i++){
           MPI_Recv(& res->errors, 5, MPI_LONG_LONG_INT, i, 4712, of->pfind_com, MPI_STATUS_IGNORE);
           fprintf(fd, "%d, %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64, i, res->errors, res->unknown_file, res->found_files, res->total_files, res->checked_dirents);
@@ -98,7 +98,7 @@ double run_find(const char * phase_name, opt_find * of){
       ERROR("Failed to run find command: \"%s\" error: %s\n", of->command, strerror(errno));
       return -1;
     }
-    char line[1024];
+    char line[PATH_MAX];
     uint64_t hits = 0;
     *line = '\0';
     while (fgets(line, sizeof(line), fp) != NULL) {
@@ -175,16 +175,7 @@ static void validate(void){
     u_argv_push(argv, "-name");
     u_argv_push(argv, "*01*");
     u_argv_push(argv, "-C");
-    if(of.pfind_steal_from_next){
-      u_argv_push(argv, "-N");
-    }
-    if(of.pfind_par_single_dir_access_hash){
-      u_argv_push(argv, "-H");
-      u_argv_push(argv, "1");
-    }
-    u_argv_push(argv, "-q");
-    u_argv_push_printf(argv, "%d", of.pfind_queue_length);
-
+    
     pfind_prepare_arguments(argv, & of);
   }
 }
@@ -226,6 +217,16 @@ void pfind_prepare_arguments(u_argv_t * argv, opt_find * of){
         com = MPI_COMM_NULL;
       }
     }
+    
+    if(of->pfind_steal_from_next){
+      u_argv_push(argv, "-N");
+    }
+    if(of->pfind_par_single_dir_access_hash){
+      u_argv_push(argv, "-H");
+      u_argv_push(argv, "1");
+    }
+    u_argv_push(argv, "-q");
+    u_argv_push_printf(argv, "%d", of->pfind_queue_length);
 
     of->command = u_flatten_argv(argv);
     of->pfind_com = com;
diff --git a/src/phase_find_easy.c b/src/phase_find_easy.c
deleted file mode 100644
index b2f3236..0000000
--- a/src/phase_find_easy.c
+++ /dev/null
@@ -1,64 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-#include <mpi.h>
-#include <string.h>
-#include <errno.h>
-
-#include <pfind-options.h>
-#include <io500-phase.h>
-#include "phase_find.h"
-
-static opt_find of;
-
-static double run(void){
-  if(of.run == 0) return 0.0;
-  return run_find("find-easy", & of);
-}
-
-static ini_option_t option[] = {
-  {"external-script", "Set to an external script to perform the find phase", 0, INI_STRING, NULL, & of.ext_find},
-  {"external-mpi-args", "Startup arguments for external scripts, some MPI's may not support this!", 0, INI_STRING, "", & of.ext_mpi},
-  {"external-extra-args", "Extra arguments for the external scripts", 0, INI_STRING, "", & of.ext_args},
-  {"nproc", "Set the number of processes for pfind/the external script", 0, INI_UINT, NULL, & of.nproc},
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & of.run},
-  {"pfind-queue-length", "Pfind queue length", 0, INI_INT, "10000", & of.pfind_queue_length},
-  {"pfind-steal-next", "Pfind Steal from next", 0, INI_BOOL, "FALSE", & of.pfind_steal_from_next},
-  {"pfind-parallelize-single-dir-access-using-hashing", "Parallelize the readdir by using hashing. Your system must support this!", 0, INI_BOOL, "FALSE", &  of.pfind_par_single_dir_access_hash},
-  {NULL} };
-
-static void validate(void){
-  if(of.run == 0) return;
-  if(of.ext_find){
-    char args[1024];
-    sprintf(args, "%s/mdtest-easy/ -name \"*01*\"", opt.datadir);
-    external_find_prepare_arguments(args, & of);
-  }else{
-    u_argv_t * argv = u_argv_create();
-    u_argv_push(argv, "./pfind");
-    u_argv_push_printf(argv, "%s/mdtest-easy/", opt.datadir);
-    u_argv_push(argv, "-name");
-    u_argv_push(argv, "*01*");
-    u_argv_push(argv, "-C");
-    if(of.pfind_steal_from_next){
-      u_argv_push(argv, "-N");
-    }
-    if(of.pfind_par_single_dir_access_hash){
-      u_argv_push(argv, "-H");
-      u_argv_push(argv, "1");
-    }
-    u_argv_push(argv, "-q");
-    u_argv_push_printf(argv, "%d", of.pfind_queue_length);
-
-    pfind_prepare_arguments(argv, & of);
-  }
-}
-
-u_phase_t p_find_easy = {
-  "find-easy",
-  IO500_PHASE_READ | IO500_PHASE_FLAG_OPTIONAL,
-  option,
-  validate,
-  run,
-  0,
-  .group = IO500_SCORE_MD
-};
diff --git a/src/phase_find_hard.c b/src/phase_find_hard.c
deleted file mode 100644
index 2481a32..0000000
--- a/src/phase_find_hard.c
+++ /dev/null
@@ -1,68 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-#include <mpi.h>
-#include <string.h>
-#include <errno.h>
-
-#include <pfind-options.h>
-#include <io500-phase.h>
-#include "phase_find.h"
-
-static opt_find of;
-
-static double run(void){
-  if(of.run == 0) return 0.0;
-  return run_find("find-hard", & of);;
-}
-
-static ini_option_t option[] = {
-  {"external-script", "Set to an external script to perform the find phase", 0, INI_STRING, NULL, & of.ext_find},
-  {"external-mpi-args", "Startup arguments for external scripts, some MPI's may not support this!", 0, INI_STRING, "", & of.ext_mpi},
-  {"external-extra-args", "Extra arguments for the external scripts", 0, INI_STRING, "", & of.ext_args},
-  {"nproc", "Set the number of processes for pfind/the external script", 0, INI_UINT, NULL, & of.nproc},
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & of.run},
-  {"pfind-queue-length", "Pfind queue length", 0, INI_INT, "10000", & of.pfind_queue_length},
-  {"pfind-steal-next", "Pfind Steal from next", 0, INI_BOOL, "FALSE", & of.pfind_steal_from_next},
-  {"pfind-parallelize-single-dir-access-using-hashing", "Parallelize the readdir by using hashing. Your system must support this!", 0, INI_BOOL, "FALSE", &  of.pfind_par_single_dir_access_hash},
-  {NULL} };
-
-static void validate(void){
-  if(of.run == 0) return;
-  if(of.ext_find){
-    char args[1024];
-    sprintf(args, "%s/mdtest-hard/ -newer %s/timestampfile -size 3901c -name \"*01*\"", opt.datadir, opt.resdir);
-    external_find_prepare_arguments(args, & of);
-  }else{
-    u_argv_t * argv = u_argv_create();
-    u_argv_push(argv, "./pfind");
-    u_argv_push_printf(argv, "%s/mdtest-hard/", opt.datadir);
-    u_argv_push(argv, "-newer");
-    u_argv_push_printf(argv, "%s/timestampfile", opt.resdir);
-    u_argv_push(argv, "-size");
-    u_argv_push(argv, "3901c");
-    u_argv_push(argv, "-name");
-    u_argv_push(argv, "*01*");
-    u_argv_push(argv, "-C");
-    if(of.pfind_steal_from_next){
-      u_argv_push(argv, "-N");
-    }
-    if(of.pfind_par_single_dir_access_hash){
-      u_argv_push(argv, "-H");
-      u_argv_push(argv, "1");
-    }
-    u_argv_push(argv, "-q");
-    u_argv_push_printf(argv, "%d", of.pfind_queue_length);
-
-    pfind_prepare_arguments(argv, & of);
-  }
-}
-
-u_phase_t p_find_hard = {
-  "find-hard",
-  IO500_PHASE_READ | IO500_PHASE_FLAG_OPTIONAL,
-  option,
-  validate,
-  run,
-  0,
-  .group = IO500_SCORE_MD
-};
diff --git a/src/phase_ior.c b/src/phase_ior.c
index 2a0ed18..494eacb 100644
--- a/src/phase_ior.c
+++ b/src/phase_ior.c
@@ -2,8 +2,8 @@
 #include <assert.h>
 
 
-double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){
-  IOR_test_t * test = ior_run(argv->size, argv->vector, MPI_COMM_WORLD, out);
+double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com){
+  IOR_test_t * test = ior_run(argv->size, argv->vector, com, out);
   assert(test);
   IOR_results_t * res = test->results;
   assert(res);
@@ -31,8 +31,8 @@ double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){
   return tp;
 }
 
-double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){
-  IOR_results_t * res = ior_run(argv->size, argv->vector, MPI_COMM_WORLD, out)->results;
+double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com){
+  IOR_results_t * res = ior_run(argv->size, argv->vector, com, out)->results;
   u_res_file_close(out);
   u_argv_free(argv);
 
diff --git a/src/phase_ior.h b/src/phase_ior.h
index 3df6888..c4f56b0 100644
--- a/src/phase_ior.h
+++ b/src/phase_ior.h
@@ -34,6 +34,7 @@ typedef struct{
 
   int random_prefill_bytes;
   uint64_t block_size;
+  int segments;
   int verbosity;
 } opt_ior_rnd;
 
@@ -41,13 +42,13 @@ extern opt_ior_rnd ior_rnd4K_o;
 extern opt_ior_rnd ior_rnd1MB_o;
 
 
-void ior_easy_add_params(u_argv_t * argv, int addStdFlags);
+void ior_easy_add_params(u_argv_t * argv, int useStatusFile);
 void ior_hard_add_params(u_argv_t * argv);
 void ior_rnd4K_add_params(u_argv_t * argv);
 void ior_rnd1MB_add_params(u_argv_t * argv);
 
 // Generic helpers
-double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out);
-double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out);
+double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com);
+double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com);
 
 #endif
diff --git a/src/phase_ior_easy.c b/src/phase_ior_easy.c
index 3b224a2..a4ee5b1 100644
--- a/src/phase_ior_easy.c
+++ b/src/phase_ior_easy.c
@@ -21,7 +21,7 @@ static void validate(void){
 }
 
 static void cleanup(void){
-  if (opt.dry_run) return;
+  if (opt.dry_run || ! ior_easy_o.run) return;
   
   if(opt.rank == 0){
     char filename[PATH_MAX];
@@ -42,7 +42,7 @@ static void cleanup(void){
   }
 }
 
-void ior_easy_add_params(u_argv_t * argv, int addStdFlags){
+void ior_easy_add_params(u_argv_t * argv, int useStatusFile){
   opt_ior_easy d = ior_easy_o;
 
   u_argv_push(argv, "./ior");
@@ -50,9 +50,12 @@ void ior_easy_add_params(u_argv_t * argv, int addStdFlags){
   for(int i=0; i < ior_easy_o.verbosity; i++){
     u_argv_push(argv, "-v");	/* verbose */
   }
-  if(opt.io_buffers_on_gpu){
+  if(opt.allocateBufferDevice){
     u_argv_push(argv, "-O");
-    u_argv_push(argv, "allocateBufferOnGPU=1");
+    u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
   u_argv_push(argv, "-C");	/* reorder tasks in constant order for read */
   u_argv_push(argv, "-Q");	/* task per node offset */
@@ -66,14 +69,14 @@ void ior_easy_add_params(u_argv_t * argv, int addStdFlags){
   u_argv_push(argv, "-e");	/* fsync upon write close */
   u_argv_push(argv, "-o");	/* filename for output file */
   u_argv_push_printf(argv, "%s/ior-easy/ior_file_easy", opt.datadir);
-  if(addStdFlags){
+  if(useStatusFile){
     u_argv_push(argv, "-O");	/* additional IOR options */
     u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-easy.stonewall", opt.resdir);
-    u_argv_push(argv, "-t");	/* transfer size */
-    u_argv_push(argv, d.transferSize);
-    u_argv_push(argv, "-b");	/* blocksize in bytes */
-    u_argv_push(argv, d.blockSize);
   }
+  u_argv_push(argv, "-t");	/* transfer size */
+  u_argv_push(argv, d.transferSize);
+  u_argv_push(argv, "-b");	/* blocksize in bytes */
+  u_argv_push(argv, d.blockSize);
 
   if(ior_easy_o.uniqueDir){
     u_argv_push(argv, "-u");	/* unique directory for each output file */
diff --git a/src/phase_ior_easy_read.c b/src/phase_ior_easy_read.c
index 543885d..59eddab 100644
--- a/src/phase_ior_easy_read.c
+++ b/src/phase_ior_easy_read.c
@@ -49,8 +49,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_easy_read.name);
-  return ior_process_read(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_easy_read.name, opt.rank);
+  return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 
diff --git a/src/phase_ior_easy_write.c b/src/phase_ior_easy_write.c
index 87e97b5..5d16b2b 100644
--- a/src/phase_ior_easy_write.c
+++ b/src/phase_ior_easy_write.c
@@ -51,8 +51,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_easy_write.name);
-  return ior_process_write(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_easy_write.name, opt.rank);
+  return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 u_phase_t p_ior_easy_write = {
diff --git a/src/phase_ior_hard.c b/src/phase_ior_hard.c
index 9162943..74425de 100644
--- a/src/phase_ior_hard.c
+++ b/src/phase_ior_hard.c
@@ -21,7 +21,7 @@ static void validate(void){
 }
 
 static void cleanup(void){
-  if (opt.dry_run) return;
+  if (opt.dry_run || ! ior_hard_o.run) return;
   
   if( opt.rank == 0){
     char filename[PATH_MAX];
@@ -40,9 +40,12 @@ void ior_hard_add_params(u_argv_t * argv){
   for(int i=0; i < ior_hard_o.verbosity; i++){
     u_argv_push(argv, "-v");	/* verbose */
   }
-  if(opt.io_buffers_on_gpu){
+  if(opt.allocateBufferDevice){
     u_argv_push(argv, "-O");
-    u_argv_push(argv, "allocateBufferOnGPU=1");
+    u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
   u_argv_push(argv, "-C");	/* reorder tasks in constant order for read */
   u_argv_push(argv, "-Q");	/* task per node offset */
diff --git a/src/phase_ior_hard_read.c b/src/phase_ior_hard_read.c
index 5b35ee7..8c59650 100644
--- a/src/phase_ior_hard_read.c
+++ b/src/phase_ior_hard_read.c
@@ -51,8 +51,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_hard_read.name);
-  return ior_process_read(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_hard_read.name, opt.rank);
+  return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 
diff --git a/src/phase_ior_hard_write.c b/src/phase_ior_hard_write.c
index 91ea568..660c2d4 100644
--- a/src/phase_ior_hard_write.c
+++ b/src/phase_ior_hard_write.c
@@ -52,8 +52,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_hard_write.name);
-  return ior_process_write(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_hard_write.name, opt.rank);
+  return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 u_phase_t p_ior_hard_write = {
diff --git a/src/phase_ior_rnd1MB.c b/src/phase_ior_rnd1MB.c
deleted file mode 100644
index 956c803..0000000
--- a/src/phase_ior_rnd1MB.c
+++ /dev/null
@@ -1,68 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include <phase_ior.h>
-#include <io500-phase.h>
-
-opt_ior_rnd ior_rnd1MB_o;
-
-static ini_option_t option[] = {
-  {"API", "The API to be used", 0, INI_STRING, NULL, & ior_rnd1MB_o.api},
-  {"blockSize", "Size of a random block, change only if explicitly allowed", 0, INI_UINT64, "1073741824", & ior_rnd1MB_o.block_size},
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_rnd1MB_o.run},
-  {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_rnd1MB_o.verbosity},
-  {"randomPrefill", "Prefill the file with this blocksize in bytes, e.g., 2097152", 0, INI_INT, "0", & ior_rnd1MB_o.random_prefill_bytes},
-  {NULL} };
-
-
-static void validate(void){
-  opt_ior_rnd d = ior_rnd1MB_o;
-  if(d.block_size < 1024){
-    FATAL("Random blocksize must be larger than 1024\n");
-  }
-  if(d.random_prefill_bytes > 0 && (d.block_size % d.random_prefill_bytes) != 0){
-    FATAL("Random prefill bytes must divide blocksize\n");
-  }
-  if(d.random_prefill_bytes > 0 && d.block_size < d.random_prefill_bytes){
-    FATAL("Random prefill bytes must be < blocksize\n");
-  }
-}
-
-
-void ior_rnd1MB_add_params(u_argv_t * argv){
-  opt_ior_rnd d = ior_rnd1MB_o;
-
-  u_argv_push(argv, "./ior");
-  u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType);
-  for(int i=0; i < ior_rnd1MB_o.verbosity; i++){
-    u_argv_push(argv, "-v");
-  }
-  if(opt.io_buffers_on_gpu){
-    u_argv_push(argv, "-O");
-    u_argv_push(argv, "allocateBufferOnGPU=1");
-  }
-  u_argv_push(argv, "-Q=1");
-  //u_argv_push(argv, "-F");
-  u_argv_push(argv, "-g");
-  int hash = u_phase_unique_random_number("ior-random1MB");
-  u_argv_push_printf(argv, "-G=%d", hash);
-  u_argv_push(argv, "-z");
-  u_argv_push(argv, "--random-offset-seed=11");
-  u_argv_push(argv, "-e");
-  u_argv_push_printf(argv, "-o=%s/ior-rnd1MB/file", opt.datadir);
-  u_argv_push(argv, "-O");
-  u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-rnd1MB.stonewall", opt.resdir );
-  u_argv_push(argv, "-k");
-  u_argv_push(argv, "-t=1048576");
-  u_argv_push_printf(argv, "-b=%ld", d.block_size);
-  u_argv_push_printf(argv, "-s=%d", 10000000);
-}
-
-u_phase_t p_ior_rnd1MB = {
-  "ior-rnd1MB",
-  IO500_PHASE_DUMMY,
-  option,
-  validate,
-  NULL,
-  NULL
-};
diff --git a/src/phase_ior_rnd4K.c b/src/phase_ior_rnd4K.c
deleted file mode 100644
index 6e6b773..0000000
--- a/src/phase_ior_rnd4K.c
+++ /dev/null
@@ -1,68 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include <phase_ior.h>
-#include <io500-phase.h>
-
-opt_ior_rnd ior_rnd4K_o;
-
-static ini_option_t option[] = {
-  {"API", "The API to be used", 0, INI_STRING, NULL, & ior_rnd4K_o.api},
-  {"blockSize", "Size of a random block, change only if explicitly allowed", 0, INI_UINT64, "1073741824", & ior_rnd4K_o.block_size},
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_rnd4K_o.run},
-  {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_rnd4K_o.verbosity},
-  {"randomPrefill", "Prefill the file with this blocksize in bytes, e.g., 2097152", 0, INI_INT, "0", & ior_rnd4K_o.random_prefill_bytes},
-  {NULL} };
-
-
-static void validate(void){
-  opt_ior_rnd d = ior_rnd4K_o;
-  if(d.block_size < 1024){
-    FATAL("Random blocksize must be larger than 1024\n");
-  }
-  if(d.random_prefill_bytes > 0 && (d.block_size % d.random_prefill_bytes) != 0){
-    FATAL("Random prefill bytes must divide blocksize\n");
-  }
-  if(d.random_prefill_bytes > 0 && d.block_size < d.random_prefill_bytes){
-    FATAL("Random prefill bytes must be < blocksize\n");
-  }
-}
-
-
-void ior_rnd4K_add_params(u_argv_t * argv){
-  opt_ior_rnd d = ior_rnd4K_o;
-
-  u_argv_push(argv, "./ior");
-  u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType);
-  for(int i=0; i < ior_rnd4K_o.verbosity; i++){
-    u_argv_push(argv, "-v");
-  }
-  if(opt.io_buffers_on_gpu){
-    u_argv_push(argv, "-O");
-    u_argv_push(argv, "allocateBufferOnGPU=1");
-  }
-  u_argv_push(argv, "-Q=1");
-  //u_argv_push(argv, "-F");
-  u_argv_push(argv, "-g");
-  int hash = u_phase_unique_random_number("ior-random4K");
-  u_argv_push_printf(argv, "-G=%d", hash);
-  u_argv_push(argv, "-z");
-  u_argv_push(argv, "--random-offset-seed=123");
-  u_argv_push(argv, "-e");
-  u_argv_push_printf(argv, "-o=%s/ior-rnd4K/file", opt.datadir);
-  u_argv_push(argv, "-O");
-  u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-rnd4K.stonewall", opt.resdir );
-  u_argv_push(argv, "-k");
-  u_argv_push(argv, "-t=4096");
-  u_argv_push_printf(argv, "-b=%ld", d.block_size);
-  u_argv_push_printf(argv, "-s=%d", 10000000);
-}
-
-u_phase_t p_ior_rnd4K = {
-  "ior-rnd4K",
-  IO500_PHASE_DUMMY,
-  option,
-  validate,
-  NULL,
-  NULL
-};
diff --git a/src/phase_ior_rnd_read1MB.c b/src/phase_ior_rnd_read1MB.c
deleted file mode 100644
index 0693d41..0000000
--- a/src/phase_ior_rnd_read1MB.c
+++ /dev/null
@@ -1,60 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-#include <mpi.h>
-
-#include <io500-phase.h>
-#include <phase_ior.h>
-
-typedef struct{
-  int run;
-  char * api;
-
-  char * command;
-  IOR_point_t * res;
-} opt_ior_rnd_read;
-
-static opt_ior_rnd_read o;
-
-static ini_option_t option[] = {
-  {"API", "The API to be used", 0, INI_STRING, NULL, & o.api},
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},  
-  {NULL} };
-
-
-static void validate(void){
-
-}
-
-
-static double run(void){
-  opt_ior_rnd d = ior_rnd1MB_o;
-
-  u_argv_t * argv = u_argv_create();
-  ior_rnd1MB_add_params(argv);
-  u_argv_push(argv, "-r");
-  u_argv_push(argv, "-R");
-  u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api);
-  u_argv_push(argv, "-O");
-  u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd1MB-read.csv", opt.resdir);
-  
-  o.command = u_flatten_argv(argv);
-
-  PRINT_PAIR("exe", "%s\n", o.command);
-  if(opt.dry_run || d.run == 0 || o.run == 0){
-    u_argv_free(argv);
-    return 0;
-  }
-  FILE * out = u_res_file_prep(p_ior_rnd1MB_read.name);
-  return ior_process_read(argv, out, & o.res);
-}
-
-
-u_phase_t p_ior_rnd1MB_read = {
-  "ior-rnd1MB-read",
-  IO500_PHASE_READ | IO500_PHASE_FLAG_OPTIONAL,
-  option,
-  validate,
-  run,
-  0,
-  .group = IO500_SCORE_BW,
-};
diff --git a/src/phase_ior_rnd_read4K.c b/src/phase_ior_rnd_read4K.c
deleted file mode 100644
index 275b88c..0000000
--- a/src/phase_ior_rnd_read4K.c
+++ /dev/null
@@ -1,60 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-#include <mpi.h>
-
-#include <io500-phase.h>
-#include <phase_ior.h>
-
-typedef struct{
-  int run;
-  char * api;
-
-  char * command;
-  IOR_point_t * res;
-} opt_ior_rnd_read;
-
-static opt_ior_rnd_read o;
-
-static ini_option_t option[] = {
-  {"API", "The API to be used", 0, INI_STRING, NULL, & o.api},
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},  
-  {NULL} };
-
-
-static void validate(void){
-
-}
-
-
-static double run(void){
-  opt_ior_rnd d = ior_rnd4K_o;
-
-  u_argv_t * argv = u_argv_create();
-  ior_rnd4K_add_params(argv);
-  u_argv_push(argv, "-r");
-  u_argv_push(argv, "-R");
-  u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api);
-  u_argv_push(argv, "-O");
-  u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd4K-read.csv", opt.resdir);
-  
-  o.command = u_flatten_argv(argv);
-
-  PRINT_PAIR("exe", "%s\n", o.command);
-  if(opt.dry_run || d.run == 0 || o.run == 0){
-    u_argv_free(argv);
-    return 0;
-  }
-  FILE * out = u_res_file_prep(p_ior_rnd4K_read.name);
-  return ior_process_read(argv, out, & o.res);
-}
-
-
-u_phase_t p_ior_rnd4K_read = {
-  "ior-rnd4K-read",
-  IO500_PHASE_READ | IO500_PHASE_FLAG_OPTIONAL,
-  option,
-  validate,
-  run,
-  0,
-  .group = IO500_SCORE_BW,
-};
diff --git a/src/phase_ior_rnd_read4k-easywrite.c b/src/phase_ior_rnd_read4k-easywrite.c
index 8715bc0..ad3ef27 100644
--- a/src/phase_ior_rnd_read4k-easywrite.c
+++ b/src/phase_ior_rnd_read4k-easywrite.c
@@ -41,9 +41,7 @@ static double run(void){
   u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd4K-easy-read.csv", opt.resdir);
   u_argv_push(argv, "-t=4096");
   u_argv_push(argv, "-b=4096");
-  u_argv_push(argv, "-s=10000000");
-  u_argv_push(argv, "-O");
-  u_argv_push(argv, "stoneWallingWearOut=1");
+  u_argv_push(argv, "-s=1000000");
   
   o.command = u_flatten_argv(argv);
 
@@ -52,8 +50,8 @@ static double run(void){
     u_argv_free(argv);
     return 0;
   }
-  FILE * out = u_res_file_prep(p_ior_rnd4K_read_easywrite.name);
-  return ior_process_read(argv, out, & o.res);
+  FILE * out = u_res_file_prep(p_ior_rnd4K_read_easywrite.name, opt.rank);
+  return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD);
 }
 
 
@@ -64,5 +62,5 @@ u_phase_t p_ior_rnd4K_read_easywrite = {
   validate,
   run,
   0,
-  .group = IO500_NO_SCORE,
+  .group = IO500_SCORE_BW,
 };
diff --git a/src/phase_ior_rnd_write1MB.c b/src/phase_ior_rnd_write1MB.c
deleted file mode 100644
index e256f2a..0000000
--- a/src/phase_ior_rnd_write1MB.c
+++ /dev/null
@@ -1,78 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include <ior.h>
-#include <io500-phase.h>
-
-#include <phase_ior.h>
-
-typedef struct{
-  int run;
-  char * api;
-
-  char * command;
-  IOR_point_t * res;
-} opt_ior_rnd_write;
-
-static opt_ior_rnd_write o;
-
-static ini_option_t option[] = {
-  {"API", "The API to be used", 0, INI_STRING, NULL, & o.api},
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},  
-  {NULL} };
-
-static void validate(void){
-  if( ! opt.dry_run && opt.rank == 0){
-    u_create_datadir("ior-rnd1MB");
-  }
-}
-
-static void cleanup(void){
-  if (opt.dry_run) return;
-
-  if(opt.rank == 0){
-    char filename[PATH_MAX];
-    sprintf(filename, "%s/ior-rnd1MB.stonewall", opt.resdir);
-    unlink(filename);
-    u_purge_file("ior-rnd1MB/file");
-    u_purge_datadir("ior-rnd1MB");
-  }
-}
-
-static double run(void){
-  opt_ior_rnd d = ior_rnd1MB_o;
-
-  u_argv_t * argv = u_argv_create();
-  ior_rnd1MB_add_params(argv);
-  u_argv_push(argv, "-w");
-  u_argv_push(argv, "-D");
-  u_argv_push_printf(argv, "%d", opt.stonewall);
-  u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api);
-  if(d.random_prefill_bytes > 0){
-    u_argv_push_printf(argv, "--randomPrefill=%u", d.random_prefill_bytes);
-  }
-  u_argv_push(argv, "-O");
-  u_argv_push(argv, "stoneWallingWearOut=1");
-  u_argv_push(argv, "-O");
-  u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd1MB-write.csv", opt.resdir);
-  o.command = u_flatten_argv(argv);
-
-  PRINT_PAIR("exe", "%s\n", o.command);
-  if(opt.dry_run || d.run == 0 || o.run == 0){
-    u_argv_free(argv);
-    return 0;
-  }
-  FILE * out = u_res_file_prep(p_ior_rnd1MB_write.name);
-  return ior_process_write(argv, out, & o.res);
-}
-
-u_phase_t p_ior_rnd1MB_write = {
-  "ior-rnd1MB-write",
-  IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL,
-  option,
-  validate,
-  run,
-  .verify_stonewall = 1,
-  .group = IO500_SCORE_BW,
-  .cleanup = cleanup,
-};
diff --git a/src/phase_ior_rnd_write4K.c b/src/phase_ior_rnd_write4K.c
deleted file mode 100644
index d21d6b8..0000000
--- a/src/phase_ior_rnd_write4K.c
+++ /dev/null
@@ -1,78 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include <ior.h>
-#include <io500-phase.h>
-
-#include <phase_ior.h>
-
-typedef struct{
-  int run;
-  char * api;
-
-  char * command;
-  IOR_point_t * res;
-} opt_ior_rnd_write;
-
-static opt_ior_rnd_write o;
-
-static ini_option_t option[] = {
-  {"API", "The API to be used", 0, INI_STRING, NULL, & o.api},
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},  
-  {NULL} };
-
-static void validate(void){
-  if( ! opt.dry_run && opt.rank == 0){
-    u_create_datadir("ior-rnd4K");
-  }
-}
-
-static void cleanup(void){
-  if (opt.dry_run) return;
-
-  if(opt.rank == 0){
-    char filename[PATH_MAX];
-    sprintf(filename, "%s/ior-rnd4K.stonewall", opt.resdir);
-    unlink(filename);
-    u_purge_file("ior-rnd4K/file");
-    u_purge_datadir("ior-rnd4K");
-  }
-}
-
-static double run(void){
-  opt_ior_rnd d = ior_rnd4K_o;
-
-  u_argv_t * argv = u_argv_create();
-  ior_rnd4K_add_params(argv);
-  u_argv_push(argv, "-w");
-  u_argv_push(argv, "-D");
-  u_argv_push_printf(argv, "%d", opt.stonewall);
-  u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api);
-  if(d.random_prefill_bytes > 0){
-    u_argv_push_printf(argv, "--randomPrefill=%u", d.random_prefill_bytes);
-  }
-  u_argv_push(argv, "-O");
-  u_argv_push(argv, "stoneWallingWearOut=1");
-  u_argv_push(argv, "-O");
-  u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd4K-write.csv", opt.resdir);
-  o.command = u_flatten_argv(argv);
-
-  PRINT_PAIR("exe", "%s\n", o.command);
-  if(opt.dry_run || d.run == 0 || o.run == 0){
-    u_argv_free(argv);
-    return 0;
-  }
-  FILE * out = u_res_file_prep(p_ior_rnd4K_write.name);
-  return ior_process_write(argv, out, & o.res);
-}
-
-u_phase_t p_ior_rnd4K_write = {
-  "ior-rnd4K-write",
-  IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL,
-  option,
-  validate,
-  run,
-  .verify_stonewall = 1,
-  .group = IO500_SCORE_BW,
-  .cleanup = cleanup,
-};
diff --git a/src/phase_mdtest.c b/src/phase_mdtest.c
index 2b8b2e3..8428f1f 100644
--- a/src/phase_mdtest.c
+++ b/src/phase_mdtest.c
@@ -7,6 +7,10 @@ void p_mdtest_run(u_argv_t * argv, FILE * out, mdtest_generic_res * d, mdtest_te
   d->items = res->items[test];
   d->rate = res->rate[test] / 1000;
   d->rate_stonewall = res->stonewall_item_sum[test] / res->stonewall_time[test] / 1000;
+  INFO_PAIR("errors", "%.0f\n", (float) res->total_errors);
+  if(res->total_errors){
+    INVALID("Verification errors %.0f \n", (float) res->total_errors);
+  }
   u_res_file_close(out);
   u_argv_free(argv);
   free(res);
diff --git a/src/phase_mdtest_easy.c b/src/phase_mdtest_easy.c
index 4b9a424..6862a35 100644
--- a/src/phase_mdtest_easy.c
+++ b/src/phase_mdtest_easy.c
@@ -32,8 +32,11 @@ void mdtest_easy_add_params(u_argv_t * argv){
 
   u_argv_push(argv, "./mdtest");
   u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType);
-  if(opt.io_buffers_on_gpu){
-    u_argv_push(argv, "--allocateBufferOnGPU");
+  if(opt.allocateBufferDevice){
+    u_argv_push_printf(argv, "--allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
   u_argv_push(argv, "-n");	/* number of files per process */
   u_argv_push_printf(argv, "%"PRIu64, d.g.files_per_proc);
diff --git a/src/phase_mdtest_easy_delete.c b/src/phase_mdtest_easy_delete.c
index 7504750..92b85bf 100644
--- a/src/phase_mdtest_easy_delete.c
+++ b/src/phase_mdtest_easy_delete.c
@@ -35,7 +35,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_easy_delete.name);
+  FILE * out = u_res_file_prep(p_mdtest_easy_delete.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_REMOVE_NUM);
 
   return o.res.rate;
diff --git a/src/phase_mdtest_easy_stat.c b/src/phase_mdtest_easy_stat.c
index 5fa2c81..db39181 100644
--- a/src/phase_mdtest_easy_stat.c
+++ b/src/phase_mdtest_easy_stat.c
@@ -35,7 +35,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_easy_stat.name);
+  FILE * out = u_res_file_prep(p_mdtest_easy_stat.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_STAT_NUM);
   return o.res.rate;
 }
diff --git a/src/phase_mdtest_easy_write.c b/src/phase_mdtest_easy_write.c
index f0f2b65..f1db4a3 100644
--- a/src/phase_mdtest_easy_write.c
+++ b/src/phase_mdtest_easy_write.c
@@ -42,7 +42,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_easy_write.name);
+  FILE * out = u_res_file_prep(p_mdtest_easy_write.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_CREATE_NUM);
 
   PRINT_PAIR("rate-stonewall", "%f\n", o.res.rate_stonewall);
diff --git a/src/phase_mdtest_hard.c b/src/phase_mdtest_hard.c
index 5c5244c..a1376dc 100644
--- a/src/phase_mdtest_hard.c
+++ b/src/phase_mdtest_hard.c
@@ -35,9 +35,13 @@ void mdtest_hard_add_params(u_argv_t * argv){
 
   u_argv_push(argv, "./mdtest");
   u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType);
-  if(opt.io_buffers_on_gpu){
-    u_argv_push(argv, "--allocateBufferOnGPU");
+  if(opt.allocateBufferDevice){
+    u_argv_push_printf(argv, "--allocateBufferOnGPU=%d", opt.allocateBufferDevice);
+    if(opt.gpuDirect){
+      u_argv_push(argv, "--gpuDirect");
+    }
   }
+
   u_argv_push(argv, "-n");	/* number of files per process */
   u_argv_push_printf(argv, "%"PRIu64, d.g.files_per_proc);
 
diff --git a/src/phase_mdtest_hard_delete.c b/src/phase_mdtest_hard_delete.c
index 2a53ca7..9699569 100644
--- a/src/phase_mdtest_hard_delete.c
+++ b/src/phase_mdtest_hard_delete.c
@@ -35,7 +35,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_hard_delete.name);
+  FILE * out = u_res_file_prep(p_mdtest_hard_delete.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_REMOVE_NUM);
 
   return o.res.rate;
diff --git a/src/phase_mdtest_hard_read.c b/src/phase_mdtest_hard_read.c
index 6152ca5..86238af 100644
--- a/src/phase_mdtest_hard_read.c
+++ b/src/phase_mdtest_hard_read.c
@@ -36,7 +36,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_hard_read.name);
+  FILE * out = u_res_file_prep(p_mdtest_hard_read.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_READ_NUM);
 
   return o.res.rate;
diff --git a/src/phase_mdtest_hard_stat.c b/src/phase_mdtest_hard_stat.c
index cb3f7ec..2b48944 100644
--- a/src/phase_mdtest_hard_stat.c
+++ b/src/phase_mdtest_hard_stat.c
@@ -35,7 +35,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_hard_stat.name);
+  FILE * out = u_res_file_prep(p_mdtest_hard_stat.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_STAT_NUM);
   return o.res.rate;
 }
diff --git a/src/phase_mdtest_hard_write.c b/src/phase_mdtest_hard_write.c
index fce389b..590ea6e 100644
--- a/src/phase_mdtest_hard_write.c
+++ b/src/phase_mdtest_hard_write.c
@@ -45,7 +45,7 @@ static double run(void){
     return 0;
   }
 
-  FILE * out = u_res_file_prep(p_mdtest_hard_write.name);
+  FILE * out = u_res_file_prep(p_mdtest_hard_write.name, opt.rank);
   p_mdtest_run(argv, out, & o.res, MDTEST_FILE_CREATE_NUM);
 
   PRINT_PAIR("rate-stonewall", "%f\n", o.res.rate_stonewall);
diff --git a/src/phase_mdworkbench.c b/src/phase_mdworkbench.c
deleted file mode 100644
index 1b14970..0000000
--- a/src/phase_mdworkbench.c
+++ /dev/null
@@ -1,132 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include <phase_mdworkbench.h>
-#include <phase_mdtest.h>
-#include <io500-phase.h>
-
-/*
-This phase adjusts the precreate based on the rate of mdtest.
-It attempts to run pre-create for 60s.
-*/
-
-opt_mdworkbench mdworkbench_o;
-
-static ini_option_t option[] = {
-  {"API", "The API to be used", 0, INI_STRING, NULL, & mdworkbench_o.api},
-  {"waitingTime", "Waiting time of an IO operation relative to runtime (1.0 is 100%%)", 0, INI_FLOAT, "0.0", & mdworkbench_o.waiting_time},
-  {"precreatePerSet", "Files to precreate per set (always 10 sets), this is normally dynamically determined", 0, INI_UINT64, NULL, & mdworkbench_o.precreate_per_set},
-  {"filesPerProc", "Files to run per iteration and set (always 10 sets), this is normally dynamically determined", 0, INI_UINT64, NULL, & mdworkbench_o.files_per_proc},
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & mdworkbench_o.run},
-  {"verbosity", "The verbosity level", 0, INI_INT, 0, & mdworkbench_o.verbosity},
-  {NULL} };
-
-static void validate(void){
-  opt_mdworkbench d = mdworkbench_o;
-
-  if(d.precreate_per_set != INI_UNSET_UINT64 || d.files_per_proc != INI_UNSET_UINT64){
-    if(d.precreate_per_set == INI_UNSET_UINT64 || d.files_per_proc == INI_UNSET_UINT64){
-      FATAL("MDWorkbench, must set either both options or none of: filesPerProc and precreatePerSet\n");
-    }
-  }
-}
-
-
-void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out){
-  mdworkbench_results_t * res = md_workbench_run(argv->size, argv->vector, MPI_COMM_WORLD, out);
-  u_res_file_close(out);
-  u_argv_free(argv);
-
-  if(res->errors != 0){
-    INVALID("Errors (%d) occured during the md-workbench phase. This invalidates your run.\n", res->errors);
-  }
-  *res_out = res;
-}
-
-
-
-void mdworkbench_add_params(u_argv_t * argv, int is_create){
-  opt_mdworkbench * d = & mdworkbench_o;
-
-  u_argv_push(argv, "./md-workbench");
-  u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType);
-  for(int i=0; i < mdworkbench_o.verbosity; i++){
-    u_argv_push(argv, "-v");
-  }
-  if(opt.io_buffers_on_gpu){
-    u_argv_push(argv, "--allocateBufferOnGPU");
-  }  
-  u_argv_push(argv, "--process-reports");
-  u_argv_push_default_if_set_api_options(argv, "-a", d->api, d->api);
-  u_argv_push_printf(argv, "-o=%s/mdworkbench", opt.datadir);
-  u_argv_push_printf(argv, "-t=%f", d->waiting_time);
-  u_argv_push(argv, "-O=1");
-  u_argv_push_printf(argv, "--run-info-file=%s/mdworkbench.status", opt.resdir );
-
-  // determine the number of objects to precreate and the number of objects per iteration
-
-  uint64_t precreate_per_set;
-  uint64_t files_per_proc;
-
-  if (d->precreate_per_set != INI_UNSET_UINT64 && d->files_per_proc != INI_UNSET_UINT64){
-    precreate_per_set = d->precreate_per_set;
-    files_per_proc = d->files_per_proc;
-  }else{
-    mdtest_generic_res* mdtest = mdtest_easy_write_get_result();
-    MPI_Bcast(& mdtest->rate, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
-
-    char file[PATH_MAX];
-    sprintf(file, "%s/mdworkbench-size", opt.resdir);
-    if (is_create && opt.rank == 0){
-      // store the actual processed size, allows easy deletion
-      FILE * f = fopen(file, "w");
-      if(! f){
-        WARNING("Couldn't open mdworkbench-file: %s\n", file);
-      }else{
-        fwrite(& mdtest->rate, sizeof(mdtest->rate), 1, f);
-      }
-      fclose(f);
-    }
-    if(! is_create && mdtest->rate <= 0.0){
-      // read the size back as this is a deletion run
-      if(opt.rank == 0){
-        FILE * f = fopen(file, "r");
-        if(! f){
-          WARNING("Couldn't open mdworkbench-file: %s\n", file);
-        }else{
-          fread(& mdtest->rate, sizeof(mdtest->rate), 1, f);
-        }
-        fclose(f);
-      }
-      MPI_Bcast(& mdtest->rate, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
-    }
-    if( mdtest->rate <= 0.0 ){
-      if(opt.rank == 0){
-        WARNING("MDWorkbench uses the MDTest rates to determine suitable options but MDTest didn't run, will use a low (and sane) default instead\n");
-      }
-      mdtest->rate = 10.0;
-    }
-    // run for 60s
-    int time = opt.stonewall < 60 ? opt.stonewall : 60;
-    precreate_per_set = (uint64_t) (mdtest->rate * time * 1000 / opt.mpi_size) / 10;
-
-    files_per_proc = precreate_per_set;
-  }
-  // we have 10 sets
-  u_argv_push(argv, "-D=10");
-  PRINT_PAIR("filesPerProc", "%"PRIu64"\n", files_per_proc);
-  PRINT_PAIR("precreatePerSet", "%"PRIu64"\n", precreate_per_set);
-  int hash = u_phase_unique_random_number("md-workbench");
-  u_argv_push_printf(argv, "-G=%d", hash);
-  u_argv_push_printf(argv, "-P=%"PRIu64, precreate_per_set);
-  u_argv_push_printf(argv, "-I=%"PRIu64, files_per_proc);
-}
-
-u_phase_t p_mdworkbench = {
-  "mdworkbench",
-  IO500_PHASE_DUMMY,
-  option,
-  validate,
-  NULL,
-  NULL,
-};
diff --git a/src/phase_mdworkbench.h b/src/phase_mdworkbench.h
deleted file mode 100644
index 5ff0550..0000000
--- a/src/phase_mdworkbench.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef IO500_PHASE_MDWORKBENCH_H
-#define IO500_PHASE_MDWORKBENCH_H
-
-#include <md-workbench.h>
-#include <io500-util.h>
-
-typedef struct{
-  int run;
-  char * api;
-  int verbosity;
-  float waiting_time;
-  uint64_t precreate_per_set;
-  uint64_t files_per_proc;
-} opt_mdworkbench;
-
-extern opt_mdworkbench mdworkbench_o;
-
-void mdworkbench_add_params(u_argv_t * argv, int is_create);
-void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out);
-
-#endif
diff --git a/src/phase_mdworkbench_bench.c b/src/phase_mdworkbench_bench.c
deleted file mode 100644
index 9e7010a..0000000
--- a/src/phase_mdworkbench_bench.c
+++ /dev/null
@@ -1,63 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-#include <mpi.h>
-
-#include <io500-phase.h>
-#include <phase_mdworkbench.h>
-
-typedef struct{
-  int run;
-  char * command;
-  mdworkbench_results_t * res;
-} opt_mdworkbench_bench;
-
-static opt_mdworkbench_bench o;
-
-static ini_option_t option[] = {
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},
-  {NULL} };
-
-
-static double run(void){
-  opt_mdworkbench d = mdworkbench_o;
-
-  u_argv_t * argv = u_argv_create();
-  mdworkbench_add_params(argv, 0);
-  u_argv_push(argv, "-2");
-  u_argv_push(argv, "-R=2");
-  u_argv_push(argv, "-X"); /* turn read verification on */
-
-  o.command = u_flatten_argv(argv);
-  PRINT_PAIR("exe", "%s\n", o.command);
-  if(opt.dry_run || d.run == 0){
-    u_argv_free(argv);
-    return 0;
-  }
-  FILE * out = u_res_file_prep(p_mdworkbench_bench.name);
-  mdworkbench_process(argv, out, & o.res);
-  if(o.res->count != 2){
-    INVALID("During the md-workbench phase not two iterations are performed but %d This invalidates your run.\n", o.res->count);
-    return 0.0;
-  }
-
-  double rate = o.res->result[1].rate;
-  if(   o.res->result[0].rate       < rate * 0.5
-     || o.res->result[0].rate * 0.5 > rate){
-    WARNING("The results of both md-workbench differs by 2x.\n");
-  }
-  PRINT_PAIR("maxOpTime", "%f\n", o.res->result[1].max_op_time);
-  PRINT_PAIR("scoreIteration0", "%f\n", o.res->result[0].rate / 1000.0);
-  PRINT_PAIR("maxOpTime0", "%f\n", o.res->result[0].max_op_time);
-  return rate / 1000.0;
-}
-
-
-u_phase_t p_mdworkbench_bench = {
-  "mdworkbench-bench",
-  IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL,
-  option,
-  NULL,
-  run,
-  0,
-  .group = IO500_SCORE_MD,
-};
diff --git a/src/phase_mdworkbench_create.c b/src/phase_mdworkbench_create.c
deleted file mode 100644
index eff9041..0000000
--- a/src/phase_mdworkbench_create.c
+++ /dev/null
@@ -1,51 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-#include <mpi.h>
-
-#include <io500-phase.h>
-#include <phase_mdworkbench.h>
-
-typedef struct{
-  int run;
-  char * command;
-  mdworkbench_results_t * res;
-} opt_mdworkbench_create;
-
-static opt_mdworkbench_create o;
-
-static ini_option_t option[] = {
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},
-  {NULL} };
-
-
-static double run(void){
-  opt_mdworkbench d = mdworkbench_o;
-
-  u_argv_t * argv = u_argv_create();
-  mdworkbench_add_params(argv, 1);
-  u_argv_push(argv, "-1");
-
-  o.command = u_flatten_argv(argv);
-  PRINT_PAIR("exe", "%s\n", o.command);
-  if(opt.dry_run || d.run == 0){
-    u_argv_free(argv);
-    return 0;
-  }
-  FILE * out = u_res_file_prep(p_mdworkbench_create.name);
-  mdworkbench_process(argv, out, & o.res);
-  PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time);
-
-  double rate = o.res->result[0].rate;
-  return rate / 1000.0;
-}
-
-
-u_phase_t p_mdworkbench_create = {
-  "mdworkbench-create",
-  IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL ,
-  option,
-  NULL,
-  run,
-  0,
-  .group = IO500_NO_SCORE,
-};
diff --git a/src/phase_mdworkbench_delete.c b/src/phase_mdworkbench_delete.c
deleted file mode 100644
index af79b4f..0000000
--- a/src/phase_mdworkbench_delete.c
+++ /dev/null
@@ -1,50 +0,0 @@
-#include <sys/stat.h>
-#include <unistd.h>
-#include <mpi.h>
-
-#include <io500-phase.h>
-#include <phase_mdworkbench.h>
-
-typedef struct{
-  int run;
-  char * command;
-  mdworkbench_results_t * res;
-} opt_mdworkbench_delete;
-
-static opt_mdworkbench_delete o;
-
-static ini_option_t option[] = {
-  {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run},
-  {NULL} };
-
-static double run(void){
-  opt_mdworkbench d = mdworkbench_o;
-
-  u_argv_t * argv = u_argv_create();
-  mdworkbench_add_params(argv, 0);
-  u_argv_push(argv, "-3");
-
-  o.command = u_flatten_argv(argv);
-  PRINT_PAIR("exe", "%s\n", o.command);
-  if(opt.dry_run || d.run == 0){
-    u_argv_free(argv);
-    return 0;
-  }
-  FILE * out = u_res_file_prep(p_mdworkbench_delete.name);
-  mdworkbench_process(argv, out, & o.res);
-  PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time);
-
-  double rate = o.res->result[0].rate;
-  return rate / 1000.0;
-}
-
-
-u_phase_t p_mdworkbench_delete = {
-  "mdworkbench-delete",
-  IO500_PHASE_REMOVE | IO500_PHASE_FLAG_OPTIONAL,
-  option,
-  NULL,
-  run,
-  0,
-  .group = IO500_NO_SCORE,
-};
diff --git a/src/phase_opt.c b/src/phase_opt.c
index ea8f176..fc018e4 100644
--- a/src/phase_opt.c
+++ b/src/phase_opt.c
@@ -10,7 +10,8 @@ static ini_option_t option[] = {
   {"api", "The general API for the tests (to create/delete the datadir, extra options will be passed to IOR/mdtest)", 0, INI_STRING, "POSIX", & opt.api},
   {"drop-caches", "Purge the caches, this is useful for testing and needed for single node runs", 0, INI_BOOL, "FALSE", & opt.drop_caches},
   {"drop-caches-cmd", "Cache purging command, invoked before each I/O phase", 0, INI_STRING, "sudo -n bash -c \"echo 3 > /proc/sys/vm/drop_caches\"", & opt.drop_caches_cmd},
-  {"io-buffers-on-gpu", "Allocate the I/O buffers on the GPU", 0, INI_BOOL, "FALSE", & opt.io_buffers_on_gpu},
+  {"allocateBufferDevice", "Allocate the I/O buffers on 0=CPU, 1=Managed Check CPU, 2=Managed Check GPU, 3=GPU", 0, INI_INT, "0", & opt.allocateBufferDevice},
+  {"gpuDirect", "Use GPUDirect only useful with Buffers > 0", 0, INI_BOOL, "FALSE", & opt.gpuDirect},
   {"verbosity", "The verbosity level between 1 and 10", 0, INI_UINT, "1", & opt.verbosity},
   {"scc", "Use the rules for the Student Cluster Competition", 0, INI_BOOL, "FALSE", & opt.scc},
   {"dataPacketType", "Type of packet that will be created [timestamp|offset|incompressible|random]", 0, INI_STRING, "timestamp", & opt.dataPacketType},
diff --git a/src/util.c b/src/util.c
index b104f7b..e6f82a7 100644
--- a/src/util.c
+++ b/src/util.c
@@ -158,10 +158,11 @@ static void push_api_args(u_argv_t * argv, char const * var){
       if(! t){
         break;
       }
-      if(strncasecmp(buff, t, len) != 0){
-        WARNING("Provided API option %s starts with a different prefix and might be not supported - don't worry if you checked!\n", t);
+      if(strncasecmp(buff, t, len) == 0){
+        u_argv_push(argv, t);
+      }else{
+        FATAL("Provided API option %s appears to be no API supported version\n", t);
       }
-      u_argv_push(argv, t);
     }
   }
   free(str);
@@ -235,9 +236,9 @@ void u_print_timestamp(FILE * out){
   fprintf(out, "%s", buffer);
 }
 
-FILE * u_res_file_prep(char const * name){
+FILE * u_res_file_prep(char const * name, int rank){
   FILE * out = stdout;
-  if(opt.rank == 0){
+  if(rank == 0){
     char fname[PATH_MAX];
     sprintf(fname, "%s/%s.txt", opt.resdir, name);
     INFO_PAIR("result-file", "%s\n", fname);
@@ -250,7 +251,7 @@ FILE * u_res_file_prep(char const * name){
 }
 
 void u_res_file_close(FILE * out){
-  if(opt.rank == 0){
+  if(out != stdout){
     fclose(out);
     out_logfile = stdout;
   }