Phase concurrent

Makefile

@@ -22,7 +22,7 @@ vpath %.h $(SEARCHPATH)
 DEPS += io500-util.h io500-debug.h io500-opt.h
 OBJSO += util.o
 OBJSO += ini-parse.o phase_dbg.o phase_opt.o phase_timestamp.o
-OBJSO += phase_find.o phase_find_easy.o phase_find_hard.o phase_ior_easy.o phase_ior_easy_read.o phase_mdtest.o phase_ior.o phase_ior_easy_write.o phase_ior_hard.o phase_ior_hard_read.o phase_ior_hard_write.o phase_mdtest_easy.o phase_mdtest_easy_delete.o phase_mdtest_easy_stat.o phase_mdtest_easy_write.o phase_mdtest_hard.o phase_mdtest_hard_delete.o phase_mdtest_hard_read.o phase_mdtest_hard_stat.o phase_mdtest_hard_write.o phase_ior_rnd1MB.o phase_ior_rnd4K.o phase_ior_rnd_write4K.o phase_ior_rnd_read4K.o phase_ior_rnd_write1MB.o phase_ior_rnd_read1MB.o phase_mdworkbench.o phase_mdworkbench_create.o phase_mdworkbench_delete.o phase_mdworkbench_bench.o phase_ior_rnd_read4k-easywrite.o
+OBJSO += phase_find.o phase_find_easy.o phase_find_hard.o phase_ior_easy.o phase_ior_easy_read.o phase_mdtest.o phase_ior.o phase_ior_easy_write.o phase_ior_hard.o phase_ior_hard_read.o phase_ior_hard_write.o phase_mdtest_easy.o phase_mdtest_easy_delete.o phase_mdtest_easy_stat.o phase_mdtest_easy_write.o phase_mdtest_hard.o phase_mdtest_hard_delete.o phase_mdtest_hard_read.o phase_mdtest_hard_stat.o phase_mdtest_hard_write.o phase_ior_rnd1MB.o phase_ior_rnd4K.o phase_ior_rnd_write4K.o phase_ior_rnd_read4K.o phase_ior_rnd_write1MB.o phase_ior_rnd_read1MB.o phase_mdworkbench.o phase_mdworkbench_create.o phase_mdworkbench_delete.o phase_mdworkbench_bench.o phase_ior_rnd_read4k-easywrite.o  phase_concurrent.o
 
 OBJS = $(patsubst %,./build/%,$(OBJSO))
 

find/sfind.sh

@@ -9,6 +9,10 @@
 # When you are done pring 'x/y' where x is matched files and y is total files searched.
 # There is a parallel version also install in bin that might be better
 
+# The script should support the following extra options
+# -e for deletion of found files 
+# -E for deletion of directories
+
 function parse_rates {
   #find -D rates gives a weird thing like this:
   #Predicate success rates after completion
@@ -18,7 +22,20 @@ function parse_rates {
   echo $rates | tr " " "\n" | grep '/' | $1 -1 | cut -d \/ -f 2 | cut -d = -f 1
 }
 
-rates=`find -D rates $* 2>&1 | grep -A1 Predicate | tail -1` 
+IN=""
+for var in "$@"
+do
+    if [[ $var == "-e" ]] ; then 
+      continue
+    fi
+    if [[ $var == "-E" ]] ; then 
+      IN="$IN -exec rm -f {} ;"
+      continue
+    fi
+    IN="$IN $var"
+done
+
+rates=$(find -D rates $IN 2>&1 | grep -A1 Predicate | tail -1)
 total_files=$(parse_rates 'head')
 match_files=$(parse_rates 'tail')
 echo "MATCHED $match_files/$total_files"

include/io500-opt.h

@@ -27,7 +27,8 @@ typedef struct{
 
   int rank;
   int mpi_size;
-  int io_buffers_on_gpu; /* are the I/O buffers to be allocated on a GPU */
+  int allocateBufferDevice; /* where are the I/O buffers allocated and processed */
+  int gpuDirect; /* useGPUDirect */
 
   char * api;
   char * apiArgs; // for IOR and mdtest

include/io500-phase.h

@@ -8,9 +8,10 @@
 #define DEBUG_OPTION "INVALIDATES RUN; FOR DEBUGGING."
 
 typedef enum {
-  IO500_NO_SCORE,
+  IO500_NO_SCORE = 0,
   IO500_SCORE_MD,
   IO500_SCORE_BW,
+  IO500_SCORE_CONCURRENT,
   IO500_SCORE_LAST
 } io500_phase_score_group;
 
@@ -37,7 +38,7 @@ typedef struct{
   io500_phase_score_group group;
 } u_phase_t;
 
-#define IO500_PHASES (2 + 1 + 2*3 + 3 + 3 + 4 + 5 + 3 + 4 + 1)
+#define IO500_PHASES (2 + 1 + 2*3 + 3 + 3 + 4 + 5 + 3 + 4 + 1 + 1)
 
 extern u_phase_t p_opt;
 extern u_phase_t p_debug;
@@ -65,6 +66,8 @@ extern u_phase_t p_mdworkbench_create;
 extern u_phase_t p_mdworkbench_bench;
 extern u_phase_t p_mdworkbench_delete;
 
+extern u_phase_t p_concurrent;
+
 extern u_phase_t p_ior_easy;
 extern u_phase_t p_ior_easy_write;
 extern u_phase_t p_ior_easy_read;

include/io500-util.h

@@ -82,7 +82,7 @@ void u_print_timestamp(FILE * out);
 void * u_malloc(int size);
 
 
-FILE * u_res_file_prep(char const * name);
+FILE * u_res_file_prep(char const * name, int rank);
 void u_res_file_close(FILE * out);
 
 uint32_t u_phase_unique_random_number(char const * phase_name);

src/main.c

@@ -22,7 +22,16 @@ FILE* file_out = NULL;
 static char const * io500_phase_str[IO500_SCORE_LAST] = {
   "NO SCORE",
   "MD",
-  "BW"};
+  "BW", 
+  "CONCURRENT"
+  };
+
+static char const * io500_unit_str[IO500_SCORE_LAST] = {
+  "     ",
+  "kIOPS",
+  "GiB/s",
+  "score"
+};
 
 extern int rank;
 
@@ -127,14 +136,19 @@ static void print_cfg_hash(FILE * out, ini_section_t ** cfg){
 
 static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t * hash){
   double overall_score = 1;
-  for(io500_phase_score_group g=1; g < IO500_SCORE_LAST; g++){
+  int groups = IO500_SCORE_BW;
+  if(extended){
+    groups = IO500_SCORE_CONCURRENT;
+  }
+  memset(scores, 0, sizeof(double) * IO500_SCORE_LAST);
+  for(io500_phase_score_group g=1; g <= groups; g++){
     char score_string[2048];
     char *p = score_string;
     double score = 1;
     int numbers = 0;
-    p += sprintf(p, " %s = (", io500_phase_str[g]);
+    p += sprintf(p, " %s = (1.0", io500_phase_str[g]);
     for(int i=0; i < IO500_PHASES; i++){
-      if(phases[i]->group == g && (extended || ! (phases[i]->type & IO500_PHASE_FLAG_OPTIONAL)) ){
+      if( phases[i]->group == g && (extended || ! (phases[i]->type & IO500_PHASE_FLAG_OPTIONAL)) ){
         double t = phases[i]->score;
         if(t <= 0){
           continue;
@@ -157,7 +171,7 @@ static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t
 
     overall_score *= score;
   }
-  return sqrt(overall_score);
+  return pow(overall_score, 1.0 / groups);
 }
 
 static const char * io500_mode_str(io500_mode mode){
@@ -169,6 +183,92 @@ static const char * io500_mode_str(io500_mode mode){
   }
 }
 
+typedef struct {
+  int run;
+  int valid;
+  double score;
+  double time;
+} io500_phase_result_t;
+
+typedef struct{
+  double score[IO500_SCORE_LAST];
+  double scoreX[IO500_SCORE_LAST];
+  double runtime;
+} io500_overall_results_t;
+
+static io500_phase_result_t io500_results[IO500_PHASES];
+static io500_overall_results_t io500_overall_results;
+
+static void dump_io500_phase(FILE * o, int i){
+  io500_phase_result_t * r = & io500_results[i];
+  if(! r->run) return;
+  u_phase_t * phase = phases[i];
+  char score_str[40];
+  sprintf(score_str, "%f", r->score);
+  fprintf(o, "%25s\t%15s\t%s\t%.3fs\t%s\n", phase->name, score_str, io500_unit_str[phase->group], r->time, r->valid ? "" : "INVALID");
+}
+
+static void dump_io500_results(void){
+    FILE * res;
+    char file[PATH_MAX];
+    sprintf(file, "%s/result_pretty.txt", opt.resdir);
+    res = fopen(file, "w");
+    if(! res){
+      FATAL("Could not open \"%s\" for writing (%s)\n", file, strerror(errno));
+    } 
+
+    fprintf(res, "%25s\t%fs\n", "RUNTIME", io500_overall_results.runtime);
+    fprintf(res, "%25s", "SCORE");
+    for(int i=0; i < IO500_SCORE_LAST; i++){
+      fprintf(res, "\t%f\t%s\t", io500_overall_results.score[i], io500_unit_str[i]);
+    }
+    fprintf(res, "\n");
+
+    if(opt.mode == IO500_MODE_EXTENDED){
+      fprintf(res, "%25s", "SCOREX");
+      for(int i=0; i < IO500_SCORE_LAST; i++){
+        fprintf(res, "\t%f\t%s\t", io500_overall_results.scoreX[i], io500_unit_str[i]);
+      }
+      fprintf(res, "\n");
+    }
+
+    // ior easy
+    dump_io500_phase(res, 3);
+    dump_io500_phase(res, 24);
+    // ior hard
+    dump_io500_phase(res, 15);
+    dump_io500_phase(res, 26);
+    // rand4k
+    dump_io500_phase(res, 5);
+    dump_io500_phase(res, 19);
+    // rand1M
+    dump_io500_phase(res, 8);
+    dump_io500_phase(res, 20);
+    // concurrent
+    dump_io500_phase(res, 23);
+
+    // MD easy
+    dump_io500_phase(res, 7);
+    dump_io500_phase(res, 25);
+    dump_io500_phase(res, 29);
+    // MD hard
+    dump_io500_phase(res, 17);
+    dump_io500_phase(res, 27);
+    dump_io500_phase(res, 30);
+    dump_io500_phase(res, 31);
+    // MDW
+    dump_io500_phase(res, 22);
+    dump_io500_phase(res, 28);
+    // find
+    dump_io500_phase(res, 18);
+    // find easy
+    dump_io500_phase(res, 13);    
+    // find hard
+    dump_io500_phase(res, 21);
+
+    fclose(res);
+}
+
 int main(int argc, char ** argv){
   int mpi_init = 0;
   file_out = stdout;
@@ -201,6 +301,9 @@ int main(int argc, char ** argv){
     }
     goto out;
   }
+
+  memset(io500_results, 0, sizeof(io500_results));
+  double t_io500_start = GetTimeStamp();
 
   mpi_init = 1;
   MPI_Init(& argc, & argv);
@@ -372,11 +475,12 @@ int main(int argc, char ** argv){
 
   for(int i=0; i < IO500_PHASES; i++){
     u_phase_t * phase = phases[i];
-    if(! phase->run) continue;
+    if(! phase->run) continue;        
     if( cleanup_only && ! (phase->type & IO500_PHASE_REMOVE) ) continue;
     if(! RUN_PHASE(phase)){
       continue;
     }
+    io500_results[i].run = 1;
 
     if(opt.drop_caches && ! (phase->type & IO500_PHASE_DUMMY) ){
       DEBUG_INFO("Dropping cache\n");
@@ -448,6 +552,11 @@ int main(int argc, char ** argv){
           opt.is_valid_run = 0;
         }
       }
+
+      io500_results[i].score = score;
+      io500_results[i].time = runtime;
+      io500_results[i].valid = opt.is_valid_phase;
+
       if(! (phase->type & IO500_PHASE_DUMMY)){
         PRINT_PAIR("score", "%f\n", score);
       }
@@ -459,7 +568,12 @@ int main(int argc, char ** argv){
       }else{
         dupprintf("[      ]");
       }
-      dupprintf(" %20s %15s %s : time %.3f seconds%s\n", phase->name, score_str, phase->name[0] == 'i' ? "GiB/s" : "kIOPS", runtime, valid_str);
+      dupprintf(" %25s %15s %s : time %.3f seconds%s\n", phase->name, score_str, io500_unit_str[phase->group], runtime, valid_str);
+
+      PRINT_PAIR("t_delta", "%.4f\n", runtime);
+      PRINT_PAIR_HEADER("t_end");
+      u_print_timestamp(file_out);
+      fprintf(file_out, "\n");
     }
     if(phase->group > IO500_NO_SCORE){
       if(phase->type & IO500_PHASE_FLAG_OPTIONAL){
@@ -469,14 +583,6 @@ int main(int argc, char ** argv){
       }
     }
     phases[i]->score = score;
-
-
-    if(opt.verbosity > 0 && opt.rank == 0){
-      PRINT_PAIR("t_delta", "%.4f\n", runtime);
-      PRINT_PAIR_HEADER("t_end");
-      u_print_timestamp(file_out);
-      fprintf(file_out, "\n");
-    }
   }
 
   MPI_Barrier(MPI_COMM_WORLD);
@@ -494,7 +600,10 @@ int main(int argc, char ** argv){
     PRINT_PAIR("hash", "%X\n", (int) score_hash);
     dupprintf("[SCORE ] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n",
     scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_score, valid_str);
-
+
+    scores[IO500_NO_SCORE] = overall_score;
+    memcpy(io500_overall_results.score, scores, sizeof(scores));
+
     // extended run
     if(opt.mode == IO500_MODE_EXTENDED){
       valid_str = opt.is_valid_extended_run ? "" : " [INVALID]";
@@ -507,8 +616,13 @@ int main(int argc, char ** argv){
       PRINT_PAIR("SCORE", "%f%s\n", overall_extended_score, valid_str);
       PRINT_PAIR("hash", "%X\n", (int) score_extended_hash);
       dupprintf("[SCOREX] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n", scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_extended_score, valid_str);
+
+      scores[IO500_NO_SCORE] = overall_score;
+      memcpy(io500_overall_results.scoreX, scores, sizeof(scores));
     }
-
+
+    io500_overall_results.runtime = GetTimeStamp() - t_io500_start;
+    dump_io500_results();
     printf("\nThe result files are stored in the directory: %s\n", opt.resdir);
   }
 

src/phase-definitions.h

@@ -36,6 +36,8 @@ static u_phase_t * phases[IO500_PHASES] = {
 
   & p_mdworkbench_bench,
 
+  & p_concurrent,
+
   & p_ior_easy_read,
   & p_mdtest_easy_stat,