Fix/plot improvements

bin/radical-analytics-inspect

@@ -107,26 +107,31 @@ EOT
                       why a specific resource element has *not* been used for
                       task execution at that point in time (overhead).
 
-         utilization: (\$sid.\$pid.util.png)
-                      Plot the resource utilization for each pilot is plotted
-                      as percentage over time.
+         utilization: (\$sid.\$pid.util2.png)
+                      alternative utilization plot, showing the utilization
+                      as percentage over time, including the different
+                      distributions to that utilization.
 EOT
 }
 
 test -z "$ARGS"          && usage "missing session ID(s)"
 test    "$ARGS" = "-h"   && usage
 test    "$ARGS" = "help" && usage
 
+ret=0
+base="$bin/rp_inspect/"
 for session in $ARGS
 do
     sid=$(basename $session)
     echo -n "$sid "
-    echo 'STATE:'; $bin/rp_inspect/plot_state.py  "$session" && echo -n .
-    echo 'DUR  :'; $bin/rp_inspect/plot_dur.py    "$session" && echo -n .
-    echo 'CONC :'; $bin/rp_inspect/plot_conc.py   "$session" && echo -n .
-    echo 'RATE :'; $bin/rp_inspect/plot_rate.py   "$session" && echo -n .
-    echo 'UTIL :'; $bin/rp_inspect/plot_util.py   "$session" && echo -n .
-    echo 'UTIL2:'; $bin/rp_inspect/plot_util_2.py "$session" && echo -n .
+    echo 'STATE:'; $base/plot_state.py  "$session"; ret=$((ret + $?))
+    echo 'DUR  :'; $base/plot_dur.py    "$session"; ret=$((ret + $?))
+    echo 'CONC :'; $base/plot_conc.py   "$session"; ret=$((ret + $?))
+    echo 'RATE :'; $base/plot_rate.py   "$session"; ret=$((ret + $?))
+    echo 'UTIL :'; $base/plot_util.py   "$session";
+    echo 'UTIL2:'; $base/plot_util_2.py "$session"; ret=$((ret + $?))
     echo " done"
 done
 
+exit $ret
+

bin/radical-analytics-plot.py

@@ -38,6 +38,8 @@
 FNAME     = None
 SAVE_AS   = 'x11'   # 'svg', 'png', 'x11', 'pdf'
 WIDTH     = 500
+HEIGHT    = None
+STDEV     = False
 
 
 # ------------------------------------------------------------------------------
@@ -63,13 +65,15 @@ def usage(msg=None):
         -m, --match      <pattern>             : use lines matching pattern
         -x, --x-column   <col>                 : source column for x-values
         -y, --y-columns  <col_1,col_2,...>     : list of columns columns to plot
+        -S, --stdev                            : plot stdev from `col + 1`
         -L, --legend     <label_1,label_2,...> : name of plots specified in '-y'
         -u, --x-ticks    <tick_1,tick_2,...>   : not yet supported
         -v, --y-ticks    <tick_1,tick_2,...>   : not yet supported
         -r, --range      <xmin,xmax,ymin,ymax> : axis range
         -s, --style      <point | line | step | bar | hist | lp>
                                                : plot type
-        -w, --width      <252>                 : canvas width
+        -W, --width      <pixels>              : canvas width (default: 500)
+        -H, --height     <pixels>              : canvas height
         -l, --log        <x | y | x,y>         : log-scale for x and/or y axis
         -g, --grid                             : grid lines (default: no)
         -a, --save-as    <png | svg | x11>     : save fig in format (x11: show)
@@ -93,10 +97,12 @@ def usage(msg=None):
 parser.add_option('-y', '--y-columns', dest='ycols')
 parser.add_option('-u', '--x-ticks',   dest='xticks')
 parser.add_option('-v', '--y-ticks',   dest='yticks')
+parser.add_option('-S', '--stdev',     dest='stdev', action='store_true')
 parser.add_option('-r', '--range',     dest='range')
 parser.add_option('-L', '--legend',    dest='legend')
 parser.add_option('-s', '--style',     dest='style')
-parser.add_option('-w', '--width',     dest='width')
+parser.add_option('-W', '--width',     dest='width')
+parser.add_option('-H', '--height',    dest='height')
 parser.add_option('-l', '--log',       dest='log')
 parser.add_option('-g', '--grid',      dest='grid', action='store_true')
 parser.add_option('-a', '--save-as',   dest='save')
@@ -124,11 +130,13 @@ def usage(msg=None):
 if options.yticks : TICKS_Y      = [str(x) for x in options.yticks.split(',')]
 if options.legend : LEGEND       = [str(x) for x in options.legend.split(',')]
 if options.width  : WIDTH        =  int(options.width)
+if options.height : HEIGHT       =  int(options.height)
 if options.log    : LOG          =  str(options.log)
-if options.grid   : GRID         =  str(options.grid)
 if options.style  : STYLE        =  str(options.style)
 if options.save   : SAVE_AS      =  str(options.save)
 if options.fname  : FNAME        =  str(options.fname)
+if options.stdev  : STDEV        =  True
+if options.grid   : GRID         =  True
 
 if options.range  :
     RANGE = options.range .split(',')
@@ -196,7 +204,9 @@ def get_elems(line):
         for idx in range(len(elems)):
             elem = elems[idx]
             if idx == COLUMN_X:
-                if not LABEL_X:
+                if LABEL_X == '-':
+                    LABEL_X = None
+                else:
                     LABEL_X = elem
             else:
                 if not LEGEND:
@@ -234,75 +244,109 @@ def get_elems(line):
 # pprint.pprint(data)
 try:
 
-    fig, ax = plt.subplots(figsize=ra.get_plotsize(WIDTH))
-    cnum    = 0
+    fig, ax = plt.subplots(figsize=ra.get_plotsize(width=WIDTH, height=HEIGHT))
+    cnum = 0
     for col in COLUMNS_Y:
 
-        if LEGEND: label = to_latex(LEGEND[cnum])
-        else     : label = to_latex(str(cnum))
+        data_y_err = None
+
+        if   LEGEND[0] == ['-']: label = None,
+        elif LEGEND            : label = to_latex(LEGEND[cnum])
+        else                   : label = to_latex(str(cnum))
+
         cnum += 1
 
         if COLUMN_X == 'count':
             data_x = list(range(len(data[0])))
         else:
             data_x = np.array(data[COLUMN_X])
 
+
         if '+' in col:
-            cols = col.split('+', 1)
-            cols = [int(cols[0]), int(cols[1])]
-            data_y = np.array(data[cols[0]]) + np.array(data[cols[1]])
+            cols   = [int(c) for c in col.split('+')]
+            data_y = sum([np.array(data[c]) for c in cols])
+            if STDEV:
+                data_y_err = sum([np.array(data[c + 1]) for c in cols])
 
         elif '-' in col:
-            cols = col.split('-', 1)
-            cols = [int(cols[0]), int(cols[1])]
-            data_y = np.array(data[cols[0]]) - np.array(data[cols[1]])
+            cols   = [int(c) for c in col.split('-')]
+            data_y = np.array(data[cols[0]])
+            if STDEV:
+                data_y_err = np.array(data[cols[0] + 1])
+            for c in cols[1:]:
+                data_y -= np.array(data[c])
+                if STDEV:
+                    data_y_err += np.array(data[c + 1])
 
         elif '*' in col:
-            cols = col.split('*', 1)
-            cols = [int(cols[0]), int(cols[1])]
-            data_y = np.array(data[cols[0]]) * np.array(data[cols[1]])
+            cols   = [int(c) for c in col.split('*')]
+            data_y = np.array(data[cols[0]])
+            if STDEV:
+                data_y_err = np.array(data[cols[0] + 1])
+            for c in cols[1:]:
+                data_y *= np.array(data[c])
+                if STDEV:
+                    data_y_err += np.array(data[c + 1])
 
         elif '/' in col:
-            cols = col.split('/', 1)
-            cols = [int(cols[0]), int(cols[1])]
-            data_y = np.array(data[cols[0]]) / np.array(data[cols[1]])
+            cols   = [int(c) for c in col.split('/')]
+            data_y = np.array(data[cols[0]])
+            if STDEV:
+                data_y_err = np.array(data[cols[0] + 1])
+            for c in cols[1:]:
+                data_y /= np.array(data[c])
+                if STDEV:
+                    data_y_err += np.array(data[c + 1])
 
         else:
             col = int(col)
             time.sleep(1)
-            data_y = np.array(data[col])
+            data_y  = np.array(data[col])
+            if STDEV:
+                data_y_err = np.array(data[col + 1])
+
+        if STDEV and data_y_err is None:
+            raise ValueError('stdev not available with column spec %s' % col)
 
         colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
         color = colors[cnum]
 
         # if cnum == 3:
         #     color = colors[4]
+        if   STYLE == 'point': ax.scatter(data_x, data_y, c=color, label=label, s=10)
+        elif STYLE == 'line' : ax.plot   (data_x, data_y, c=color, label=label)
+        elif STYLE == 'step' : ax.step   (data_x, data_y, c=color, label=label)
+        elif STYLE == 'bar'  : ax.bar    (data_x, data_y, c=color, label=label)
+        elif STYLE == 'hist' : ax.hist   (data_y,  150,   color=color, label=label)
+        elif STYLE == 'lp'   : ax.plot   (data_x, data_y, c=color, label=label, marker='.')
 
-        if   STYLE == 'point': ax.scatter(data_x, data_y,      c=color, label=label, s=10)
-        elif STYLE == 'line' : ax.plot   (data_x, data_y, 'b', c=color, label=label)
-        elif STYLE == 'step' : ax.step   (data_x, data_y, 'b', c=color, label=label)
-        elif STYLE == 'bar'  : ax.bar    (data_x, data_y,      c=color, label=label)
-        elif STYLE == 'hist' : ax.hist   (data_y,  150,        color=color, label=label)
-        elif STYLE == 'lp'   : ax.plot   (data_x, data_y, 'b', c=color, label=label, marker='.')
-
+      # if STDEV:
+      #     ax.fill_between(data_x, data_y - data_y_err, data_y + data_y_err,
+      #                     color=color, alpha=0.1)
 
+        if STDEV:
+            ax.errorbar(data_x, data_y, yerr=data_y_err, fmt='.', color=color)
 
 except IndexError:
     print('index error')
     for i,e in enumerate(data[0]):
         print('    %2d: %s' % (i, e))
     raise
 
-if LEGEND != ['-']:
-    plt.legend(ncol=1, fancybox=True)
+print(LEGEND)
+if LEGEND[0] != '-':
+    plt.legend(ncol=len(LEGEND), bbox_to_anchor=(1.05, 1.25),
+          fancybox=True)
+
+# ax.ytick.label_format(axis='both', style='sci', scilimits=(2,None))
 
 if TITLE   : ax.set_title(TITLE, loc='center')
 if LOG_X   : ax.set_xscale('log')
 if LOG_Y   : ax.set_yscale('log')
 if LABEL_X : ax.set_xlabel(to_latex(LABEL_X))
 if LABEL_Y : ax.set_ylabel(to_latex(LABEL_Y))
 if TICKS_X : ax.set_xticks([int(t) for t in TICKS_X], TICKS_X)
-if TICKS_Y : ax.set_yticks([int(t) for t in TICKS_Y], TICKS_Y)
+if TICKS_Y : ax.set_yticks([int(t) for t in TICKS_Y], TICKS_Y, )
 if GRID    : ax.grid(True)
 
 xmin, xmax, ymin, ymax = RANGE

bin/rp_inspect/plot_state.py

@@ -124,7 +124,7 @@
 
     # FIXME: how to do the legend now?  With the large font size, I don't see
     plt.legend()
-    plt.savefig('%s.state.png' % session.uid)
+    plt.savefig('%s_state.png' % session.uid)
   # plt.show()
 
 

bin/rp_inspect/plot_util.py

@@ -123,6 +123,14 @@
     fig, axes = plt.subplots(2, figsize=ra.get_plotsize(500))
     session   = ra.Session(src, stype=stype)
 
+    # this script does not really work for many tasks
+    tasks = session.get(etype='task')
+    if len(tasks) > 1024:
+        if 'RA_MANY_TASKS_OK' not in os.environ:
+            print('too many tasks for this plot type.  Set "RA_MANY_TASKS_OK"')
+            print('in the your environment to allow this script to run anyway.')
+            sys.exit(1)
+
     # this script only works for one pilot
     pilots = session.get(etype='pilot')
     assert(len(pilots) == 1), len(pilots)

bin/rp_inspect/plot_util_2.py

@@ -270,7 +270,7 @@ def main():
     fig.suptitle(to_latex('%s Tasks - %s Nodes' % (n_tasks, n_nodes)))
 
     # Save a publication quality plot
-    fig.savefig('%s.util2.png' % sid, dpi=600, bbox_inches='tight')
+    fig.savefig('%s_util2.png' % sid, dpi=600, bbox_inches='tight')
 
 
 # ------------------------------------------------------------------------------

src/radical/analytics/utils/plot.py

@@ -12,15 +12,15 @@
 
 # ------------------------------------------------------------------------------
 #
-def get_plotsize(width, fraction=1, subplots=(1, 1)):
+def get_plotsize(width, height=None, fraction=1, subplots=(1, 1)):
     """ Sets aesthetic figure dimensions to avoid scaling in latex.
 
     Parameters
     ----------
     width   : float
               Width in points (pts).
     fraction: float
-              Fraction of the width which you wish the figure to occupy.
+              Fraction of the width and hight the figure to occupies
     subplots: tuple
               Number of raws and number of columns of the plot.
 
@@ -35,14 +35,17 @@ def get_plotsize(width, fraction=1, subplots=(1, 1)):
     # Convert from pt to inches
     inches_per_pt = 1 / 72.27
 
-    # Golden ratio to set aesthetic figure height
-    golden_ratio = (5 ** 0.5 - 1) / 2
-
     # Figure width in inches
     fig_width_in = fig_width_pt * inches_per_pt
 
     # Figure height in inches
-    fig_height_in = fig_width_in * golden_ratio * (subplots[0] / subplots[1])
+    if height:
+        fig_height_in = height * fraction * inches_per_pt
+
+    else:
+        # Golden ratio to set aesthetic figure height
+        golden_ratio  = (5 ** 0.5 - 1) / 2
+        fig_height_in = fig_width_in * golden_ratio * (subplots[0] / subplots[1])
 
     return fig_width_in, fig_height_in