diff --git a/bin/radical-analytics-inspect b/bin/radical-analytics-inspect index 0d1065e..42ce4ed 100755 --- a/bin/radical-analytics-inspect +++ b/bin/radical-analytics-inspect @@ -107,9 +107,10 @@ EOT why a specific resource element has *not* been used for task execution at that point in time (overhead). - utilization: (\$sid.\$pid.util.png) - Plot the resource utilization for each pilot is plotted - as percentage over time. + utilization: (\$sid.\$pid.util2.png) + alternative utilization plot, showing the utilization + as percentage over time, including the different + distributions to that utilization. EOT } @@ -117,16 +118,20 @@ test -z "$ARGS" && usage "missing session ID(s)" test "$ARGS" = "-h" && usage test "$ARGS" = "help" && usage +ret=0 +base="$bin/rp_inspect/" for session in $ARGS do sid=$(basename $session) echo -n "$sid " - echo 'STATE:'; $bin/rp_inspect/plot_state.py "$session" && echo -n . - echo 'DUR :'; $bin/rp_inspect/plot_dur.py "$session" && echo -n . - echo 'CONC :'; $bin/rp_inspect/plot_conc.py "$session" && echo -n . - echo 'RATE :'; $bin/rp_inspect/plot_rate.py "$session" && echo -n . - echo 'UTIL :'; $bin/rp_inspect/plot_util.py "$session" && echo -n . - echo 'UTIL2:'; $bin/rp_inspect/plot_util_2.py "$session" && echo -n . + echo 'STATE:'; $base/plot_state.py "$session"; ret=$((ret + $?)) + echo 'DUR :'; $base/plot_dur.py "$session"; ret=$((ret + $?)) + echo 'CONC :'; $base/plot_conc.py "$session"; ret=$((ret + $?)) + echo 'RATE :'; $base/plot_rate.py "$session"; ret=$((ret + $?)) + echo 'UTIL :'; $base/plot_util.py "$session"; + echo 'UTIL2:'; $base/plot_util_2.py "$session"; ret=$((ret + $?)) echo " done" done +exit $ret + diff --git a/bin/radical-analytics-plot.py b/bin/radical-analytics-plot.py index 02d70a4..df6d6e8 100755 --- a/bin/radical-analytics-plot.py +++ b/bin/radical-analytics-plot.py @@ -38,6 +38,8 @@ FNAME = None SAVE_AS = 'x11' # 'svg', 'png', 'x11', 'pdf' WIDTH = 500 +HEIGHT = None +STDEV = False # ------------------------------------------------------------------------------ @@ -63,13 +65,15 @@ def usage(msg=None): -m, --match : use lines matching pattern -x, --x-column : source column for x-values -y, --y-columns : list of columns columns to plot + -S, --stdev : plot stdev from `col + 1` -L, --legend : name of plots specified in '-y' -u, --x-ticks : not yet supported -v, --y-ticks : not yet supported -r, --range : axis range -s, --style : plot type - -w, --width <252> : canvas width + -W, --width : canvas width (default: 500) + -H, --height : canvas height -l, --log : log-scale for x and/or y axis -g, --grid : grid lines (default: no) -a, --save-as : save fig in format (x11: show) @@ -93,10 +97,12 @@ def usage(msg=None): parser.add_option('-y', '--y-columns', dest='ycols') parser.add_option('-u', '--x-ticks', dest='xticks') parser.add_option('-v', '--y-ticks', dest='yticks') +parser.add_option('-S', '--stdev', dest='stdev', action='store_true') parser.add_option('-r', '--range', dest='range') parser.add_option('-L', '--legend', dest='legend') parser.add_option('-s', '--style', dest='style') -parser.add_option('-w', '--width', dest='width') +parser.add_option('-W', '--width', dest='width') +parser.add_option('-H', '--height', dest='height') parser.add_option('-l', '--log', dest='log') parser.add_option('-g', '--grid', dest='grid', action='store_true') parser.add_option('-a', '--save-as', dest='save') @@ -124,11 +130,13 @@ def usage(msg=None): if options.yticks : TICKS_Y = [str(x) for x in options.yticks.split(',')] if options.legend : LEGEND = [str(x) for x in options.legend.split(',')] if options.width : WIDTH = int(options.width) +if options.height : HEIGHT = int(options.height) if options.log : LOG = str(options.log) -if options.grid : GRID = str(options.grid) if options.style : STYLE = str(options.style) if options.save : SAVE_AS = str(options.save) if options.fname : FNAME = str(options.fname) +if options.stdev : STDEV = True +if options.grid : GRID = True if options.range : RANGE = options.range .split(',') @@ -196,7 +204,9 @@ def get_elems(line): for idx in range(len(elems)): elem = elems[idx] if idx == COLUMN_X: - if not LABEL_X: + if LABEL_X == '-': + LABEL_X = None + else: LABEL_X = elem else: if not LEGEND: @@ -234,12 +244,16 @@ def get_elems(line): # pprint.pprint(data) try: - fig, ax = plt.subplots(figsize=ra.get_plotsize(WIDTH)) - cnum = 0 + fig, ax = plt.subplots(figsize=ra.get_plotsize(width=WIDTH, height=HEIGHT)) + cnum = 0 for col in COLUMNS_Y: - if LEGEND: label = to_latex(LEGEND[cnum]) - else : label = to_latex(str(cnum)) + data_y_err = None + + if LEGEND[0] == ['-']: label = None, + elif LEGEND : label = to_latex(LEGEND[cnum]) + else : label = to_latex(str(cnum)) + cnum += 1 if COLUMN_X == 'count': @@ -247,45 +261,71 @@ def get_elems(line): else: data_x = np.array(data[COLUMN_X]) + if '+' in col: - cols = col.split('+', 1) - cols = [int(cols[0]), int(cols[1])] - data_y = np.array(data[cols[0]]) + np.array(data[cols[1]]) + cols = [int(c) for c in col.split('+')] + data_y = sum([np.array(data[c]) for c in cols]) + if STDEV: + data_y_err = sum([np.array(data[c + 1]) for c in cols]) elif '-' in col: - cols = col.split('-', 1) - cols = [int(cols[0]), int(cols[1])] - data_y = np.array(data[cols[0]]) - np.array(data[cols[1]]) + cols = [int(c) for c in col.split('-')] + data_y = np.array(data[cols[0]]) + if STDEV: + data_y_err = np.array(data[cols[0] + 1]) + for c in cols[1:]: + data_y -= np.array(data[c]) + if STDEV: + data_y_err += np.array(data[c + 1]) elif '*' in col: - cols = col.split('*', 1) - cols = [int(cols[0]), int(cols[1])] - data_y = np.array(data[cols[0]]) * np.array(data[cols[1]]) + cols = [int(c) for c in col.split('*')] + data_y = np.array(data[cols[0]]) + if STDEV: + data_y_err = np.array(data[cols[0] + 1]) + for c in cols[1:]: + data_y *= np.array(data[c]) + if STDEV: + data_y_err += np.array(data[c + 1]) elif '/' in col: - cols = col.split('/', 1) - cols = [int(cols[0]), int(cols[1])] - data_y = np.array(data[cols[0]]) / np.array(data[cols[1]]) + cols = [int(c) for c in col.split('/')] + data_y = np.array(data[cols[0]]) + if STDEV: + data_y_err = np.array(data[cols[0] + 1]) + for c in cols[1:]: + data_y /= np.array(data[c]) + if STDEV: + data_y_err += np.array(data[c + 1]) else: col = int(col) time.sleep(1) - data_y = np.array(data[col]) + data_y = np.array(data[col]) + if STDEV: + data_y_err = np.array(data[col + 1]) + + if STDEV and data_y_err is None: + raise ValueError('stdev not available with column spec %s' % col) colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] color = colors[cnum] # if cnum == 3: # color = colors[4] + if STYLE == 'point': ax.scatter(data_x, data_y, c=color, label=label, s=10) + elif STYLE == 'line' : ax.plot (data_x, data_y, c=color, label=label) + elif STYLE == 'step' : ax.step (data_x, data_y, c=color, label=label) + elif STYLE == 'bar' : ax.bar (data_x, data_y, c=color, label=label) + elif STYLE == 'hist' : ax.hist (data_y, 150, color=color, label=label) + elif STYLE == 'lp' : ax.plot (data_x, data_y, c=color, label=label, marker='.') - if STYLE == 'point': ax.scatter(data_x, data_y, c=color, label=label, s=10) - elif STYLE == 'line' : ax.plot (data_x, data_y, 'b', c=color, label=label) - elif STYLE == 'step' : ax.step (data_x, data_y, 'b', c=color, label=label) - elif STYLE == 'bar' : ax.bar (data_x, data_y, c=color, label=label) - elif STYLE == 'hist' : ax.hist (data_y, 150, color=color, label=label) - elif STYLE == 'lp' : ax.plot (data_x, data_y, 'b', c=color, label=label, marker='.') - + # if STDEV: + # ax.fill_between(data_x, data_y - data_y_err, data_y + data_y_err, + # color=color, alpha=0.1) + if STDEV: + ax.errorbar(data_x, data_y, yerr=data_y_err, fmt='.', color=color) except IndexError: print('index error') @@ -293,8 +333,12 @@ def get_elems(line): print(' %2d: %s' % (i, e)) raise -if LEGEND != ['-']: - plt.legend(ncol=1, fancybox=True) +print(LEGEND) +if LEGEND[0] != '-': + plt.legend(ncol=len(LEGEND), bbox_to_anchor=(1.05, 1.25), + fancybox=True) + +# ax.ytick.label_format(axis='both', style='sci', scilimits=(2,None)) if TITLE : ax.set_title(TITLE, loc='center') if LOG_X : ax.set_xscale('log') @@ -302,7 +346,7 @@ def get_elems(line): if LABEL_X : ax.set_xlabel(to_latex(LABEL_X)) if LABEL_Y : ax.set_ylabel(to_latex(LABEL_Y)) if TICKS_X : ax.set_xticks([int(t) for t in TICKS_X], TICKS_X) -if TICKS_Y : ax.set_yticks([int(t) for t in TICKS_Y], TICKS_Y) +if TICKS_Y : ax.set_yticks([int(t) for t in TICKS_Y], TICKS_Y, ) if GRID : ax.grid(True) xmin, xmax, ymin, ymax = RANGE diff --git a/bin/rp_inspect/plot_state.py b/bin/rp_inspect/plot_state.py index ca70321..0b8b3d4 100755 --- a/bin/rp_inspect/plot_state.py +++ b/bin/rp_inspect/plot_state.py @@ -124,7 +124,7 @@ # FIXME: how to do the legend now? With the large font size, I don't see plt.legend() - plt.savefig('%s.state.png' % session.uid) + plt.savefig('%s_state.png' % session.uid) # plt.show() diff --git a/bin/rp_inspect/plot_util.py b/bin/rp_inspect/plot_util.py index 432ded1..18882ab 100755 --- a/bin/rp_inspect/plot_util.py +++ b/bin/rp_inspect/plot_util.py @@ -123,6 +123,14 @@ fig, axes = plt.subplots(2, figsize=ra.get_plotsize(500)) session = ra.Session(src, stype=stype) + # this script does not really work for many tasks + tasks = session.get(etype='task') + if len(tasks) > 1024: + if 'RA_MANY_TASKS_OK' not in os.environ: + print('too many tasks for this plot type. Set "RA_MANY_TASKS_OK"') + print('in the your environment to allow this script to run anyway.') + sys.exit(1) + # this script only works for one pilot pilots = session.get(etype='pilot') assert(len(pilots) == 1), len(pilots) diff --git a/bin/rp_inspect/plot_util_2.py b/bin/rp_inspect/plot_util_2.py index 693f073..9a72a67 100755 --- a/bin/rp_inspect/plot_util_2.py +++ b/bin/rp_inspect/plot_util_2.py @@ -270,7 +270,7 @@ def main(): fig.suptitle(to_latex('%s Tasks - %s Nodes' % (n_tasks, n_nodes))) # Save a publication quality plot - fig.savefig('%s.util2.png' % sid, dpi=600, bbox_inches='tight') + fig.savefig('%s_util2.png' % sid, dpi=600, bbox_inches='tight') # ------------------------------------------------------------------------------ diff --git a/src/radical/analytics/utils/plot.py b/src/radical/analytics/utils/plot.py index a9c9128..bf8c905 100644 --- a/src/radical/analytics/utils/plot.py +++ b/src/radical/analytics/utils/plot.py @@ -12,7 +12,7 @@ # ------------------------------------------------------------------------------ # -def get_plotsize(width, fraction=1, subplots=(1, 1)): +def get_plotsize(width, height=None, fraction=1, subplots=(1, 1)): """ Sets aesthetic figure dimensions to avoid scaling in latex. Parameters @@ -20,7 +20,7 @@ def get_plotsize(width, fraction=1, subplots=(1, 1)): width : float Width in points (pts). fraction: float - Fraction of the width which you wish the figure to occupy. + Fraction of the width and hight the figure to occupies subplots: tuple Number of raws and number of columns of the plot. @@ -35,14 +35,17 @@ def get_plotsize(width, fraction=1, subplots=(1, 1)): # Convert from pt to inches inches_per_pt = 1 / 72.27 - # Golden ratio to set aesthetic figure height - golden_ratio = (5 ** 0.5 - 1) / 2 - # Figure width in inches fig_width_in = fig_width_pt * inches_per_pt # Figure height in inches - fig_height_in = fig_width_in * golden_ratio * (subplots[0] / subplots[1]) + if height: + fig_height_in = height * fraction * inches_per_pt + + else: + # Golden ratio to set aesthetic figure height + golden_ratio = (5 ** 0.5 - 1) / 2 + fig_height_in = fig_width_in * golden_ratio * (subplots[0] / subplots[1]) return fig_width_in, fig_height_in