Skip to content

Commit 08ce025

Browse files
GosstyGossty
authored
Resource allocation colormap (#3453)
* Update to DB qiita.slurm_resource_allocations * added colormap, create equation table * Fixed styling * Using qiita.allocation_equations table in util.py * Debug * Updates to @antgonza comments * Changes to @antgonza comments * Back to np.log
1 parent ef26847 commit 08ce025

File tree

7 files changed

+210
-133
lines changed

7 files changed

+210
-133
lines changed

‎qiita_db/meta_util.py‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -593,7 +593,7 @@ def update_resource_allocation_redis(active=True):
593593
if len(df) == 0:
594594
continue
595595

596-
fig, axs = resource_allocation_plot(df, cname, sname, col_name)
596+
fig, axs = resource_allocation_plot(df, col_name)
597597
titles = [0, 0]
598598
images = [0, 0]
599599

@@ -605,21 +605,18 @@ def update_resource_allocation_redis(active=True):
605605
# only time
606606
new_fig = plt.figure()
607607
new_ax = new_fig.add_subplot(111)
608-
609-
scatter_data = ax.collections[0]
610-
new_ax.scatter(scatter_data.get_offsets()[:, 0],
611-
scatter_data.get_offsets()[:, 1],
612-
s=scatter_data.get_sizes(), label="data")
613-
614608
line = ax.lines[0]
615609
new_ax.plot(line.get_xdata(), line.get_ydata(),
616610
linewidth=1, color='orange')
617-
618-
if len(ax.collections) > 1:
619-
failure_data = ax.collections[1]
620-
new_ax.scatter(failure_data.get_offsets()[:, 0],
621-
failure_data.get_offsets()[:, 1],
622-
color='red', s=3, label="failures")
611+
handles, labels = ax.get_legend_handles_labels()
612+
for handle, label, scatter_data in zip(handles,
613+
labels,
614+
ax.collections):
615+
color = handle.get_facecolor()
616+
new_ax.scatter(scatter_data.get_offsets()[:, 0],
617+
scatter_data.get_offsets()[:, 1],
618+
s=scatter_data.get_sizes(), label=label,
619+
color=color)
623620

624621
new_ax.set_xscale('log')
625622
new_ax.set_yscale('log')

‎qiita_db/support_files/patches/94.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
-- Jan 13, 2025
2+
-- Adding a table for formulas for resource allocations
3+
CREATE TABLE qiita.allocation_equations (
4+
equation_id SERIAL PRIMARY KEY,
5+
equation_name TEXT NOT NULL,
6+
expression TEXT NOT NULL
7+
);

‎qiita_db/support_files/patches/test_db_sql/94.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
INSERT INTO qiita.allocation_equations(equation_name, expression)
2+
VALUES
3+
('mem_model1', '(k * (np.log(x))) + (x * a) + b'),
4+
('mem_model2', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + a'),
5+
('mem_model3', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.np.log(x))**3))'),
6+
('mem_model4', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**2.5))'),
7+
('time_model1', 'a + b + ((np.log(x)) * k)'),
8+
('time_model2', 'a + (b * x) + ((np.log(x)) * k)'),
9+
('time_model3', 'a + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'),
10+
('time_model4', '(a * ((np.log(x))**3)) + (b * ((np.log(x))**2)) + ((np.log(x)) * k)');

‎qiita_db/test/test_meta_util.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -529,16 +529,19 @@ def test_update_resource_allocation_redis(self):
529529
cname, sname, version, col_name, 'title_mem')
530530
title_mem = str(r_client.get(title_mem_str))
531531
self.assertTrue(
532-
"model: "
533-
"k * log(x) + "
534-
"b * log(x)^2 + "
535-
"a * log(x)^3" in title_mem
532+
"model: (k * (np.log(x))) + "
533+
"(b * ((np.log(x))**2)) + "
534+
"(a * ((np.log(x))**2.5))" in title_mem
536535
)
537536

538537
title_time_str = 'resources$#%s$#%s$#%s$#%s:%s' % (
539538
cname, sname, version, col_name, 'title_time')
540539
title_time = str(r_client.get(title_time_str))
541-
self.assertTrue("model: a + b + log(x) * k" in title_time)
540+
self.assertTrue(
541+
"model: (a * ((np.log(x))**3)) + "
542+
"(b * ((np.log(x))**2)) + "
543+
"((np.log(x)) * k)" in title_time
544+
)
542545

543546

544547
if __name__ == '__main__':

‎qiita_db/test/test_util.py‎

Lines changed: 22 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1329,8 +1329,7 @@ def setUp(self):
13291329

13301330
def test_plot_return(self):
13311331
# check the plot returns correct objects
1332-
fig1, axs1 = qdb.util.resource_allocation_plot(
1333-
self.df, self.cname, self.sname, self.col_name)
1332+
fig1, axs1 = qdb.util.resource_allocation_plot(self.df, self.col_name)
13341333
self.assertIsInstance(
13351334
fig1, Figure,
13361335
"Returned object fig1 is not a Matplotlib Figure")
@@ -1346,46 +1345,46 @@ def test_minimize_const(self):
13461345
self.df[self.col_name] = self.df.samples * self.df['columns']
13471346
fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False)
13481347

1349-
bm, options = qdb.util._resource_allocation_plot_helper(
1350-
self.df, axs[0], self.cname, self.sname, 'MaxRSSRaw',
1351-
qdb.util.MODELS_MEM, self.col_name)
1348+
mem_models, time_models = qdb.util.retrieve_equations()
1349+
bm_name, bm, options = qdb.util._resource_allocation_plot_helper(
1350+
self.df, axs[0], 'MaxRSSRaw', mem_models, self.col_name)
13521351
# check that the algorithm chooses correct model for MaxRSSRaw and
13531352
# has 0 failures
13541353
k, a, b = options.x
1355-
failures_df = qdb.util._resource_allocation_failures(
1356-
self.df, k, a, b, bm, self.col_name, 'MaxRSSRaw')
1354+
failures_df = qdb.util._resource_allocation_success_failures(
1355+
self.df, k, a, b, bm, self.col_name, 'MaxRSSRaw')[-1]
13571356
failures = failures_df.shape[0]
1358-
self.assertEqual(bm, qdb.util.mem_model3,
1357+
1358+
self.assertEqual(bm_name, 'mem_model4',
1359+
msg=f"""Best memory model
1360+
doesn't match
1361+
{bm_name} != 'mem_model4'""")
1362+
self.assertEqual(bm, mem_models['mem_model4']['equation'],
13591363
msg=f"""Best memory model
13601364
doesn't match
13611365
Coefficients:{k} {a} {b}
1362-
{qdb.util.mem_model1}, "qdb.util.mem_model1"
1363-
{qdb.util.mem_model2}, "qdb.util.mem_model2"
1364-
{qdb.util.mem_model3}, "qdb.util.mem_model3"
1365-
{qdb.util.mem_model4}, "qdb.util.mem_model4"
13661366
""")
13671367
self.assertEqual(failures, 0, "Number of failures must be 0")
13681368

13691369
# check that the algorithm chooses correct model for ElapsedRaw and
13701370
# has 1 failure
1371-
bm, options = qdb.util._resource_allocation_plot_helper(
1372-
self.df, axs[1], self.cname, self.sname, 'ElapsedRaw',
1373-
qdb.util.MODELS_TIME, self.col_name)
1371+
bm_name, bm, options = qdb.util._resource_allocation_plot_helper(
1372+
self.df, axs[1], 'ElapsedRaw', time_models, self.col_name)
13741373
k, a, b = options.x
1375-
failures_df = qdb.util._resource_allocation_failures(
1376-
self.df, k, a, b, bm, self.col_name, 'ElapsedRaw')
1374+
failures_df = qdb.util._resource_allocation_success_failures(
1375+
self.df, k, a, b, bm, self.col_name, 'ElapsedRaw')[-1]
13771376
failures = failures_df.shape[0]
1377+
self.assertEqual(bm_name, 'time_model4',
1378+
msg=f"""Best time model
1379+
doesn't match
1380+
{bm_name} != 'time_model4'""")
13781381

1379-
self.assertEqual(bm, qdb.util.time_model1,
1382+
self.assertEqual(bm, time_models[bm_name]['equation'],
13801383
msg=f"""Best time model
13811384
doesn't match
13821385
Coefficients:{k} {a} {b}
1383-
{qdb.util.time_model1}, "qdb.util.time_model1"
1384-
{qdb.util.time_model2}, "qdb.util.time_model2"
1385-
{qdb.util.time_model3}, "qdb.util.time_model3"
1386-
{qdb.util.time_model4}, "qdb.util.time_model4"
13871386
""")
1388-
self.assertEqual(failures, 1, "Number of failures must be 1")
1387+
self.assertEqual(failures, 0, "Number of failures must be 0")
13891388

13901389
def test_MaxRSS_helper(self):
13911390
tests = [

0 commit comments

Comments
 (0)