Skip to content

Commit 707d94b

Browse files
DanInciDaniel Incicau
DanInci
and
Daniel Incicau
authored
Create github actions for continously running benchmark example (#11)
Co-authored-by: Daniel Incicau <[email protected]>
1 parent d53e15a commit 707d94b

File tree

2 files changed

+186
-159
lines changed

2 files changed

+186
-159
lines changed

‎.github/workflows/benchmark.yml

+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
name: Run Benchmark
2+
concurrency:
3+
group: ${{ github.head_ref || github.run_id }}
4+
cancel-in-progress: true
5+
6+
on:
7+
push:
8+
branches:
9+
- main
10+
pull_request:
11+
workflow_dispatch:
12+
13+
permissions:
14+
id-token: write
15+
pages: write
16+
17+
jobs:
18+
run-benchmark:
19+
name: Run Benchmark
20+
## runs-on: ubuntu-latest
21+
runs-on: self-hosted
22+
steps:
23+
- name: Check out repository
24+
uses: actions/checkout@v4
25+
26+
- name: Install Mambaforge
27+
uses: conda-incubator/setup-miniconda@v3
28+
with:
29+
miniforge-variant: Miniforge3
30+
use-mamba: true
31+
activate-environment: omnibenchmark-env
32+
python-version: "3.12"
33+
auto-update-conda: true
34+
channels: conda-forge
35+
36+
- name: Cache environment
37+
id: cache-env
38+
uses: actions/cache@v3
39+
with:
40+
path: |
41+
~/.conda/pkgs
42+
~/.conda/envs/omnibenchmark-env
43+
~/.cache/pip
44+
key: ${{ runner.os }}-conda-pip-${{ hashFiles('requirements.txt') }}
45+
restore-keys: |
46+
${{ runner.os }}-conda-pip-
47+
48+
- name: Install omnibenchmark CLI
49+
shell: bash -l {0}
50+
run: |
51+
mamba install -y pip
52+
pip install git+https://github.com/omnibenchmark/omnibenchmark.git@dev
53+
54+
- name: Load benchmark cache
55+
id: cache-benchmark
56+
uses: actions/cache@v3
57+
with:
58+
path: out/
59+
key: benchmark-${{ runner.os }}-${{ hashFiles('Clustering.yaml') }}
60+
61+
- name: Run benchmark
62+
shell: bash -l {0}
63+
continue-on-error: true
64+
run: |
65+
echo "y" | ob run benchmark -b Clustering.yaml --local --threads 20 --continue-on-error
66+
67+
upload-artifact:
68+
name: Benchmark Artifact
69+
## runs-on: ubuntu-latest
70+
runs-on: self-hosted
71+
needs: run-benchmark
72+
if: always()
73+
steps:
74+
- name: Check out repository
75+
uses: actions/checkout@v4
76+
77+
- name: Load cached output
78+
uses: actions/cache@v3
79+
with:
80+
path: out/
81+
key: benchmark-${{ runner.os }}-${{ hashFiles('Clustering.yaml') }}
82+
83+
- name: Prepare output
84+
run: |
85+
zip -r benchmark_output.zip out/
86+
mkdir -p gh-pages
87+
cp out/plotting/plotting_report.html gh-pages/index.html
88+
89+
- name: Upload zipped output
90+
uses: actions/upload-artifact@v4
91+
with:
92+
name: benchmark-output
93+
path: benchmark_output.zip
94+
retention-days: 7
95+
96+
- name: Upload Pages Artifact
97+
uses: actions/upload-pages-artifact@v3
98+
with:
99+
path: gh-pages
100+
101+
- name: Deploy to GitHub Pages
102+
uses: actions/deploy-pages@v4
103+
104+
- name: Create Job Summary
105+
if: always()
106+
run: |
107+
echo "### Reports" >> $GITHUB_STEP_SUMMARY
108+
echo "- [Plotting Report](https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }})" >> $GITHUB_STEP_SUMMARY
109+
echo "### All Outputs" >> $GITHUB_STEP_SUMMARY
110+
echo "- [Complete Benchmark Output](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts)" >> $GITHUB_STEP_SUMMARY
111+

‎Clustering.yaml

+75-159
Original file line numberDiff line numberDiff line change
@@ -23,27 +23,29 @@ software_environments:
2323
conda: envs/r.yml
2424
apptainer: envs/r.sif
2525
envmodule: fcps # not true, but
26+
rmarkdown:
27+
description: "R with some plotting dependencies"
28+
conda: envs/rmarkdown.yml
29+
apptainer: envs/r.sif # not true, but
30+
envmodule: fcps # not true, but
2631
fcps:
2732
description: "CRAN's FCPS"
2833
conda: envs/fcps.yml
2934
apptainer: envs/fcps.sif
3035
envmodule: fcps
3136
metric_collectors:
32-
- id: biometrics
33-
name: "Biologically-relevant performance metrics gathering and postprocessing."
34-
software_environment: "R"
37+
- id: plotting
38+
name: "Single-backend metric collector."
39+
software_environment: "rmarkdown"
3540
repository:
36-
url: https://github.com/omnibenchmark-example/metric-collector.git
37-
commit: ef4a601
41+
url: https://github.com/imallona/clustering_report
42+
commit: 0a4ddff
3843
inputs:
3944
- metrics.scores
4045
outputs:
41-
- id: biometrics.report.html
42-
path: "{input}/{name}/biometrics_report.html"
43-
- id: biometrics.tsv
44-
path: "{input}/{name}/biometrics.tsv"
46+
- id: plotting.html
47+
path: "{input}/{name}/plotting_report.html"
4548
stages:
46-
4749
## clustbench data ##########################################################
4850

4951
- id: data
@@ -55,68 +57,68 @@ stages:
5557
url: https://github.com/imallona/clustbench_data
5658
commit: 366c5a2
5759
parameters: # comments depict the possible cardinalities and the number of curated labelsets
58-
- values: ["--dataset_generator", "fcps", "--dataset_name", "atom"] # 2 1
59-
- values: ["--dataset_generator", "fcps", "--dataset_name", "chainlink"] # 2 1
60-
- values: ["--dataset_generator", "fcps", "--dataset_name", "engytime"] # 2 2
61-
- values: ["--dataset_generator", "fcps", "--dataset_name", "hepta"] # 7 1
62-
- values: ["--dataset_generator", "fcps", "--dataset_name", "lsun"] # 3 1
63-
- values: ["--dataset_generator", "fcps", "--dataset_name", "target"] # 2, 6 2
64-
- values: ["--dataset_generator", "fcps", "--dataset_name", "tetra"] # 4 1
65-
- values: ["--dataset_generator", "fcps", "--dataset_name", "twodiamonds"] # 2 1
66-
- values: ["--dataset_generator", "fcps", "--dataset_name", "wingnut"] # 2 1
67-
- values: ["--dataset_generator", "graves", "--dataset_name", "dense"] # 2 1
68-
- values: ["--dataset_generator", "graves", "--dataset_name", "fuzzyx"] # 2, 4, 5 6
69-
- values: ["--dataset_generator", "graves", "--dataset_name", "line"] # 2 1
70-
- values: ["--dataset_generator", "graves", "--dataset_name", "parabolic"] # 2, 4 2
71-
- values: ["--dataset_generator", "graves", "--dataset_name", "ring"] # 2 1
72-
- values: ["--dataset_generator", "graves", "--dataset_name", "ring_noisy"] # 2 1
73-
- values: ["--dataset_generator", "graves", "--dataset_name", "ring_outliers"] # 2, 5 2
74-
- values: ["--dataset_generator", "graves", "--dataset_name", "zigzag"] # 3, 5 2
75-
- values: ["--dataset_generator", "graves", "--dataset_name", "zigzag_noisy"] # 3, 5 2
76-
- values: ["--dataset_generator", "graves", "--dataset_name", "zigzag_outliers"] # 3, 5 2
77-
- values: ["--dataset_generator", "other", "--dataset_name", "chameleon_t4_8k"] # 6 1
78-
- values: ["--dataset_generator", "other", "--dataset_name", "chameleon_t5_8k"] # 6 1
79-
- values: ["--dataset_generator", "other", "--dataset_name", "hdbscan"] # 6 1
80-
- values: ["--dataset_generator", "other", "--dataset_name", "iris"] # 3 1
81-
- values: ["--dataset_generator", "other", "--dataset_name", "iris5"] # 3 1
82-
- values: ["--dataset_generator", "other", "--dataset_name", "square"] # 2 1
83-
- values: ["--dataset_generator", "sipu", "--dataset_name", "aggregation"] # 7 1
84-
- values: ["--dataset_generator", "sipu", "--dataset_name", "compound"] # 4, 5, 6 5
85-
- values: ["--dataset_generator", "sipu", "--dataset_name", "flame"] # 2 2
86-
- values: ["--dataset_generator", "sipu", "--dataset_name", "jain"] # 2 1
87-
- values: ["--dataset_generator", "sipu", "--dataset_name", "pathbased"] # 3, 4 2
88-
- values: ["--dataset_generator", "sipu", "--dataset_name", "r15"] # 8, 9, 15 3
89-
- values: ["--dataset_generator", "sipu", "--dataset_name", "spiral"] # 3 1
90-
- values: ["--dataset_generator", "sipu", "--dataset_name", "unbalance"] # 8 1
91-
- values: ["--dataset_generator", "uci", "--dataset_name", "ecoli"] # 8 1
92-
- values: ["--dataset_generator", "uci", "--dataset_name", "ionosphere"] # 2 1
93-
- values: ["--dataset_generator", "uci", "--dataset_name", "sonar"] # 2 1
94-
- values: ["--dataset_generator", "uci", "--dataset_name", "statlog"] # 7 1
95-
- values: ["--dataset_generator", "uci", "--dataset_name", "wdbc"] # 2 1
96-
- values: ["--dataset_generator", "uci", "--dataset_name", "wine"] # 3 1
97-
- values: ["--dataset_generator", "uci", "--dataset_name", "yeast"] # 10 1
98-
- values: ["--dataset_generator", "wut", "--dataset_name", "circles"] # 4 1
99-
- values: ["--dataset_generator", "wut", "--dataset_name", "cross"] # 4 1
100-
- values: ["--dataset_generator", "wut", "--dataset_name", "graph"] # 10 1
101-
- values: ["--dataset_generator", "wut", "--dataset_name", "isolation"] # 3 1
102-
- values: ["--dataset_generator", "wut", "--dataset_name", "labirynth"] # 6 1
103-
- values: ["--dataset_generator", "wut", "--dataset_name", "mk1"] # 3 1
104-
- values: ["--dataset_generator", "wut", "--dataset_name", "mk2"] # 2 1
105-
- values: ["--dataset_generator", "wut", "--dataset_name", "mk3"] # 3 1
106-
- values: ["--dataset_generator", "wut", "--dataset_name", "mk4"] # 3 1
107-
- values: ["--dataset_generator", "wut", "--dataset_name", "olympic"] # 5 1
108-
- values: ["--dataset_generator", "wut", "--dataset_name", "smile"] # 4, 6 2
109-
- values: ["--dataset_generator", "wut", "--dataset_name", "stripes"] # 2 1
110-
- values: ["--dataset_generator", "wut", "--dataset_name", "trajectories"] # 4 1
111-
- values: ["--dataset_generator", "wut", "--dataset_name", "trapped_lovers"] # 3 1
112-
- values: ["--dataset_generator", "wut", "--dataset_name", "twosplashes"] # 2 1
113-
- values: ["--dataset_generator", "wut", "--dataset_name", "windows"] # 5 1
114-
- values: ["--dataset_generator", "wut", "--dataset_name", "x1"] # 3 1
115-
- values: ["--dataset_generator", "wut", "--dataset_name", "x2"] # 3 1
116-
- values: ["--dataset_generator", "wut", "--dataset_name", "x3"] # 4 1
117-
- values: ["--dataset_generator", "wut", "--dataset_name", "z1"] # 3 1
118-
- values: ["--dataset_generator", "wut", "--dataset_name", "z2"] # 5 1
119-
- values: ["--dataset_generator", "wut", "--dataset_name", "z3"] # 4 1
60+
- values: ["--dataset_generator", "fcps", "--dataset_name", "atom"] # 2 1
61+
# - values: ["--dataset_generator", "fcps", "--dataset_name", "chainlink"] # 2 1
62+
# - values: ["--dataset_generator", "fcps", "--dataset_name", "engytime"] # 2 2
63+
# - values: ["--dataset_generator", "fcps", "--dataset_name", "hepta"] # 7 1
64+
# - values: ["--dataset_generator", "fcps", "--dataset_name", "lsun"] # 3 1
65+
# - values: ["--dataset_generator", "fcps", "--dataset_name", "target"] # 2, 6 2
66+
# - values: ["--dataset_generator", "fcps", "--dataset_name", "tetra"] # 4 1
67+
# - values: ["--dataset_generator", "fcps", "--dataset_name", "twodiamonds"] # 2 1
68+
# - values: ["--dataset_generator", "fcps", "--dataset_name", "wingnut"] # 2 1
69+
# - values: ["--dataset_generator", "graves", "--dataset_name", "dense"] # 2 1
70+
# - values: ["--dataset_generator", "graves", "--dataset_name", "fuzzyx"] # 2, 4, 5 6
71+
# - values: ["--dataset_generator", "graves", "--dataset_name", "line"] # 2 1
72+
# - values: ["--dataset_generator", "graves", "--dataset_name", "parabolic"] # 2, 4 2
73+
# - values: ["--dataset_generator", "graves", "--dataset_name", "ring"] # 2 1
74+
# - values: ["--dataset_generator", "graves", "--dataset_name", "ring_noisy"] # 2 1
75+
# - values: ["--dataset_generator", "graves", "--dataset_name", "ring_outliers"] # 2, 5 2
76+
# - values: ["--dataset_generator", "graves", "--dataset_name", "zigzag"] # 3, 5 2
77+
# - values: ["--dataset_generator", "graves", "--dataset_name", "zigzag_noisy"] # 3, 5 2
78+
# - values: ["--dataset_generator", "graves", "--dataset_name", "zigzag_outliers"] # 3, 5 2
79+
# - values: ["--dataset_generator", "other", "--dataset_name", "chameleon_t4_8k"] # 6 1
80+
# - values: ["--dataset_generator", "other", "--dataset_name", "chameleon_t5_8k"] # 6 1
81+
# - values: ["--dataset_generator", "other", "--dataset_name", "hdbscan"] # 6 1
82+
# - values: ["--dataset_generator", "other", "--dataset_name", "iris"] # 3 1
83+
# - values: ["--dataset_generator", "other", "--dataset_name", "iris5"] # 3 1
84+
# - values: ["--dataset_generator", "other", "--dataset_name", "square"] # 2 1
85+
# - values: ["--dataset_generator", "sipu", "--dataset_name", "aggregation"] # 7 1
86+
# - values: ["--dataset_generator", "sipu", "--dataset_name", "compound"] # 4, 5, 6 5
87+
# - values: ["--dataset_generator", "sipu", "--dataset_name", "flame"] # 2 2
88+
# - values: ["--dataset_generator", "sipu", "--dataset_name", "jain"] # 2 1
89+
# - values: ["--dataset_generator", "sipu", "--dataset_name", "pathbased"] # 3, 4 2
90+
# - values: ["--dataset_generator", "sipu", "--dataset_name", "r15"] # 8, 9, 15 3
91+
# - values: ["--dataset_generator", "sipu", "--dataset_name", "spiral"] # 3 1
92+
# - values: ["--dataset_generator", "sipu", "--dataset_name", "unbalance"] # 8 1
93+
# - values: ["--dataset_generator", "uci", "--dataset_name", "ecoli"] # 8 1
94+
# - values: ["--dataset_generator", "uci", "--dataset_name", "ionosphere"] # 2 1
95+
# - values: ["--dataset_generator", "uci", "--dataset_name", "sonar"] # 2 1
96+
# - values: ["--dataset_generator", "uci", "--dataset_name", "statlog"] # 7 1
97+
# - values: ["--dataset_generator", "uci", "--dataset_name", "wdbc"] # 2 1
98+
# - values: ["--dataset_generator", "uci", "--dataset_name", "wine"] # 3 1
99+
# - values: ["--dataset_generator", "uci", "--dataset_name", "yeast"] # 10 1
100+
# - values: ["--dataset_generator", "wut", "--dataset_name", "circles"] # 4 1
101+
# - values: ["--dataset_generator", "wut", "--dataset_name", "cross"] # 4 1
102+
# - values: ["--dataset_generator", "wut", "--dataset_name", "graph"] # 10 1
103+
# - values: ["--dataset_generator", "wut", "--dataset_name", "isolation"] # 3 1
104+
# - values: ["--dataset_generator", "wut", "--dataset_name", "labirynth"] # 6 1
105+
# - values: ["--dataset_generator", "wut", "--dataset_name", "mk1"] # 3 1
106+
# - values: ["--dataset_generator", "wut", "--dataset_name", "mk2"] # 2 1
107+
# - values: ["--dataset_generator", "wut", "--dataset_name", "mk3"] # 3 1
108+
# - values: ["--dataset_generator", "wut", "--dataset_name", "mk4"] # 3 1
109+
# - values: ["--dataset_generator", "wut", "--dataset_name", "olympic"] # 5 1
110+
# - values: ["--dataset_generator", "wut", "--dataset_name", "smile"] # 4, 6 2
111+
# - values: ["--dataset_generator", "wut", "--dataset_name", "stripes"] # 2 1
112+
# - values: ["--dataset_generator", "wut", "--dataset_name", "trajectories"] # 4 1
113+
# - values: ["--dataset_generator", "wut", "--dataset_name", "trapped_lovers"] # 3 1
114+
# - values: ["--dataset_generator", "wut", "--dataset_name", "twosplashes"] # 2 1
115+
# - values: ["--dataset_generator", "wut", "--dataset_name", "windows"] # 5 1
116+
# - values: ["--dataset_generator", "wut", "--dataset_name", "x1"] # 3 1
117+
# - values: ["--dataset_generator", "wut", "--dataset_name", "x2"] # 3 1
118+
- values: ["--dataset_generator", "wut", "--dataset_name", "x3"] # 4 1
119+
# - values: ["--dataset_generator", "wut", "--dataset_name", "z1"] # 3 1
120+
# - values: ["--dataset_generator", "wut", "--dataset_name", "z2"] # 5 1
121+
# - values: ["--dataset_generator", "wut", "--dataset_name", "z3"] # 4 1
120122
outputs:
121123
- id: data.matrix
122124
path: "{input}/{stage}/{module}/{params}/{dataset}.data.gz"
@@ -226,90 +228,4 @@ stages:
226228
- data.true_labels
227229
outputs:
228230
- id: metrics.scores
229-
path: "{input}/{stage}/{module}/{params}/{dataset}.scores.gz"
230-
231-
# ## daniel's data ###########################################################################
232-
233-
# - id: danielsdata
234-
# modules:
235-
# - id: iris_manual
236-
# name: "Iris Dataset"
237-
# software_environment: "sklearn"
238-
# repository:
239-
# url: https://github.com/omnibenchmark-example/iris.git
240-
# commit: 47c63f0
241-
# - id: penguins
242-
# name: "Penguins Dataset"
243-
# software_environment: "sklearn"
244-
# repository:
245-
# url: https://github.com/omnibenchmark-example/penguins.git
246-
# commit: 9032478
247-
# outputs:
248-
# - id: data.features
249-
# path: "{input}/{stage}/{module}/{params}/{dataset}.features.csv"
250-
# - id: data.labels
251-
# path: "{input}/{stage}/{module}/{params}/{dataset}.labels.csv"
252-
253-
# ## daniel's distances ########################################################################
254-
255-
# - id: distances
256-
# modules:
257-
# - id: D1
258-
# software_environment: "sklearn"
259-
# parameters:
260-
# - values: ["--measure", "cosine"]
261-
# - values: ["--measure", "euclidean"]
262-
# - values: ["--measure", "manhattan"]
263-
# - values: ["--measure", "chebyshev"]
264-
# repository:
265-
# url: https://github.com/omnibenchmark-example/distance.git
266-
# commit: dd99d4f
267-
# inputs:
268-
# - entries:
269-
# - data.features
270-
# outputs:
271-
# - id: distances
272-
# path: "{input}/{stage}/{module}/{params}/{dataset}.distances.csv"
273-
274-
# ## daniel's methods ###################################################################
275-
276-
# - id: danielmethods
277-
# modules:
278-
# - id: kmeans
279-
# software_environment: "sklearn"
280-
# repository:
281-
# url: https://github.com/omnibenchmark-example/kmeans.git
282-
# commit: 049c8b1
283-
# - id: ward
284-
# software_environment: "R"
285-
# repository:
286-
# url: https://github.com/omnibenchmark-example/ward.git
287-
# commit: 976e3f3
288-
# inputs:
289-
# - entries:
290-
# - distances
291-
# outputs:
292-
# - id: methods.clusters
293-
# path: "{input}/{stage}/{module}/{params}/{dataset}.clusters.csv"
294-
295-
# ## daniel's metrics ###################################################################
296-
297-
# - id: danielsmetrics
298-
# modules:
299-
# - id: ari
300-
# software_environment: "R"
301-
# repository:
302-
# url: https://github.com/omnibenchmark-example/ari.git
303-
# commit: 72708f0
304-
# - id: accuracy
305-
# software_environment: "R"
306-
# repository:
307-
# url: https://github.com/omnibenchmark-example/accuracy.git
308-
# commit: e26b32f
309-
# inputs:
310-
# - entries:
311-
# - methods.clusters
312-
# - data.labels
313-
# outputs:
314-
# - id: metrics.mapping
315-
# path: "{input}/{stage}/{module}/{params}/{dataset}.metrics.txt"
231+
path: "{input}/{stage}/{module}/{params}/{dataset}.scores.gz"

0 commit comments

Comments
 (0)