From e76ab96be7e7ddf58e4e81c1d6faf3fd805db989 Mon Sep 17 00:00:00 2001 From: Yuri Chiucconi Date: Fri, 14 Nov 2025 16:09:41 +0100 Subject: [PATCH 01/16] empty commit From 2dc90c936dcf2de928f11c1268f426d0eb81adb1 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 08:45:34 +0100 Subject: [PATCH 02/16] add error handling --- benchmarks/api_bench.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index d137c50adb..dd060503db 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -120,10 +120,14 @@ def make_md_diff(self, agg_values_main: dict, agg_values_curr: dict): ) template = env.get_template("bench_diff_template.md") - rendered_md = template.render( - zip=zip(agg_values_main.items(), agg_values_curr.items()), - exceptions=self.exceptions, - ) + try: + rendered_md = template.render( + zip=zip(agg_values_main.items(), agg_values_curr.items()), + exceptions=self.exceptions, + ) + except Exception as e: + print(f"ERROR in template.render. Original error {str(e)}") + rendered_md = f"error in `template.rendere`.\n{str(e)}" with open("bench_diff.md", "w") as output_file: output_file.write(rendered_md) From 8b99d469d348f61aa12ee1d38c2bbe069a078896 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 08:53:42 +0100 Subject: [PATCH 03/16] improve error message --- benchmarks/api_bench.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index dd060503db..5d3adae66b 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -127,7 +127,13 @@ def make_md_diff(self, agg_values_main: dict, agg_values_curr: dict): ) except Exception as e: print(f"ERROR in template.render. Original error {str(e)}") - rendered_md = f"error in `template.rendere`.\n{str(e)}" + rendered_md = ( + "There was an exception in `template.render`, within " + "`make_md_diff`.\n" + f"Original error:s{str(e)}\n" + f"{agg_values_main=}\n" + f"{agg_values_curr=}\n" + ) with open("bench_diff.md", "w") as output_file: output_file.write(rendered_md) From fa9036b33fe12be1e3b71584a0054319d2e27d2c Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 08:59:25 +0100 Subject: [PATCH 04/16] improve error message --- benchmarks/api_bench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 5d3adae66b..4d14649cc8 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -131,8 +131,8 @@ def make_md_diff(self, agg_values_main: dict, agg_values_curr: dict): "There was an exception in `template.render`, within " "`make_md_diff`.\n" f"Original error:s{str(e)}\n" - f"{agg_values_main=}\n" - f"{agg_values_curr=}\n" + f"```json\n{json.dumps(agg_values_main,indent=2)}```\n" + f"```json\n{json.dumps(agg_values_curr,indent=2)}```\n" ) with open("bench_diff.md", "w") as output_file: From 2d8dae7eea7b8622d156232b9fc07a788a45bf6a Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:00:02 +0100 Subject: [PATCH 05/16] improve error message --- benchmarks/api_bench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 4d14649cc8..70099e6119 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -131,8 +131,8 @@ def make_md_diff(self, agg_values_main: dict, agg_values_curr: dict): "There was an exception in `template.render`, within " "`make_md_diff`.\n" f"Original error:s{str(e)}\n" - f"```json\n{json.dumps(agg_values_main,indent=2)}```\n" - f"```json\n{json.dumps(agg_values_curr,indent=2)}```\n" + f"Main branch:\n```json\n{json.dumps(agg_values_main,indent=2)}```\n" + f"Current branch:\n```json\n{json.dumps(agg_values_curr,indent=2)}```\n" ) with open("bench_diff.md", "w") as output_file: From 1d34db2356a90b4f149f3d3f85d238118ab93684 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:00:16 +0100 Subject: [PATCH 06/16] improve error message --- benchmarks/api_bench.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 70099e6119..5e6ee46d28 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -131,8 +131,10 @@ def make_md_diff(self, agg_values_main: dict, agg_values_curr: dict): "There was an exception in `template.render`, within " "`make_md_diff`.\n" f"Original error:s{str(e)}\n" - f"Main branch:\n```json\n{json.dumps(agg_values_main,indent=2)}```\n" - f"Current branch:\n```json\n{json.dumps(agg_values_curr,indent=2)}```\n" + "Main branch:\n" + f"```json\n{json.dumps(agg_values_main,indent=2)}```\n" + "Current branch:\n" + f"```json\n{json.dumps(agg_values_curr,indent=2)}```\n" ) with open("bench_diff.md", "w") as output_file: From cecd3adee4a56c77c97ac07db3ac55bd58677241 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:03:09 +0100 Subject: [PATCH 07/16] precommit --- benchmarks/api_bench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 5e6ee46d28..399e724824 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -132,9 +132,9 @@ def make_md_diff(self, agg_values_main: dict, agg_values_curr: dict): "`make_md_diff`.\n" f"Original error:s{str(e)}\n" "Main branch:\n" - f"```json\n{json.dumps(agg_values_main,indent=2)}```\n" + f"```json\n{json.dumps(agg_values_main, indent=2)}```\n" "Current branch:\n" - f"```json\n{json.dumps(agg_values_curr,indent=2)}```\n" + f"```json\n{json.dumps(agg_values_curr, indent=2)}```\n" ) with open("bench_diff.md", "w") as output_file: From 1398daedff83e7417a85eeda8e5ded9c58192946 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:05:48 +0100 Subject: [PATCH 08/16] messagge --- benchmarks/api_bench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 399e724824..89ecdd0115 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -132,9 +132,9 @@ def make_md_diff(self, agg_values_main: dict, agg_values_curr: dict): "`make_md_diff`.\n" f"Original error:s{str(e)}\n" "Main branch:\n" - f"```json\n{json.dumps(agg_values_main, indent=2)}```\n" + f"```json\n{json.dumps(agg_values_main, indent=2)}\n```\n" "Current branch:\n" - f"```json\n{json.dumps(agg_values_curr, indent=2)}```\n" + f"```json\n{json.dumps(agg_values_curr, indent=2)}\n```\n" ) with open("bench_diff.md", "w") as output_file: From 234e5e4350afb21e214d6fb51497027e03d51c1d Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:28:51 +0100 Subject: [PATCH 09/16] edit benchmark json --- benchmarks/bench.json | 41 ++++++----------------------------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/benchmarks/bench.json b/benchmarks/bench.json index 1afbe04d6a..edc9d8974a 100644 --- a/benchmarks/bench.json +++ b/benchmarks/bench.json @@ -35,41 +35,12 @@ "size": 0.0 }, { - "path": "/api/v2/dataset/", - "verb": "GET", - "username": "vanilla", - "time": 7.9, - "size": 0.6 - }, - { - "path": "/api/v2/dataset/", - "verb": "GET", - "username": "power", - "time": 281.0, - "size": 389.0 - }, - { - "path": "/api/v2/dataset/", - "verb": "GET", - "username": "dataset", - "time": 108.1, - "size": 145.6 - }, - { - "path": "/api/v2/dataset/", - "verb": "GET", - "username": "project", - "time": 34.0, - "size": 50.9 - }, - { - "path": "/api/v2/dataset/", - "verb": "GET", - "username": "job", - "time": 20.0, - "size": 35.3 - }, - { + + + + + + "path": "/api/v2/job/", "verb": "GET", "username": "vanilla", From 2ecc9578331593b8e55d8ad689f5feca26ad1b56 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:29:34 +0100 Subject: [PATCH 10/16] improve error message --- benchmarks/api_bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 89ecdd0115..6d53bdf1c1 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -130,7 +130,7 @@ def make_md_diff(self, agg_values_main: dict, agg_values_curr: dict): rendered_md = ( "There was an exception in `template.render`, within " "`make_md_diff`.\n" - f"Original error:s{str(e)}\n" + f"Original error:\n{str(e)}\n" "Main branch:\n" f"```json\n{json.dumps(agg_values_main, indent=2)}\n```\n" "Current branch:\n" From 88046dbb6846183c66f47890c63d2a045f48e880 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:36:47 +0100 Subject: [PATCH 11/16] edit benchmark json --- benchmarks/bench.json | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/benchmarks/bench.json b/benchmarks/bench.json index edc9d8974a..0f036a4352 100644 --- a/benchmarks/bench.json +++ b/benchmarks/bench.json @@ -145,41 +145,6 @@ "time": 4.6, "size": 0.7 }, - { - "path": "/api/v2/workflow/", - "verb": "GET", - "username": "vanilla", - "time": 10.0, - "size": 0.8 - }, - { - "path": "/api/v2/workflow/", - "verb": "GET", - "username": "power", - "time": 18.1, - "size": 27.2 - }, - { - "path": "/api/v2/workflow/", - "verb": "GET", - "username": "dataset", - "time": 14.2, - "size": 15.8 - }, - { - "path": "/api/v2/workflow/", - "verb": "GET", - "username": "project", - "time": 16.9, - "size": 19.9 - }, - { - "path": "/api/v2/workflow/", - "verb": "GET", - "username": "job", - "time": 9.2, - "size": 0.8 - }, { "path": "/api/v2/project/3/dataset/420/images/query/", "verb": "POST", From 7a440415cd03afda91f29abc1183f12143470ae3 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:43:43 +0100 Subject: [PATCH 12/16] provisionally edit link to benchmark data --- benchmarks/api_bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 6d53bdf1c1..2c68b164c5 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -275,7 +275,7 @@ def run_benchmark(self, n_requests: int) -> list: # get the bench_diff.json from the bechmark-api branch url = ( "https://raw.githubusercontent.com/fractal-analytics-platform/" - "fractal-server/benchmark-api/benchmarks/bench.json" + "fractal-server/test-benchmark/benchmarks/bench.json" ) response = httpx.get(url) From 3319895af3b00cb06269c5d59437ad966d31b9a9 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:44:39 +0100 Subject: [PATCH 13/16] Revert "provisionally edit link to benchmark data" This reverts commit 7a440415cd03afda91f29abc1183f12143470ae3. --- benchmarks/api_bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 2c68b164c5..6d53bdf1c1 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -275,7 +275,7 @@ def run_benchmark(self, n_requests: int) -> list: # get the bench_diff.json from the bechmark-api branch url = ( "https://raw.githubusercontent.com/fractal-analytics-platform/" - "fractal-server/test-benchmark/benchmarks/bench.json" + "fractal-server/benchmark-api/benchmarks/bench.json" ) response = httpx.get(url) From 5a6a712905209a9d961f40d963ef0ba61b26152e Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:47:14 +0100 Subject: [PATCH 14/16] more logging --- benchmarks/api_bench.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 6d53bdf1c1..2e2e6f8c7a 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -180,6 +180,8 @@ def _replace_path_params(self, headers: dict, path: str): updated_path = re.sub(pattern, lambda x: next(id_list), path) else: updated_path = path + + print(f"OLD PATH: {path}\nNEW PATH: {updated_path}") return updated_path def make_user_metrics( From bab035d484953b4a2792250279f2de19a76d30db Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:53:36 +0100 Subject: [PATCH 15/16] only test dataset-heavy user for dataset endpoint; --- benchmarks/api_bench.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/benchmarks/api_bench.py b/benchmarks/api_bench.py index 2e2e6f8c7a..b940fdab6f 100644 --- a/benchmarks/api_bench.py +++ b/benchmarks/api_bench.py @@ -240,6 +240,13 @@ def make_user_metrics( ) def run_benchmark(self, n_requests: int) -> list: + """ + + Note that the following endpoints are only tested for the + `dataset@example.org` user: + 1. POST /api/v2/project/$project_id$/dataset/$dataset_id$/images/query/ + 2. GET /api/v2/project/$project_id$/dataset/ + """ # time and size are the two keys to extract and make the average keys_to_sum = ["time", "size"] user_metrics: list[dict] = [] @@ -252,6 +259,11 @@ def run_benchmark(self, n_requests: int) -> list: and user.name != "dataset@example.org" ): pass + elif ( + endpoint["path"] == "/api/v2/project/$project_id$/dataset/" + and user.name != "dataset@example.org" + ): + pass else: user_metrics.append( self.make_user_metrics( From f2e5c69d788fb8d4fc63af1aa8ea39578b7433ec Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:55:31 +0100 Subject: [PATCH 16/16] Drop benchmarks/runner --- .github/workflows/benchmarks.yaml | 34 ----- CHANGELOG.md | 3 + benchmarks/runner/README.md | 1 - benchmarks/runner/benchmark_runner.py | 192 ------------------------- benchmarks/runner/mocks.py | 65 --------- benchmarks/runner/profile_results | Bin 24191 -> 0 bytes benchmarks/runner/runner_benchmark.txt | 12 -- 7 files changed, 3 insertions(+), 304 deletions(-) delete mode 100644 benchmarks/runner/README.md delete mode 100644 benchmarks/runner/benchmark_runner.py delete mode 100644 benchmarks/runner/mocks.py delete mode 100644 benchmarks/runner/profile_results delete mode 100644 benchmarks/runner/runner_benchmark.txt diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 4713cdf1d6..0eb55bce12 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -77,37 +77,3 @@ jobs: push_options: '--force' skip_checkout: true file_pattern: 'benchmarks/bench.html benchmarks/bench.json' - - benchmark_runner: - if: false - name: Benchmark Runner - runs-on: ubuntu-24.04 - - steps: - - - uses: actions/checkout@v5 - with: - persist-credentials: false - - - name: Start Postgres - run: | - sudo systemctl start postgresql.service - sudo --login -u postgres psql -c "ALTER USER postgres WITH PASSWORD 'postgres';" - sudo --login -u postgres createdb fractal_test - - - name: Install poetry - run: pipx install poetry==2.2.1 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: "poetry" - - - name: Install dependencies - run: poetry install --with dev --no-interaction - - - name: Benchmark - run: | - cd benchmarks/runner/ - poetry run python benchmark_runner.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 410f3d9f26..05305f69b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ * Major review of documentation, including making it up-to-date with v2.17.0 and relying more on autogenerated contents (\#2949, \#2983). * Development: * Add `shellcheck` to precommit, for `fractal-server/` files (\#2986). +* Testing: + * Update benchmarks (\#2990). + * Drop `benchmarks_runner` (\#2990). # 2.17.1 diff --git a/benchmarks/runner/README.md b/benchmarks/runner/README.md deleted file mode 100644 index b08bfd6a70..0000000000 --- a/benchmarks/runner/README.md +++ /dev/null @@ -1 +0,0 @@ -`benchmark/runner` diff --git a/benchmarks/runner/benchmark_runner.py b/benchmarks/runner/benchmark_runner.py deleted file mode 100644 index dfd7b460b1..0000000000 --- a/benchmarks/runner/benchmark_runner.py +++ /dev/null @@ -1,192 +0,0 @@ -import cProfile -import json -import os -import pstats -import shlex -import shutil -import subprocess # nosec -import sys -import tempfile -from pathlib import Path - -import benchmarks.runner -import tests.v2.fractal_tasks_mock.dist as dist -from benchmarks.runner.mocks import DatasetV2Mock -from benchmarks.runner.mocks import TaskV2Mock -from benchmarks.runner.mocks import WorkflowTaskV2Mock -from fractal_server.runner.v2._local import LocalRunner -from fractal_server.runner.v2.runner import execute_tasks_v2 - - -def _run_cmd(cmd: str) -> str: - res = subprocess.run( # nosec - shlex.split(cmd), - capture_output=True, - encoding="utf8", - ) - if not res.returncode == 0: - raise ValueError(res) - return res.stdout - - -def mock_venv(tmp_path: str) -> dict: - venv = f"{tmp_path}/venv" - python = f"{venv}/bin/python" - - if not os.path.isdir(venv): - # Create venv - _run_cmd(f"{sys.executable} -m venv {tmp_path}/venv") - _run_cmd( - f"{python} -m pip install " - f"{dist.__path__[0]}/fractal_tasks_mock-0.0.1-py3-none-any.whl" - ) - - # Extract installed-package folder - out = _run_cmd(f"{python} -m pip show fractal_tasks_mock") - location = next( - line for line in out.split("\n") if line.startswith("Location:") - ) - location = location.replace("Location: ", "") - src_dir = Path(location) / "fractal_tasks_mock/" - - with (src_dir / "__FRACTAL_MANIFEST__.json").open("r") as f: - manifest = json.load(f) - - task_dict = {} - for ind, task in enumerate(manifest["task_list"]): - args = {} - if task.get("executable_non_parallel"): - args[ - "command_non_parallel" - ] = f"{python} {src_dir / task['executable_non_parallel']}" - args["meta_non_parallel"] = task.get("meta_non_parallel") - if task.get("executable_parallel"): - args[ - "command_parallel" - ] = f"{python} {src_dir / task['executable_parallel']}" - args["meta_parallel"] = task.get("meta_parallel") - - t = TaskV2Mock( - id=ind, - name=task["name"], - source=task["name"].replace(" ", "_"), - input_types=task.get("input_types", {}), - output_types=task.get("output_types", {}), - **args, - ) - task_dict[t.name] = t - - return task_dict - - -venv_dir = tempfile.mkdtemp() -fractal_tasks_mock_venv = mock_venv(venv_dir) - - -def benchmark(N: int, tmp_path: str): - WORKING_DIR = Path(f"{tmp_path}/job") - ZARR_DIR = Path(f"{tmp_path}/zarr").as_posix().rstrip("/") - execute_tasks_v2( - wf_task_list=[ - # compound - WorkflowTaskV2Mock( - task=fractal_tasks_mock_venv["create_ome_zarr_compound"], - args_non_parallel=dict(image_dir=ZARR_DIR, num_images=N), - id=0, - order=0, - ), - # parallel - WorkflowTaskV2Mock( - task=fractal_tasks_mock_venv["illumination_correction"], - args_parallel=dict(overwrite_input=True), - id=1, - order=1, - ), - # compound - WorkflowTaskV2Mock( - task=fractal_tasks_mock_venv["MIP_compound"], - id=2, - order=2, - ), - # parallel - WorkflowTaskV2Mock( - task=fractal_tasks_mock_venv["cellpose_segmentation"], - id=3, - order=3, - ), - ], - dataset=DatasetV2Mock(name="dataset", zarr_dir=ZARR_DIR), - workflow_dir_local=WORKING_DIR, - workflow_dir_remote=WORKING_DIR, - runner=LocalRunner(), - ) - - -if __name__ == "__main__": - results = [] - - for N in [100, 200, 300]: - tmp_path = tempfile.mkdtemp() - cProfile.run(f"benchmark({N}, '{tmp_path}')", "profile_results") - stats = pstats.Stats("profile_results") - stats.sort_stats("tottime") - - thread_time = stats.stats[ - "~", 0, "" - ][2] - total_time = stats.total_tt - - list_dirs = {} - for path, key in [ - (f"{tmp_path}/job", "job_dir"), - (f"{tmp_path}/zarr", "zarr_dir"), - ]: - size = 0 - count = 0 - for file in os.listdir(path): - if os.path.isfile(f"{path}/{file}"): - count += 1 - size += os.path.getsize(f"{path}/{file}") - list_dirs[key] = dict(count=count, size=f"{size / 1024:.2f} KB") - - shutil.rmtree(tmp_path) - - results.append( - dict( - N=N, - thread_time=thread_time, - total_time=total_time, - list_dirs=list_dirs, - ) - ) - - runner = os.path.dirname(benchmarks.runner.__path__[0]) - with open(f"{runner}/runner/runner_benchmark.txt", "w") as file: - # Headers - to_write = ( - "\n\n\n" - "Parallel Tasks: 2\n" - "Compound Tasks: 2\n" - "Images: N\n" - "\n" - "|\tN\t" - "|\texecutor\t" - "|\ttotal\t" - "|\toverhead\t" - "|\tjob_dir\t\t\t" - "|\n" - "|\t---\t|\t---\t|\t---\t|\t---\t\t|\t---\t\t\t|" - "\n" - ) - # Results - for result in results: - to_write += ( - f"|\t{result['N']}\t" - f"|\t{result['thread_time']:.4f}\t" - f"|\t{result['total_time']:.4f}\t" - f"|\t{(result['total_time'] - result['thread_time']):.4f}\t\t" - f"|\t{tuple(result['list_dirs']['job_dir'].values())}\t" - "\n" - ) - print(to_write) - file.write(to_write) diff --git a/benchmarks/runner/mocks.py b/benchmarks/runner/mocks.py deleted file mode 100644 index be036fb065..0000000000 --- a/benchmarks/runner/mocks.py +++ /dev/null @@ -1,65 +0,0 @@ -from typing import Any -from typing import Optional - -from pydantic import BaseModel -from pydantic import Field -from pydantic import model_validator - - -class DatasetV2Mock(BaseModel): - id: Optional[int] = None - name: str - zarr_dir: str - images: list[dict[str, Any]] = Field(default_factory=list) - history: list = Field(default_factory=list) - - @property - def image_zarr_urls(self) -> list[str]: - return [image["zarr_urls"] for image in self.images] - - -class TaskV2Mock(BaseModel): - id: int - name: str - source: str - input_types: dict[str, bool] = Field(default_factory=dict) - output_types: dict[str, bool] = Field(default_factory=dict) - - command_non_parallel: Optional[str] = None - command_parallel: Optional[str] = None - meta_non_parallel: Optional[dict[str, Any]] = Field(default_factory=dict) - meta_parallel: Optional[dict[str, Any]] = Field(default_factory=dict) - type: Optional[str] = None - - @model_validator(mode="before") - @classmethod - def _not_both_commands_none(cls, values): - _command_non_parallel = values.get("command_non_parallel") - _command_parallel = values.get("command_parallel") - if _command_non_parallel is None and _command_parallel is None: - raise ValueError( - "Both command_non_parallel and command_parallel are None" - ) - elif _command_non_parallel is None: - values["type"] = "parallel" - elif _command_parallel is None: - values["type"] = "non_parallel" - else: - values["type"] = "compound" - - return values - - -class WorkflowTaskV2Mock(BaseModel): - args_non_parallel: dict[str, Any] = Field(default_factory=dict) - args_parallel: dict[str, Any] = Field(default_factory=dict) - meta_non_parallel: dict[str, Any] = Field(default_factory=dict) - meta_parallel: dict[str, Any] = Field(default_factory=dict) - meta_parallel: Optional[dict[str, Any]] = Field(None) - meta_non_parallel: Optional[dict[str, Any]] = Field(None) - task: TaskV2Mock = None - type_filters: dict[str, bool] = Field(default_factory=dict) - order: int - id: int - workflow_id: int = 0 - task_id: int diff --git a/benchmarks/runner/profile_results b/benchmarks/runner/profile_results deleted file mode 100644 index aa055b7d96089b2c50c83bfb7cfdf77f8802d7f2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24191 zcmb_EXJ8Y@(gqhy$EJs7FwGR38oCY$VAHYbB?wuT*N2P5O2P&cdMKeh0t7+=flxvT zJ&-_vP(lwKLJ_?qLJK{V@XhRLZ%;nkFW)17hQT*CGdnvwJ3Bk4*I~Y|+DCMdW2uP_ ziD*Mj$(B?rO|(aJ8RAH=ri91yzeMz~P`hkRu}9PmuU#u5Nsf)MF%tG)vo+3Qwujq> zxPSKX@o@zjjdF_YFdD=B-TB}T)DHkjTe%>2arw4EMbel1zREG&4bOaDm24X8l#?7a z<&=J4`8Ni#fq!PQ+oz zyI}|nka3nPhM^Z4OJvxn6?Y<2+eP$(2Cxh#FFw~?RYEsCOW-&V*7Nf&tPqZ5c*9r4;7l8+~ z5CGD8Yqv*!({30@_t?2JCsy^7K#v~(jvmPtN-RdJ(_wQu*tF(Cb@F@tIm}O9i$*~} z7H|K!_^SqBl%87#blzUCorLq!>HX)$X0^*{Hj^U(Y$8YJK;KHS(q#M#Ov^Uwyyjr(3B((wHV>=FzRfiDx?0Vt`KWU zF()LOaJhM0K>dqR`2%`F4^$$z`UI8~n8AmZ1wi`r9I~U;?=2-#bD*Qzj%oFqr-%8v z0-M?$R8AoVMg#v%w#GS=EVz*Hr7Ye=)nnd&!Uo!{DG_}6qxxy6zF(X(*%s!nS;)Dc z7Ylk6_oYPSvaR}>Kie*)Q;ZZ|i(N;9I<|d(9W5!&Wb`M({K;rEo05`nk<*fB9t5P` zwWJOE?>sS(Hef`gtz7Ia>AH%8I38H)C_#51=csd>$%@RI06%< zyo1WnN{R`D3{IxfqX{~^zX{A%+Yo9YI6-kB)0KxQYmFVqg53sid>H|`R4Aot!#OP^ zBK_2O=(5XEY+d~eQ-Jbu3Vd-2@)@bc3|otXI?WDPVA+Vr;YYv!@R^sw>Vs{Iq74$j z2Gkdgul3O=^8DSNosSKKBM|9nc_giXAQb7fn-eEf>Cqu-0+ss^09PK_VWEmpFEM=$ zi=D2NLK2DImZ#QVQ%b^8h{(uVN5fgmMbUzVjDoi@Nwzy6I0lx?;llvAvoiTVBj0@? zksA+l9iKUDJx(IiOP+@>=heIn*3afWRcJ?@&^Q^U&R=efLc8tG;Y02j3(ffY+r<)D z`}T(u*VC2@p{Oyf!$v4SDgpEe$m}2Gku&bikx2aWwp}k+m&0zR=!mjteTDpk@4+!E zECa_3KA7tBeD+j{_^c>3w@vH}iPmK#?so7GZ3uwW9TfGvU9`VM%FQTn>hA)QyXWPKh(JN5J1;VqG+|+?g{FSAN?Z8P#}V2b}F@II_;jAx~5E zVY4;a2AdXpL3!&<%)BerW9mp`H|XKx4gf%spH{O}-}BT!mJS|p?K8I@^HuiJb$qFSk-ueRZtkbq zD4ut0aAD$Smks2}nApf|6Ym-LHr5*6qzK5B7L6|i?Qsk4>NNg{+zScrCWkdyHbZa_ zFPFZPw7c2G83w`;T)U_m{RlxT0N^v((5u1@pfJr)lrW)52`+uTe`epKKZBmP?~ac7 zavSV<9TfH=ZP3wh_B0CBX^Zs!jHlZda58J{H_Lx*dmeJsFJ~3}==2l86DL|pMRYWr zr~3)eKzK@oIWs~AHrN8Z#oqXnU%mNPy1EQx%bVYV&Yig|cxOYyd9$5|)?oT-Hk`Nb z{B31EHc4b@i^uKc1X=Y2+8Fh1g7n%AN=Fa;|Bd}Q)nZDdmU!GES1K_Y4h#+m^2scr z!2Dl11jP0KrCPTsRV2a@JdN~!{~Z$cW3!XOb_E9*r^88cl_yY(DGs;G4ItiC#Au2e z=(IbG%HNGCljx}kBpyuWTnk9EKzpbagJ+W4B{;=ntEVP-x!yyG$nkw}Z! zzPY>A3&olIH%=tVXd%7ExoHAUuH+5caqC2*(_x^Yc#L`#p*!jtF&_z6QUhbG$!<{+ zc%#W~mSqEk_+MeQFhK4Hk1gVNK3P(cblL!zf>UNg<1N2dQ_l&zN}Oyr#U@#*gjnN4 zsu+{(HfoIz?*c$hvQYSI>_CgzVFv{?QKE0z8rOD}|HeQ#l8&?bFVqkmLu?+a)xW4}}d#*To@8(;X#PJhJ1*5yH9|AaHkKRs@F4&vIAw;c}n$tHV*aWI6JI0(O# zo%;gOqXOV>Fb`naY&1a2;!#~OPzcV1)>zA1TuV6H2udD~njhWpoX<3#P)_FwtOx?t zhj2Xkd!N{Jyim3gB+74LUv~urA;jW< zoiE?Y$1nY5Z*g<=RXC4*cCr2MM{XF<>HdXt5Ec@kmW4sB zGHR9IC=)pETZ|S;t>Aqc$FFidtx5a7M-3#p=E&CvOC2}hPAsGS{r%DYmNdv&?4Y&S zEp@Npl=XvG+z3`EvX2&xbnt5c*7HFvZLn`s7gA2h8oG=fiv(>CPpBIxbjv_hy9x)FlI(x9AH3*|-KX2ilIhG2uxTVt1bx6Ymz-_0tKNp%jEo)P3=X&_TP z5a?aGVX>sZX^gFxwA?U!1Z2d-j)@CyL-M$FL)CFRzM;Z8Y!l99G7oaf)B+(v%vf;e z*)1lwjFL$2YRPT#+q+;U%LH`<3%UGKO|ti<7`o!zZ#iS{8A!;duZG?1_1pmTcAS8P zs8|4Ow6Gw&Bow+(s@K;)n_(xr9P&-nnItimy!Z+fTfy~Mv6YQN8A`phoad7#f&to( zxi)iYF*_`MEN*%+p7D+E(1UfFOQh?H&9_dJYAw-JMxxn8U)jprCl!aY>5(6kpBmZ* zOLRy^F1bFvbLH>X10_;Hg!2izM>ace)2iPF_N_#&Z5tYS^6z@hRn0_1(1yW!tIAcM z_Ss1TSsU`C+?lFw76!3K=}rg0LxPAPI(n5qt|>Yg=Rg|)S9D(Y$F$er?H?G1b4*A}nch!Fu+^uGAH!B+}{8l@A*o21lcP8BdL;>u6M-VYrY3VCo^? z^hz;X-AME+ckXus;Rx;sZ0*m4iFEnH9Rl@U{G3pILluJ2Lh*R5;wK(QRd+p)kjRUI zXD9V62$dT;CZpmy8kMKWPPIJSmTh$F?cYrxoRm&5Js;goHL%7QIvSNHj*O0Flcu3> zQfqb9DYrq-gQwSZxZ4W#R3^A;g3`rRn9DwQ^0dt z@|=de*k2195IkoD6rwH!w3xWBMqp;{r=vmkm|eKBmgUN8Gf@bk03pTjcSEtqhkgct z{23c?G%UH8MCwnvc`V1_5EesOR?-OFHZQ&cFPbR9)C6D2;yqPff0P$A&?=`W@5XVd z6sE~@m3QV`-zl@&T{94lsJRePq_Q%YIYCZ}1F6}(r%HW@QghpFNwR}lt+>vFu6sVQ z^!$0#a|4+=XW-Z|#+Pd5MJJ#}69IUQO-Bz>UVZeBb6WMPMZK0uhBS>G3@{;O1sf%8 zg>w=l5ilW|oUh-oGgCX{SYjX?LDQ5%Jxe1fqbMulBPUboxk+T4Ksp)ewZ^Er{o0%V zw1Nq5n6_wr;1@N;gtLSruQ8rllRZEa7U{!UyB{Qd6OHz4=x9_C15iW}V?18w zIWl^-@U^}_2QHm)E(dI?$103$o-el;Q!+AuOC2~9LfWWZOS*Sdp|~38G{tVh&f|YL zT0ZTIK#9zse|e;7Yb_z8JMJ5KjYhZ$vBb%PnVf9iQ{|v;bS{(QE(rh7Zvc>hgJ)+| za0W}HK%Kj%TfQzUxolr4i9Et}P$_*S6F zZe+KzU;^RA?=F-~>~{NzfjDE9hkm~ABA(BbmrxX)ub4oXS+39}wEZ7{8pw_lJKG#N z1Yry4MTw-e?qJMBhQp5P@IDns`Hg%N@NL*edFgfpq*lSBa84%Zc@YmZ9G&gA_)o_TAJv zO}6YakX#i%G(N6*3mkT~!ZRBh)o|zwbe_Uf=&)^N!i!TQK!?O?J&yI?T^sh2d>QrB z(WpGdy42Q~d#^F&iVV91<61Z3`&w{OqUtHuu8v0K*$R|%`QglX#X*a-V|IzLa9noX zEppD#AqOx`$~br(jmmU7GWBQELQLi%Mk_U@u-!|PE|l{B zIf#;}hB^M>7ljvOL!+`E4(#bHMP)%>Y5!ltJ~{}V#SwMJvY|nqVu^*SRX}J#fvSsA z1jI`LZQ)NoI|B|12uOI1jl>iTr2~%O##R(GV|^L480zRJ-x&#iKtSrI3|#Qn+(b!5 zfSg@ibOA~l&B!&I!}Ux1W^Qgi*FZQzK`3x^vu$%F;H1-lJdQmd9W=XLZQ^V1+V3F1%TlEC2E$kd1ke<-V_a|KsE!lOm!hcqy1M-kD-{V6POG zl`uPT6@e$vzQg)>aPNVJ0}Hm2hzqfiHJ(U2gK|rRBVeu7@Tz;C?tQ!$gIYUeR+{&} z+S&72V*5N2;RvKW|I(=9EOAcA%giQdL_+_-QRP9c3T1o(8j0qZ&P(pUyVbOzL^zU_ z|5?%;S^Hl?ZkRU&#NK=I_?wNIN-C1hZ~q1$4Fb}rbN?E>$~BTy1m?EqzcaVkf(3nH zknhEt%oel(dc5lcrA3Tia@rx&hCZPbd5{yzM4AF-O4C2z^*q1QKsW-L_4;=-^ORYY zUNI}{XTVEkmv6Gk=B2}p$+jVH2SDDz5$_oZQ6qxQYRe)Vh>kDEjxxKKb%c9lj$k-o zg{(f9CKOZ(S?nqt>+0Yd*}UtmNCj09xJb67Wub~#JKgde`F7H&z7pX`I&BID@LC`I zkxJt&vZ5-ITA)kM?zt1m7!KXRz?bd;K;m1PK8x-Pbr6oEdjgi`;IiyYWDoMmUIXC> z-O2RQ_(kZdp}#a~w8KC+;;omM6DXG}hQKRmuMiyqrwbVYF;#BT?$()(P*C$A=`=<$ z#4%7@MnFc-yfl_A6;j(}|azWkke zk>49s1VYby^+WPo8EA;azA+X?@%q)3pDn_{wQ-xpW=JMqQNnk_!_n3T8w`Xa;Ek-I zpd!AQ4|0BUf>AlIfnU^v8y*J8tJF_>K8uHnHAlSdUtaZ#M9UC+786hv8!!-%jwio* z>0@posR%9#6KLQrk81eBl@dq~L3#w_WK`@Zb047R2v6pOb$@(PWYB}g>kNb=-U@2g z3S16nrQrhDcx41+>Gim$jox;H3vLgh-XBK8UM?dbKP9evv32)Hl8Vqs&>qa)8UA5_ zT$g zZe;{S>i27C*gTjTj_~xKJ;({FFJ=1n19a72wM+0b}PET=9a zxTIGELMc@n!`xMI4lV#7_XAzePx`}+14r;=*c%_SL=Z2|z!SxlQ~bQdRtwEg9O8(+ z@>u1g;lG_e=O+=4c&k@gRZq6VjW|}mJfSv&mYT3#@mcfU37t0?2uJulWJlvIwt?S-9o+jzJPw|MT~Lrjj4T4}XwS?boQTens%YzNo{v6=x=|HY25Q>DY9{O3W?T5wnH zLzzXxn6)(hE#v0YiQHKdDxDmmUn&L_C%wyuYFKuJLBQb%aFl_FtvvdbLM}FoC8I|+ zyjL7fPaN?cWewH1M!1%VbDFUTYcwiepfnh09YLdNmjffm6<*RyA{Al^!eGy=Ul zbDj30P6)`?=A@cUuD~9_kxX>DKYh{OCEo!JN4S?@osE9280|CoM=3WS>5&lk!$G+9 z0;=^5&0+BqXY9cezTl%A!F?8;lzxt2x*}efhm)!FS&66Fh!-x< zX6M*V1rI?}9!J#UcQ!QMA~kN3<$&dfg<(n%kZt#x9yn!eC#eXobCy`JNO+75I6Hc8 z=_2H@J1Tps>`2JmD@IlV6%dfSE9Ttn_#Ir#aYXeBTXCR4K;nl4%kRRvLKCkCq3~DS zhQ+aP$<^JD3oh;=5svVERpj&0r!NmRKdlZ8(H_K0c^Zc-wI%oKp9m?+=xk`bMe0kV zs!`-PW)I80`qT2ToBwGKg%^(SwltAe=3QFz#l9}U zh9mss!SXc(q{ZI$T`$>I7*qtTS{)wHjtDv_Ld6 z!*hgwuW%7US0C$pF#g>t1L26bp(!WJPTARThP>sD|Dl~jbc!DDwj zy$^uIjO#$Y$L@QVn1L0N3PWA$#Aaa2tPs!&ze-L_U|3f!)-4| zfTP-oMlFmZYd}-F&>9ptz54N?ToU03FWX^JkFv_%o8Z=`$KL`SIYLh;Ttu?HV9g5W z2W22AaD<*#NCI=AF#wQ&&?c9B@RD_N1KT-xGF7Abq!+T@l^dccRE9LZ$IDSW>F*ojSnh9f)hUdZ~&H{yRZPKpkX)D%tSn}ALAfF>@2cGDAWoSF(2s%?gP)x>7G&(+`ikwWP_m&?aDm1*kRXGasDP~>jUTa!(k0zWi zcPZ>7)Soz_8af*qZ}}ez`5F3m|G65)hQ#!)+}jDE#e-y`27jc|c&kxYsG)6G{^vR@ z_q(|B%}F@ka3m8Q_#>6ZTL;Zl@c#7wCVxHbWy+@mrhq(?`!tGy%O4MdH4)|(*z!&t z%q=vH>zqub_tqs`*aIqd*kAtNRN0HFARuFN-P_V=CYXXFsw&vAPSZ2061=2Z|26}F zo+J7$*W(S>U0MAXbWw1`TU$|P^!SjsvKS%YFR}c4Cb<>FT?R;cl|%Ed`PXl*BI;_w zrgA!5(N#2|KN~u`d*@OWB*GEC_^?Hsj>t$}yWpJ1<(E~H2uFC=`EES6ARw`03L6fg z1{}$1Xdb`(FGCZmE^Tjj6JO}(qC&i@Dt70p+!H??^kfh;NDlZ%~T2c}IBL~^h zc*__52fs?=kq+6W(-PiQPPQdkq7l~)U*lV|O7ER^el3(NIFe2=6v2Fo0D86K^Df(@qBb@G{<_~KXv-!%^UkNXoBSkDs6{1G+!oZzi$|@^6+u{qTUi& zG_P5ZZRk+&0-Yv08kOf*P#S(O!-%PY=Tjjq^Csn2r>|+(6Ea*6lCJ+=hd)wjROa{v z5*^CiR%4dE9m-#FO`a~1sqRg_+aOM*d%74j4ILk^uC$@u1fWUl-Rp9{&Gnk=&)eoj zLmn(`4nL;xxqrnHxl5d6&6CVdJ#vCrcAD0fLZ{5@nk^#WwcdH|lGN4Bq=VU($|xFs!YNr>6X)qfvPZ9ke?8 zl%78Q#0wvZTzc#p_lyt8q;ClT0*1Jh z(b22?1--wf+avmesxP;Xb4aAsqk4!GXZr8`du<1ot9$Z7-(~f%$#yo%)mC_PG%C-0FeI8E$EDS?+3+(z7CA~K5*yt4 z@vGsRScqT+x)9WX(Q5;|j$Y-@@}XeHUvqlxK6uyczayY3omTl$l)d>w(YY(me>xhK zXHV3j7miV@^Tm=qZ9c>LL#>n^Dbt|uMzDh}{Of3trwQ2|Md0yU;WLjT>j!Tqu(!Sr zjIz}{^A+%f-V^kJxtQi6AU9$nUA z+uJ=H!6NiNl(-+k0c5F-c7}pmF4m7d0pA9{8T?C-?F^Vl5vwUy>u7K&Cz0K609?W1 zx7BzDC0I*T^y61Wl`g&tZrK{G+#XcA#4Ir~aWSyPci7ko2I%>W?9p{jM43+|PdS)< zB&mEUnd92^E=BV}ZnxGs!8a(?st#ThKnle#%KLTEIuaRiV8-+b1BbK0GevPNy9x+J ztyG3+sQOH6 z*XederUEPYfj+8kOf+!BZ38Kix4`?o$|kA?Ij?*t_c%Ulnd5 z_5d9X@+^Vu*v5&4XfV0BJR8wtRsNZ=;Ho`_E~xT4c>!Dbs$0=gxRT+Qtni4%ZiaF{ z`hH~xTy;m{FZTMOZ)9QkFxnRG+;gC7Dw1zTZEz;EzQBHm zP}`GMBx&Nv4~N5>fw@Z-X&IEtCNV#}<;I=_PnM(covM!QldJTE#EBz(4P{4z#$i0y zqdIw-V!xR3V@duRr_VDpe5GGk_~4l%Dr@0${Ey56?V%7*Y?q zRXm8wGaDMtlfqHVhZY3jwUa-)7T)xH(dD40VbqBl zeX^dP@WLKw*rr<}Cj>;kzmiVm@XH;fggqZy;j?7NK51Mu2-ZOPq_ZLNPDdYwOsDptSo z&CJB+7&B>oo{NVKWXAKP7T3&2Rk30o2b8o#J{@+(K(-Bce=tJq2U&E-$b4|PplMmc}!oD0~UY>;hUJo-}obyhV!Ht z|L6}H^}PP8Z_el~^We7=E_51RXL;q!^wiNH&ke|KBZ40lo?@o|ShaujH?CRmc~0E^ zveTQ)1|QblD7!-+jB~UeA(38dkIo!>bp-!)I?YnJ+V+E8ql2g7{#x~kY1~w{P=iro zVFX1KvBEP7mgvyB>ZEBBskrsao%VK1K&;NeA}98Bt+kJW^LfDX7F)lCUqaMrwzlJt z0TODqWzd8c)t}5_VsS%rV|x~#6(P3et(Rx_j43G*j->O;@oZ?&g2Ur1x3<^8#R6a zFZAcHsPA%Fv^uVJ+#|j8513GKS@hax(D7Sk)7#mF;3IU;=SLczIvU^!KX~>YKTzfQ zwv2u9l>#?G_$>rGRk;eoeItAvV{};I`WZe5!>6!Ui;rxs+?f>cplCCRaD@AgW>1@N zVBTzFe&{3Ph(3jE7&z29tSpdlM70x3OUp1jX;_xd@qA=#4XCsQZ4RlrBoAi8y0lbB zgFLt46n}uW*B$g;$C&Fc427+C_56zI6Hh7!Jv-3LqIRVHzQ|;`6`T7hq_HjXF1|N- z%ce}0gwNr@2Kmr!1DTmTbL77