From 4991917fb7f8772de5e6218013e7cf7893ed30ed Mon Sep 17 00:00:00 2001 From: Dmitry Livchak Date: Tue, 12 Mar 2024 13:55:46 +0000 Subject: [PATCH 1/3] Support explicit runtime platform architecture selection for ECS and Fargate clusters --- dask_cloudprovider/aws/ecs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dask_cloudprovider/aws/ecs.py b/dask_cloudprovider/aws/ecs.py index 02db1d55..b8391b69 100644 --- a/dask_cloudprovider/aws/ecs.py +++ b/dask_cloudprovider/aws/ecs.py @@ -484,6 +484,10 @@ class ECSCluster(SpecCluster, ConfigMixin): The docker image to use for the scheduler and worker tasks. Defaults to ``daskdev/dask:latest`` or ``rapidsai/rapidsai:latest`` if ``worker_gpu`` is set. + cpu_architecture: str (optional) + Runtime platform CPU architecture + + Defaults to ``X86_64``. scheduler_cpu: int (optional) The amount of CPU to request for the scheduler in milli-cpu (1/1024). @@ -712,6 +716,7 @@ def __init__( fargate_workers=None, fargate_spot=None, image=None, + cpu_architecture="X86_64", scheduler_cpu=None, scheduler_mem=None, scheduler_port=8786, @@ -754,6 +759,7 @@ def __init__( mount_volumes_on_scheduler=False, **kwargs, ): + self._cpu_architecture = cpu_architecture self._fargate_scheduler = fargate_scheduler self._fargate_workers = fargate_workers self._fargate_spot = fargate_spot @@ -1223,6 +1229,7 @@ async def _create_scheduler_task_definition_arn(self): if self._volumes and self._mount_volumes_on_scheduler else [], requiresCompatibilities=["FARGATE"] if self._fargate_scheduler else [], + runtimePlatform={"cpuArchitecture": self._cpu_architecture}, cpu=str(self._scheduler_cpu), memory=str(self._scheduler_mem), tags=dict_to_aws(self.tags), @@ -1297,6 +1304,7 @@ async def _create_worker_task_definition_arn(self): ], volumes=self._volumes if self._volumes else [], requiresCompatibilities=["FARGATE"] if self._fargate_workers else [], + runtimePlatform={"cpuArchitecture": self._cpu_architecture}, cpu=str(self._worker_cpu), memory=str(self._worker_mem), tags=dict_to_aws(self.tags), From 08ec063652f0843ec5810e0a2b257ed2bb2d250d Mon Sep 17 00:00:00 2001 From: Dmitry Livchak Date: Tue, 12 Mar 2024 14:11:45 +0000 Subject: [PATCH 2/3] Include CPU architecture into yaml config --- dask_cloudprovider/cloudprovider.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dask_cloudprovider/cloudprovider.yaml b/dask_cloudprovider/cloudprovider.yaml index 1a17ff45..aa99b102 100755 --- a/dask_cloudprovider/cloudprovider.yaml +++ b/dask_cloudprovider/cloudprovider.yaml @@ -15,6 +15,7 @@ cloudprovider: scheduler_timeout: "5 minutes" # Length of inactivity to wait before closing the cluster image: "daskdev/dask:latest" # Docker image to use for non GPU tasks + cpu_architecture: "X86_64" # Runtime platform CPU architecture gpu_image: "rapidsai/rapidsai:latest" # Docker image to use for GPU tasks cluster_name_template: "dask-{uuid}" # Template to use when creating a cluster cluster_arn: "" # ARN of existing ECS cluster to use (if not set one will be created) From c7b0c3318a5e1f761c436e2e848a15a9d1cbc4b6 Mon Sep 17 00:00:00 2001 From: Dmitry Livchak Date: Thu, 14 Mar 2024 14:47:34 +0000 Subject: [PATCH 3/3] Capitalise CPU architecture, as AWS requires upper case, but some libraries output lower vase --- dask_cloudprovider/aws/ecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_cloudprovider/aws/ecs.py b/dask_cloudprovider/aws/ecs.py index b8391b69..1999a601 100644 --- a/dask_cloudprovider/aws/ecs.py +++ b/dask_cloudprovider/aws/ecs.py @@ -759,11 +759,11 @@ def __init__( mount_volumes_on_scheduler=False, **kwargs, ): - self._cpu_architecture = cpu_architecture self._fargate_scheduler = fargate_scheduler self._fargate_workers = fargate_workers self._fargate_spot = fargate_spot self.image = image + self._cpu_architecture = cpu_architecture.upper() self._scheduler_cpu = scheduler_cpu self._scheduler_mem = scheduler_mem self._scheduler_port = scheduler_port