Merge remote-tracking branch 'upstream/main'

Author: macarronesc

ci/scripts/test_imports.sh

@@ -20,3 +20,4 @@ test_import "azure"             "import dask_cloudprovider.azure"
 test_import "digitalocean"      "import dask_cloudprovider.digitalocean"
 test_import "gcp"               "import dask_cloudprovider.gcp"
 test_import "ibm"               "import dask_cloudprovider.ibm"
+test_import "openstack"         "import dask_cloudprovider.openstack"

dask_cloudprovider/cloudprovider.yaml

@@ -130,4 +130,18 @@ cloudprovider:
     worker_cpu: "2.0"
     worker_mem: 8G
     worker_threads: 1
-    
+
+  openstack:
+    region: "RegionOne" # The name of the region where resources will be allocated in OpenStack. List available regions using: `openstack region list`.
+    size: null # Openstack flavors define the compute, memory, and storage capacity of computing instances. List available flavors using: `openstack flavor list`
+    auth_url: null # The authentication URL for the OpenStack Identity service (Keystone). Example: https://cloud.example.com:5000
+    application_credential_id: null # The application credential id created in OpenStack. Create application credentials using: openstack application credential create
+    application_credential_secret: null # The secret associated with the application credential ID for authentication.
+    auth_type: "v3applicationcredential" # The type of authentication used, typically "v3applicationcredential" for using OpenStack application credentials.
+    network_id: null # The unique identifier for the internal/private network in OpenStack where the cluster VMs will be connected. List available networks using: `openstack network list`
+    image: null # The OS image name or id to use for the VM. List available images using: `openstack image list` 
+    keypair_name: null # The name of the SSH keypair used for instance access. Ensure you have created a keypair or use an existing one. List available keypairs using: `openstack keypair list`
+    security_group: null # The security group name that defines firewall rules for instances.  List available security groups using: `openstack security group list`
+    external_network_id: null # The ID of the external network used for assigning floating IPs. List available external networks using: `openstack network list --external`
+    create_floating_ip: false # Specifies whether to assign a floating IP to each instance, enabling external access. Set to `True` if external connectivity is needed.
+    docker_image: "daskdev/dask:latest" # docker image to use

dask_cloudprovider/gcp/instances.py

@@ -4,6 +4,7 @@
 import json
 
 import sqlite3
+from typing import Optional, Any, Dict
 
 import dask
 from dask.utils import tmpfile
@@ -106,7 +107,6 @@ def __init__(
         self.instance_labels = _instance_labels
 
         self.general_zone = "-".join(self.zone.split("-")[:2])  # us-east1-c -> us-east1
-
         self.service_account = service_account or self.config.get("service_account")
 
     def create_gcp_config(self):
@@ -494,6 +494,8 @@ class GCPCluster(VMCluster):
     service_account: str
         Service account that all VMs will run under.
         Defaults to the default Compute Engine service account for your GCP project.
+    service_account_credentials: Optional[Dict[str, Any]]
+        Service account credentials to create the compute engine Vms
 
     Examples
     --------
@@ -587,9 +589,10 @@ def __init__(
         debug=False,
         instance_labels=None,
         service_account=None,
+        service_account_credentials: Optional[Dict[str, Any]] = None,
         **kwargs,
     ):
-        self.compute = GCPCompute()
+        self.compute = GCPCompute(service_account_credentials)
 
         self.config = dask.config.get("cloudprovider.gcp", {})
         self.auto_shutdown = (
@@ -641,9 +644,17 @@ def __init__(
 
 
 class GCPCompute:
-    """Wrapper for the ``googleapiclient`` compute object."""
+    """
+    Wrapper for the ``googleapiclient`` compute object.
+
+    Attributes
+    ----------
+    service_account_credentials: Optional[dict]
+        Service account credentials to create the compute engine Vms
+    """
 
-    def __init__(self):
+    def __init__(self, service_account_credentials: Optional[dict[str, Any]] = None):
+        self.service_account_credentials = service_account_credentials or {}
         self._compute = self.refresh_client()
 
     def refresh_client(self):
@@ -654,6 +665,13 @@ def refresh_client(self):
                 os.environ["GOOGLE_APPLICATION_CREDENTIALS"],
                 scopes=["https://www.googleapis.com/auth/cloud-platform"],
             )
+        elif self.service_account_credentials:
+            import google.oauth2.service_account  # google-auth
+
+            creds = google.oauth2.service_account.Credentials.from_service_account_info(
+                self.service_account_credentials,
+                scopes=["https://www.googleapis.com/auth/cloud-platform"],
+            )
         else:
             import google.auth.credentials  # google-auth
 

dask_cloudprovider/openstack/__init__.py

@@ -0,0 +1 @@
+from .instances import OpenStackCluster