Merge pull request #93 from BerkeleyAutomation/datacenter_selection

Datacenter selection 

Former-commit-id: a8fb166

Author: SimeonOA

fogros2/fogros2/aws_cloud_instance.py

@@ -107,6 +107,7 @@ def __init__(
 
         self.create()
 
+
     def create(self):
         self.logger.info(f"Creating new EC2 instance with name {self._name}")
         self.create_security_group()

fogros2/fogros2/cloud_instance.py

@@ -159,7 +159,7 @@ def install_ros(self):
         self.scp.execute_cmd("export LANG=en_US.UTF-8")
 
         # install ros2 packages
-        self.apt_install(f"ros-{self.ros_distro}-desktop")
+        # self.apt_install(f"ros-{self.ros_distro}-desktop")
 
         # source environment
         self.scp.execute_cmd(f"source /opt/ros/{self.ros_distro}/setup.bash")

fogros2/utils/__init__.py

@@ -0,0 +1,75 @@
+import boto3
+import json
+from pkg_resources import resource_filename
+from .region_ami_selection import haversine, aws_regions
+
+FLT = '[{{"Field": "tenancy", "Value": "shared", "Type": "TERM_MATCH"}},'\
+      '{{"Field": "operatingSystem", "Value": "{o}", "Type": "TERM_MATCH"}},'\
+      '{{"Field": "preInstalledSw", "Value": "NA", "Type": "TERM_MATCH"}},'\
+      '{{"Field": "instanceType", "Value": "{t}", "Type": "TERM_MATCH"}},'\
+      '{{"Field": "location", "Value": "{r}", "Type": "TERM_MATCH"}},'\
+      '{{"Field": "capacitystatus", "Value": "Used", "Type": "TERM_MATCH"}}]'
+
+
+def get_price(region_name, instance, os):
+    f = FLT.format(r=get_region(region_name), t=instance, o=os)
+    pricing_client = boto3.client('pricing', 
+        region_name=region_name, 
+    )
+    data = pricing_client.get_products(ServiceCode='AmazonEC2', Filters=json.loads(f))
+    od = json.loads(data['PriceList'][0])['terms']['OnDemand']
+    id1 = list(od)[0]
+    id2 = list(od[id1]['priceDimensions'])[0]
+    price = od[id1]['priceDimensions'][id2]['pricePerUnit']['USD']
+    return price
+
+def get_region(region_name):
+    default_region = 'US East (N. Virginia)'
+    endpoint_file = resource_filename('botocore', 'data/endpoints.json')
+    try:
+        with open(endpoint_file, 'r') as f:
+            data = json.load(f)
+        return data['partitions'][0]['regions'][region_name]['description'].replace('Europe', 'EU')
+    except IOError:
+        return default_region
+
+def ec2_instance_types(region_name, cpu_architecture="x86_64", default_cores=2, default_threads_per_core=1, gpu=True):
+    '''Yield all available EC2 instance types in region <region_name>'''
+    ec2 = boto3.client('ec2', 
+        region_name=region_name,
+    )
+
+    describe_args = {
+        "Filters": [
+            {
+                'Name': 'processor-info.supported-architecture',
+                'Values': [
+                    cpu_architecture,
+                ]
+            },
+            {
+                'Name': 'vcpu-info.default-cores',
+                'Values': [
+                    str(default_cores),
+                ]
+            },
+            {
+                'Name': 'vcpu-info.default-threads-per-core',
+                'Values': [
+                    str(default_threads_per_core),
+                ]
+            },
+        ],
+    }
+
+    while True:
+        describe_result = ec2.describe_instance_types(**describe_args)
+        yield from [i["InstanceType"] for i in describe_result['InstanceTypes'] if not gpu or "GpuInfo" in i]
+        if 'NextToken' not in describe_result:
+            break
+        describe_args['NextToken'] = describe_result['NextToken']
+
+def find_cheapest_ec2_instance_type(region_name, cpu_architecture="x86_64", default_cores=2, default_threads_per_core=1, gpu=False):
+    (lat, lon) = aws_regions[region_name]
+    region_name = min(['us-east-1', 'ap-south-1'], key= lambda region: haversine(region, lat, lon))
+    return min(ec2_instance_types(region_name, cpu_architecture, default_cores, default_threads_per_core, gpu), key=lambda instance_type: get_price(region_name, instance_type, 'Linux'))

fogros2/utils/ec2_instance_type_selection.py

@@ -0,0 +1,46 @@
+import json
+import requests
+from math import radians, cos, sin, asin, sqrt
+
+aws_regions = {
+    "us-east-2": (40.4167, -82.9167),
+    "us-east-1": (38.0339, -78.4860),
+    "us-west-1": (37.7749, -122.4194),
+    "us-west-2": (45.5200, -122.6819),
+    "af-south-1": (-33.9249, 18.4241),
+    "ap-east-1": (22.2800, 114.1588),
+    "ap-southeast-3": (-6.2315, 106.8275),
+    "ap-south-1": (19.0760, 72.8777),
+    "ap-northeast-3": (34.6723, 135.4848),
+    "ap-northeast-2": (37.5665, 126.9780),
+    "ap-southeast-1": (1.3521, 103.8198),
+    "ap-southeast-2": (-33.8688, 151.2093),
+    "ap-northeast-1": (35.6895, 139.6917),
+    "ca-central-1": (43.6532, -79.3832),
+    "eu-central-1": (50.1147, 8.6821),
+    "eu-west-1": (53.4129, -8.2439),
+    "eu-west-2": (51.5074, -0.1278),
+    "eu-south-1": (45.4642, 9.1900),
+    "eu-west-3": (48.8566, 2.3522),
+    "eu-north-1": (59.3293, 18.0686),
+    "me-south-1": (26.0667, 50.5577),
+    "me-central-1": (23.4241, 53.8478),
+    "sa-east-1": (-23.5505, -46.6333),
+}
+
+def find_nearest_region_and_ami(regions):
+        ip = json.loads(requests.get("https://ip.seeip.org/jsonip?").text)["ip"]
+        response = requests.get("http://ip-api.com/json/" + ip).json()
+        lat = response["lat"]
+        long = response["lon"]
+        closest_region = min(regions, key=lambda region: haversine(region, lat, long))
+        return closest_region, regions[closest_region]["ami_image"]
+
+def haversine(region, lat, lon):
+    lon1, lat1, lon2, lat2 = map(radians, [aws_regions[region][1], aws_regions[region][0], lon, lat])
+    dlon = lon2 - lon1 
+    dlat = lat2 - lat1 
+    a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
+    c = 2 * asin(sqrt(a)) 
+    km = 6371 * c
+    return km

fogros2/utils/region_ami_selection.py

@@ -0,0 +1,72 @@
+# Copyright 2022 The Regents of the University of California (Regents)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright ©2022. The Regents of the University of California (Regents).
+# All Rights Reserved. Permission to use, copy, modify, and distribute this
+# software and its documentation for educational, research, and not-for-profit
+# purposes, without fee and without a signed licensing agreement, is hereby
+# granted, provided that the above copyright notice, this paragraph and the
+# following two paragraphs appear in all copies, modifications, and
+# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150
+# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201,
+# [email protected], http://ipira.berkeley.edu/industry-info for commercial
+# licensing opportunities. IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY
+# FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES,
+# INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+# DOCUMENTATION, EVEN IF REGENTS HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE. REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY,
+# PROVIDED HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE
+# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+
+from launch_ros.actions import Node
+import fogros2
+from utils import region_ami_selection, ec2_instance_type_selection
+
+
+def generic_ubuntu_ami():
+    return {
+        "us-west-1": { "ami_image": "ami-01154c8b2e9a14885" },
+        "us-west-2": { "ami_image": "ami-0ddf424f81ddb0720" },
+        "us-east-1": { "ami_image": "ami-08d4ac5b634553e16" },
+        "us-east-2": { "ami_image": "ami-0960ab670c8bb45f3" },
+    }
+
+def generate_launch_description():
+    """Talker example that launches the listener on AWS."""
+    ld = fogros2.FogROSLaunchDescription()
+    
+    region, ami = region_ami_selection.find_nearest_region_and_ami(generic_ubuntu_ami())
+
+    ec2_instance_type = ec2_instance_type_selection.find_cheapest_ec2_instance_type(region)
+
+    print(region, ami, ec2_instance_type)
+    machine1 = fogros2.AWSCloudInstance(
+        region=region, ec2_instance_type=ec2_instance_type, ami_image=ami
+    )
+
+    listener_node = Node(
+        package="fogros2_examples", executable="listener", output="screen"
+    )
+
+    talker_node = fogros2.CloudNode(
+        package="fogros2_examples",
+        executable="talker",
+        output="screen",
+        machine=machine1,
+    )
+    ld.add_action(talker_node)
+    ld.add_action(listener_node)
+    return ld