Fixes for installing conda environment on EC2

.gitignore

@@ -118,5 +118,8 @@ ipython_config.py
 *.ipynb
 
 # Terraform
-covalent_ec2_plugin/infra/.terraform.lock.hcl
-covalent_ec2_plugin/infra/.terraform
+**/.terraform/**
+**/*.tfstate*
+**/.terraform.lock.hcl
+**/*.tfvars
+**/*.plan

CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+## Changed
+
+- Moved the terraform files into **covalent_ec2_plugin/assets/infra** folder.
+- Minor modifications on the plugin core to handle the above folder changes.
+
+## Fixed
+
+- Fixed security group, enabling ec2 instances to connect to internet and download conda installation file.
+
+## Added
+
+- Added tftpl file to project configuration value
+
 ## [0.13.2] - 2023-11-01
 
 ## Changed

MANIFEST.in

@@ -1,3 +1,5 @@
 include VERSION
 include requirements.txt
-recursive-include covalent_ec2_plugin/infra/ *
+include covalent_ec2_plugin/assets/infra/*
+exclude covalent_ec2_plugin/assets/infra/*.swp
+include covalent_ec2_plugin/assets/infra/*.tf

covalent_ec2_plugin/assets/infra/ec2.conf.tftpl

@@ -0,0 +1,8 @@
+[ec2]
+credentials   = ${credentials}
+profile       = ${profile}
+region        = ${region}
+key_name      = ${key_name}
+volume_size   = ${volume_size}
+vpc           = ${vpc}
+subnet        = ${subnet}

covalent_ec2_plugin/infra/main.tf → covalent_ec2_plugin/assets/infra/main.tf

@@ -44,8 +44,8 @@ resource "aws_instance" "covalent_ec2_instance" {
   subnet_id                   = var.vpc_id == "" ? module.vpc.public_subnets[0] : var.subnet_id
   associate_public_ip_address = true
 
-  key_name             = var.key_name # Name of a valid key pair
-  monitoring           = true
+  key_name   = var.key_name # Name of a valid key pair
+  monitoring = true
 
   root_block_device {
     volume_type = "gp2"
@@ -78,7 +78,7 @@ resource "null_resource" "deps_install" {
       "echo 'Installing Covalent...'",
 
       # TODO: Update to a variable version
-      "pip install \"covalent${var.covalent_version}\"",
+      "pip install \"covalent==${var.covalent_version}\"",
       "chmod +x /tmp/script.sh",
       "sudo bash /tmp/script.sh",
       "echo ok"
@@ -92,3 +92,22 @@ resource "null_resource" "deps_install" {
     host        = aws_instance.covalent_ec2_instance.public_ip
   }
 }
+
+data "template_file" "executor_config" {
+  template = file("${path.module}/ec2.conf.tftpl")
+
+  vars = {
+    credentials = var.aws_credentials
+    profile     = var.aws_profile
+    region      = var.aws_region
+    key_name    = var.key_name
+    volume_size = var.disk_size
+    vpc         = var.vpc_id == "" ? module.vpc.vpc_id : var.vpc_id
+    subnet      = var.vpc_id == "" ? module.vpc.public_subnets[0] : var.subnet_id
+  }
+}
+
+resource "local_file" "executor_config" {
+  content  = data.template_file.executor_config.rendered
+  filename = "${path.module}/ec2.conf"
+}

covalent_ec2_plugin/infra/networking.tf → covalent_ec2_plugin/assets/infra/networking.tf

@@ -49,7 +49,15 @@ resource "aws_security_group" "covalent_firewall" {
     from_port   = 22
     to_port     = 22
     protocol    = "tcp"
-    cidr_blocks = ["${chomp(data.http.myip.body)}/32"]
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "Allow HTTPS Access For Conda Installation"
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
   }
 
   egress {

covalent_ec2_plugin/ec2.py

@@ -29,6 +29,7 @@
 from covalent_aws_plugins import AWSExecutor
 from covalent_ssh_plugin.ssh import _EXECUTOR_PLUGIN_DEFAULTS as _SSH_EXECUTOR_PLUGIN_DEFAULTS
 from covalent_ssh_plugin.ssh import SSHExecutor
+from pydantic import BaseModel
 
 executor_plugin_name = "EC2Executor"
 
@@ -50,6 +51,31 @@
     }
 )
 
+
+class ExecutorPluginDefaults(BaseModel):
+    profile: str = ""
+    credentials_file: str = ""
+    region: str = ""
+    instance_type: str = "t2.micro"
+    volume_size: str = "8"
+    vpc: str = ""
+    subnet: str = ""
+    key_name: str = ""
+    conda_env: str = "covalent"
+
+
+class ExecutorInfraDefaults(BaseModel):
+    profile: str = ""
+    credentials_file: str = ""
+    region: str = ""
+    instance_type: str = "t2.micro"
+    volume_size: str = "8"
+    vpc: str = ""
+    subnet: str = ""
+    key_name: str = ""
+    conda_env: str = "covalent"
+
+
 EC2_KEYPAIR_NAME = "covalent-ec2-executor-keypair"
 EC2_SSH_DIR = "~/.ssh/covalent"
 
@@ -77,7 +103,7 @@ class EC2Executor(SSHExecutor, AWSExecutor):
             it can also include the extras if needed as "[qiskit, braket]==0.220.0.post2"
     """
 
-    _TF_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "infra"))
+    _TF_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "assets", "infra"))
 
     def __init__(
         self,
@@ -175,10 +201,8 @@ async def _run_async_subprocess(self, cmd: List[str], cwd=None, log_output: bool
 
         return proc, stdout, stderr
 
-    def _get_tf_statefile_path(self, task_metadata: Dict) -> str:
-        state_file = (
-            f"{self._TF_DIR}/ec2-{task_metadata['dispatch_id']}-{task_metadata['node_id']}.tfstate"
-        )
+    def _get_tf_statefile_path(self) -> str:
+        state_file = f"{self._TF_DIR}/terraform.tfstate"
         return state_file
 
     async def _get_tf_output(self, var: str, state_file: str) -> str:
@@ -197,7 +221,7 @@ async def setup(self, task_metadata: Dict) -> None:
         # locks or to ensure that terraform init is run just once)
         subprocess.run(["terraform init"], cwd=self._TF_DIR, shell=True, check=True)
 
-        state_file = self._get_tf_statefile_path(task_metadata)
+        state_file = self._get_tf_statefile_path()
 
         boto_session = boto3.Session(**self.boto_session_options())
         profile = boto_session.profile_name
@@ -279,8 +303,7 @@ async def teardown(self, task_metadata: Dict) -> None:
         """
         Invokes Terraform to terminate the instance and teardown supporting resources
         """
-
-        state_file = self._get_tf_statefile_path(task_metadata)
+        state_file = self._get_tf_statefile_path()
 
         if not os.path.exists(state_file):
             raise FileNotFoundError(