Added terraform files to lambda plugin (#89)

* Added terraform files to lambda plugin

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* minor code changes

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* skipping misplaced test

* removing misplaced test

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: sankalp <[email protected]>

.gitignore

@@ -5,4 +5,11 @@ __pycache__
 dist
 results
 scratch
-*.ipynb
+*.ipynb
+
+# Terraform
+**/.terraform/**
+**/*.tfstate*
+**/.terraform.lock.hcl
+**/*.tfvars
+**/*.plan

CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Added
+
+- Added the lambda terraform files to `covalent_awslambda_plugin` folder from terraform repo ( including license blocks )
+
+### Changed
+
+- Modified the MANIFEST.in to include the newly added plugin files
+- Added terraform ignore file paths to `.gitignore`
+
 ## [0.32.0] - 2023-04-26
 
 ### Changed

MANIFEST.in

@@ -1,2 +1,5 @@
 include VERSION
 include requirements.txt
+include covalent_awslambda_plugin/assets/infra/*
+exclude covalent_awslambda_plugin/assets/infra/*.swp
+include covalent_awslambda_plugin/assets/infra/*.tf

covalent_awslambda_plugin/assets/infra/functional_tests.tfvars

@@ -0,0 +1,17 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name = "lambda-ci-ft-test"

covalent_awslambda_plugin/assets/infra/main.tf

@@ -0,0 +1,74 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+provider "aws" {
+  region = var.aws_region
+}
+
+data "aws_caller_identity" "current" {}
+
+resource "aws_s3_bucket" "s3_bucket" {
+  bucket = "${var.name}-covalent-artifact-bucket"
+  force_destroy = true
+}
+
+resource "aws_iam_role" "lambda_iam_role" {
+  name = "${var.name}-lambda-iam-role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = ""
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        }
+      },
+    ]
+  })
+  managed_policy_arns = [ "arn:aws:iam::aws:policy/AWSLambdaExecute" ]
+
+  tags = {
+    "Terraform" = "true"
+  }
+}
+
+resource "aws_ecr_repository" "ecr_repository" {
+  name                 = "${var.name}-lambda-executor-base-ecr-repo"
+  image_tag_mutability = "MUTABLE"
+
+  force_delete = true
+  image_scanning_configuration {
+    scan_on_push = false
+  }
+
+  provisioner "local-exec" {
+    command = "docker pull public.ecr.aws/covalent/covalent-lambda-executor:${var.executor_base_image_tag_name} && aws ecr get-login-password --region ${var.aws_region} | docker login --username AWS --password-stdin ${data.aws_caller_identity.current.account_id}.dkr.ecr.${var.aws_region}.amazonaws.com && docker tag public.ecr.aws/covalent/covalent-lambda-executor:${var.executor_base_image_tag_name} ${aws_ecr_repository.ecr_repository.repository_url}:${var.executor_base_image_tag_name} && docker push ${aws_ecr_repository.ecr_repository.repository_url}:${var.executor_base_image_tag_name}"
+  }
+}
+
+resource aws_lambda_function lambda {
+    function_name = "${var.name}-lambda-fn"
+    role = aws_iam_role.lambda_iam_role.arn
+    package_type = "Image"
+    timeout = var.timeout
+    memory_size = var.memory_size
+    image_uri = "${aws_ecr_repository.ecr_repository.repository_url}:${var.executor_base_image_tag_name}"
+    ephemeral_storage {
+      size = var.ephemeral_storage  # Min 512 MB and the Max 10240 MB
+    }
+}

covalent_awslambda_plugin/assets/infra/outputs.tf

@@ -0,0 +1,25 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+output "s3_bucket_name" {
+    value = aws_s3_bucket.s3_bucket.id
+    description = "Allocated AWS S3 bucket name for storing lambda files"
+}
+
+output "function_name" {
+    value = aws_lambda_function.lambda.function_name
+    description = "AWS Lambda function name"
+}

covalent_awslambda_plugin/assets/infra/variables.tf

@@ -0,0 +1,45 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+variable "name" {
+  default = "covalent-lambda"
+  description = "Prefix to use for all provisioned resources"
+}
+
+variable "executor_base_image_tag_name" {
+  default = "latest"
+  description = "Image tag for image in provisioned ecr repo to be used for lambda invocations"
+}
+
+variable "aws_region" {
+  default = "us-east-1"
+  description = "The aws region"
+}
+
+variable "timeout" {
+  default = 900
+  description = "The amount of time your Lambda Function has to run in seconds"
+}
+
+variable "memory_size" {
+  default = 1024
+  description = "The amount of memory in MB your Lambda Function can use at runtime"
+}
+
+variable "ephemeral_storage" {
+  default = 1024
+  description = "Size of the ephemeral storage in MB"
+}

covalent_awslambda_plugin/assets/infra/versions.tf

@@ -0,0 +1,24 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.17.0"
+    }
+  }
+}

tests/lambda_executor_test.py

@@ -61,26 +61,6 @@ def test_init():
     assert awslambda.poll_freq == 30
 
 
-@pytest.mark.asyncio
-async def test_setup_and_teardown_are_invoked(lambda_executor, mocker):
-    "Simply assert that the setup, run and teardown methods are invoked when execute is called"
-    lambda_executor.get_session = MagicMock()
-    lambda_executor._is_lambda_active = MagicMock()
-    lambda_executor._create_lambda = MagicMock()
-    lambda_executor.submit_task = MagicMock()
-    lambda_executor.get_status = MagicMock()
-    lambda_executor.query_result = MagicMock()
-    lambda_executor.setup = AsyncMock()
-    lambda_executor.run = AsyncMock()
-    lambda_executor.teardown = AsyncMock()
-
-    await lambda_executor.execute(MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock())
-
-    lambda_executor.setup.assert_awaited_once()
-    lambda_executor.run.assert_awaited_once()
-    lambda_executor.teardown.assert_awaited_once()
-
-
 @pytest.mark.asyncio
 async def test_function_pickle_dump(lambda_executor, mocker):
     def f(x):