From 7c4b32f72f79ef84ed80c435b529bd37a584b2da Mon Sep 17 00:00:00 2001 From: jrzeszutek Date: Mon, 20 Aug 2018 08:19:18 +0200 Subject: [PATCH 1/5] version 0.4 - added possibility of using zip archives, directories and jinja2 templates; changed executor for central_deployment_agent --- examples/blueprint.yaml | 37 ++-- examples/resources/ansible.zip | Bin 0 -> 653 bytes examples/resources/ansible/helloworld.yml | 2 +- examples/resources/ansible/hosts.tmp | 2 +- exec_plugin/tasks.py | 218 +++++++++++++++++++++- plugin.yaml | 18 +- setup.py | 2 +- 7 files changed, 246 insertions(+), 33 deletions(-) create mode 100644 examples/resources/ansible.zip diff --git a/examples/blueprint.yaml b/examples/blueprint.yaml index bae348c..bbb96f3 100644 --- a/examples/blueprint.yaml +++ b/examples/blueprint.yaml @@ -2,7 +2,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/4.3/types.yaml - - https://raw.githubusercontent.com/EarthmanT/cloudify-execution-plugin/0.3/plugin.yaml + - plugin:cloudify-execution-plugin?version=0.4 inputs: @@ -16,9 +16,7 @@ inputs: resource_list: # For Ansible. default: - - resources/ansible/exec - - resources/ansible/helloworld.yml - - resources/ansible/hosts.tmp + - hosts.tmp node_templates: @@ -26,10 +24,17 @@ node_templates: type: cloudify.nodes.Execution properties: resource_config: + resource_dir: resources/ansible.zip resource_list: { get_input: resource_list } - relationships: - - type: cloudify.relationships.contained_in - target: host + template_variables: + host_ip: { get_input: host_ip } + # interfaces: + # cloudify.interfaces.lifecycle: + # create: + # executor: central_deployment_agent + # relationships: + # - type: cloudify.relationships.contained_in + # target: host # For HELM. # interfaces: # cloudify.interfaces.lifecycle: @@ -44,12 +49,12 @@ node_templates: # resource_config: { get_property: [ SELF, resource_config ] } # subprocess_args_overrides: {'env': {'SCRIPT_NAME': 'uninstall.sh'}} - host: - type: cloudify.nodes.Compute - properties: - ip: { get_input: host_ip } - agent_config: - install_method: remote - port: 22 - user: { get_input: username } - key: { get_secret: agent_key_private } + # host: + # type: cloudify.nodes.Compute + # properties: + # ip: { get_input: host_ip } + # agent_config: + # install_method: remote + # port: 22 + # user: { get_input: username } + # key: { get_secret: agent_key_private } diff --git a/examples/resources/ansible.zip b/examples/resources/ansible.zip new file mode 100644 index 0000000000000000000000000000000000000000..558e0678003baa1a9cce550e1620fcbf5644379a GIT binary patch literal 653 zcmWIWW@Zs#U|`^2C&8;iH&gWKjW(mAY3l{_X?*v!8a5;g0>G=ZG6Oe^HxKiQDwrrz;bVtQKNnS`Rx zyw&m)Up`~=Qd965}j`6AiDm>8XupRi>v?Ie>$HR7IEZ`UYw}V&pG=# z!etKKeitrzT;)?k>x|UN=YAS#wW|lfoyt@o;F~ArC*=Tiq7xAF0dYoZPELM#eo;<} zUS)2M?5T~M2Ml;zF2)7RyxZC^jZ;lRL@D6G9m)4{xA*@#8XOu}vFUPvGRupp`o0^2 zE=0F1ygN_hw{Wsdy{-H+olndlySRX+GBD_u^7_tNfB&`ykPX6|K%9|ZTvDu8l3P$+ ztpH-jXBH^b)+!|C6=x>pq{b)b=jEj)mt^MW+2-UYC+6^S1$Z+u*)!k@XBD84AfNyy zA)(F%R~vZOId|@)a!gw_E9@uCY9pKH%2J$U45DEe{rvWho0|4eh B#1a4i literal 0 HcmV?d00001 diff --git a/examples/resources/ansible/helloworld.yml b/examples/resources/ansible/helloworld.yml index 9692690..c9760e6 100644 --- a/examples/resources/ansible/helloworld.yml +++ b/examples/resources/ansible/helloworld.yml @@ -1,4 +1,4 @@ --- - hosts: all tasks: - - shell: echo "hello world" + - shell: echo "hello {{ host_ip }}" diff --git a/examples/resources/ansible/hosts.tmp b/examples/resources/ansible/hosts.tmp index 05614f6..458a80a 100644 --- a/examples/resources/ansible/hosts.tmp +++ b/examples/resources/ansible/hosts.tmp @@ -1 +1 @@ -localhost ansible_connection=local \ No newline at end of file +{{ host_ip }} ansible_connection=local diff --git a/exec_plugin/tasks.py b/exec_plugin/tasks.py index 2bdb830..f6f4d25 100644 --- a/exec_plugin/tasks.py +++ b/exec_plugin/tasks.py @@ -2,6 +2,9 @@ import os import subprocess import tempfile +import errno +import zipfile +import copy from cloudify import ctx from cloudify.exceptions import ( @@ -9,17 +12,201 @@ OperationRetry) -def get_package_dir(resource_list, template_variables): +def get_package_dir(resource_dir='', resource_list=[], template_variables={}): """ Download resources and return the path. """ work_dir = tempfile.mkdtemp() - for resource_path in resource_list: - resource_name = os.path.basename(resource_path) - download_to = os.path.join(work_dir, resource_name) - ctx.download_resource_and_render( - resource_path, - download_to, - template_variables) + blueprint_resources_path = os.path.join('/opt/manager/resources/blueprints/default_tenant/', ctx.blueprint.id) + + if resource_dir and resource_list: + # Case, when user defines a directory with files, which need to be + # downloaded, but doesn't want to render all of them - only these + # defined in resource_list. + + ctx.logger.debug('resource_dir and resource_list params are not empty.') + + # Deal with ZIP files + filename, extension = os.path.splitext(resource_dir) + if extension == '.zip': + with zipfile.ZipFile(os.path.join(blueprint_resources_path, resource_dir)) as archive: + resource_dir = filename + archive.extractall(os.path.join(blueprint_resources_path, resource_dir)) + + # Deal with ZIP files in resource_list + for template_path in copy.copy(resource_list): + + filename, extension = os.path.splitext(template_path) + + if extension == '.zip': + + resource_list.remove(template_path) + + with zipfile.ZipFile(os.path.join(blueprint_resources_path, resource_dir, template_path)) as archive: + extracted_templates_dir = os.path.join(blueprint_resources_path, resource_dir, filename) + archive.extractall(os.path.join(blueprint_resources_path, resource_dir, filename)) + + for extracted_template in os.walk(extracted_templates_dir): + extracted_template_path = extracted_template[0][len(os.path.join(blueprint_resources_path, resource_dir)) + 1:] + if extracted_template[2]: + for filename in extracted_template[2]: + resource_list.append(os.path.join(extracted_template_path, filename)) + elif not extracted_template[1] and not extracted_template[2]: + resource_list.append(extracted_template_path) + + merged_list = [] + + # This loop goes through a directory defined in resource_dir parameter + # and prepares a list of paths inside it. + for resource_path in os.walk(os.path.join(blueprint_resources_path, resource_dir)): + trimmed_resource_path = resource_path[0][len(os.path.join(blueprint_resources_path)) + 1:] + + if resource_path[2]: + for filename in resource_path[2]: + merged_list.append(os.path.join(trimmed_resource_path, filename)) + elif not resource_path[1] and not resource_path[2]: + merged_list.append(trimmed_resource_path) + + # This loop goes through a templates list defined in resource_list + # parameter. For each template it renders it (resolves all of variables + # defined in template_variables parameter) and downloads it to working + # directory. Finally, it removes a path to this file from the merged_list, + # because it should be ommitted at the next step, which is copying the rest + # of the files, which are not templates. + for template_path in resource_list: + template_dirname = os.path.join(resource_dir, os.path.dirname(template_path)) + download_from_file = os.path.join(resource_dir, template_path) + download_to_directory = os.path.join(work_dir, template_dirname) + download_to_file = os.path.join(work_dir, download_from_file) + + try: + os.makedirs(download_to_directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + ctx.download_resource_and_render( + download_from_file, + download_to_file, + template_variables.copy()) + + if os.path.splitext(download_to_file)[1] == '.py': + os.chmod(download_to_file, 0755) + + if download_from_file in merged_list: + merged_list.remove(download_from_file) + + # This loop goes through the merged_list and downloads the rest of + # files to our working directory. + for resource_path in merged_list: + resource_dirname = os.path.dirname(resource_path) + download_to_directory = os.path.join(work_dir, resource_dirname) + download_to_file = os.path.join(work_dir, resource_path) + + try: + os.makedirs(download_to_directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + try: + ctx.download_resource( + resource_path, + download_to_file) + except OSError as e: + if e.errno != errno.EISDIR: + raise + + if os.path.splitext(download_to_file)[1] == '.py': + os.chmod(download_to_file, 0755) + + + elif resource_dir and not resource_list: + # Case, when user defines path to a directory, where files, which need to + # be downloaded and rendered, reside. + + ctx.logger.debug('only resource_dir is not empty.') + + # Deal with ZIP files + filename, extension = os.path.splitext(resource_dir) + if extension == '.zip': + with zipfile.ZipFile(os.path.join(blueprint_resources_path, resource_dir)) as archive: + resource_dir = filename + archive.extractall(os.path.join(blueprint_resources_path, resource_dir)) + + merged_list = [] + + # This loop goes through a directory defined in resource_dir parameter + # and prepares a list of paths inside it. + for resource_path in os.walk(os.path.join(blueprint_resources_path, resource_dir)): + trimmed_resource_path = resource_path[0][len(os.path.join(blueprint_resources_path)) + 1:] + + if resource_path[2]: + for filename in resource_path[2]: + merged_list.append(os.path.join(trimmed_resource_path, filename)) + elif not resource_path[1] and not resource_path[2]: + merged_list.append(trimmed_resource_path) + + # This loop goes through the merged_list and downloads the rest of + # files to our working directory. + for template_path in merged_list: + template_dirname = os.path.dirname(template_path) + download_from_file = template_path + download_to_directory = os.path.join(work_dir, template_dirname) + download_to_file = os.path.join(work_dir, download_from_file) + + try: + os.makedirs(download_to_directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + ctx.download_resource_and_render( + download_from_file, + download_to_file, + template_variables.copy()) + + if os.path.splitext(download_to_file)[1] == '.py': + os.chmod(download_to_file, 0755) + + elif not resource_dir and resource_list: + # Case, when user defines a list of files in resource_list, which need to be + # downloaded and rendered. + + ctx.logger.debug('only resource_list is not empty.') + + # Deal with ZIP files in resource_list + for template_path in copy.copy(resource_list): + + filename, extension = os.path.splitext(template_path) + + if extension == '.zip': + + resource_list.remove(template_path) + + with zipfile.ZipFile(os.path.join(blueprint_resources_path, template_path)) as archive: + extracted_templates_dir = os.path.join(blueprint_resources_path, filename) + archive.extractall(extracted_templates_dir) + + for extracted_template in os.walk(extracted_templates_dir): + extracted_template_path = extracted_template[0][len(os.path.join(blueprint_resources_path)) + 1:] + if extracted_template[2]: + for filename in extracted_template[2]: + resource_list.append(os.path.join(extracted_template_path, filename)) + elif not extracted_template[1] and not extracted_template[2]: + resource_list.append(extracted_template_path) + + for template_path in resource_list: + resource_name = os.path.basename(template_path) + download_to = os.path.join(work_dir, resource_name) + ctx.download_resource_and_render( + template_path, + download_to, + template_variables.copy()) + + else: + raise NonRecoverableError("At least one of the two properties, \ + resource_dir or resource_list, has to be defined.") + return work_dir @@ -54,13 +241,24 @@ def execute(resource_config, resource_config = \ resource_config or ctx.node.properties['resource_config'] + resource_dir = resource_config.get('resource_dir', '') resource_list = resource_config.get('resource_list', []) template_variables = resource_config.get('template_variables', {}) - if not isinstance(resource_list, list) or not len(resource_list) > 0: + if not isinstance(resource_dir, basestring): + raise NonRecoverableError("'resource_dir' must be a string.") + + if not isinstance(resource_list, list): raise NonRecoverableError("'resource_list' must be a list.") - cwd = get_package_dir(resource_list, template_variables) + if not isinstance(template_variables, dict): + raise NonRecoverableError("'template_variables' must be a dictionary.") + + if resource_dir: + tmp_dir = get_package_dir(resource_dir, resource_list, template_variables) + cwd = os.path.join(tmp_dir, os.path.splitext(resource_dir)[0]) # in case of resource_dir is zip + else: + cwd = get_package_dir(resource_dir, resource_list, template_variables) command = ['bash', '-c', 'source {0}'.format(file_to_source)] subprocess_args = \ diff --git a/plugin.yaml b/plugin.yaml index 288079d..becd44f 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -1,17 +1,23 @@ plugins: exec: - executor: host_agent + executor: central_deployment_agent package_name: cloudify-execution-plugin - package_version: '0.3' - source: https://github.com/EarthmanT/cloudify-execution-plugin/archive/0.3.zip + package_version: '0.4' data_types: cloudify.types.exec.Package: properties: + resource_dir: + description: > + A directory under the blueprint root, + which content needs to be copied and kept unchanged. + default: '' resource_list: - description: A list of file paths under the blueprint root. + description: > + A list of file paths under the blueprint root, + which need to be filled with values of template_variables. default: [] template_variables: description: A dict containing variables as key-values. @@ -24,6 +30,7 @@ node_types: properties: resource_config: type: cloudify.types.exec.Package + required: false interfaces: cloudify.interfaces.lifecycle: create: &exec_op @@ -32,3 +39,6 @@ node_types: resource_config: type: cloudify.types.exec.Package default: { get_property: [ SELF, resource_config ] } + file_to_source: + type: string + default: exec diff --git a/setup.py b/setup.py index 846718f..9c29134 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name='cloudify-execution-plugin', - version='0.3', + version='0.4', packages=['exec_plugin'], license='LICENSE', description='Execute something.', From a7f49f9dc890be570315cf1f46490762a46b5dfc Mon Sep 17 00:00:00 2001 From: jrzeszutek Date: Mon, 3 Sep 2018 14:50:08 +0200 Subject: [PATCH 2/5] 0.4-build: README update --- README.md | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b37e340..07d983c 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,38 @@ source ./my_file **What happens** -The plugin creates a temporary directory. It copies all of the files to it. +The plugin creates a temporary directory. What happens then, depends on defined properties. -It will then execute the `exec` file relative to the temporary directory. + * 1st use case: both `resource_dir` and `resource_list` are defined + + Plugin renders all of the files defined in `resource_list` with the values \ + defined in `template_variables` (if empty, it just copies the files unchanged \ + to temporary working directory) and copies rendered files to temporary \ + directory. The rest of files inside `resource_dir` directory are copied \ + unchanged. + + If `resource_dir` is a zip file, its content is being extracted first. If \ + there are any zip files defined in `resource_list`, they are also being \ + extracted. + + * 2nd use case: `resource_dir` is defined and `resource_list` is not + + Plugin renders all of the files inside of a directory (or files extracted \ + from zip archive) \ + defined in `resource_dir` with the values defined in `template_variables`. \ + If `template_variables` is not empty, files extracted from `resource_dir` are \ + being rendered with them and copied to temporary directory. If `template_variables` \ + is empty, then files are just being copied (without rendering). + + * 3rd use case: `resource_dir` is not defined and `resource_list` is defined + + Plugin renders all of the files in `resource_list` (including these extracted \ + from zip files being a part of a `resource_list`) with the values defined in \ + `template_variables` and copies them to temporary directory. If \ + `template_variables` is empty, then files are just being copied (without \ + rendering). + +Plugin will then execute the `exec` file relative to the temporary directory. _Note: By default the plugin looks to execute a file named `exec` in the temporary directory. This may be overridden._ @@ -47,7 +76,7 @@ _Note: By default the plugin looks to execute a file named `exec` in the tempora ## Operations -There is only one task defined in the plugin `execute`. This function wraps a `subprocess.Popen` [constructor](https://docs.python.org/2/library/subprocess.html#subprocess.Popen). +There is only one task defined in the plugin `execute`. This function wraps a `subprocess.Popen` [constructor](https://docs.python.org/2/library/subprocess.html#subprocess.Popen). * `cloudify.interfaces.lifecycle.create` * `implementation: exec.exec_plugin.tasks.execute` From 8aede91c63777c3f1ee469cd5fddb83adede95f8 Mon Sep 17 00:00:00 2001 From: jrzeszutek Date: Thu, 27 Sep 2018 09:41:57 +0200 Subject: [PATCH 3/5] View #7: added requested changes --- examples/blueprint.yaml | 8 +- exec_plugin/tasks.py | 426 ++++++++++++++++++++++++---------------- test-requirements.txt | 4 + 3 files changed, 262 insertions(+), 176 deletions(-) create mode 100644 test-requirements.txt diff --git a/examples/blueprint.yaml b/examples/blueprint.yaml index bbb96f3..a3ce863 100644 --- a/examples/blueprint.yaml +++ b/examples/blueprint.yaml @@ -16,6 +16,8 @@ inputs: resource_list: # For Ansible. default: + # - exec + # - helloworld.yml - hosts.tmp node_templates: @@ -24,14 +26,16 @@ node_templates: type: cloudify.nodes.Execution properties: resource_config: - resource_dir: resources/ansible.zip + resource_dir: resources/ansible resource_list: { get_input: resource_list } template_variables: host_ip: { get_input: host_ip } # interfaces: # cloudify.interfaces.lifecycle: # create: - # executor: central_deployment_agent + # inputs: + # resource_config: { get_property: [ SELF, resource_config ] } + # file_to_source: exec # relationships: # - type: cloudify.relationships.contained_in # target: host diff --git a/exec_plugin/tasks.py b/exec_plugin/tasks.py index f6f4d25..fab8281 100644 --- a/exec_plugin/tasks.py +++ b/exec_plugin/tasks.py @@ -12,203 +12,281 @@ OperationRetry) -def get_package_dir(resource_dir='', resource_list=[], template_variables={}): - """ Download resources and return the path. """ +def verify_os_file_path(os_file_path): + if not os.path.exists(os_file_path): + return + return os_file_path - work_dir = tempfile.mkdtemp() - blueprint_resources_path = os.path.join('/opt/manager/resources/blueprints/default_tenant/', ctx.blueprint.id) - if resource_dir and resource_list: - # Case, when user defines a directory with files, which need to be - # downloaded, but doesn't want to render all of them - only these - # defined in resource_list. +def get_directory_by_property_name(property_name, + creation_action=None, + creation_action_args=None, + creation_action_kwargs=None): - ctx.logger.debug('resource_dir and resource_list params are not empty.') - # Deal with ZIP files - filename, extension = os.path.splitext(resource_dir) - if extension == '.zip': - with zipfile.ZipFile(os.path.join(blueprint_resources_path, resource_dir)) as archive: - resource_dir = filename - archive.extractall(os.path.join(blueprint_resources_path, resource_dir)) - - # Deal with ZIP files in resource_list - for template_path in copy.copy(resource_list): - - filename, extension = os.path.splitext(template_path) - - if extension == '.zip': - - resource_list.remove(template_path) - - with zipfile.ZipFile(os.path.join(blueprint_resources_path, resource_dir, template_path)) as archive: - extracted_templates_dir = os.path.join(blueprint_resources_path, resource_dir, filename) - archive.extractall(os.path.join(blueprint_resources_path, resource_dir, filename)) - - for extracted_template in os.walk(extracted_templates_dir): - extracted_template_path = extracted_template[0][len(os.path.join(blueprint_resources_path, resource_dir)) + 1:] - if extracted_template[2]: - for filename in extracted_template[2]: - resource_list.append(os.path.join(extracted_template_path, filename)) - elif not extracted_template[1] and not extracted_template[2]: - resource_list.append(extracted_template_path) - - merged_list = [] - - # This loop goes through a directory defined in resource_dir parameter - # and prepares a list of paths inside it. - for resource_path in os.walk(os.path.join(blueprint_resources_path, resource_dir)): - trimmed_resource_path = resource_path[0][len(os.path.join(blueprint_resources_path)) + 1:] - - if resource_path[2]: - for filename in resource_path[2]: - merged_list.append(os.path.join(trimmed_resource_path, filename)) - elif not resource_path[1] and not resource_path[2]: - merged_list.append(trimmed_resource_path) - - # This loop goes through a templates list defined in resource_list - # parameter. For each template it renders it (resolves all of variables - # defined in template_variables parameter) and downloads it to working - # directory. Finally, it removes a path to this file from the merged_list, - # because it should be ommitted at the next step, which is copying the rest - # of the files, which are not templates. - for template_path in resource_list: - template_dirname = os.path.join(resource_dir, os.path.dirname(template_path)) - download_from_file = os.path.join(resource_dir, template_path) - download_to_directory = os.path.join(work_dir, template_dirname) - download_to_file = os.path.join(work_dir, download_from_file) - - try: - os.makedirs(download_to_directory) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - ctx.download_resource_and_render( - download_from_file, - download_to_file, - template_variables.copy()) - - if os.path.splitext(download_to_file)[1] == '.py': - os.chmod(download_to_file, 0755) - - if download_from_file in merged_list: - merged_list.remove(download_from_file) - - # This loop goes through the merged_list and downloads the rest of - # files to our working directory. - for resource_path in merged_list: - resource_dirname = os.path.dirname(resource_path) - download_to_directory = os.path.join(work_dir, resource_dirname) - download_to_file = os.path.join(work_dir, resource_path) - - try: - os.makedirs(download_to_directory) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - try: - ctx.download_resource( - resource_path, - download_to_file) - except OSError as e: - if e.errno != errno.EISDIR: - raise - - if os.path.splitext(download_to_file)[1] == '.py': - os.chmod(download_to_file, 0755) + directory = \ + str(ctx.instance.runtime_properties.get( + property_name)) + if not verify_os_file_path(directory): + ctx.logger.warn( + 'No runtime property "{0}" set. ' + 'Creating new directory.'.format(property_name)) + if creation_action_args and creation_action_kwargs: + directory = creation_action( + *creation_action_args, **creation_action_kwargs) + elif creation_action_args: + directory = creation_action(*creation_action_args) + elif creation_action_kwargs: + directory = creation_action(**creation_action_kwargs) + else: + directory = creation_action() + ctx.instance.runtime_properties[property_name] = \ + directory + + return verify_os_file_path(directory) + + +def get_current_working_directory(): + return get_directory_by_property_name( + 'current_working_directory', + tempfile.mkdtemp) + + +def get_blueprint_directory(): + return get_directory_by_property_name( + 'blueprint_directory', + lambda: '/opt/manager/resources/blueprints/{0}/{1}'.format( + ctx.tenant_name, ctx.blueprint.id)) + + +def extract_archive_from_path(archive_path, target_directory, intermediate_actions=None): + return_value = None + with zipfile.ZipFile(archive_path) as archive: + if intermediate_actions: + return_value = intermediate_actions() + archive.extractall(target_directory) + return return_value + + +def get_resource_relative_path(resource, relative_dir): + """ + :param resource: Entry of the list returned by os.walk() containing absolute path of a resource + :param relative_dir: Directory, which the resource is relative to + :return: String object containing relative path of the resource + """ + return resource[0][len(relative_dir) + 1:] - elif resource_dir and not resource_list: - # Case, when user defines path to a directory, where files, which need to - # be downloaded and rendered, reside. - ctx.logger.debug('only resource_dir is not empty.') +def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_variables={}): + # Case, when user defines a directory with files, which need to be + # downloaded, but doesn't want to render all of them - only these + # defined in resource_list. + + ctx.logger.debug('resource_dir and resource_list params are not empty.') + + # Deal with ZIP files + filename, extension = os.path.splitext(resource_dir) + if extension == '.zip': + archive_path = os.path.join(get_blueprint_directory(), resource_dir) + target_directory = os.path.join(get_blueprint_directory(), filename) + resource_dir = filename + extract_archive_from_path(archive_path, target_directory) + + # Deal with ZIP files in resource_list + for template_path in copy.copy(resource_list): + + filename, extension = os.path.splitext(template_path) - # Deal with ZIP files - filename, extension = os.path.splitext(resource_dir) if extension == '.zip': - with zipfile.ZipFile(os.path.join(blueprint_resources_path, resource_dir)) as archive: - resource_dir = filename - archive.extractall(os.path.join(blueprint_resources_path, resource_dir)) - - merged_list = [] - - # This loop goes through a directory defined in resource_dir parameter - # and prepares a list of paths inside it. - for resource_path in os.walk(os.path.join(blueprint_resources_path, resource_dir)): - trimmed_resource_path = resource_path[0][len(os.path.join(blueprint_resources_path)) + 1:] - - if resource_path[2]: - for filename in resource_path[2]: - merged_list.append(os.path.join(trimmed_resource_path, filename)) - elif not resource_path[1] and not resource_path[2]: - merged_list.append(trimmed_resource_path) - - # This loop goes through the merged_list and downloads the rest of - # files to our working directory. - for template_path in merged_list: - template_dirname = os.path.dirname(template_path) - download_from_file = template_path - download_to_directory = os.path.join(work_dir, template_dirname) - download_to_file = os.path.join(work_dir, download_from_file) - - try: - os.makedirs(download_to_directory) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - ctx.download_resource_and_render( - download_from_file, - download_to_file, - template_variables.copy()) - - if os.path.splitext(download_to_file)[1] == '.py': - os.chmod(download_to_file, 0755) + resource_list.remove(template_path) + archive_path = os.path.join(get_blueprint_directory(), resource_dir, template_path) + target_directory = os.path.join(get_blueprint_directory(), resource_dir, filename) + extract_archive_from_path(archive_path, target_directory) + + for extracted_template in os.walk(target_directory): + extracted_template_path = get_resource_relative_path(extracted_template, os.path.join(get_blueprint_directory(), resource_dir)) + if extracted_template[2]: + for filename in extracted_template[2]: + resource_list.append(os.path.join(extracted_template_path, filename)) + elif not extracted_template[1] and not extracted_template[2]: + resource_list.append(extracted_template_path) + + merged_list = [] + + # This loop goes through a directory defined in resource_dir parameter + # and prepares a list of paths inside it. + for resource_path in os.walk(os.path.join(get_blueprint_directory(), resource_dir)): + trimmed_resource_path = get_resource_relative_path(resource_path, os.path.join(get_blueprint_directory())) + if resource_path[2]: + for filename in resource_path[2]: + merged_list.append(os.path.join(trimmed_resource_path, filename)) + elif not resource_path[1] and not resource_path[2]: + merged_list.append(trimmed_resource_path) + + # This loop goes through a templates list defined in resource_list + # parameter. For each template it renders it (resolves all of variables + # defined in template_variables parameter) and downloads it to working + # directory. Finally, it removes a path to this file from the merged_list, + # because it should be ommitted at the next step, which is copying the rest + # of the files, which are not templates. + for template_path in resource_list: + template_dirname = os.path.join(resource_dir, os.path.dirname(template_path)) + download_from_file = os.path.join(resource_dir, template_path) + download_to_directory = os.path.join(get_current_working_directory(), template_dirname) + download_to_file = os.path.join(get_current_working_directory(), download_from_file) + + try: + os.makedirs(download_to_directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + ctx.download_resource_and_render( + download_from_file, + download_to_file, + template_variables.copy()) + + if os.path.splitext(download_to_file)[1] == '.py': + os.chmod(download_to_file, 0755) + + if download_from_file in merged_list: + merged_list.remove(download_from_file) + + # This loop goes through the merged_list and downloads the rest of + # files to our working directory. + for resource_path in merged_list: + resource_dirname = os.path.dirname(resource_path) + download_to_directory = os.path.join(get_current_working_directory(), resource_dirname) + download_to_file = os.path.join(get_current_working_directory(), resource_path) + + try: + os.makedirs(download_to_directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + try: + ctx.download_resource( + resource_path, + download_to_file) + except OSError as e: + if e.errno != errno.EISDIR: + raise + + if os.path.splitext(download_to_file)[1] == '.py': + os.chmod(download_to_file, 0755) + + return get_current_working_directory() + + +def get_package_dir_from_dir(resource_dir, template_variables={}): + # Case, when user defines path to a directory, where files, which need to + # be downloaded and rendered, reside. + + ctx.logger.debug('only resource_dir is not empty.') + + # Deal with ZIP files + filename, extension = os.path.splitext(resource_dir) + if extension == '.zip': + archive_path = os.path.join(get_blueprint_directory(), resource_dir) + target_directory = os.path.join(get_blueprint_directory(), filename) + resource_dir = filename + extract_archive_from_path(archive_path, target_directory) + + merged_list = [] + + # This loop goes through a directory defined in resource_dir parameter + # and prepares a list of paths inside it. + for resource_path in os.walk(os.path.join(get_blueprint_directory(), resource_dir)): + trimmed_resource_path = get_resource_relative_path(resource_path, os.path.join(get_blueprint_directory())) + + if resource_path[2]: + for filename in resource_path[2]: + merged_list.append(os.path.join(trimmed_resource_path, filename)) + elif not resource_path[1] and not resource_path[2]: + merged_list.append(trimmed_resource_path) + + # This loop goes through the merged_list and downloads the rest of + # files to our working directory. + for template_path in merged_list: + template_dirname = os.path.dirname(template_path) + download_from_file = template_path + download_to_directory = os.path.join(get_current_working_directory(), template_dirname) + download_to_file = os.path.join(get_current_working_directory(), download_from_file) + + try: + os.makedirs(download_to_directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + ctx.download_resource_and_render( + download_from_file, + download_to_file, + template_variables.copy()) + + if os.path.splitext(download_to_file)[1] == '.py': + os.chmod(download_to_file, 0755) + + return get_current_working_directory() + + +def get_package_dir_from_list(resource_list, template_variables={}): + # Case, when user defines a list of files in resource_list, which need to be + # downloaded and rendered. - elif not resource_dir and resource_list: - # Case, when user defines a list of files in resource_list, which need to be - # downloaded and rendered. + ctx.logger.debug('only resource_list is not empty.') + + # Deal with ZIP files in resource_list + for template_path in copy.copy(resource_list): + + filename, extension = os.path.splitext(template_path) - ctx.logger.debug('only resource_list is not empty.') + if extension == '.zip': - # Deal with ZIP files in resource_list - for template_path in copy.copy(resource_list): + resource_list.remove(template_path) - filename, extension = os.path.splitext(template_path) + archive_path = os.path.join(get_blueprint_directory(), template_path) + target_directory = os.path.join(get_blueprint_directory(), filename) + extract_archive_from_path(archive_path, target_directory) - if extension == '.zip': + for extracted_template in os.walk(target_directory): + extracted_template_path = get_resource_relative_path(extracted_template, get_blueprint_directory()) + if extracted_template[2]: + for filename in extracted_template[2]: + resource_list.append(os.path.join(extracted_template_path, filename)) + elif not extracted_template[1] and not extracted_template[2]: + resource_list.append(extracted_template_path) - resource_list.remove(template_path) + for template_path in resource_list: + resource_name = os.path.basename(template_path) + download_to = os.path.join(get_current_working_directory(), resource_name) + ctx.download_resource_and_render( + template_path, + download_to, + template_variables.copy()) - with zipfile.ZipFile(os.path.join(blueprint_resources_path, template_path)) as archive: - extracted_templates_dir = os.path.join(blueprint_resources_path, filename) - archive.extractall(extracted_templates_dir) + return get_current_working_directory() - for extracted_template in os.walk(extracted_templates_dir): - extracted_template_path = extracted_template[0][len(os.path.join(blueprint_resources_path)) + 1:] - if extracted_template[2]: - for filename in extracted_template[2]: - resource_list.append(os.path.join(extracted_template_path, filename)) - elif not extracted_template[1] and not extracted_template[2]: - resource_list.append(extracted_template_path) - for template_path in resource_list: - resource_name = os.path.basename(template_path) - download_to = os.path.join(work_dir, resource_name) - ctx.download_resource_and_render( - template_path, - download_to, - template_variables.copy()) +def get_package_dir(resource_dir='', resource_list=[], template_variables={}): + """ Download resources and return the path. """ + if resource_dir and resource_list: + return get_package_dir_from_dir_and_list(resource_dir = resource_dir, + resource_list = resource_list, + template_variables = template_variables) + elif resource_dir and not resource_list: + return get_package_dir_from_dir(resource_dir = resource_dir, + template_variables = template_variables) + elif not resource_dir and resource_list: + return get_package_dir_from_list(resource_list = resource_list, + template_variables = template_variables) else: raise NonRecoverableError("At least one of the two properties, \ resource_dir or resource_list, has to be defined.") - return work_dir - def handle_overrides(overrides, current): if not isinstance(overrides, dict): diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..0f481f4 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,4 @@ +nose +mock +tox +testtools From 75f39adb9a75fd04fe8b816d084eb68ab2de5ab1 Mon Sep 17 00:00:00 2001 From: jrzeszutek Date: Wed, 24 Oct 2018 10:55:12 +0200 Subject: [PATCH 4/5] added downloading resources from deployment directory --- exec_plugin/tasks.py | 100 +++++++++++++++++++++++++++---------------- setup.py | 3 +- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/exec_plugin/tasks.py b/exec_plugin/tasks.py index fab8281..ea12cbb 100644 --- a/exec_plugin/tasks.py +++ b/exec_plugin/tasks.py @@ -60,6 +60,23 @@ def get_blueprint_directory(): ctx.tenant_name, ctx.blueprint.id)) +def get_deployment_directory(): + # In order to use update workflow, scripts should be taken from + # deployment directory. But to assure the possibility of using the + # plugin in install workflow (before the deployment directory is + # created), it returns this method returns blueprint directory, + # if deployment directory doesn't exist yet. + deployment_dir = get_directory_by_property_name( + 'deployment_directory', + lambda: '/opt/manager/resources/deployments/{0}/{1}'.format( + ctx.tenant_name, ctx.deployment.id)) + + if deployment_dir: + return deployment_dir + else: + return get_blueprint_directory() + + def extract_archive_from_path(archive_path, target_directory, intermediate_actions=None): return_value = None with zipfile.ZipFile(archive_path) as archive: @@ -88,8 +105,8 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari # Deal with ZIP files filename, extension = os.path.splitext(resource_dir) if extension == '.zip': - archive_path = os.path.join(get_blueprint_directory(), resource_dir) - target_directory = os.path.join(get_blueprint_directory(), filename) + archive_path = os.path.join(get_deployment_directory(), resource_dir) + target_directory = os.path.join(get_deployment_directory(), filename) resource_dir = filename extract_archive_from_path(archive_path, target_directory) @@ -100,12 +117,12 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari if extension == '.zip': resource_list.remove(template_path) - archive_path = os.path.join(get_blueprint_directory(), resource_dir, template_path) - target_directory = os.path.join(get_blueprint_directory(), resource_dir, filename) + archive_path = os.path.join(get_deployment_directory(), resource_dir, template_path) + target_directory = os.path.join(get_deployment_directory(), resource_dir, filename) extract_archive_from_path(archive_path, target_directory) for extracted_template in os.walk(target_directory): - extracted_template_path = get_resource_relative_path(extracted_template, os.path.join(get_blueprint_directory(), resource_dir)) + extracted_template_path = get_resource_relative_path(extracted_template, os.path.join(get_deployment_directory(), resource_dir)) if extracted_template[2]: for filename in extracted_template[2]: resource_list.append(os.path.join(extracted_template_path, filename)) @@ -116,8 +133,8 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari # This loop goes through a directory defined in resource_dir parameter # and prepares a list of paths inside it. - for resource_path in os.walk(os.path.join(get_blueprint_directory(), resource_dir)): - trimmed_resource_path = get_resource_relative_path(resource_path, os.path.join(get_blueprint_directory())) + for resource_path in os.walk(os.path.join(get_deployment_directory(), resource_dir)): + trimmed_resource_path = get_resource_relative_path(resource_path, os.path.join(get_deployment_directory())) if resource_path[2]: for filename in resource_path[2]: merged_list.append(os.path.join(trimmed_resource_path, filename)) @@ -142,13 +159,17 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari if e.errno != errno.EEXIST: raise - ctx.download_resource_and_render( - download_from_file, - download_to_file, - template_variables.copy()) - - if os.path.splitext(download_to_file)[1] == '.py': - os.chmod(download_to_file, 0755) + try: + ctx.download_resource_and_render( + download_from_file, + download_to_file, + template_variables.copy()) + + if os.path.splitext(download_to_file)[1] == '.py': + os.chmod(download_to_file, 0755) + except IOError as e: + if e.errno != errno.EISDIR: + raise if download_from_file in merged_list: merged_list.remove(download_from_file) @@ -170,13 +191,12 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari ctx.download_resource( resource_path, download_to_file) - except OSError as e: + if os.path.splitext(download_to_file)[1] == '.py': + os.chmod(download_to_file, 0755) + except IOError as e: if e.errno != errno.EISDIR: raise - if os.path.splitext(download_to_file)[1] == '.py': - os.chmod(download_to_file, 0755) - return get_current_working_directory() @@ -189,8 +209,8 @@ def get_package_dir_from_dir(resource_dir, template_variables={}): # Deal with ZIP files filename, extension = os.path.splitext(resource_dir) if extension == '.zip': - archive_path = os.path.join(get_blueprint_directory(), resource_dir) - target_directory = os.path.join(get_blueprint_directory(), filename) + archive_path = os.path.join(get_deployment_directory(), resource_dir) + target_directory = os.path.join(get_deployment_directory(), filename) resource_dir = filename extract_archive_from_path(archive_path, target_directory) @@ -198,8 +218,8 @@ def get_package_dir_from_dir(resource_dir, template_variables={}): # This loop goes through a directory defined in resource_dir parameter # and prepares a list of paths inside it. - for resource_path in os.walk(os.path.join(get_blueprint_directory(), resource_dir)): - trimmed_resource_path = get_resource_relative_path(resource_path, os.path.join(get_blueprint_directory())) + for resource_path in os.walk(os.path.join(get_deployment_directory(), resource_dir)): + trimmed_resource_path = get_resource_relative_path(resource_path, os.path.join(get_deployment_directory())) if resource_path[2]: for filename in resource_path[2]: @@ -221,13 +241,17 @@ def get_package_dir_from_dir(resource_dir, template_variables={}): if e.errno != errno.EEXIST: raise - ctx.download_resource_and_render( - download_from_file, - download_to_file, - template_variables.copy()) - - if os.path.splitext(download_to_file)[1] == '.py': - os.chmod(download_to_file, 0755) + try: + ctx.download_resource_and_render( + download_from_file, + download_to_file, + template_variables.copy()) + + if os.path.splitext(download_to_file)[1] == '.py': + os.chmod(download_to_file, 0755) + except IOError as e: + if e.errno != errno.EISDIR: + raise return get_current_working_directory() @@ -247,12 +271,12 @@ def get_package_dir_from_list(resource_list, template_variables={}): resource_list.remove(template_path) - archive_path = os.path.join(get_blueprint_directory(), template_path) - target_directory = os.path.join(get_blueprint_directory(), filename) + archive_path = os.path.join(get_deployment_directory(), template_path) + target_directory = os.path.join(get_deployment_directory(), filename) extract_archive_from_path(archive_path, target_directory) for extracted_template in os.walk(target_directory): - extracted_template_path = get_resource_relative_path(extracted_template, get_blueprint_directory()) + extracted_template_path = get_resource_relative_path(extracted_template, get_deployment_directory()) if extracted_template[2]: for filename in extracted_template[2]: resource_list.append(os.path.join(extracted_template_path, filename)) @@ -262,10 +286,14 @@ def get_package_dir_from_list(resource_list, template_variables={}): for template_path in resource_list: resource_name = os.path.basename(template_path) download_to = os.path.join(get_current_working_directory(), resource_name) - ctx.download_resource_and_render( - template_path, - download_to, - template_variables.copy()) + try: + ctx.download_resource_and_render( + template_path, + download_to, + template_variables.copy()) + except IOError as e: + if e.errno != errno.EISDIR: + raise return get_current_working_directory() diff --git a/setup.py b/setup.py index 9c29134..36532f7 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ license='LICENSE', description='Execute something.', install_requires=[ - 'cloudify-plugins-common' + 'cloudify-plugins-common', + 'Jinja2>=2.9.0' ] ) From 24439d0ecba40eaa98655fb16d5076381f9b1fdb Mon Sep 17 00:00:00 2001 From: jrzeszutek Date: Wed, 24 Oct 2018 15:16:44 +0200 Subject: [PATCH 5/5] resolved conflicts --- CHANGELOG.txt | 3 + constraints.txt | 1 + examples/blueprint.yaml | 11 ++- examples/resources/ansible/exec | 14 +-- examples/resources/ansible/helloworld.yml | 4 +- examples/resources/ansible/hosts.tmp | 7 +- examples/resources/ansible/id_rsa | 27 ++++++ examples/resources/ansible/id_rsa.pub | 1 + exec_plugin/tasks.py | 109 ++++++++++++++-------- host-agent-plugin.yaml | 45 +++++++++ plugin.yaml | 2 +- setup.py | 4 +- 12 files changed, 169 insertions(+), 59 deletions(-) create mode 100644 CHANGELOG.txt create mode 100644 constraints.txt create mode 100644 examples/resources/ansible/id_rsa create mode 100644 examples/resources/ansible/id_rsa.pub create mode 100644 host-agent-plugin.yaml diff --git a/CHANGELOG.txt b/CHANGELOG.txt new file mode 100644 index 0000000..fb0c7bb --- /dev/null +++ b/CHANGELOG.txt @@ -0,0 +1,3 @@ +releases: +- 0.3.0: First functional release. +- 0.4.0: Add support for folders and nested zip archives. diff --git a/constraints.txt b/constraints.txt new file mode 100644 index 0000000..2109599 --- /dev/null +++ b/constraints.txt @@ -0,0 +1 @@ +cloudify-common<=4.4 \ No newline at end of file diff --git a/examples/blueprint.yaml b/examples/blueprint.yaml index a3ce863..18706c4 100644 --- a/examples/blueprint.yaml +++ b/examples/blueprint.yaml @@ -6,7 +6,7 @@ imports: inputs: - host_ip: + ansible_host: type: string username: @@ -16,9 +16,10 @@ inputs: resource_list: # For Ansible. default: - # - exec - # - helloworld.yml + - exec + - helloworld.yml - hosts.tmp + - id_rsa node_templates: @@ -29,7 +30,9 @@ node_templates: resource_dir: resources/ansible resource_list: { get_input: resource_list } template_variables: - host_ip: { get_input: host_ip } + ansible_host: { get_input: ansible_host } + ansible_user: { get_input: username } + ansible_ssh_private_key_file: id_rsa # interfaces: # cloudify.interfaces.lifecycle: # create: diff --git a/examples/resources/ansible/exec b/examples/resources/ansible/exec index c28221c..6d8c441 100644 --- a/examples/resources/ansible/exec +++ b/examples/resources/ansible/exec @@ -1,12 +1,6 @@ #!/bin/bash -sudo apt-get update -y -sudo apt-get install software-properties-common -y -sudo apt-add-repository ppa:ansible/ansible -y - -sudo apt-get update -y -sudo apt-get install ansible -y - -sudo cp hosts.tmp /etc/ansible/hosts - -ansible-playbook helloworld.yml +/opt/cfy/embedded/bin/virtualenv ansible +source ansible/bin/activate +pip install ansible +ansible-playbook --inventory hosts.tmp helloworld.yml diff --git a/examples/resources/ansible/helloworld.yml b/examples/resources/ansible/helloworld.yml index c9760e6..a700be0 100644 --- a/examples/resources/ansible/helloworld.yml +++ b/examples/resources/ansible/helloworld.yml @@ -1,4 +1,4 @@ --- -- hosts: all +- hosts: jumper tasks: - - shell: echo "hello {{ host_ip }}" + - shell: echo "hello {{ ansible_host }}" diff --git a/examples/resources/ansible/hosts.tmp b/examples/resources/ansible/hosts.tmp index 458a80a..dbe6958 100644 --- a/examples/resources/ansible/hosts.tmp +++ b/examples/resources/ansible/hosts.tmp @@ -1 +1,6 @@ -{{ host_ip }} ansible_connection=local +hosts: + jumper: + ansible_host={{ ansible_host }} + ansible_connection: ssh + ansible_user: {{ ansible_user }} + ansible_ssh_private_key_file: {{ ansible_ssh_private_key_file }} diff --git a/examples/resources/ansible/id_rsa b/examples/resources/ansible/id_rsa new file mode 100644 index 0000000..9e73f52 --- /dev/null +++ b/examples/resources/ansible/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAoLHaq/BKxQwPpbnCZJyZxFVikaGIfszcU0RLFaaZh+dtCAAB +M4xpQEBUHm2ZDscNp+qzUYpvVu4VmEd15wAZ20sB9Oy3eHK2hp95hbLIMQAG8fvP +cw6d0tyltdBypNkgeDg4e9S0skc5Z9LPI6RPJ83AiO12Vex8TcJ6s5Im6e723Igf +G5+1GhH0Ajzeu3j5zGudiaZR8rc0s7gzG/ifPRhr/V5vE0oOyRg4CvwU/nWpIN4g +V5AnyKmJbwRBGxLhxXiGZmyQSWpeuGXX7y+dakXVX9tPs5NQTMcKN9XBXv0a6COE +TmZgVd9Ijgda8O6LckY65QOQPEyKuYJBimfmfwIDAQABAoIBAQCcs/OfOetf7lwP +627iuEfGytEdXN2DcXjlhjKGQXl0qzyK0Z5aS3GT42dkaNIaNAhgGCKXwJzEs9XB +MuaJxnJwg9u7oV9lCNBvxXI269oG1pTvQvTQqY9EQkEiZMx3bvkX4xNJxmoTdBtJ +4Kx4gFFkamN348FDFyM8nQsUs9BYzFCMr+Da08vy6SVdQ07pJWgkwJ88UFPcov0A +NEyY06dI8iyISdwnxPev8mInDTnLL3rEccS5F1RXGETH9pKrNwij8wS7aqoiYFMV +Mvm2/EDr7c1lAVEq216mQqk92WJDJUfSh6p1m5qCAa+hKCZ3bSJCyaEweWm1g45U +YtPaLhgRAoGBAMRmfApeIv4E0XAeGnEgJPNCboqBkT6QNb8ANYvvQPr0b4vonIXL +p9PZ5CNLIfrwH1b4qZ7wLnsUJ8OspdnBfL2afDyZYzg3eh+P1SQC9MZ4mo9O9Hfs +A+22UpstSAcXYJ4jCEmBVPO708CZO3Pu/eKYfnm4iU342ZNhXi/FVEf3AoGBANF1 +jYLJLFVyAvNYD4VEPj3NC3ud1T+ORL4t84W16VLs/dJ62mJfnH8JGp7WBZ/RCXSw +wdJeFYhbELhiIGKPWyuFvnEOFFNPpwI83KOep9na/WnHDtGslWZOdY6uH6qt70HX +yXcpakUkqEiOAdk82XSxXwu5K9ShwxrRqDRPmAO5AoGBALGUnMlfDrKxtpFylqP8 +1YwiYALaXnK8P+yAiuoJHa8LfyR44bXUS94w4KfM0l+r6BpiYjQ0pUJiGgZCP/+h +hGGHopfEdUMqXkDVc3nJNnuSJuY3FLKt82tndE+c5MiMqa6yrFaSGzb6eN9nbatI +x8Sv74OtiHC49jndSyVUff1HAoGAbYgaYx0OKPSA5H3VR+xC3BCAg7kqjZy5B/aB +lRdmlu3FYa7KyoyTCr9ZA1yJ/mPSnQndoUN/4Rk16iWuJ7wtgdnjl700FcjeXcYz +3wAVaFAjGpB0iGLGkYRP5zfY6WkhZ34PJVgr8VJVCfPwtYgjculZ+YhwijW5ECo4 +2noQ1UkCgYAl7/Puy/l1LJK49zqD/Bo6q/rkspKYBCOfAuHQoCyboYNKmAk9aebe +XKnecMl2b98MYdfcXe+EFRr5/AyCrjehZAmsqi/CnD1mpJFEe/pgs4+JzdEuU/kx +xZTG5PH3KB99ZWbIphA3NKZkk3PA7JuctqC3N5XhAuj+dSiTLGbQMQ== +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/examples/resources/ansible/id_rsa.pub b/examples/resources/ansible/id_rsa.pub new file mode 100644 index 0000000..076e76d --- /dev/null +++ b/examples/resources/ansible/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCgsdqr8ErFDA+lucJknJnEVWKRoYh+zNxTREsVppmH520IAAEzjGlAQFQebZkOxw2n6rNRim9W7hWYR3XnABnbSwH07Ld4craGn3mFssgxAAbx+89zDp3S3KW10HKk2SB4ODh71LSyRzln0s8jpE8nzcCI7XZV7HxNwnqzkibp7vbciB8bn7UaEfQCPN67ePnMa52JplHytzSzuDMb+J89GGv9Xm8TSg7JGDgK/BT+dakg3iBXkCfIqYlvBEEbEuHFeIZmbJBJal64ZdfvL51qRdVf20+zk1BMxwo31cFe/RroI4ROZmBV30iOB1rw7otyRjrlA5A8TIq5gkGKZ+Z/ \ No newline at end of file diff --git a/exec_plugin/tasks.py b/exec_plugin/tasks.py index ea12cbb..bf1d3b0 100644 --- a/exec_plugin/tasks.py +++ b/exec_plugin/tasks.py @@ -23,7 +23,6 @@ def get_directory_by_property_name(property_name, creation_action_args=None, creation_action_kwargs=None): - directory = \ str(ctx.instance.runtime_properties.get( property_name)) @@ -77,7 +76,9 @@ def get_deployment_directory(): return get_blueprint_directory() -def extract_archive_from_path(archive_path, target_directory, intermediate_actions=None): +def extract_archive_from_path(archive_path, + target_directory, + intermediate_actions=None): return_value = None with zipfile.ZipFile(archive_path) as archive: if intermediate_actions: @@ -88,14 +89,17 @@ def extract_archive_from_path(archive_path, target_directory, intermediate_actio def get_resource_relative_path(resource, relative_dir): """ - :param resource: Entry of the list returned by os.walk() containing absolute path of a resource + :param resource: Entry of the list returned by os.walk() + containing absolute path of a resource :param relative_dir: Directory, which the resource is relative to :return: String object containing relative path of the resource """ return resource[0][len(relative_dir) + 1:] -def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_variables={}): +def get_package_dir_from_dir_and_list(resource_dir, + resource_list, + template_variables={}): # Case, when user defines a directory with files, which need to be # downloaded, but doesn't want to render all of them - only these # defined in resource_list. @@ -117,15 +121,20 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari if extension == '.zip': resource_list.remove(template_path) - archive_path = os.path.join(get_deployment_directory(), resource_dir, template_path) - target_directory = os.path.join(get_deployment_directory(), resource_dir, filename) + archive_path = os.path.join( + get_deployment_directory(), resource_dir, template_path) + target_directory = os.path.join( + get_deployment_directory(), resource_dir, filename) extract_archive_from_path(archive_path, target_directory) for extracted_template in os.walk(target_directory): - extracted_template_path = get_resource_relative_path(extracted_template, os.path.join(get_deployment_directory(), resource_dir)) + extracted_template_path = get_resource_relative_path( + extracted_template, + os.path.join(get_deployment_directory(), resource_dir)) if extracted_template[2]: for filename in extracted_template[2]: - resource_list.append(os.path.join(extracted_template_path, filename)) + resource_list.append( + os.path.join(extracted_template_path, filename)) elif not extracted_template[1] and not extracted_template[2]: resource_list.append(extracted_template_path) @@ -133,11 +142,14 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari # This loop goes through a directory defined in resource_dir parameter # and prepares a list of paths inside it. - for resource_path in os.walk(os.path.join(get_deployment_directory(), resource_dir)): - trimmed_resource_path = get_resource_relative_path(resource_path, os.path.join(get_deployment_directory())) + for resource_path in os.walk(os.path.join( + get_deployment_directory(), resource_dir)): + trimmed_resource_path = get_resource_relative_path( + resource_path, os.path.join(get_deployment_directory())) if resource_path[2]: for filename in resource_path[2]: - merged_list.append(os.path.join(trimmed_resource_path, filename)) + merged_list.append( + os.path.join(trimmed_resource_path, filename)) elif not resource_path[1] and not resource_path[2]: merged_list.append(trimmed_resource_path) @@ -148,10 +160,13 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari # because it should be ommitted at the next step, which is copying the rest # of the files, which are not templates. for template_path in resource_list: - template_dirname = os.path.join(resource_dir, os.path.dirname(template_path)) + template_dirname = os.path.join( + resource_dir, os.path.dirname(template_path)) download_from_file = os.path.join(resource_dir, template_path) - download_to_directory = os.path.join(get_current_working_directory(), template_dirname) - download_to_file = os.path.join(get_current_working_directory(), download_from_file) + download_to_directory = os.path.join( + get_current_working_directory(), template_dirname) + download_to_file = os.path.join( + get_current_working_directory(), download_from_file) try: os.makedirs(download_to_directory) @@ -164,7 +179,6 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari download_from_file, download_to_file, template_variables.copy()) - if os.path.splitext(download_to_file)[1] == '.py': os.chmod(download_to_file, 0755) except IOError as e: @@ -178,8 +192,10 @@ def get_package_dir_from_dir_and_list(resource_dir, resource_list, template_vari # files to our working directory. for resource_path in merged_list: resource_dirname = os.path.dirname(resource_path) - download_to_directory = os.path.join(get_current_working_directory(), resource_dirname) - download_to_file = os.path.join(get_current_working_directory(), resource_path) + download_to_directory = os.path.join( + get_current_working_directory(), resource_dirname) + download_to_file = os.path.join( + get_current_working_directory(), resource_path) try: os.makedirs(download_to_directory) @@ -218,12 +234,15 @@ def get_package_dir_from_dir(resource_dir, template_variables={}): # This loop goes through a directory defined in resource_dir parameter # and prepares a list of paths inside it. - for resource_path in os.walk(os.path.join(get_deployment_directory(), resource_dir)): - trimmed_resource_path = get_resource_relative_path(resource_path, os.path.join(get_deployment_directory())) + for resource_path in os.walk( + os.path.join(get_deployment_directory(), resource_dir)): + trimmed_resource_path = get_resource_relative_path( + resource_path, os.path.join(get_deployment_directory())) if resource_path[2]: for filename in resource_path[2]: - merged_list.append(os.path.join(trimmed_resource_path, filename)) + merged_list.append( + os.path.join(trimmed_resource_path, filename)) elif not resource_path[1] and not resource_path[2]: merged_list.append(trimmed_resource_path) @@ -232,8 +251,10 @@ def get_package_dir_from_dir(resource_dir, template_variables={}): for template_path in merged_list: template_dirname = os.path.dirname(template_path) download_from_file = template_path - download_to_directory = os.path.join(get_current_working_directory(), template_dirname) - download_to_file = os.path.join(get_current_working_directory(), download_from_file) + download_to_directory = os.path.join( + get_current_working_directory(), template_dirname) + download_to_file = os.path.join( + get_current_working_directory(), download_from_file) try: os.makedirs(download_to_directory) @@ -246,7 +267,6 @@ def get_package_dir_from_dir(resource_dir, template_variables={}): download_from_file, download_to_file, template_variables.copy()) - if os.path.splitext(download_to_file)[1] == '.py': os.chmod(download_to_file, 0755) except IOError as e: @@ -257,8 +277,8 @@ def get_package_dir_from_dir(resource_dir, template_variables={}): def get_package_dir_from_list(resource_list, template_variables={}): - # Case, when user defines a list of files in resource_list, which need to be - # downloaded and rendered. + # Case, when user defines a list of files in resource_list, + # which need to be downloaded and rendered. ctx.logger.debug('only resource_list is not empty.') @@ -271,21 +291,26 @@ def get_package_dir_from_list(resource_list, template_variables={}): resource_list.remove(template_path) - archive_path = os.path.join(get_deployment_directory(), template_path) - target_directory = os.path.join(get_deployment_directory(), filename) + archive_path = os.path.join( + get_deployment_directory(), template_path) + target_directory = os.path.join( + get_deployment_directory(), filename) extract_archive_from_path(archive_path, target_directory) for extracted_template in os.walk(target_directory): - extracted_template_path = get_resource_relative_path(extracted_template, get_deployment_directory()) + extracted_template_path = get_resource_relative_path( + extracted_template, get_deployment_directory()) if extracted_template[2]: for filename in extracted_template[2]: - resource_list.append(os.path.join(extracted_template_path, filename)) + resource_list.append( + os.path.join(extracted_template_path, filename)) elif not extracted_template[1] and not extracted_template[2]: resource_list.append(extracted_template_path) for template_path in resource_list: resource_name = os.path.basename(template_path) - download_to = os.path.join(get_current_working_directory(), resource_name) + download_to = os.path.join( + get_current_working_directory(), resource_name) try: ctx.download_resource_and_render( template_path, @@ -302,15 +327,18 @@ def get_package_dir(resource_dir='', resource_list=[], template_variables={}): """ Download resources and return the path. """ if resource_dir and resource_list: - return get_package_dir_from_dir_and_list(resource_dir = resource_dir, - resource_list = resource_list, - template_variables = template_variables) + return get_package_dir_from_dir_and_list( + resource_dir=resource_dir, + resource_list=resource_list, + template_variables=template_variables) elif resource_dir and not resource_list: - return get_package_dir_from_dir(resource_dir = resource_dir, - template_variables = template_variables) + return get_package_dir_from_dir( + resource_dir=resource_dir, + template_variables=template_variables) elif not resource_dir and resource_list: - return get_package_dir_from_list(resource_list = resource_list, - template_variables = template_variables) + return get_package_dir_from_list( + resource_list=resource_list, + template_variables=template_variables) else: raise NonRecoverableError("At least one of the two properties, \ resource_dir or resource_list, has to be defined.") @@ -361,8 +389,11 @@ def execute(resource_config, raise NonRecoverableError("'template_variables' must be a dictionary.") if resource_dir: - tmp_dir = get_package_dir(resource_dir, resource_list, template_variables) - cwd = os.path.join(tmp_dir, os.path.splitext(resource_dir)[0]) # in case of resource_dir is zip + tmp_dir = get_package_dir( + resource_dir, resource_list, template_variables) + # in case of resource_dir is zip + cwd = os.path.join( + tmp_dir, os.path.splitext(resource_dir)[0]) else: cwd = get_package_dir(resource_dir, resource_list, template_variables) command = ['bash', '-c', 'source {0}'.format(file_to_source)] diff --git a/host-agent-plugin.yaml b/host-agent-plugin.yaml new file mode 100644 index 0000000..d2c5e70 --- /dev/null +++ b/host-agent-plugin.yaml @@ -0,0 +1,45 @@ +plugins: + + exec: + executor: host_agent + source: https://github.com/cloudify-incubator/cloudify-execution-plugin/archive/0.4.0.zip + package_name: cloudify-execution-plugin + package_version: '0.4.0' + +data_types: + + cloudify.types.exec.Package: + properties: + resource_dir: + description: > + A directory under the blueprint root, + which content needs to be copied and kept unchanged. + default: '' + resource_list: + description: > + A list of file paths under the blueprint root, + which need to be filled with values of template_variables. + default: [] + template_variables: + description: A dict containing variables as key-values. + default: {} + +node_types: + + cloudify.nodes.Execution: + derived_from: cloudify.nodes.SoftwareComponent + properties: + resource_config: + type: cloudify.types.exec.Package + required: false + interfaces: + cloudify.interfaces.lifecycle: + create: &exec_op + implementation: exec.exec_plugin.tasks.execute + inputs: + resource_config: + type: cloudify.types.exec.Package + default: { get_property: [ SELF, resource_config ] } + file_to_source: + type: string + default: exec diff --git a/plugin.yaml b/plugin.yaml index becd44f..3f0972f 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -3,7 +3,7 @@ plugins: exec: executor: central_deployment_agent package_name: cloudify-execution-plugin - package_version: '0.4' + package_version: '0.4.0' data_types: diff --git a/setup.py b/setup.py index 36532f7..ab3bd78 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,12 @@ setup( name='cloudify-execution-plugin', - version='0.4', + version='0.4.0', packages=['exec_plugin'], license='LICENSE', description='Execute something.', install_requires=[ - 'cloudify-plugins-common', + 'cloudify-common', 'Jinja2>=2.9.0' ] )