Merge pull request #358 from cloudify-cosmo/replace-wagon-builder-to-py2py3-builder

Replace wagon builder to py2py3 builder

Author: AdarShaked

.circleci/config.yml

@@ -27,6 +27,9 @@ executors:
     machine:
       image: ubuntu-1604:201903-01
 
+  cloudify-machine-510:
+    machine:
+      image: ubuntu-1604:201903-01
 
 commands:
   run_unittest:
@@ -69,16 +72,16 @@ commands:
               exit 1
             fi
 
-  generate_py27_wagon:
+  generate_py27py36_wagon:
     steps:
       - run:
           name: Create Workspace Build directory.
           command: mkdir -p workspace/build
       - run:
-          name: Build 27 Wagon
+          name: Build py27py36 Wagon
           command: |
               git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
-              docker build -t cloudify-centos-7-wagon-builder cloudify-wagon-build-containers/centos_7
+              docker build -t cloudify-centos-7-wagon-builder cloudify-wagon-build-containers/centos_7_py2py3
               docker run -v ~/project/:/packaging cloudify-centos-7-wagon-builder
       - run:
           name: copy wagon to workspace
@@ -88,16 +91,16 @@ commands:
           paths:
             - build/*
 
-  generate_rhel_py27_wagon:
+  generate_rhel_py27py36_wagon:
     steps:
       - run:
           name: Create Workspace Build directory.
           command: mkdir -p workspace/build
       - run:
-          name: Build RHEL 27 Wagon
+          name: Build RHEL py27py36 Wagon
           command: |
               git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
-              docker build -t cloudify-redhat-7-wagon-builder cloudify-wagon-build-containers/redhat_7 --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD"
+              docker build -t cloudify-redhat-7-wagon-builder cloudify-wagon-build-containers/redhat_7_py2py3 --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD"
               docker run -v ~/project/:/packaging cloudify-redhat-7-wagon-builder
       - run:
           name: copy wagon to workspace
@@ -107,7 +110,7 @@ commands:
           paths:
             - build/*
 
-  run_integration_tests:
+  run_integration_tests_505:
     steps:
       - run:
           name: "Pull Submodules"
@@ -121,16 +124,52 @@ commands:
           command: pip install -U pip
       - run:
           name: install cloudify
-          command: pip install -U cloudify==5.0.0 pytest==4.6.3 pyyaml==3.10
+          command: |
+            pip install -r test-requirements.txt
+      - run:
+          name: download manager docker image
+          command: wget http://repository.cloudifysource.org/cloudify/5.0.5/ga-release/cloudify-docker-manager-5.0.5.tar
+      - run:
+          name: load docker image
+          command: docker load -i cloudify-docker-manager-5.0.5.tar
+      - run:
+          name: retain space by dumping the tar
+          command: rm cloudify-docker-manager-5.0.5.tar
+      - run:
+          name: show images
+          command: docker images
+      - run:
+          name: start docker container
+          command: docker run --name cfy_manager -d --restart unless-stopped -v /sys/fs/cgroup:/sys/fs/cgroup:ro --tmpfs /run --tmpfs /run/lock --security-opt seccomp:unconfined --cap-add SYS_ADMIN --network host cloudifyplatform/premium-cloudify-manager-aio
+      - run:
+          name: install local project
+          command: pip install https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/latest.zip
+      - run: pytest -s .circleci/test_examples.py
+
+  run_integration_tests_510:
+    steps:
+      - run:
+          name: "Pull Submodules"
+          command: |
+            git submodule update --init --recursive --remote
+      - attach_workspace:
+          at: workspace
+      - run:
+          name: update pip
+          command: pip install -U pip
+      - run:
+          name: install cloudify
+          command: |
+            pip install -r test-requirements.txt
       - run:
           name: download manager docker image
-          command: wget http://repository.cloudifysource.org/cloudify/5.0.0/ga-release/cloudify-docker-manager-5.0.0.tar
+          command:  wget http://repository.cloudifysource.org/cloudify/5.1.0/.dev1-release/cloudify-docker-manager-5.1.0.dev1.tar
       - run:
           name: load docker image
-          command: docker load -i cloudify-docker-manager-5.0.0.tar
+          command: docker load -i cloudify-docker-manager-5.1.0.dev1.tar
       - run:
           name: retain space by dumping the tar
-          command: rm cloudify-docker-manager-5.0.0.tar
+          command: rm cloudify-docker-manager-5.1.0.dev1.tar
       - run:
           name: show images
           command: docker images
@@ -195,23 +234,32 @@ jobs:
     executor: wagon_generator
     steps:
       - checkout
-      - generate_py27_wagon
+      - generate_py27py36_wagon
 
   rhel_wagon:
     executor: wagon_generator
     steps:
       - checkout
-      - generate_rhel_py27_wagon
+      - generate_rhel_py27py36_wagon
 
-  integration_tests:
+  integration_tests_505:
     executor: cloudify-machine
     environment:
       CLOUDIFY_SSL_TRUST_ALL: true
       IAAS: openstack
       TEST_NAME: cloudformation
     steps:
       - checkout
-      - run_integration_tests
+      - run_integration_tests_505
+
+  integration_tests_510:
+    executor: cloudify-machine-510
+    environment:
+      CLOUDIFY_SSL_TRUST_ALL: true
+      IAAS: openstack
+    steps:
+      - checkout
+      - run_integration_tests_510
 
   release:
     executor: py27
@@ -222,7 +270,7 @@ jobs:
 workflows:
   version: 2
   tests:
-    jobs: &all_jobs
+    jobs: 
       - py3_compat
       - unittests_py27
       - unittests_py36
@@ -234,7 +282,14 @@ workflows:
           filters:
             branches:
               only: /([0-9\.]*\-build|master|dev)/
-      - integration_tests:
+      - integration_tests_505:
+          requires:
+            - wagon
+            - rhel_wagon
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master|dev)/
+      - integration_tests_510:
           requires:
             - wagon
             - rhel_wagon
@@ -246,9 +301,12 @@ workflows:
             branches:
               only: /master/
           requires:
+            - unittests_py27
+            - unittests_py36
             - wagon
             - rhel_wagon
-            - integration_tests
+            - integration_tests_505
+            - integration_tests_510
 
   nightly:
     triggers:
@@ -258,4 +316,29 @@ workflows:
           branches:
             only:
               - master
-    jobs: *all_jobs
+    jobs:
+      - py3_compat
+      - unittests_py27
+      - unittests_py36
+      - wagon:
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master|dev)/
+      - rhel_wagon:
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master|dev)/
+      - integration_tests_505:
+          requires:
+            - wagon
+            - rhel_wagon
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master|dev)/
+      - integration_tests_510:
+          requires:
+            - wagon
+            - rhel_wagon
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master|dev)/

.circleci/test_examples.py

@@ -22,6 +22,15 @@
     cleanup_on_failure, prepare_test
 )
 
+'''Temporary until all the plugins in the bundle will 
+released with py2py3 wagons'''
+UT_VERSION = '1.23.5'
+UT_WAGON = 'https://github.com/cloudify-incubator/cloudify-utilities-plugin/' \
+           'releases/download/{v}/cloudify_utilities_plugin-{v}-centos' \
+           '-Core-py27.py36-none-linux_x86_64.wgn'.format(v=UT_VERSION)
+UT_PLUGIN = 'https://github.com/cloudify-incubator/cloudify-utilities-' \
+            'plugin/releases/download/{v}/plugin.yaml'.format(v=UT_VERSION)
+PLUGINS_TO_UPLOAD = [(UT_WAGON, UT_PLUGIN)]
 SECRETS_TO_CREATE = {
     'openstack_username': False,
     'openstack_password': False,
@@ -35,7 +44,8 @@
     'openstack_project_domain_name': False,
 }
 
-prepare_test(secrets=SECRETS_TO_CREATE)
+prepare_test(plugins=PLUGINS_TO_UPLOAD, secrets=SECRETS_TO_CREATE,
+             execute_bundle_upload=False)
 
 blueprint_list = ['examples/blueprint-examples/virtual-machine/openstack.yaml']
 

CHANGELOG.txt

@@ -1,3 +1,6 @@
+3.2.16:
+  - Update wagon builder to py2py3 wagon.
+  - Added 5.1.0 integration tests.
 3.2.14: Support Python 3.
 3.2.13:
   - Support quota operations on Project [ get_quota/update_quota ].

constraints.txt

@@ -1 +0,0 @@
-cloudify-common==4.4

plugin.yaml

@@ -2,9 +2,9 @@ plugins:
 
   openstack:
     executor: central_deployment_agent
-    source: https://github.com/cloudify-cosmo/cloudify-openstack-plugin/archive/3.2.15.zip
+    source: https://github.com/cloudify-cosmo/cloudify-openstack-plugin/archive/3.2.16.zip
     package_name: cloudify-openstack-plugin
-    package_version: '3.2.15'
+    package_version: '3.2.16'
 
 dsl_definitions:
 

setup.py

@@ -20,7 +20,7 @@
 
 setup(
     name='cloudify-openstack-plugin',
-    version='3.2.15',
+    version='3.2.16',
     author='Cloudify',
     author_email='[email protected]',
     license='LICENSE',

test-requirements.txt

@@ -4,8 +4,12 @@ nose-cov>=1.3
 mock>=1.0
 flake8==3.7.9
 tox==3.14.6
-https://github.com/cloudify-cosmo/cloudify-common/archive/master.zip
-https://github.com/cloudify-cosmo/cloudify-cli/archive/master.zip
 sh==1.11
 IPy==0.81
 pylint
+
+# For integration tests
+git+https://github.com/cloudify-cosmo/cloudify-common@master#egg=cloudify-common[dispatcher]==master
+git+https://github.com/cloudify-cosmo/cloudify-cli@master#egg=cloudify-cli==master
+pytest==4.6.3
+pyyaml==3.12