Merge branch 'master' into update-circle

__NODOCS__

Author: EarthmanT

.circleci/config.yml

@@ -27,6 +27,9 @@ executors:
     machine:
       image: ubuntu-1604:201903-01
 
+  cloudify-machine-510:
+    machine:
+      image: ubuntu-1604:201903-01
 
 commands:
 
@@ -108,16 +111,16 @@ commands:
               exit 1
             fi
 
-  generate_py27_wagon:
+  generate_py27py36_wagon:
     steps:
       - run:
           name: Create Workspace Build directory.
           command: mkdir -p workspace/build
       - run:
-          name: Build py27 Wagon
+          name: Build py27py36 Wagon
           command: |
               git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
-              docker build -t cloudify-centos-7-wagon-builder cloudify-wagon-build-containers/centos_7
+              docker build -t cloudify-centos-7-wagon-builder cloudify-wagon-build-containers/centos_7_py2py3
               docker run -v ~/project/:/packaging cloudify-centos-7-wagon-builder
       - run:
           name: copy wagon to workspace
@@ -127,36 +130,16 @@ commands:
           paths:
             - build/*
 
-  generate_py3_wagon:
+  generate_rhel_py27py36_wagon:
     steps:
       - run:
           name: Create Workspace Build directory.
           command: mkdir -p workspace/build
       - run:
-          name: Build py3 Wagon
+          name: Build RHEL py27py36 Wagon
           command: |
               git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
-              docker build -t cloudify-centos-7-py3-wagon-builder cloudify-wagon-build-containers/centos_7_py3
-              docker run -v ~/project/:/packaging cloudify-centos-7-py3-wagon-builder
-      - run:
-          name: copy wagon to workspace
-          command: cp *.wgn workspace/build/
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            - build/*
-
-
-  generate_rhel_py27_wagon:
-    steps:
-      - run:
-          name: Create Workspace Build directory.
-          command: mkdir -p workspace/build
-      - run:
-          name: Build RHEL 27 Wagon
-          command: |
-              git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
-              docker build -t cloudify-redhat-7-wagon-builder cloudify-wagon-build-containers/redhat_7 --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD"
+              docker build -t cloudify-redhat-7-wagon-builder cloudify-wagon-build-containers/redhat_7_py2py3 --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD"
               docker run -v ~/project/:/packaging cloudify-redhat-7-wagon-builder
       - run:
           name: copy wagon to workspace
@@ -166,7 +149,7 @@ commands:
           paths:
             - build/*
 
-  run_integration_tests:
+  run_integration_tests_505:
     steps:
       - run:
           name: "Pull Submodules"
@@ -180,7 +163,8 @@ commands:
           command: pip install -U pip
       - run:
           name: install cloudify
-          command: pip install -U cloudify==5.0.0 pytest==4.6.3 pyyaml==3.10
+          command: |
+            pip install -r test-requirements.txt
       - run:
           name: download manager docker image
           command: wget http://repository.cloudifysource.org/cloudify/5.0.5/ga-release/cloudify-docker-manager-5.0.5.tar
@@ -201,6 +185,41 @@ commands:
           command: pip install https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/latest.zip
       - run: pytest -s .circleci/test_examples.py
 
+  run_integration_tests_510:
+    steps:
+      - run:
+          name: "Pull Submodules"
+          command: |
+            git submodule update --init --recursive --remote
+      - attach_workspace:
+          at: workspace
+      - run:
+          name: update pip
+          command: pip install -U pip
+      - run:
+          name: install cloudify
+          command: |
+            pip install -r test-requirements.txt
+      - run:
+          name: download manager docker image
+          command:  wget http://repository.cloudifysource.org/cloudify/5.1.0/.dev1-release/cloudify-manager-aio-docker-5.1.0.dev1.tar
+      - run:
+          name: load docker image
+          command: docker load -i cloudify-manager-aio-docker-5.1.0.dev1.tar
+      - run:
+          name: retain space by dumping the tar
+          command: rm cloudify-manager-aio-docker-5.1.0.dev1.tar
+      - run:
+          name: show images
+          command: docker images
+      - run:
+          name: start docker container
+          command: docker run --name cfy_manager -d --restart unless-stopped -v /sys/fs/cgroup:/sys/fs/cgroup:ro --tmpfs /run --tmpfs /run/lock --security-opt seccomp:unconfined --cap-add SYS_ADMIN --network host cloudify-manager-aio:latest
+      - run:
+          name: install local project
+          command: pip install https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/latest.zip
+      - run: pytest -s .circleci/test_examples.py
+
   release_plugin:
     steps:
       - run:
@@ -299,24 +318,32 @@ jobs:
     executor: wagon_generator
     steps:
       - checkout
-      - generate_py27_wagon
+      - generate_py27py36_wagon
       # - generate_py3_wagon  # TODO: When we have Python3 Manager Integration Test.
 
   rhel_wagon:
     executor: wagon_generator
     steps:
       - checkout
-      - generate_rhel_py27_wagon
-
+      - generate_rhel_py27py36_wagon
 
-  integration_tests:
+  integration_tests_505:
     executor: cloudify-machine
     environment:
       CLOUDIFY_SSL_TRUST_ALL: true
-      IAAS: gcp
+      IAAS: openstack
     steps:
       - checkout
-      - run_integration_tests
+      - run_integration_tests_505
+
+  integration_tests_510:
+    executor: cloudify-machine-510
+    environment:
+      CLOUDIFY_SSL_TRUST_ALL: true
+      IAAS: openstack
+    steps:
+      - checkout
+      - run_integration_tests_510
 
   release:
     executor: py27
@@ -369,6 +396,7 @@ workflows:
 #            - release
 #            - validate_documentation
 
+
   nightly:
     triggers:
     - schedule:

.circleci/test_examples.py

@@ -24,14 +24,29 @@
     cleanup_on_failure, prepare_test
 )
 
-OS_VERSION = '3.2.15'
+OS_VERSION = '3.2.16'
 OS_WAGON = 'https://github.com/cloudify-cosmo/cloudify-openstack-plugin/' \
-           'releases/download/{v}/cloudify_openstack_plugin-{v}-py27-none-' \
-           'linux_x86_64-centos-Core.wgn'.format(v=OS_VERSION)
-OS_PLUGIN = 'https://github.com/cloudify-cosmo/' \
-            'cloudify-openstack-plugin/releases/download/' \
-            '{v}/plugin.yaml'.format(v=OS_VERSION)
-PLUGINS_TO_UPLOAD = [(OS_WAGON, OS_PLUGIN)]
+           'releases/download/{v}/cloudify_openstack_plugin-{v}-' \
+           'centos-Core-py27.py36-none-linux_x86_64.wgn'.format(v=OS_VERSION)
+OS_PLUGIN = 'https://github.com/cloudify-cosmo/cloudify-openstack-plugin/' \
+            'releases/download/{v}/plugin.yaml'.format(v=OS_VERSION)
+'''Temporary until all the plugins in the bundle will 
+released with py2py3 wagons'''
+UT_VERSION = '1.23.5'
+UT_WAGON = 'https://github.com/cloudify-incubator/cloudify-utilities-plugin/' \
+           'releases/download/{v}/cloudify_utilities_plugin-{v}-centos' \
+           '-Core-py27.py36-none-linux_x86_64.wgn'.format(v=UT_VERSION)
+UT_PLUGIN = 'https://github.com/cloudify-incubator/cloudify-utilities-' \
+            'plugin/releases/download/{v}/plugin.yaml'.format(v=UT_VERSION)
+FAB_VERSION = '2.0.4'
+FAB_WAGON = 'https://github.com/cloudify-cosmo/cloudify-fabric-plugin/' \
+            'releases/download/{v}/cloudify_fabric_plugin-{v}-centos-' \
+            'Core-py27.py36-none-linux_x86_64.wgn'.format(v=FAB_VERSION)
+FAB_PLUGIN = 'https://github.com/cloudify-cosmo/cloudify-fabric-plugin/' \
+             'releases/download/{v}/plugin.yaml'.format(v=FAB_VERSION)
+
+PLUGINS_TO_UPLOAD = [(OS_WAGON, OS_PLUGIN), (UT_WAGON, UT_PLUGIN),
+                     (FAB_WAGON, FAB_PLUGIN)]
 
 
 SECRETS_TO_CREATE = {
@@ -54,7 +69,7 @@
 
 prepare_test(plugins=PLUGINS_TO_UPLOAD,
              secrets=SECRETS_TO_CREATE,
-             plugin_test=False)
+             plugin_test=False, execute_bundle_upload=False)
 
 virtual_machine_list = ['examples/blueprint.yaml']
 infra_path = 'examples/blueprint-examples/virtual-machine/{0}.yaml'

examples/scripts/configure.py

@@ -24,7 +24,7 @@
 import logging
 from tempfile import mkstemp
 
-from fabric.api import run, put, sudo
+from fabric2 import task
 RUN_WITH = 'source /home/centos/host_pool_service/bin/activate &&'
 
 from cloudify import ctx
@@ -34,14 +34,14 @@
 POOL_CFG_PATH = os.path.join(BASE_DIR, 'pool.yaml')
 
 
-def get_key_content(key_file, logger):
+def get_key_content(key_file, logger, connection):
     '''Downloads a key and returns the key contents'''
     tfd, target_path = mkstemp()
     os.close(tfd)
     logger.debug('Downloading key file "{0}" to path "{1}"'
                  .format(key_file, target_path))
     local_key_file = ctx.download_resource(key_file)
-    put(local_key_file, target_path)
+    connection.put(local_key_file, target_path)
     keycontent = None
     with open(target_path, 'r') as f_key:
         keycontent = f_key.read()
@@ -51,7 +51,7 @@ def get_key_content(key_file, logger):
     return keycontent
 
 
-def set_host_key_content(cfg, logger):
+def set_host_key_content(cfg, logger, connection):
     '''Replaces host key file string with key content
 
     :param dict cfg: Host-Pool configuration data
@@ -67,7 +67,7 @@ def set_host_key_content(cfg, logger):
         # Default key file was specified, let's convert and use
         logger.debug('Default key file: "{0}"'.format(default_key_file))
         cfg['default']['credentials']['key'] = \
-            get_key_content(default_key_file, logger)
+            get_key_content(default_key_file, logger, connection)
         del cfg['default']['credentials']['key_file']
 
     # Get the host keys
@@ -81,7 +81,7 @@ def set_host_key_content(cfg, logger):
             # Default key file was specified, let's convert and use
             logger.debug('Host key file: "{0}"'.format(host_key_file))
             host['credentials']['key'] = \
-                get_key_content(host_key_file, logger)
+                get_key_content(host_key_file, logger, connection)
             del host['credentials']['key_file']
     return cfg
 
@@ -130,7 +130,8 @@ def get_hostpool_logger(mod, debug=False,
     return logger
 
 
-def main():
+@task
+def main(connection):
     '''Entry point'''
 
     logger = get_hostpool_logger('configure',
@@ -146,7 +147,7 @@ def main():
                  .format(ctx.node.properties['pool'], POOL_CFG_PATH))
     local_file = ctx.download_resource(ctx.node.properties['pool'],
                                        target_path=POOL_CFG_PATH)
-    put(local_file, POOL_CFG_PATH)
+    connection.put(local_file, POOL_CFG_PATH)
 
     # if not os.path.exists(POOL_CFG_PATH):
     #     raise NonRecoverableError('Configuration file for the Host-Pool '
@@ -162,7 +163,7 @@ def main():
             raise NonRecoverableError('Configuration file for the Host-Pool '
                                       'service is not valid YAML')
         logger.info('Converting host key files from blueprint')
-        seed_config = set_host_key_content(cfg, logger)
+        seed_config = set_host_key_content(cfg, logger, connection)
         ctx.instance.runtime_properties['seed_config'] = seed_config
 
 # main()

examples/scripts/create.py

@@ -20,8 +20,7 @@
 
 import os
 
-from fabric.api import run, put, sudo
-from fabric.context_managers import cd
+from fabric2 import task
 
 RUN_WITH = 'source /home/centos/host_pool_service/bin/activate &&'
 
@@ -31,15 +30,15 @@
 RUN_WITH = 'source /home/centos/host_pool_service/bin/activate &&'
 
 
-def install_requirements():
+def install_requirements(connection):
     '''Install required Python packages'''
 
-    with cd('/home/centos'):
-        run('curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py')
-        run('python get-pip.py')
-        run('pip install virtualenv')
-        run('virtualenv /home/centos/host_pool_service')
-        ctx.instance.runtime_properties['virtualenv'] = \
+    with connection.cd("/home/centos"):
+        connection.run('curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py')
+        connection.run('python get-pip.py')
+    connection.sudo('yum install -y python-virtualenv')
+    connection.run('virtualenv /home/centos/host_pool_service')
+    ctx.instance.runtime_properties['virtualenv'] = \
             '/home/centos/host_pool_service'
 
     reqs = [
@@ -51,7 +50,7 @@ def install_requirements():
 
     for req in reqs:
         ctx.logger.info('Installing Python package "{0}"'.format(req))
-        run(RUN_WITH + 'pip install {0}'.format(req))
+        connection.run(RUN_WITH + 'pip install {0}'.format(req))
         # install(req)
 
 
@@ -63,7 +62,8 @@ def create_virtualenv():
     os.system('source {0}/bin/activate'.format(venv_name))
 
 
-def main():
+@task
+def main(connection):
     '''Entry point'''
 
     # create_virtualenv()
@@ -73,15 +73,15 @@ def main():
     ctx.logger.info('UNAME {0}'.format(os.uname()))
     try:
         ctx.logger.info('Creating working directory: "{0}"'.format(base_dir))
-        run('mkdir -p {0}'.format(base_dir))
+        connection.run('mkdir -p {0}'.format(base_dir))
         # if not os.path.isdir(base_dir):
         #     os.makedirs(base_dir)
     except OSError as ex:
         ctx.logger.error('Error making directory "{0}"'.format(base_dir))
         raise RecoverableError(message=ex, retry_after=2)
 
     ctx.logger.info('Installing required Python packages')
-    install_requirements()
+    install_requirements(connection)
 
     ctx.logger.info('Setting runtime_property "working_directory" to "{0}"'
                     .format(base_dir))