CH-194 remove dependency on current path in deployment

Author: filippomc

tools/deployment-cli-tools/ch_cli_tools/codefresh.py

@@ -1,5 +1,5 @@
 import os
-from os.path import join, relpath, exists, dirname, basename
+from os.path import join, relpath, exists, dirname, basename, abspath
 from cloudharness_model.models.git_dependency_config import GitDependencyConfig
 
 import logging
@@ -105,6 +105,9 @@ def create_codefresh_deployment_scripts(root_paths, envs=(), include=(), exclude
                       for app in helm_values['apps'].values() if 'harness' in app]
     out_filename = f"codefresh-{'-'.join(envs)}.yaml"
 
+    if base_image_name is None:
+        base_image_name = helm_values['name']
+
     if build_included:
         logging.info(
             'Including the following subpaths to the build: %s.', ', '
@@ -125,10 +128,12 @@ def create_codefresh_deployment_scripts(root_paths, envs=(), include=(), exclude
 
     steps = {}
     build_steps = {}
-    for root_path in root_paths:
-        if '.overrides' not in root_path:
-            base_image_name = helm_values['name'] if root_path == root_paths[0] else\
-                clean_image_name(os.path.basename(os.path.abspath(root_path)))
+    has_overrides = any(DEFAULT_MERGE_PATH in root_path for root_path in root_paths)
+    for i in range(len(root_paths)):
+            
+        root_path = root_paths[i]
+        base_name = clean_image_name(basename(abspath(root_path))) if i < len(root_paths) - (2 if has_overrides else 1) else base_image_name
+
         for e in envs:
 
             template_name = f"codefresh-template-{e}.yaml"
@@ -245,7 +250,7 @@ def codefresh_steps_from_base_path(base_path, fixed_context=None, include=build_
                                 relpath(
                                     dockerfile_path, root_path) if fixed_context else '',
                                 "Dockerfile"),
-                            base_name=base_image_name,
+                            base_name=base_name,
                             helm_values=helm_values,
                             dependencies=dependencies
                         )
@@ -259,7 +264,7 @@ def codefresh_steps_from_base_path(base_path, fixed_context=None, include=build_
                         steps[CD_STEP_PUBLISH]['steps']['publish_' + app_name] = codefresh_app_publish_spec(
                             app_name=app_name,
                             build_tag=build and build['tag'],
-                            base_name=base_image_name
+                            base_name=base_name
                         )
                         found = True
 
@@ -314,7 +319,7 @@ def add_unit_test_step(app_config: ApplicationHarnessConfig):
                     steps[CD_UNIT_TEST_STEP]['steps'][f"{app_name}_ut"] = dict(
                         title=f"Unit tests for {app_name}",
                         commands=test_config.unit.commands,
-                        image=image_tag_with_variables(app_name, tag, base_image_name),
+                        image=image_tag_with_variables(app_name, tag, base_name),
                     )
 
             if helm_values[KEY_TASK_IMAGES]:
@@ -325,15 +330,15 @@ def add_unit_test_step(app_config: ApplicationHarnessConfig):
                 codefresh_steps_from_base_path(join(root_path, APPS_PATH), include=helm_values[KEY_TASK_IMAGES].keys())
                 codefresh_steps_from_base_path(join(root_path, APPS_PATH), include=build_included)
 
-            if CD_E2E_TEST_STEP:
+            if CD_E2E_TEST_STEP in steps:
                 name = "test-e2e"
                 if codefresh_steps_from_base_path(join(root_path, TEST_IMAGES_PATH), include=(name,), publish=False):
-                    steps[CD_E2E_TEST_STEP]["image"] = image_tag_with_variables(name, app_specific_tag_variable(name), base_name=base_image_name)
+                    steps[CD_E2E_TEST_STEP]["image"] = image_tag_with_variables(name, app_specific_tag_variable(name), base_name=base_name)
 
             if CD_API_TEST_STEP in steps:
                 name = "test-api"
                 if codefresh_steps_from_base_path(join(root_path, TEST_IMAGES_PATH), include=(name,), fixed_context=relpath(root_path, os.getcwd()), publish=False):
-                    steps[CD_API_TEST_STEP]["image"] = image_tag_with_variables(name, app_specific_tag_variable(name), base_name=base_image_name)
+                    steps[CD_API_TEST_STEP]["image"] = image_tag_with_variables(name, app_specific_tag_variable(name), base_name=base_name)
 
     if build_steps:
 

tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py

@@ -96,7 +96,7 @@ def _adjust_missing_values(self, helm_values):
     def _process_applications(self, helm_values, base_image_name=None):
         for i in range(len(self.root_paths)):
             root_path = self.root_paths[i]
-            base_name = clean_image_name(root_path.name) if i > 0 else base_image_name
+            base_name = clean_image_name(root_path.name) if i < len(self.root_paths) - 1 else base_image_name
             app_values = init_app_values(
                 root_path, exclude=self.exclude, values=helm_values[KEY_APPS])
             helm_values[KEY_APPS] = dict_merge(helm_values[KEY_APPS],
@@ -152,7 +152,7 @@ def collect_app_values(self, app_base_path, base_image_name=None, helm_values=No
     def _init_static_images(self, base_image_name):
         for i in range(len(self.root_paths)):
             root_path = self.root_paths[i]
-            base_name = clean_image_name(root_path.name) if i > 0 else base_image_name
+            base_name = clean_image_name(root_path.name) if i < len(self.root_paths) - 1  else base_image_name
             for static_img_dockerfile in find_dockerfiles_paths(os.path.join(root_path, STATIC_IMAGES_PATH)):
                 self.static_images.add(static_img_dockerfile)
 
@@ -185,7 +185,7 @@ def _assign_static_build_dependencies(self, helm_values):
     def _init_base_images(self, base_image_name):
         for i in range(len(self.root_paths)):
             root_path = self.root_paths[i]
-            base_name = clean_image_name(root_path.name) if i > 0 else base_image_name
+            base_name = clean_image_name(root_path.name) if i < len(self.root_paths) - 1 else base_image_name
 
             for base_img_dockerfile in self.__find_static_dockerfile_paths(root_path):
                 img_name = image_name_from_dockerfile_path(
@@ -201,7 +201,7 @@ def _init_test_images(self, base_image_name):
         test_images = {}
         for i in range(len(self.root_paths)):
             root_path = self.root_paths[i]
-            base_name = clean_image_name(root_path.name) if i > 0 else base_image_name
+            base_name = clean_image_name(root_path.name) if i < len(self.root_paths) - 1 else base_image_name
 
             for base_img_dockerfile in find_dockerfiles_paths(os.path.join(root_path, TEST_IMAGES_PATH)):
                 img_name = image_name_from_dockerfile_path(

tools/deployment-cli-tools/tests/resources/deployment-configuration/values-template.yaml

@@ -1,5 +1,6 @@
 mainapp: myapp
 domain: myapp.local
+name: testprojectname
 a: b
 database:
   auto: true

tools/deployment-cli-tools/tests/resources/infrastructure/base-images/cloudharness-base/Dockerfile

@@ -8,7 +8,7 @@
 RESOURCES = os.path.join(HERE, 'resources')
 OUT = '/tmp/deployment'
 CLOUDHARNESS_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
-BUILD_MERGE_DIR = "./build/test_deployment"
+BUILD_MERGE_DIR = "./build/.overrides"
 
 myapp_path = os.path.join(HERE, "resources/applications/myapp")
 if not os.path.exists(os.path.join(myapp_path, "dependencies/a/.git")):
@@ -61,8 +61,10 @@ def test_create_codefresh_configuration():
         steps = l1_steps[STEP_0]["steps"]
         assert len(steps) == 7, "all images that do not depend on othe builds should be included in the first step"
         assert "cloudharness-base" in steps, "cloudharness-base image should be included as dependency"
+        assert "testprojectname/" in steps["cloudharness-base"]['image_name'], "cloudharness-base image should be overridden and take the main name"
         assert "cloudharness-base-debian" not in steps, "cloudharness-base image should not be included"
         assert "cloudharness-frontend-build" in steps, "cloudharness-frontend-build image should be included as dependency"
+        assert "cloud-harness/" in steps["cloudharness-frontend-build"]['image_name'], "cloudharness-frontend-build image is not overridden and should keep the cloud-harness prefix"
 
         step = steps["cloudharness-frontend-build"]
         assert os.path.samefile(step['working_directory'], CLOUDHARNESS_ROOT)
@@ -117,6 +119,7 @@ def test_create_codefresh_configuration():
 
         step = steps["myapp"]
         assert step['dockerfile'] == "Dockerfile"
+        assert "testprojectname/" in step['image_name'], f"myapp image should have the project name coming from the chart in its path, is {step['image_name']}"
         for build_argument in step['build_arguments']:
             if build_argument.startswith("CLOUDHARNESS_FLASK="):
                 assert "cloud-harness" in build_argument, "Cloudharness flask image should have cloud-harness in its path"
@@ -132,7 +135,7 @@ def test_create_codefresh_configuration():
                    ) == 2, "Two unit test steps are expected"
         assert 'myapp_ut' in l1_steps[CD_UNIT_TEST_STEP]['steps'], "Myapp test step is expected"
         tstep = l1_steps[CD_UNIT_TEST_STEP]['steps']['myapp_ut']
-        assert tstep['image'] == r"${{REGISTRY}}/resources/myapp:${{MYAPP_TAG}}", "The test image should be the one built for the current app"
+        assert tstep['image'] == r"${{REGISTRY}}/testprojectname/myapp:${{MYAPP_TAG}}", "The test image should be the one built for the current app"
         assert len(
             tstep['commands']) == 2, "Unit test commands are not properly loaded from the unit test configuration file"
         assert tstep['commands'][0] == "tox", "Unit test commands are not properly loaded from the unit test configuration file"
@@ -175,7 +178,7 @@ def test_create_codefresh_configuration_multienv():
         for cmd in cf['steps']['prepare_deployment']['commands']:
             if 'harness-deployment' in cmd:
                 assert '-e dev-test' in cmd
-                assert "test_deployment" in cmd
+                assert "test-${{NAMESPACE_BASENAME}}" in cmd
                 assert "-i samples" in cmd
 
     finally:

tools/deployment-cli-tools/tests/test_codefresh.py

@@ -22,8 +22,8 @@ def test_collect_compose_values(tmp_path):
                                                  namespace='test', env='dev', local=False, tag=1, registry='reg')
 
     # Auto values
-    assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'reg/resources/myapp:1'
-    assert values.apps['myapp'].harness.deployment.image == 'reg/resources/myapp:1'
+    assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'reg/testprojectname/myapp:1'
+    assert values.apps['myapp'].harness.deployment.image == 'reg/testprojectname/myapp:1'
     assert values[KEY_APPS]['myapp'][KEY_HARNESS]['name'] == 'myapp'
     assert values[KEY_APPS]['legacy'][KEY_HARNESS]['name'] == 'legacy'
     assert values[KEY_APPS]['accounts'][KEY_HARNESS]['deployment']['image'] == 'reg/cloud-harness/accounts:1'
@@ -87,8 +87,8 @@ def test_collect_compose_values(tmp_path):
     # Checl base and task images
     assert values[KEY_TASK_IMAGES]
     assert 'cloudharness-base' in values[KEY_TASK_IMAGES]
-    assert values[KEY_TASK_IMAGES]['cloudharness-base'] == 'reg/cloud-harness/cloudharness-base:1'
-    assert values[KEY_TASK_IMAGES]['myapp-mytask'] == 'reg/resources/myapp-mytask:1'
+    assert values[KEY_TASK_IMAGES]['cloudharness-base'] == 'reg/testprojectname/cloudharness-base:1', "cloudharness-base image is overridden"
+    assert values[KEY_TASK_IMAGES]['myapp-mytask'] == 'reg/testprojectname/myapp-mytask:1'
     # Not indicated as a build dependency
     assert 'cloudharness-base-debian' not in values[KEY_TASK_IMAGES]
 
@@ -99,7 +99,7 @@ def test_collect_compose_values_noreg_noinclude(tmp_path):
                                                  namespace='test', env='dev', local=False, tag=1)
 
     # Auto values
-    assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'resources/myapp:1'
+    assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'testprojectname/myapp:1'
     assert values[KEY_APPS]['myapp'][KEY_HARNESS]['name'] == 'myapp'
     assert values[KEY_APPS]['legacy'][KEY_HARNESS]['name'] == 'legacy'
     assert values[KEY_APPS]['accounts'][KEY_HARNESS]['deployment']['image'] == 'cloud-harness/accounts:1'
@@ -157,8 +157,8 @@ def test_collect_compose_values_noreg_noinclude(tmp_path):
 
     assert values[KEY_TASK_IMAGES]
     assert 'cloudharness-base' in values[KEY_TASK_IMAGES]
-    assert values[KEY_TASK_IMAGES]['cloudharness-base'] == 'cloud-harness/cloudharness-base:1'
-    assert values[KEY_TASK_IMAGES]['myapp-mytask'] == 'resources/myapp-mytask:1'
+    assert values[KEY_TASK_IMAGES]['cloudharness-base'] == 'testprojectname/cloudharness-base:1'
+    assert values[KEY_TASK_IMAGES]['myapp-mytask'] == 'testprojectname/myapp-mytask:1'
 
 
 def test_collect_compose_values_precedence(tmp_path):
@@ -208,7 +208,6 @@ def test_collect_compose_values_build_dependencies(tmp_path):
 
     assert 'cloudharness-flask' in values[KEY_TASK_IMAGES], "Cloudharness-flask is included in the build dependencies"
     assert 'cloudharness-base' in values[KEY_TASK_IMAGES], "Cloudharness-base is included in cloudharness-flask Dockerfile and it should be guessed"
-    assert 'cloudharness-base-debian' not in values[KEY_TASK_IMAGES], "Cloudharness-base-debian is not included in any dependency"
     assert 'cloudharness-frontend-build' not in values[KEY_TASK_IMAGES], "cloudharness-frontend-build is not included in any dependency"
 
 
@@ -219,7 +218,6 @@ def test_collect_compose_values_build_dependencies_nodeps(tmp_path):
 
     assert 'cloudharness-flask' not in values[KEY_TASK_IMAGES], "Cloudharness-flask is not included in the build dependencies"
     assert 'cloudharness-base' not in values[KEY_TASK_IMAGES], "Cloudharness-base is not included in the build dependencies"
-    assert 'cloudharness-base-debian' not in values[KEY_TASK_IMAGES], "Cloudharness-base-debian is not included in any dependency"
     assert 'cloudharness-frontend-build' not in values[KEY_TASK_IMAGES], "cloudharness-frontend-build is not included in any dependency"
 
 
@@ -316,9 +314,9 @@ def create():
     values = create()
 
     # Auto values are set by using the directory hash
-    assert 'reg/resources/myapp:' in values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image']
-    assert 'reg/resources/myapp:' in values.apps['myapp'].harness.deployment.image
-    assert 'resources/myapp-mytask' in values[KEY_TASK_IMAGES]['myapp-mytask']
+    assert 'reg/testprojectname/myapp:' in values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image']
+    assert 'reg/testprojectname/myapp:' in values.apps['myapp'].harness.deployment.image
+    assert 'testprojectname/myapp-mytask' in values[KEY_TASK_IMAGES]['myapp-mytask']
     assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == values.apps['myapp'].harness.deployment.image
     v1 = values.apps['myapp'].harness.deployment.image
     c1 = values["task-images"]["my-common"]
@@ -372,7 +370,7 @@ def create():
     finally:
         fname.unlink()
 
-    fname = Path(CLOUDHARNESS_ROOT) / 'atestfile'
+    fname = Path(RESOURCES) / 'atestfile'
     try:
         fname.write_text('a')
 

tools/deployment-cli-tools/tests/test_dockercompose.py

@@ -35,9 +35,9 @@ def test_collect_helm_values(tmp_path):
     assert 'events' not in values[KEY_APPS]
 
     # Auto values
-    assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'reg/resources/myapp:1'
+    assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'reg/testprojectname/myapp:1'
     assert values[KEY_APPS]['myapp']['build'] == True
-    assert values.apps['myapp'].harness.deployment.image == 'reg/resources/myapp:1'
+    assert values.apps['myapp'].harness.deployment.image == 'reg/testprojectname/myapp:1'
     assert values[KEY_APPS]['myapp'][KEY_HARNESS]['name'] == 'myapp'
     assert values[KEY_APPS]['legacy'][KEY_HARNESS]['name'] == 'legacy'
     assert values[KEY_APPS]['accounts'][KEY_HARNESS]['deployment']['image'] == 'reg/cloud-harness/accounts:1'
@@ -74,8 +74,9 @@ def test_collect_helm_values(tmp_path):
     # Checl base and task images
     assert values[KEY_TASK_IMAGES]
     assert 'cloudharness-base' in values[KEY_TASK_IMAGES]
-    assert values[KEY_TASK_IMAGES]['cloudharness-base'] == 'reg/cloud-harness/cloudharness-base:1'
-    assert values[KEY_TASK_IMAGES]['myapp-mytask'] == 'reg/resources/myapp-mytask:1'
+    assert values[KEY_TASK_IMAGES]['cloudharness-base'] == 'reg/testprojectname/cloudharness-base:1', "Cloudharness base image is overridden, so takes the main project name prefix"
+    assert values[KEY_TASK_IMAGES]['myapp-mytask'] == 'reg/testprojectname/myapp-mytask:1'
+    assert values[KEY_TASK_IMAGES]['cloudharness-flask'] == 'reg/cloud-harness/cloudharness-flask:1'
     # Not indicated as a build dependency
     assert 'cloudharness-base-debian' not in values[KEY_TASK_IMAGES]
 
@@ -99,7 +100,7 @@ def test_collect_helm_values_noreg_noinclude(tmp_path):
                                namespace='test', env='dev', local=False, tag=1)
 
     # Auto values
-    assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'resources/myapp:1'
+    assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'testprojectname/myapp:1'
     assert values[KEY_APPS]['myapp'][KEY_HARNESS]['name'] == 'myapp'
     assert values[KEY_APPS]['legacy'][KEY_HARNESS]['name'] == 'legacy'
     assert values[KEY_APPS]['accounts'][KEY_HARNESS]['deployment']['image'] == 'cloud-harness/accounts:1'
@@ -145,8 +146,10 @@ def test_collect_helm_values_noreg_noinclude(tmp_path):
 
     assert values[KEY_TASK_IMAGES]
     assert 'cloudharness-base' in values[KEY_TASK_IMAGES]
-    assert values[KEY_TASK_IMAGES]['cloudharness-base'] == 'cloud-harness/cloudharness-base:1'
-    assert values[KEY_TASK_IMAGES]['myapp-mytask'] == 'resources/myapp-mytask:1'
+    assert values[KEY_TASK_IMAGES]['cloudharness-base'] == 'testprojectname/cloudharness-base:1'
+    assert values[KEY_TASK_IMAGES]['myapp-mytask'] == 'testprojectname/myapp-mytask:1'
+    assert values[KEY_TASK_IMAGES]['my-common'] == 'testprojectname/my-common:1'
+    
 
 
 def test_collect_helm_values_precedence(tmp_path):
@@ -304,9 +307,9 @@ def create():
     values = create()
 
     # Auto values are set by using the directory hash
-    assert 'reg/resources/myapp:' in values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image']
-    assert 'reg/resources/myapp:' in values.apps['myapp'].harness.deployment.image
-    assert 'resources/myapp-mytask' in values[KEY_TASK_IMAGES]['myapp-mytask']
+    assert 'reg/testprojectname/myapp:' in values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image']
+    assert 'reg/testprojectname/myapp:' in values.apps['myapp'].harness.deployment.image
+    assert 'testprojectname/myapp-mytask' in values[KEY_TASK_IMAGES]['myapp-mytask']
     assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == values.apps['myapp'].harness.deployment.image
     v1 = values.apps['myapp'].harness.deployment.image
     c1 = values["task-images"]["my-common"]
@@ -360,7 +363,7 @@ def create():
     finally:
         fname.unlink()
 
-    fname = Path(CLOUDHARNESS_ROOT) / 'atestfile'
+    fname = Path(RESOURCES) / 'atestfile'
     try:
         fname.write_text('a')