From 73278a7eb0fe8b97460c8378d00b2945062bcf59 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 25 Jan 2024 15:01:49 +0100 Subject: [PATCH 01/17] Move from alpine to debian image --- applications/common/server/Dockerfile | 5 ++--- .../tasks/extract-download/Dockerfile | 6 +++--- .../base-images/cloudharness-base/Dockerfile | 21 +++++++------------ 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/applications/common/server/Dockerfile b/applications/common/server/Dockerfile index 2d2515a87..962f117ea 100644 --- a/applications/common/server/Dockerfile +++ b/applications/common/server/Dockerfile @@ -6,8 +6,7 @@ ENV MODULE_NAME=common ENV WORKERS=2 ENV PORT=8080 -RUN apk update -RUN apk add postgresql-dev +RUN apt update && apt install -y postgresql COPY ./requirements.txt /usr/src/app/ @@ -18,4 +17,4 @@ COPY . /usr/src/app ENV FLASK_ENV=production ENV APP_SETTINGS=common.config.ProductionConfig RUN pip3 install -e /usr/src/app -ENTRYPOINT gunicorn --workers=$WORKERS --bind=0.0.0.0:$PORT $MODULE_NAME.__main__:app \ No newline at end of file +ENTRYPOINT gunicorn --workers=$WORKERS --bind=0.0.0.0:$PORT $MODULE_NAME.__main__:app diff --git a/applications/workflows/tasks/extract-download/Dockerfile b/applications/workflows/tasks/extract-download/Dockerfile index 5a48923c2..f89460709 100644 --- a/applications/workflows/tasks/extract-download/Dockerfile +++ b/applications/workflows/tasks/extract-download/Dockerfile @@ -1,6 +1,6 @@ -FROM alpine -RUN apk update -RUN apk add file unzip wget +ARG CLOUDHARNESS_BASE +FROM $CLOUDHARNESS_BASE as base +RUN apt update && apt install -y file unzip wget ADD . / diff --git a/infrastructure/base-images/cloudharness-base/Dockerfile b/infrastructure/base-images/cloudharness-base/Dockerfile index 782c7d55d..18975b717 100644 --- a/infrastructure/base-images/cloudharness-base/Dockerfile +++ b/infrastructure/base-images/cloudharness-base/Dockerfile @@ -1,17 +1,10 @@ -ARG PARENT=python:3.9.10-alpine +ARG PARENT=python:3.9.10 FROM ${PARENT} -RUN apk update -RUN apk upgrade - -# Add bash for k8s console -RUN apk add bash -# dev tools needed by some python libraries -RUN apk add gcc libc-dev g++ python3-dev libffi-dev openssl-dev rust musl-dev cargo +RUN apt-get update && apt-get install -y nfs-common && rm -rf /var/lib/apt/lists/* RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ pip install pytest --prefer-binary - COPY libraries/models/requirements.txt /libraries/models/requirements.txt RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ pip install -r /libraries/models/requirements.txt --prefer-binary @@ -23,14 +16,14 @@ RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip & pip install -r /libraries/client/cloudharness_cli/requirements.txt --prefer-binary COPY libraries/models /libraries/models -RUN pip install -e /libraries/models +RUN pip install -e /libraries/models --no-cache-dir COPY libraries/cloudharness-common /libraries/cloudharness-common COPY libraries/client/cloudharness_cli /libraries/client/cloudharness_cli # +RUN pip install -e /libraries/models --no-cache-dir +RUN pip install -e /libraries/cloudharness-common --no-cache-dir +RUN pip install -e /libraries/client/cloudharness_cli --no-cache-dir -RUN pip install -e /libraries/cloudharness-common -RUN pip install -e /libraries/client/cloudharness_cli - -WORKDIR / +WORKDIR / \ No newline at end of file From f446bd9d8d274e5ad8a370923c1dbed707da9b26 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Sat, 10 Feb 2024 02:37:57 +0100 Subject: [PATCH 02/17] CH-95: remove alpine base, migration script that perform migration of the applications based on a keyword and a dictionary of words to replace if the occurrence is found --- docs/base-common-images.md | 18 ++--- .../cloudharness-base-debian/Dockerfile | 29 ------- .../cloudharness-django/Dockerfile | 4 +- .../cloudharness-django/README.md | 2 +- .../cloudharness-fastapi/Dockerfile | 4 +- .../cloudharness-fastapi/README.md | 2 +- .../cloudharness_utils/constants.py | 4 +- .../ch_cli_tools/config/migration.json | 17 ++++ .../ch_cli_tools/migration.py | 81 +++++++++++++++++++ .../ch_cli_tools/utils.py | 34 ++++++++ tools/deployment-cli-tools/harness-deployment | 12 ++- .../applications/migration_app/Dockerfile | 4 + .../migration_app/tasks/sub-task/Dockerfile | 4 + .../applications/migration_app/Dockerfile | 4 + .../migration_app/tasks/sub-task/Dockerfile | 4 + .../tests/test_migration.py | 9 +++ .../deployment-cli-tools/tests/test_utils.py | 9 ++- 17 files changed, 192 insertions(+), 49 deletions(-) delete mode 100644 infrastructure/base-images/cloudharness-base-debian/Dockerfile create mode 100644 tools/deployment-cli-tools/ch_cli_tools/config/migration.json create mode 100644 tools/deployment-cli-tools/ch_cli_tools/migration.py create mode 100644 tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile create mode 100644 tools/deployment-cli-tools/tests/resources/applications/migration_app/tasks/sub-task/Dockerfile create mode 100644 tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/Dockerfile create mode 100644 tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/tasks/sub-task/Dockerfile create mode 100644 tools/deployment-cli-tools/tests/test_migration.py diff --git a/docs/base-common-images.md b/docs/base-common-images.md index 314995f8b..f3484f485 100644 --- a/docs/base-common-images.md +++ b/docs/base-common-images.md @@ -1,9 +1,10 @@ # Define and use base and common images ## Relevant files and directory structure - - `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks. - - `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks. - + +- `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks. +- `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks. + ## Base images and common images The main difference between the base images and common images is that base images are built in the root context, while @@ -15,7 +16,7 @@ a specific purpose (e.g. enable widely used application stacks to inherit from). After generating the codeChange the Dockerfile in order to inherit from the main Docker image need to: -1. Add the image as a build dependency to the values.yaml file of your application. The name of the image corresponds to the directory name where the Dockerfile is located +1. Add the image as a build dependency to the values.yaml file of your application. The name of the image corresponds to the directory name where the Dockerfile is located ``` harness: @@ -25,6 +26,7 @@ harness: ``` 2. Refer to the base image with the uppercased-underscored name of the dependency as an argument + ```dockerfile ARG CLOUDHARNESS_BASE FROM $CLOUDHARNESS_BASE @@ -39,19 +41,18 @@ In workflow tasks, the build dependency must be specified in the main applicatio ## Default images CloudHarness defines the following base images: + - `cloudharness-base`: python-alpine with cloudharness common libraries preinstalled -- `cloudharness-base-debian`: python-debian with cloudharness common libraries preinstalled - `cloudharness-django`: cloudharness-base with cloudharness django fastapi libraries preinstalled -- `cloudharness-django-debian`: cloudharness-base-debian with cloudharness django fastapi libraries preinstalled - `cloudharness-fastapi`: cloudharness-base with fastapi libraries preinstalled -- `cloudharness-fastapi-debian`: cloudharness-base-debian with fastapi libraries preinstalled Also the following common images are defined: + - `cloudharness-flask`: common ground image to create Flask backends ## Override base and common images from CloudHarness -To override a base or common image just create the same directory path in your +To override a base or common image just create the same directory path in your solution. The overriding can be used to replace files used in the build process or the Dockerfile itself. For example, overriding `cloudharness-base` could be useful to change some behaviour in the CloudHarness @@ -59,4 +60,3 @@ libraries or to provide new libraries to share within all applications. To override cloudharness-base, create a directory `MY_SOLUTION/infrastructure/base-images/cloudharness-base` then run `harness-deployment cloudharness MY_SOLUTION` - diff --git a/infrastructure/base-images/cloudharness-base-debian/Dockerfile b/infrastructure/base-images/cloudharness-base-debian/Dockerfile deleted file mode 100644 index 18975b717..000000000 --- a/infrastructure/base-images/cloudharness-base-debian/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -ARG PARENT=python:3.9.10 -FROM ${PARENT} - -RUN apt-get update && apt-get install -y nfs-common && rm -rf /var/lib/apt/lists/* - -RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ - pip install pytest --prefer-binary -COPY libraries/models/requirements.txt /libraries/models/requirements.txt -RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ - pip install -r /libraries/models/requirements.txt --prefer-binary -COPY libraries/cloudharness-common/requirements.txt /libraries/cloudharness-common/requirements.txt -RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ - pip install -r /libraries/cloudharness-common/requirements.txt --prefer-binary -COPY libraries/client/cloudharness_cli/requirements.txt /libraries/client/cloudharness_cli/requirements.txt -RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ - pip install -r /libraries/client/cloudharness_cli/requirements.txt --prefer-binary - -COPY libraries/models /libraries/models -RUN pip install -e /libraries/models --no-cache-dir - -COPY libraries/cloudharness-common /libraries/cloudharness-common -COPY libraries/client/cloudharness_cli /libraries/client/cloudharness_cli - -# -RUN pip install -e /libraries/models --no-cache-dir -RUN pip install -e /libraries/cloudharness-common --no-cache-dir -RUN pip install -e /libraries/client/cloudharness_cli --no-cache-dir - -WORKDIR / \ No newline at end of file diff --git a/infrastructure/common-images/cloudharness-django/Dockerfile b/infrastructure/common-images/cloudharness-django/Dockerfile index 941dad578..89a090891 100644 --- a/infrastructure/common-images/cloudharness-django/Dockerfile +++ b/infrastructure/common-images/cloudharness-django/Dockerfile @@ -1,5 +1,5 @@ -ARG CLOUDHARNESS_BASE_DEBIAN -FROM $CLOUDHARNESS_BASE_DEBIAN +ARG CLOUDHARNESS_BASE +FROM $CLOUDHARNESS_BASE ENV MODULE_NAME=backend ENV PORT=8080 diff --git a/infrastructure/common-images/cloudharness-django/README.md b/infrastructure/common-images/cloudharness-django/README.md index 1150631fb..e7abf0078 100644 --- a/infrastructure/common-images/cloudharness-django/README.md +++ b/infrastructure/common-images/cloudharness-django/README.md @@ -1,3 +1,3 @@ -# CloudHarness-Django Base Debian image +# CloudHarness-Django Base image Use this image to bring the package cloudharness-django into your image. diff --git a/infrastructure/common-images/cloudharness-fastapi/Dockerfile b/infrastructure/common-images/cloudharness-fastapi/Dockerfile index 257438faa..00de6ecc6 100644 --- a/infrastructure/common-images/cloudharness-fastapi/Dockerfile +++ b/infrastructure/common-images/cloudharness-fastapi/Dockerfile @@ -1,5 +1,5 @@ -ARG CLOUDHARNESS_BASE_DEBIAN -FROM $CLOUDHARNESS_BASE_DEBIAN +ARG CLOUDHARNESS_BASE +FROM $CLOUDHARNESS_BASE ENV MODULE_NAME=backend ENV PORT=8080 diff --git a/infrastructure/common-images/cloudharness-fastapi/README.md b/infrastructure/common-images/cloudharness-fastapi/README.md index 96949d87d..da4516e89 100644 --- a/infrastructure/common-images/cloudharness-fastapi/README.md +++ b/infrastructure/common-images/cloudharness-fastapi/README.md @@ -1,3 +1,3 @@ -# CloudHarness-FastAPI Base Debian image +# CloudHarness-FastAPI Base image Use this image for FastAPI based microservices. diff --git a/libraries/cloudharness-utils/cloudharness_utils/constants.py b/libraries/cloudharness-utils/cloudharness_utils/constants.py index 4b42761a6..c4fb4f6c3 100644 --- a/libraries/cloudharness-utils/cloudharness_utils/constants.py +++ b/libraries/cloudharness-utils/cloudharness_utils/constants.py @@ -29,7 +29,7 @@ VALUES_MANUAL_PATH = 'values.yaml' VALUE_TEMPLATE_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/value-template.yaml' -CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10-alpine', 'cloudharness-base-debian': 'python:3.9.10'} +CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10'} CD_BUILD_STEP_BASE = 'build_base_images' @@ -48,4 +48,4 @@ E2E_TESTS_DIRNAME = 'e2e' API_TESTS_DIRNAME = 'api' -E2E_TESTS_PROJECT_PATH = "test/test-e2e" \ No newline at end of file +E2E_TESTS_PROJECT_PATH = "test/test-e2e" diff --git a/tools/deployment-cli-tools/ch_cli_tools/config/migration.json b/tools/deployment-cli-tools/ch_cli_tools/config/migration.json new file mode 100644 index 000000000..d38d91df3 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/config/migration.json @@ -0,0 +1,17 @@ +{ + "deprecated": [ + { + "keyword": "CLOUDHARNESS_BASE_DEBIAN", + "to_be_replaced": [ + { + "old": "CLOUDHARNESS_BASE_DEBIAN", + "new": "CLOUDHARNESS_BASE" + }, + { + "old": "apk", + "new": "apt" + } + ] + } + ] +} \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/migration.py b/tools/deployment-cli-tools/ch_cli_tools/migration.py new file mode 100644 index 000000000..5dded115c --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/migration.py @@ -0,0 +1,81 @@ +""" +Utilities to perform a migration of the deployment to the latest supported version. +""" +import os +import json + +from . import HERE +from cloudharness_utils.constants import APPS_PATH +from .utils import get_sub_paths, search_word_in_file, search_word_in_folder + +TO_CHECK = ['deploy', 'tasks', 'Dockerfile'] + +def perform_migration(base_roots, accept_all=False): + all_files_detected = [] + f = open(os.path.join(HERE, 'config', 'migration.json'), 'r') + migration_json = json.load(f) + + for search_root in base_roots: + app_base_path = os.path.join(search_root, APPS_PATH) + + # Iterate over all the applications to check if they need to be migrated + for app_path in get_sub_paths(app_base_path): + # Iterate the folders and files to check if they need to be migrated + for sub_path in TO_CHECK: + to_check = os.path.join(app_path, sub_path) + if os.path.isdir(to_check): + for migration_obj in migration_json['deprecated']: + files = search_word_in_folder(to_check, migration_obj['keyword']) + for file in files: + file_path = os.path.join(to_check, file) + print('#########################################') + print(f'Running migration on {file_path}') + print('#########################################') + all_files_detected.append(file_path) + for word_to_replace in migration_obj['to_be_replaced']: + read_file_and_replace(file_path, word_to_replace['old'], word_to_replace['new'], accept_all) + elif os.path.isfile(to_check): + for migration_obj in migration_json['deprecated']: + if len(search_word_in_file(to_check, migration_obj['keyword'])) > 0: + print('#########################################') + print(f'Running migration on {to_check}') + print('#########################################') + all_files_detected.append(to_check) + for word_to_replace in migration_obj['to_be_replaced']: + read_file_and_replace(to_check, word_to_replace['old'], word_to_replace['new'], accept_all) + else: + pass + # print(f'Path {to_check} does not exist') + print('=========================================') + print('=== Cloud Harness migration completed ===') + print('=========================================') + print('### Summary of the files to check post migration ###') + for file in all_files_detected: + print(f'{file}') + + +def read_file_and_replace(file, old, new, accept_all=False): + file_p = open(file, 'r+') + lines = file_p.readlines() + file_p.seek(0) + for line in lines: + if old in line: + print(f'Found {old} in {file}') + + if not accept_all: + print(f'Would you like to replace:') + print(line) + print(f'with:') + print(line.replace(old, new)) + print('y/n') + + if accept_all: + file_p.write(line.replace(old, new)) + elif input() == 'y': + file_p.write(line.replace(old, new)) + else: + file_p.write(line) + else: + file_p.write(line) + file_p.truncate() + file_p.close() diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 41b96d87d..8b7e20bef 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -376,3 +376,37 @@ def check_docker_manifest_exists(registry, image_name, tag, registry_secret=None api_url = f"https://{registry}/v2/{image_name}/manifests/{tag}" resp = requests.get(api_url) return resp.status_code == 200 + + +def filter_empty_strings(value): + return value != "" + + +def search_word_in_file(file, word): + p = subprocess.Popen('grep -l %s %s'% (word, file), shell=True, + stdout=subprocess.PIPE) + output, _ = p.communicate() + output = output.decode("utf-8") + matches = [] + if p.returncode == 1: # no matches found + pass + elif p.returncode == 0: # matches found + matches = output.split('\n') + else: + raise Exception(f'Migration Error: {file} grep failed with return code {p.returncode} and output {output}') + return list(filter(filter_empty_strings, matches)) + + +def search_word_in_folder(folder, word): + p = subprocess.Popen('grep -rl %s %s'% (word, '*'), shell=True, + stdout=subprocess.PIPE, cwd=folder) + output, _ = p.communicate() + output = output.decode("utf-8") + matches = [] + if p.returncode == 1: # no matches found + pass + elif p.returncode == 0: # matches found + matches = output.split('\n') + else: + raise Exception(f'Migration Error: {folder} grep failed with return code {p.returncode} and output {output}') + return list(filter(filter_empty_strings, matches)) diff --git a/tools/deployment-cli-tools/harness-deployment b/tools/deployment-cli-tools/harness-deployment index a9cecabbf..51b357e52 100644 --- a/tools/deployment-cli-tools/harness-deployment +++ b/tools/deployment-cli-tools/harness-deployment @@ -9,6 +9,7 @@ from ch_cli_tools.skaffold import create_skaffold_configuration, create_vscode_d from ch_cli_tools.codefresh import create_codefresh_deployment_scripts, write_env_file from ch_cli_tools.preprocessing import preprocess_build_overrides from ch_cli_tools.utils import merge_app_directories +from ch_cli_tools.migration import perform_migration from cloudharness_utils.constants import DEPLOYMENT_PATH HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/') @@ -61,7 +62,10 @@ if __name__ == "__main__": help=f'Do not generate ci/cd files') parser.add_argument('-we', '--write-env', dest='write_env', action="store_const", default=None, const=True, help=f'Write build env to .env file in {DEPLOYMENT_PATH}') - + parser.add_argument('--migrate', dest='migration', action="store_true", + help='Perform a migration of the deployment to the latest supported version.') + parser.add_argument('--migrate-accept-all', dest='migration_accept_all', action="store_true", + help='Perform a migration of the deployment to the latest supported version without asking for confirmation. Use with caution.') args, unknown = parser.parse_known_args(sys.argv[1:]) @@ -74,13 +78,17 @@ if __name__ == "__main__": print('There are unknown args. Make sure to call the script with the accepted args. Try --help') print(f'unknown: {unknown}') else: - if args.merge: logging.warn( "Merge (-m, --merge) argument is deprecated. Directory merging is now set automatically") merge_app_directories(root_paths, destination=args.merge) root_paths = [args.merge] + if args.migration_accept_all: + perform_migration(root_paths, accept_all=True) + elif args.migration: + perform_migration(root_paths, accept_all=False) + helm_values = create_helm_chart( root_paths, tag=args.tag, diff --git a/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile b/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/resources/applications/migration_app/tasks/sub-task/Dockerfile b/tools/deployment-cli-tools/tests/resources/applications/migration_app/tasks/sub-task/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/migration_app/tasks/sub-task/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/Dockerfile b/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/tasks/sub-task/Dockerfile b/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/tasks/sub-task/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/tasks/sub-task/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/test_migration.py b/tools/deployment-cli-tools/tests/test_migration.py new file mode 100644 index 000000000..39e890173 --- /dev/null +++ b/tools/deployment-cli-tools/tests/test_migration.py @@ -0,0 +1,9 @@ +from ch_cli_tools.utils import * +from ch_cli_tools.migration import perform_migration + +HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/') + +def test_migration_accept_all(): + assert len(search_word_in_folder(os.path.join(HERE, './resources/migration'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 + perform_migration(os.path.join(HERE, './resources/migration'), accept_all=True) + assert len(search_word_in_folder(os.path.join(HERE, './resources/migration'), "CLOUDHARNESS_BASE_DEBIAN")) == 0 diff --git a/tools/deployment-cli-tools/tests/test_utils.py b/tools/deployment-cli-tools/tests/test_utils.py index 507f75b91..abae71372 100644 --- a/tools/deployment-cli-tools/tests/test_utils.py +++ b/tools/deployment-cli-tools/tests/test_utils.py @@ -64,4 +64,11 @@ def test_guess_build_dependencies_from_dockerfile(): def test_check_docker_manifest_exists(): assert check_docker_manifest_exists("gcr.io/metacellllc", "cloudharness/cloudharness-base", "latest") assert not check_docker_manifest_exists("gcr.io/metacellllc", "cloudharness/cloudharness-base", "RANDOM_TAG") - \ No newline at end of file + + +def test_search_word_in_file(): + assert len(search_word_in_file(os.path.join(HERE, './resources/applications/migration_app/Dockerfile'), "CLOUDHARNESS_BASE_DEBIAN")) == 1 + + +def test_search_word_in_folder(): + assert len(search_word_in_folder(os.path.join(HERE, './resources/applications/migration_app/'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 From 4963712c57e9455182c0998385e0f0a124032560 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Tue, 13 Feb 2024 17:24:12 +0100 Subject: [PATCH 03/17] changes as per PR review --- .../ch_cli_tools/config/migration.json | 33 +++++- .../ch_cli_tools/migration.py | 100 ++++++++++-------- tools/deployment-cli-tools/harness-migrate | 17 +++ 3 files changed, 104 insertions(+), 46 deletions(-) create mode 100644 tools/deployment-cli-tools/harness-migrate diff --git a/tools/deployment-cli-tools/ch_cli_tools/config/migration.json b/tools/deployment-cli-tools/ch_cli_tools/config/migration.json index d38d91df3..3a8b7cc2f 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/config/migration.json +++ b/tools/deployment-cli-tools/ch_cli_tools/config/migration.json @@ -8,8 +8,37 @@ "new": "CLOUDHARNESS_BASE" }, { - "old": "apk", - "new": "apt" + "old": "apk add", + "new": "apt-get install" + }, + { + "old": "apk update", + "new": "apt-get update" + }, + { + "old": "apk upgrade", + "new": "apt-get upgrade" + }, + { + "old": "apk add --no-cache", + "new": "apt-get install --no-cache" + }, + { + "old": "apk update --no-cache", + "new": "apt-get update --no-cache" + }, + { + "old": "apk upgrade --no-cache", + "new": "apt-get upgrade --no-cache" + } + ] + }, + { + "keyword": "cloudharness-base-debian", + "to_be_replaced": [ + { + "old": "cloudharness-base-debian", + "new": "cloudharness-base" } ] } diff --git a/tools/deployment-cli-tools/ch_cli_tools/migration.py b/tools/deployment-cli-tools/ch_cli_tools/migration.py index 5dded115c..3beae0729 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/migration.py +++ b/tools/deployment-cli-tools/ch_cli_tools/migration.py @@ -8,70 +8,82 @@ from cloudharness_utils.constants import APPS_PATH from .utils import get_sub_paths, search_word_in_file, search_word_in_folder -TO_CHECK = ['deploy', 'tasks', 'Dockerfile'] +TO_CHECK = ["deploy", "tasks", "Dockerfile"] -def perform_migration(base_roots, accept_all=False): + +def perform_migration(base_root, accept_all=False): all_files_detected = [] - f = open(os.path.join(HERE, 'config', 'migration.json'), 'r') + f = open(os.path.join(HERE, "config", "migration.json"), "r") migration_json = json.load(f) - for search_root in base_roots: - app_base_path = os.path.join(search_root, APPS_PATH) + app_base_path = os.path.join(base_root, APPS_PATH) - # Iterate over all the applications to check if they need to be migrated - for app_path in get_sub_paths(app_base_path): - # Iterate the folders and files to check if they need to be migrated - for sub_path in TO_CHECK: - to_check = os.path.join(app_path, sub_path) - if os.path.isdir(to_check): - for migration_obj in migration_json['deprecated']: - files = search_word_in_folder(to_check, migration_obj['keyword']) - for file in files: - file_path = os.path.join(to_check, file) - print('#########################################') - print(f'Running migration on {file_path}') - print('#########################################') - all_files_detected.append(file_path) - for word_to_replace in migration_obj['to_be_replaced']: - read_file_and_replace(file_path, word_to_replace['old'], word_to_replace['new'], accept_all) - elif os.path.isfile(to_check): - for migration_obj in migration_json['deprecated']: - if len(search_word_in_file(to_check, migration_obj['keyword'])) > 0: - print('#########################################') - print(f'Running migration on {to_check}') - print('#########################################') - all_files_detected.append(to_check) - for word_to_replace in migration_obj['to_be_replaced']: - read_file_and_replace(to_check, word_to_replace['old'], word_to_replace['new'], accept_all) - else: - pass - # print(f'Path {to_check} does not exist') - print('=========================================') - print('=== Cloud Harness migration completed ===') - print('=========================================') - print('### Summary of the files to check post migration ###') + # Iterate over all the applications to check if they need to be migrated + for app_path in get_sub_paths(app_base_path): + # Iterate the folders and files to check if they need to be migrated + for sub_path in TO_CHECK: + to_check = os.path.join(app_path, sub_path) + if os.path.isdir(to_check): + for migration_obj in migration_json["deprecated"]: + files = search_word_in_folder(to_check, migration_obj["keyword"]) + for file in files: + file_path = os.path.join(to_check, file) + print("#########################################") + print(f"Running migration on {file_path}") + print("#########################################") + all_files_detected.append(file_path) + for word_to_replace in migration_obj["to_be_replaced"]: + read_file_and_replace( + file_path, + word_to_replace["old"], + word_to_replace["new"], + accept_all, + ) + elif os.path.isfile(to_check): + for migration_obj in migration_json["deprecated"]: + if len(search_word_in_file(to_check, migration_obj["keyword"])) > 0: + print("#########################################") + print(f"Running migration on {to_check}") + print("#########################################") + all_files_detected.append(to_check) + for word_to_replace in migration_obj["to_be_replaced"]: + read_file_and_replace( + to_check, + word_to_replace["old"], + word_to_replace["new"], + accept_all, + ) + else: + print(f'Path {to_check} does not exist') + print("=========================================") + print("=== Cloud Harness migration completed ===") + print("=========================================") + print("### Summary of the files to check post migration ###") for file in all_files_detected: - print(f'{file}') + print(f"{file}") + print("=========================================") + print("=== End of Summary ===") + print("=========================================") def read_file_and_replace(file, old, new, accept_all=False): - file_p = open(file, 'r+') + file_p = open(file, "r+") lines = file_p.readlines() file_p.seek(0) for line in lines: if old in line: - print(f'Found {old} in {file}') + print(f"Found {old} in {file}") if not accept_all: - print(f'Would you like to replace:') + print(f"Would you like to replace:") print(line) - print(f'with:') + print(f"with:") print(line.replace(old, new)) - print('y/n') + print("y/n") if accept_all: file_p.write(line.replace(old, new)) - elif input() == 'y': + elif input() == "y": file_p.write(line.replace(old, new)) else: file_p.write(line) diff --git a/tools/deployment-cli-tools/harness-migrate b/tools/deployment-cli-tools/harness-migrate new file mode 100644 index 000000000..1b414a573 --- /dev/null +++ b/tools/deployment-cli-tools/harness-migrate @@ -0,0 +1,17 @@ +#!/usr/bin/env python +import os +import sys +from ch_cli_tools.migration import perform_migration + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description='Perform a migration of the deployment to the latest supported version.') + + parser.add_argument('--accept-all', dest='migration_accept_all', action="store_true", + help='Perform a migration of the deployment to the latest supported version without asking for confirmation. Use with caution.') + + args, unknown = parser.parse_known_args(sys.argv[1:]) + + perform_migration(os.getcwd(), accept_all=args.migration_accept_all) From 35ec13ad9d5b86fcdb077737e4dc7882fc4bdeec Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Tue, 13 Feb 2024 17:24:24 +0100 Subject: [PATCH 04/17] adding script to setup.py --- tools/deployment-cli-tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/setup.py b/tools/deployment-cli-tools/setup.py index 15fb7f01d..625ebbe93 100644 --- a/tools/deployment-cli-tools/setup.py +++ b/tools/deployment-cli-tools/setup.py @@ -43,7 +43,7 @@ install_requires=REQUIREMENTS, packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), include_package_data=True, - scripts=['harness-deployment', 'harness-generate', 'harness-application'], + scripts=['harness-deployment', 'harness-generate', 'harness-application', 'harness-migrate'], long_description="""\ CloudHarness deploy library """ From fcb9a05de25dd2ac28ef0418c64b76d3d3f4c0fc Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 16 Feb 2024 22:17:26 +0100 Subject: [PATCH 05/17] required files for unit test --- .../resources/migration/backup/migration_app/Dockerfile | 4 ++++ .../backup/migration_app/tasks/sub-task/Dockerfile | 4 ++++ tools/deployment-cli-tools/tests/test_migration.py | 8 ++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/Dockerfile create mode 100644 tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/tasks/sub-task/Dockerfile diff --git a/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/Dockerfile b/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/tasks/sub-task/Dockerfile b/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/tasks/sub-task/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/tasks/sub-task/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/test_migration.py b/tools/deployment-cli-tools/tests/test_migration.py index 39e890173..c4a7ad02d 100644 --- a/tools/deployment-cli-tools/tests/test_migration.py +++ b/tools/deployment-cli-tools/tests/test_migration.py @@ -4,6 +4,10 @@ HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/') def test_migration_accept_all(): - assert len(search_word_in_folder(os.path.join(HERE, './resources/migration'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 + assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 perform_migration(os.path.join(HERE, './resources/migration'), accept_all=True) - assert len(search_word_in_folder(os.path.join(HERE, './resources/migration'), "CLOUDHARNESS_BASE_DEBIAN")) == 0 + assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 0 + os.system(f'cp -R {os.path.join(HERE, "resources/migration/backup/migration_app")} {os.path.join(HERE, "resources/migration/applications/")}') + # print(f'cp -R {os.path.join(HERE, "resources/migration/backup")} {os.path.join(HERE, "resources/migration/applications")}') + # subprocess.Popen(['cp', '-R', 'backup/*', 'applications/'], cwd=os.path.join(HERE, 'resources/migration')) + assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 From 963384e03d3d53a0123f689e6c49e4d5802d2f3a Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 16 Feb 2024 22:18:11 +0100 Subject: [PATCH 06/17] ignoring files for unit test to avoid to commit changes on these --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 90f643c08..69e96a9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,6 @@ skaffold.yaml /deployment.yaml .hypothesis __pycache__ -.env \ No newline at end of file +.env +tools/deployment-cli-tools/tests/resources/migration +tools/deployment-cli-tools/tests/resources/migration.bak From 002d37961d716b3362594fca8f626b950398cc4e Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 16 Feb 2024 22:30:56 +0100 Subject: [PATCH 07/17] fixing pr --- .github/workflows/trivy-analysis.yml | 2 +- infrastructure/base-images/cloudharness-base/Dockerfile | 3 +-- tools/deployment-cli-tools/tests/test_migration.py | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/trivy-analysis.yml b/.github/workflows/trivy-analysis.yml index 4b73cabcf..8a46800ae 100644 --- a/.github/workflows/trivy-analysis.yml +++ b/.github/workflows/trivy-analysis.yml @@ -42,6 +42,6 @@ jobs: severity: 'CRITICAL,HIGH' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v1 + uses: github/codeql-action/upload-sarif@v2 with: sarif_file: 'trivy-results.sarif' diff --git a/infrastructure/base-images/cloudharness-base/Dockerfile b/infrastructure/base-images/cloudharness-base/Dockerfile index 18975b717..c479cbca0 100644 --- a/infrastructure/base-images/cloudharness-base/Dockerfile +++ b/infrastructure/base-images/cloudharness-base/Dockerfile @@ -21,9 +21,8 @@ RUN pip install -e /libraries/models --no-cache-dir COPY libraries/cloudharness-common /libraries/cloudharness-common COPY libraries/client/cloudharness_cli /libraries/client/cloudharness_cli -# RUN pip install -e /libraries/models --no-cache-dir RUN pip install -e /libraries/cloudharness-common --no-cache-dir RUN pip install -e /libraries/client/cloudharness_cli --no-cache-dir -WORKDIR / \ No newline at end of file +WORKDIR / diff --git a/tools/deployment-cli-tools/tests/test_migration.py b/tools/deployment-cli-tools/tests/test_migration.py index c4a7ad02d..0aff24c82 100644 --- a/tools/deployment-cli-tools/tests/test_migration.py +++ b/tools/deployment-cli-tools/tests/test_migration.py @@ -8,6 +8,4 @@ def test_migration_accept_all(): perform_migration(os.path.join(HERE, './resources/migration'), accept_all=True) assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 0 os.system(f'cp -R {os.path.join(HERE, "resources/migration/backup/migration_app")} {os.path.join(HERE, "resources/migration/applications/")}') - # print(f'cp -R {os.path.join(HERE, "resources/migration/backup")} {os.path.join(HERE, "resources/migration/applications")}') - # subprocess.Popen(['cp', '-R', 'backup/*', 'applications/'], cwd=os.path.join(HERE, 'resources/migration')) assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 From d7a7f52b5e1e5d95fb275450a6f57c84c4fbbec8 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 16 Feb 2024 22:37:01 +0100 Subject: [PATCH 08/17] sarif v2 is deprecated --- .github/workflows/trivy-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy-analysis.yml b/.github/workflows/trivy-analysis.yml index 8a46800ae..30a95ae83 100644 --- a/.github/workflows/trivy-analysis.yml +++ b/.github/workflows/trivy-analysis.yml @@ -42,6 +42,6 @@ jobs: severity: 'CRITICAL,HIGH' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: 'trivy-results.sarif' From df87193679a73567d789adfaacdff734ba3f4f3a Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 23 Feb 2024 15:22:12 +0100 Subject: [PATCH 09/17] fixing sarif check --- .github/workflows/trivy-analysis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/trivy-analysis.yml b/.github/workflows/trivy-analysis.yml index 30a95ae83..a0305d3ea 100644 --- a/.github/workflows/trivy-analysis.yml +++ b/.github/workflows/trivy-analysis.yml @@ -36,8 +36,7 @@ jobs: uses: aquasecurity/trivy-action@2a2157eb22c08c9a1fac99263430307b8d1bc7a2 with: image-ref: 'cloudharness-base:${{ github.sha }}' - format: 'template' - template: '@/contrib/sarif.tpl' + format: 'sarif' output: 'trivy-results.sarif' severity: 'CRITICAL,HIGH' From 52d927c780df7c03d5b8b4a11794bdd71a155907 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 23 Feb 2024 15:27:02 +0100 Subject: [PATCH 10/17] fixing sarif check 2 --- .github/workflows/trivy-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy-analysis.yml b/.github/workflows/trivy-analysis.yml index a0305d3ea..44a6fe7f8 100644 --- a/.github/workflows/trivy-analysis.yml +++ b/.github/workflows/trivy-analysis.yml @@ -33,7 +33,7 @@ jobs: DOCKER_BUILDKIT=1 docker build -t cloudharness-base:${{ github.sha }} . -f ./infrastructure/base-images/cloudharness-base/Dockerfile - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@2a2157eb22c08c9a1fac99263430307b8d1bc7a2 + uses: aquasecurity/trivy-action@master with: image-ref: 'cloudharness-base:${{ github.sha }}' format: 'sarif' From ee064c96ba09ee104e8899a7cff2f4a245ab7e47 Mon Sep 17 00:00:00 2001 From: Dario Del Piano Date: Sun, 28 Jul 2024 23:58:21 +0200 Subject: [PATCH 11/17] changes per PR review --- .../ch_cli_tools/migration.py | 88 ++++++++++--------- .../1_migration.json} | 0 .../ch_cli_tools/utils.py | 28 ++---- .../applications/migration_app/Dockerfile | 2 +- 4 files changed, 57 insertions(+), 61 deletions(-) rename tools/deployment-cli-tools/ch_cli_tools/{config/migration.json => migrations/1_migration.json} (100%) diff --git a/tools/deployment-cli-tools/ch_cli_tools/migration.py b/tools/deployment-cli-tools/ch_cli_tools/migration.py index 3beae0729..862648c04 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/migration.py +++ b/tools/deployment-cli-tools/ch_cli_tools/migration.py @@ -13,48 +13,56 @@ def perform_migration(base_root, accept_all=False): all_files_detected = [] - f = open(os.path.join(HERE, "config", "migration.json"), "r") - migration_json = json.load(f) + for file in os.listdir(os.path.join(HERE, "migrations")): + f = open(os.path.join(HERE, "migrations", file), "r") + migration_json = json.load(f) - app_base_path = os.path.join(base_root, APPS_PATH) + sub_paths = [] + if isinstance(base_root, list): + app_base_path = [ + os.path.join(base_root_path, APPS_PATH) + for base_root_path in base_root + ] + for app_path in app_base_path: + sub_paths.extend(get_sub_paths(app_path)) + else: + app_base_path = os.path.join(base_root, APPS_PATH) + sub_paths = get_sub_paths(app_base_path) - # Iterate over all the applications to check if they need to be migrated - for app_path in get_sub_paths(app_base_path): - # Iterate the folders and files to check if they need to be migrated - for sub_path in TO_CHECK: - to_check = os.path.join(app_path, sub_path) - if os.path.isdir(to_check): - for migration_obj in migration_json["deprecated"]: - files = search_word_in_folder(to_check, migration_obj["keyword"]) - for file in files: - file_path = os.path.join(to_check, file) - print("#########################################") - print(f"Running migration on {file_path}") - print("#########################################") - all_files_detected.append(file_path) - for word_to_replace in migration_obj["to_be_replaced"]: - read_file_and_replace( - file_path, - word_to_replace["old"], - word_to_replace["new"], - accept_all, - ) - elif os.path.isfile(to_check): - for migration_obj in migration_json["deprecated"]: - if len(search_word_in_file(to_check, migration_obj["keyword"])) > 0: - print("#########################################") - print(f"Running migration on {to_check}") - print("#########################################") - all_files_detected.append(to_check) - for word_to_replace in migration_obj["to_be_replaced"]: - read_file_and_replace( - to_check, - word_to_replace["old"], - word_to_replace["new"], - accept_all, - ) - else: - print(f'Path {to_check} does not exist') + for app_path in sub_paths: + # Iterate the folders and files to check if they need to be migrated + for sub_path in TO_CHECK: + to_check = os.path.join(app_path, sub_path) + if os.path.isdir(to_check): + for migration_obj in migration_json["deprecated"]: + files = search_word_in_folder(to_check, migration_obj["keyword"]) + for file in files: + file_path = os.path.join(to_check, file) + print("#########################################") + print(f"Running migration on {file_path}") + print("#########################################") + all_files_detected.append(file_path) + for word_to_replace in migration_obj["to_be_replaced"]: + read_file_and_replace( + file_path, + word_to_replace["old"], + word_to_replace["new"], + accept_all, + ) + elif os.path.isfile(to_check): + for migration_obj in migration_json["deprecated"]: + if len(search_word_in_file(to_check, migration_obj["keyword"])) > 0: + print("#########################################") + print(f"Running migration on {to_check}") + print("#########################################") + all_files_detected.append(to_check) + for word_to_replace in migration_obj["to_be_replaced"]: + read_file_and_replace( + to_check, + word_to_replace["old"], + word_to_replace["new"], + accept_all, + ) print("=========================================") print("=== Cloud Harness migration completed ===") print("=========================================") diff --git a/tools/deployment-cli-tools/ch_cli_tools/config/migration.json b/tools/deployment-cli-tools/ch_cli_tools/migrations/1_migration.json similarity index 100% rename from tools/deployment-cli-tools/ch_cli_tools/config/migration.json rename to tools/deployment-cli-tools/ch_cli_tools/migrations/1_migration.json diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 375b35ec1..0bb6b5886 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -398,32 +398,20 @@ def filter_empty_strings(value): def search_word_in_file(file, word): - p = subprocess.Popen('grep -l %s %s'% (word, file), shell=True, - stdout=subprocess.PIPE) - output, _ = p.communicate() - output = output.decode("utf-8") + if os.path.isdir(file): + return [] matches = [] - if p.returncode == 1: # no matches found - pass - elif p.returncode == 0: # matches found - matches = output.split('\n') - else: - raise Exception(f'Migration Error: {file} grep failed with return code {p.returncode} and output {output}') + with open(file) as f: + if word in f.read(): + matches.append(file) return list(filter(filter_empty_strings, matches)) def search_word_in_folder(folder, word): - p = subprocess.Popen('grep -rl %s %s'% (word, '*'), shell=True, - stdout=subprocess.PIPE, cwd=folder) - output, _ = p.communicate() - output = output.decode("utf-8") matches = [] - if p.returncode == 1: # no matches found - pass - elif p.returncode == 0: # matches found - matches = output.split('\n') - else: - raise Exception(f'Migration Error: {folder} grep failed with return code {p.returncode} and output {output}') + files = glob.glob(folder + '/**/*', recursive = True) + for file in files: + matches.extend(search_word_in_file(file, word)) return list(filter(filter_empty_strings, matches)) diff --git a/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile b/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile index 12e4f487d..721ff5e5c 100644 --- a/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile +++ b/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile @@ -1,4 +1,4 @@ ARG CLOUDHARNESS_BASE_DEBIAN FROM $CLOUDHARNESS_BASE_DEBIAN -RUN apk add --no-cache bash +RUN apt install --no-cache bash From b2abdfa45d331f15a26eba96aab246a10952cfb2 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 26 Sep 2024 11:26:37 +0200 Subject: [PATCH 12/17] CH-95: review changes --- .../ch_cli_tools/migration.py | 178 +++++++++++------- .../ch_cli_tools/migrations/1_migration.json | 3 +- .../ch_cli_tools/utils.py | 16 +- 3 files changed, 128 insertions(+), 69 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/migration.py b/tools/deployment-cli-tools/ch_cli_tools/migration.py index 862648c04..fa094f632 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/migration.py +++ b/tools/deployment-cli-tools/ch_cli_tools/migration.py @@ -1,79 +1,22 @@ """ Utilities to perform a migration of the deployment to the latest supported version. """ + import os import json from . import HERE from cloudharness_utils.constants import APPS_PATH -from .utils import get_sub_paths, search_word_in_file, search_word_in_folder +from .utils import ( + get_sub_paths, + search_word_in_file, + search_word_in_folder, + search_word_by_pattern, +) TO_CHECK = ["deploy", "tasks", "Dockerfile"] -def perform_migration(base_root, accept_all=False): - all_files_detected = [] - for file in os.listdir(os.path.join(HERE, "migrations")): - f = open(os.path.join(HERE, "migrations", file), "r") - migration_json = json.load(f) - - sub_paths = [] - if isinstance(base_root, list): - app_base_path = [ - os.path.join(base_root_path, APPS_PATH) - for base_root_path in base_root - ] - for app_path in app_base_path: - sub_paths.extend(get_sub_paths(app_path)) - else: - app_base_path = os.path.join(base_root, APPS_PATH) - sub_paths = get_sub_paths(app_base_path) - - for app_path in sub_paths: - # Iterate the folders and files to check if they need to be migrated - for sub_path in TO_CHECK: - to_check = os.path.join(app_path, sub_path) - if os.path.isdir(to_check): - for migration_obj in migration_json["deprecated"]: - files = search_word_in_folder(to_check, migration_obj["keyword"]) - for file in files: - file_path = os.path.join(to_check, file) - print("#########################################") - print(f"Running migration on {file_path}") - print("#########################################") - all_files_detected.append(file_path) - for word_to_replace in migration_obj["to_be_replaced"]: - read_file_and_replace( - file_path, - word_to_replace["old"], - word_to_replace["new"], - accept_all, - ) - elif os.path.isfile(to_check): - for migration_obj in migration_json["deprecated"]: - if len(search_word_in_file(to_check, migration_obj["keyword"])) > 0: - print("#########################################") - print(f"Running migration on {to_check}") - print("#########################################") - all_files_detected.append(to_check) - for word_to_replace in migration_obj["to_be_replaced"]: - read_file_and_replace( - to_check, - word_to_replace["old"], - word_to_replace["new"], - accept_all, - ) - print("=========================================") - print("=== Cloud Harness migration completed ===") - print("=========================================") - print("### Summary of the files to check post migration ###") - for file in all_files_detected: - print(f"{file}") - print("=========================================") - print("=== End of Summary ===") - print("=========================================") - - def read_file_and_replace(file, old, new, accept_all=False): file_p = open(file, "r+") lines = file_p.readlines() @@ -99,3 +42,110 @@ def read_file_and_replace(file, old, new, accept_all=False): file_p.write(line) file_p.truncate() file_p.close() + + +def perform_migration(base_root, accept_all=False): + all_files_detected = [] + + # Iterate the list of files and replace the old word with the new word + def replace_in_files(app_path, files, words): + for file in files: + file_path = os.path.join(app_path, file) + print(f">>> Running migration on {file_path} <<<") + all_files_detected.append(file_path) + for word_to_replace in words: + read_file_and_replace( + file_path, + word_to_replace["old"], + word_to_replace["new"], + accept_all, + ) + + # Iterate migrations, patterns and search the keyword using the pattern given. + # If the keyword is found then replace the list of words given from the old to the new + def run_migrations_with_pattern(app_path, with_pattern): + for single_migration in with_pattern: + for pattern in single_migration["patterns"]: + results = search_word_by_pattern( + app_path, pattern, single_migration["keyword"] + ) + replace_in_files( + app_path, + results, + single_migration["to_be_replaced"] + ) + + # Iterate the TO_CHECK array that is set to search inside the deploy and tasks folder + # plus the Dockerfile. If the keyword is found inside one of these folders or in the + # Dockerfile then replace the list of words given from the old to the new + def run_migrations_without_pattern(app_path, without_pattern): + for sub_path in TO_CHECK: + to_check = os.path.join(app_path, sub_path) + if os.path.isdir(to_check): + for migration_obj in without_pattern: + files = search_word_in_folder( + to_check, + migration_obj["keyword"] + ) + replace_in_files( + app_path, + files, + migration_obj["to_be_replaced"] + ) + elif os.path.isfile(to_check): + for migration_obj in without_pattern: + keyword = migration_obj["keyword"] + if len(search_word_in_file(to_check, keyword)) > 0: + print(f">>> Running migration on {to_check} <<<") + all_files_detected.append(to_check) + for word_to_replace in migration_obj["to_be_replaced"]: + read_file_and_replace( + to_check, + word_to_replace["old"], + word_to_replace["new"], + accept_all, + ) + + # Parse the migration and separate the ones with patterns and without + def extract_migrations(migration_obj): + migrations_with_pattern = [] + migrations_without_pattern = [] + for single_migration in migration_obj["deprecated"]: + if "patterns" in single_migration: + migrations_with_pattern.append(single_migration) + else: + migrations_without_pattern.append(single_migration) + return migrations_with_pattern, migrations_without_pattern + + # Iterate all the migration files and run the migrations + for file in os.listdir(os.path.join(HERE, "migrations")): + with open(os.path.join(HERE, "migrations", file), "r") as f: + migration_dict = json.load(f) + + sub_paths = [] + # Find all the apps in the base root + if isinstance(base_root, list): + app_base_path = [ + os.path.join(base_root_path, APPS_PATH) + for base_root_path in base_root + ] + for app_path in app_base_path: + sub_paths.extend(get_sub_paths(app_path)) + else: + app_base_path = os.path.join(base_root, APPS_PATH) + sub_paths = get_sub_paths(app_base_path) + + with_pattern, without_pattern = extract_migrations(migration_dict) + # Iterate all the application folders and run the migrations + for app_path in sub_paths: + run_migrations_with_pattern(app_path, with_pattern) + run_migrations_without_pattern(app_path, without_pattern) + print("=========================================") + print(">>> Cloud Harness migration completed <<<") + print("=========================================") + print(">>> Migration Summary <<<") + for file in all_files_detected: + print(f"{file}") + print("=========================================") + print(">>> End of Summary <<<") + print("=========================================") diff --git a/tools/deployment-cli-tools/ch_cli_tools/migrations/1_migration.json b/tools/deployment-cli-tools/ch_cli_tools/migrations/1_migration.json index 3a8b7cc2f..ec9dd3360 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/migrations/1_migration.json +++ b/tools/deployment-cli-tools/ch_cli_tools/migrations/1_migration.json @@ -31,7 +31,8 @@ "old": "apk upgrade --no-cache", "new": "apt-get upgrade --no-cache" } - ] + ], + "patterns": ["**/Dockerfile", "**/*yaml"] }, { "keyword": "cloudharness-base-debian", diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index dadefc29b..2e66c0bfe 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -434,13 +434,13 @@ def filter_empty_strings(value): return value != "" -def search_word_in_file(file, word): - if os.path.isdir(file): +def search_word_in_file(filename, word): + if os.path.isdir(filename): return [] matches = [] - with open(file) as f: + with open(filename) as f: if word in f.read(): - matches.append(file) + matches.append(filename) return list(filter(filter_empty_strings, matches)) @@ -452,6 +452,14 @@ def search_word_in_folder(folder, word): return list(filter(filter_empty_strings, matches)) +def search_word_by_pattern(folder, pattern, word): + matches = [] + files = glob.glob(folder + pattern, recursive = True) + for file in files: + matches.extend(search_word_in_file(file, word)) + return list(filter(filter_empty_strings, matches)) + + def get_git_commit_hash(path): # return the short git commit hash in that path # if the path is not a git repo, return None From 1173465544ac444a9647142e3d095d16067dffaf Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 26 Sep 2024 11:34:55 +0200 Subject: [PATCH 13/17] CH-95: fixing linter issues --- .../cloudharness-common/cloudharness/middleware/django.py | 1 + tools/deployment-cli-tools/ch_cli_tools/utils.py | 8 ++++---- tools/deployment-cli-tools/harness-deployment | 2 +- tools/deployment-cli-tools/tests/test_migration.py | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/libraries/cloudharness-common/cloudharness/middleware/django.py b/libraries/cloudharness-common/cloudharness/middleware/django.py index e78837511..f3f6ee760 100644 --- a/libraries/cloudharness-common/cloudharness/middleware/django.py +++ b/libraries/cloudharness-common/cloudharness/middleware/django.py @@ -1,6 +1,7 @@ from cloudharness.middleware import set_authentication_token from django.http.request import HttpRequest + class CloudharnessMiddleware: def __init__(self, get_response): self.get_response = get_response diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 2e66c0bfe..d6d4df324 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -439,14 +439,14 @@ def search_word_in_file(filename, word): return [] matches = [] with open(filename) as f: - if word in f.read(): - matches.append(filename) + if word in f.read(): + matches.append(filename) return list(filter(filter_empty_strings, matches)) def search_word_in_folder(folder, word): matches = [] - files = glob.glob(folder + '/**/*', recursive = True) + files = glob.glob(folder + '/**/*', recursive=True) for file in files: matches.extend(search_word_in_file(file, word)) return list(filter(filter_empty_strings, matches)) @@ -454,7 +454,7 @@ def search_word_in_folder(folder, word): def search_word_by_pattern(folder, pattern, word): matches = [] - files = glob.glob(folder + pattern, recursive = True) + files = glob.glob(folder + pattern, recursive=True) for file in files: matches.extend(search_word_in_file(file, word)) return list(filter(filter_empty_strings, matches)) diff --git a/tools/deployment-cli-tools/harness-deployment b/tools/deployment-cli-tools/harness-deployment index 610e6cfc4..44445a068 100644 --- a/tools/deployment-cli-tools/harness-deployment +++ b/tools/deployment-cli-tools/harness-deployment @@ -63,7 +63,7 @@ if __name__ == "__main__": parser.add_argument('-N', '--no-cd', dest='no_cd_gen', action="store_const", default=None, const=True, help=f'Do not generate ci/cd files') parser.add_argument('-we', '--write-env', dest='write_env', action="store_const", default=None, const=True, - help=f'Write build env to .env file in {DEPLOYMENT_PATH}') + help=f'Write build env to .env file in {DEPLOYMENT_PATH}') parser.add_argument('--migrate', dest='migration', action="store_true", help='Perform a migration of the deployment to the latest supported version.') parser.add_argument('--migrate-accept-all', dest='migration_accept_all', action="store_true", diff --git a/tools/deployment-cli-tools/tests/test_migration.py b/tools/deployment-cli-tools/tests/test_migration.py index 0aff24c82..071e02e83 100644 --- a/tools/deployment-cli-tools/tests/test_migration.py +++ b/tools/deployment-cli-tools/tests/test_migration.py @@ -3,6 +3,7 @@ HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/') + def test_migration_accept_all(): assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 perform_migration(os.path.join(HERE, './resources/migration'), accept_all=True) From ef30b6913d3b2423e55ac123686c26d7658e1387 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Thu, 5 Jun 2025 15:12:22 +0200 Subject: [PATCH 14/17] Regenerate test pipeline --- deployment/codefresh-test.yaml | 241 +++++++++++++++++---------------- 1 file changed, 121 insertions(+), 120 deletions(-) diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index 01bf9ba6d..5092e0813 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -50,28 +50,28 @@ steps: type: parallel stage: build steps: - cloudharness-frontend-build: + cloudharness-base: type: build stage: build - dockerfile: infrastructure/base-images/cloudharness-frontend-build/Dockerfile + dockerfile: infrastructure/base-images/cloudharness-base/Dockerfile registry: '${{CODEFRESH_REGISTRY}}' buildkit: true build_arguments: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/cloudharness-frontend-build - title: Cloudharness frontend build + image_name: cloudharness/cloudharness-base + title: Cloudharness base working_directory: ./. - tag: '${{CLOUDHARNESS_FRONTEND_BUILD_TAG}}' + tag: '${{CLOUDHARNESS_BASE_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}', - '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}') == true - forceNoCache: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}', - '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}') == false - test-e2e: + buildDoesNotExist: includes('${{CLOUDHARNESS_BASE_TAG_EXISTS}}', '{{CLOUDHARNESS_BASE_TAG_EXISTS}}') + == true + forceNoCache: includes('${{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}') + == false + accounts: type: build stage: build dockerfile: Dockerfile @@ -81,39 +81,39 @@ steps: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/test-e2e - title: Test e2e - working_directory: ./test/test-e2e - tag: '${{TEST_E2E_TAG}}' + image_name: cloudharness/accounts + title: Accounts + working_directory: ./applications/accounts + tag: '${{ACCOUNTS_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{TEST_E2E_TAG_EXISTS}}', '{{TEST_E2E_TAG_EXISTS}}') + buildDoesNotExist: includes('${{ACCOUNTS_TAG_EXISTS}}', '{{ACCOUNTS_TAG_EXISTS}}') == true - forceNoCache: includes('${{TEST_E2E_TAG_FORCE_BUILD}}', '{{TEST_E2E_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{ACCOUNTS_TAG_FORCE_BUILD}}', '{{ACCOUNTS_TAG_FORCE_BUILD}}') == false - cloudharness-base: + cloudharness-frontend-build: type: build stage: build - dockerfile: infrastructure/base-images/cloudharness-base/Dockerfile + dockerfile: infrastructure/base-images/cloudharness-frontend-build/Dockerfile registry: '${{CODEFRESH_REGISTRY}}' buildkit: true build_arguments: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/cloudharness-base - title: Cloudharness base + image_name: cloudharness/cloudharness-frontend-build + title: Cloudharness frontend build working_directory: ./. - tag: '${{CLOUDHARNESS_BASE_TAG}}' + tag: '${{CLOUDHARNESS_FRONTEND_BUILD_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{CLOUDHARNESS_BASE_TAG_EXISTS}}', '{{CLOUDHARNESS_BASE_TAG_EXISTS}}') - == true - forceNoCache: includes('${{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}') - == false - accounts: + buildDoesNotExist: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}', + '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}') == true + forceNoCache: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}', + '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}') == false + test-e2e: type: build stage: build dockerfile: Dockerfile @@ -123,18 +123,23 @@ steps: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/accounts - title: Accounts - working_directory: ./applications/accounts - tag: '${{ACCOUNTS_TAG}}' + image_name: cloudharness/test-e2e + title: Test e2e + working_directory: ./test/test-e2e + tag: '${{TEST_E2E_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{ACCOUNTS_TAG_EXISTS}}', '{{ACCOUNTS_TAG_EXISTS}}') + buildDoesNotExist: includes('${{TEST_E2E_TAG_EXISTS}}', '{{TEST_E2E_TAG_EXISTS}}') == true - forceNoCache: includes('${{ACCOUNTS_TAG_FORCE_BUILD}}', '{{ACCOUNTS_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{TEST_E2E_TAG_FORCE_BUILD}}', '{{TEST_E2E_TAG_FORCE_BUILD}}') == false - workflows-extract-download: + title: Build parallel step 1 + build_application_images_1: + type: parallel + stage: build + steps: + jupyterhub: type: build stage: build dockerfile: Dockerfile @@ -144,23 +149,19 @@ steps: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/workflows-extract-download - title: Workflows extract download - working_directory: ./applications/workflows/tasks/extract-download - tag: '${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG}}' + - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} + image_name: cloudharness/jupyterhub + title: Jupyterhub + working_directory: ./applications/jupyterhub + tag: '${{JUPYTERHUB_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_EXISTS}}', - '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}', - '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}') == false - title: Build parallel step 1 - build_application_images_1: - type: parallel - stage: build - steps: - samples-secret: + buildDoesNotExist: includes('${{JUPYTERHUB_TAG_EXISTS}}', '{{JUPYTERHUB_TAG_EXISTS}}') + == true + forceNoCache: includes('${{JUPYTERHUB_TAG_FORCE_BUILD}}', '{{JUPYTERHUB_TAG_FORCE_BUILD}}') + == false + samples-sum: type: build stage: build dockerfile: Dockerfile @@ -171,18 +172,18 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/samples-secret - title: Samples secret - working_directory: ./applications/samples/tasks/secret - tag: '${{SAMPLES_SECRET_TAG}}' + image_name: cloudharness/samples-sum + title: Samples sum + working_directory: ./applications/samples/tasks/sum + tag: '${{SAMPLES_SUM_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{SAMPLES_SECRET_TAG_EXISTS}}', '{{SAMPLES_SECRET_TAG_EXISTS}}') + buildDoesNotExist: includes('${{SAMPLES_SUM_TAG_EXISTS}}', '{{SAMPLES_SUM_TAG_EXISTS}}') == true - forceNoCache: includes('${{SAMPLES_SECRET_TAG_FORCE_BUILD}}', '{{SAMPLES_SECRET_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{SAMPLES_SUM_TAG_FORCE_BUILD}}', '{{SAMPLES_SUM_TAG_FORCE_BUILD}}') == false - workflows-notify-queue: + samples-secret: type: build stage: build dockerfile: Dockerfile @@ -193,17 +194,17 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/workflows-notify-queue - title: Workflows notify queue - working_directory: ./applications/workflows/tasks/notify-queue - tag: '${{WORKFLOWS_NOTIFY_QUEUE_TAG}}' + image_name: cloudharness/samples-secret + title: Samples secret + working_directory: ./applications/samples/tasks/secret + tag: '${{SAMPLES_SECRET_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}', - '{{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}', - '{{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}') == false + buildDoesNotExist: includes('${{SAMPLES_SECRET_TAG_EXISTS}}', '{{SAMPLES_SECRET_TAG_EXISTS}}') + == true + forceNoCache: includes('${{SAMPLES_SECRET_TAG_FORCE_BUILD}}', '{{SAMPLES_SECRET_TAG_FORCE_BUILD}}') + == false cloudharness-flask: type: build stage: build @@ -226,7 +227,7 @@ steps: == true forceNoCache: includes('${{CLOUDHARNESS_FLASK_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_FLASK_TAG_FORCE_BUILD}}') == false - workflows-send-result-event: + workflows-notify-queue: type: build stage: build dockerfile: Dockerfile @@ -237,18 +238,18 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/workflows-send-result-event - title: Workflows send result event - working_directory: ./applications/workflows/tasks/send-result-event - tag: '${{WORKFLOWS_SEND_RESULT_EVENT_TAG}}' + image_name: cloudharness/workflows-notify-queue + title: Workflows notify queue + working_directory: ./applications/workflows/tasks/notify-queue + tag: '${{WORKFLOWS_NOTIFY_QUEUE_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}', - '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}', - '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}') == false - samples-print-file: + buildDoesNotExist: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}', + '{{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}') == true + forceNoCache: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}', + '{{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}') == false + workflows-extract-download: type: build stage: build dockerfile: Dockerfile @@ -259,17 +260,17 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/samples-print-file - title: Samples print file - working_directory: ./applications/samples/tasks/print-file - tag: '${{SAMPLES_PRINT_FILE_TAG}}' + image_name: cloudharness/workflows-extract-download + title: Workflows extract download + working_directory: ./applications/workflows/tasks/extract-download + tag: '${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{SAMPLES_PRINT_FILE_TAG_EXISTS}}', '{{SAMPLES_PRINT_FILE_TAG_EXISTS}}') - == true - forceNoCache: includes('${{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}', '{{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}') - == false + buildDoesNotExist: includes('${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_EXISTS}}', + '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_EXISTS}}') == true + forceNoCache: includes('${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}', + '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}') == false test-api: type: build stage: build @@ -292,7 +293,7 @@ steps: == true forceNoCache: includes('${{TEST_API_TAG_FORCE_BUILD}}', '{{TEST_API_TAG_FORCE_BUILD}}') == false - samples-sum: + samples-print-file: type: build stage: build dockerfile: Dockerfile @@ -303,18 +304,18 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/samples-sum - title: Samples sum - working_directory: ./applications/samples/tasks/sum - tag: '${{SAMPLES_SUM_TAG}}' + image_name: cloudharness/samples-print-file + title: Samples print file + working_directory: ./applications/samples/tasks/print-file + tag: '${{SAMPLES_PRINT_FILE_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{SAMPLES_SUM_TAG_EXISTS}}', '{{SAMPLES_SUM_TAG_EXISTS}}') + buildDoesNotExist: includes('${{SAMPLES_PRINT_FILE_TAG_EXISTS}}', '{{SAMPLES_PRINT_FILE_TAG_EXISTS}}') == true - forceNoCache: includes('${{SAMPLES_SUM_TAG_FORCE_BUILD}}', '{{SAMPLES_SUM_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}', '{{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}') == false - jupyterhub: + workflows-send-result-event: type: build stage: build dockerfile: Dockerfile @@ -325,23 +326,23 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/jupyterhub - title: Jupyterhub - working_directory: ./applications/jupyterhub - tag: '${{JUPYTERHUB_TAG}}' + image_name: cloudharness/workflows-send-result-event + title: Workflows send result event + working_directory: ./applications/workflows/tasks/send-result-event + tag: '${{WORKFLOWS_SEND_RESULT_EVENT_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{JUPYTERHUB_TAG_EXISTS}}', '{{JUPYTERHUB_TAG_EXISTS}}') - == true - forceNoCache: includes('${{JUPYTERHUB_TAG_FORCE_BUILD}}', '{{JUPYTERHUB_TAG_FORCE_BUILD}}') - == false + buildDoesNotExist: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}', + '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}') == true + forceNoCache: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}', + '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}') == false title: Build parallel step 2 build_application_images_2: type: parallel stage: build steps: - workflows: + common: type: build stage: build dockerfile: Dockerfile @@ -352,16 +353,16 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_FLASK=${{REGISTRY}}/cloudharness/cloudharness-flask:${{CLOUDHARNESS_FLASK_TAG}} - image_name: cloudharness/workflows - title: Workflows - working_directory: ./applications/workflows/server - tag: '${{WORKFLOWS_TAG}}' + image_name: cloudharness/common + title: Common + working_directory: ./applications/common/server + tag: '${{COMMON_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{WORKFLOWS_TAG_EXISTS}}', '{{WORKFLOWS_TAG_EXISTS}}') + buildDoesNotExist: includes('${{COMMON_TAG_EXISTS}}', '{{COMMON_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_TAG_FORCE_BUILD}}', '{{WORKFLOWS_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{COMMON_TAG_FORCE_BUILD}}', '{{COMMON_TAG_FORCE_BUILD}}') == false samples: type: build @@ -386,7 +387,7 @@ steps: == true forceNoCache: includes('${{SAMPLES_TAG_FORCE_BUILD}}', '{{SAMPLES_TAG_FORCE_BUILD}}') == false - common: + volumemanager: type: build stage: build dockerfile: Dockerfile @@ -397,18 +398,18 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_FLASK=${{REGISTRY}}/cloudharness/cloudharness-flask:${{CLOUDHARNESS_FLASK_TAG}} - image_name: cloudharness/common - title: Common - working_directory: ./applications/common/server - tag: '${{COMMON_TAG}}' + image_name: cloudharness/volumemanager + title: Volumemanager + working_directory: ./applications/volumemanager/server + tag: '${{VOLUMEMANAGER_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{COMMON_TAG_EXISTS}}', '{{COMMON_TAG_EXISTS}}') + buildDoesNotExist: includes('${{VOLUMEMANAGER_TAG_EXISTS}}', '{{VOLUMEMANAGER_TAG_EXISTS}}') == true - forceNoCache: includes('${{COMMON_TAG_FORCE_BUILD}}', '{{COMMON_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{VOLUMEMANAGER_TAG_FORCE_BUILD}}', '{{VOLUMEMANAGER_TAG_FORCE_BUILD}}') == false - volumemanager: + workflows: type: build stage: build dockerfile: Dockerfile @@ -419,16 +420,16 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_FLASK=${{REGISTRY}}/cloudharness/cloudharness-flask:${{CLOUDHARNESS_FLASK_TAG}} - image_name: cloudharness/volumemanager - title: Volumemanager - working_directory: ./applications/volumemanager/server - tag: '${{VOLUMEMANAGER_TAG}}' + image_name: cloudharness/workflows + title: Workflows + working_directory: ./applications/workflows/server + tag: '${{WORKFLOWS_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{VOLUMEMANAGER_TAG_EXISTS}}', '{{VOLUMEMANAGER_TAG_EXISTS}}') + buildDoesNotExist: includes('${{WORKFLOWS_TAG_EXISTS}}', '{{WORKFLOWS_TAG_EXISTS}}') == true - forceNoCache: includes('${{VOLUMEMANAGER_TAG_FORCE_BUILD}}', '{{VOLUMEMANAGER_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{WORKFLOWS_TAG_FORCE_BUILD}}', '{{WORKFLOWS_TAG_FORCE_BUILD}}') == false title: Build parallel step 3 tests_unit: @@ -465,12 +466,12 @@ steps: - kubectl config use-context ${{CLUSTER_NAME}} - kubectl config set-context --current --namespace=test-${{NAMESPACE_BASENAME}} - kubectl rollout status deployment/workflows - - kubectl rollout status deployment/accounts - - kubectl rollout status deployment/samples - - kubectl rollout status deployment/samples-gk - kubectl rollout status deployment/common - kubectl rollout status deployment/volumemanager + - kubectl rollout status deployment/accounts - kubectl rollout status deployment/argo-gk + - kubectl rollout status deployment/samples + - kubectl rollout status deployment/samples-gk - sleep 60 tests_api: stage: qa From f23dd654a55877bd28cda2f3a6ba6ebb4244df0c Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Thu, 5 Jun 2025 15:59:35 +0200 Subject: [PATCH 15/17] CH-95 fix unit tests --- tools/deployment-cli-tools/ch_cli_tools/utils.py | 2 ++ .../deployment-cli-tools/tests/test_migration.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 1bcef5b5d..c88bee158 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -494,6 +494,8 @@ def search_word_in_folder(folder, word): def search_word_by_pattern(folder, pattern, word): + if not folder.endswith('/'): + folder += '/' matches = [] files = glob.glob(folder + pattern, recursive=True) for file in files: diff --git a/tools/deployment-cli-tools/tests/test_migration.py b/tools/deployment-cli-tools/tests/test_migration.py index 071e02e83..532ac5df6 100644 --- a/tools/deployment-cli-tools/tests/test_migration.py +++ b/tools/deployment-cli-tools/tests/test_migration.py @@ -5,8 +5,15 @@ def test_migration_accept_all(): - assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 - perform_migration(os.path.join(HERE, './resources/migration'), accept_all=True) - assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 0 - os.system(f'cp -R {os.path.join(HERE, "resources/migration/backup/migration_app")} {os.path.join(HERE, "resources/migration/applications/")}') + found = search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN") + print(found) + assert len(found) == 2 + try: + perform_migration(os.path.join(HERE, './resources/migration'), accept_all=True) + found = search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN") + print(found) + assert len(found) == 0 + + finally: + os.system(f'cp -R {os.path.join(HERE, "resources/migration/backup/migration_app")} {os.path.join(HERE, "resources/migration/applications/")}') assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 From 213b94aac9bdcac25dce238d9642324e2341e445 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Thu, 5 Jun 2025 16:04:35 +0200 Subject: [PATCH 16/17] (chore) fix linting --- .../cloudharness-django/cloudharness_django/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py index 8f9d5c8d4..178e3b30b 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py @@ -99,4 +99,4 @@ if settings.ROOT_URLCONF: APP_URLCONF = settings.ROOT_URLCONF - ROOT_URLCONF = "cloudharness_django.urls" \ No newline at end of file + ROOT_URLCONF = "cloudharness_django.urls" From 920896a8ae9f423167f3d7f4713c44b32c4e76c4 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 6 Jun 2025 14:46:59 +0200 Subject: [PATCH 17/17] CH-95 fix unit tests --- tools/deployment-cli-tools/tests/test_codefresh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/tests/test_codefresh.py b/tools/deployment-cli-tools/tests/test_codefresh.py index 31dba2f49..92a7ce01b 100644 --- a/tools/deployment-cli-tools/tests/test_codefresh.py +++ b/tools/deployment-cli-tools/tests/test_codefresh.py @@ -59,7 +59,7 @@ def test_create_codefresh_configuration(): assert step_build_base["type"] == "parallel" steps = l1_steps[STEP_0]["steps"] - assert len(steps) == 8, "all images that do not depend on othe builds should be included in the first step" + assert len(steps) == 7, "all images that do not depend on othe builds should be included in the first step" assert "cloudharness-base" in steps, "cloudharness-base image should be included as dependency" assert "cloudharness-base-debian" not in steps, "cloudharness-base image should not be included" assert "cloudharness-frontend-build" in steps, "cloudharness-frontend-build image should be included as dependency"