diff --git a/.github/workflows/trivy-analysis.yml b/.github/workflows/trivy-analysis.yml index 4b73cabcf..44a6fe7f8 100644 --- a/.github/workflows/trivy-analysis.yml +++ b/.github/workflows/trivy-analysis.yml @@ -33,15 +33,14 @@ jobs: DOCKER_BUILDKIT=1 docker build -t cloudharness-base:${{ github.sha }} . -f ./infrastructure/base-images/cloudharness-base/Dockerfile - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@2a2157eb22c08c9a1fac99263430307b8d1bc7a2 + uses: aquasecurity/trivy-action@master with: image-ref: 'cloudharness-base:${{ github.sha }}' - format: 'template' - template: '@/contrib/sarif.tpl' + format: 'sarif' output: 'trivy-results.sarif' severity: 'CRITICAL,HIGH' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v1 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: 'trivy-results.sarif' diff --git a/.gitignore b/.gitignore index 888bbcc1c..7e6d506c1 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,7 @@ skaffold.yaml .hypothesis __pycache__ .env +tools/deployment-cli-tools/tests/resources/migration +tools/deployment-cli-tools/tests/resources/migration.bak /.venv -.history \ No newline at end of file +.history diff --git a/applications/common/server/Dockerfile b/applications/common/server/Dockerfile index 2d2515a87..962f117ea 100644 --- a/applications/common/server/Dockerfile +++ b/applications/common/server/Dockerfile @@ -6,8 +6,7 @@ ENV MODULE_NAME=common ENV WORKERS=2 ENV PORT=8080 -RUN apk update -RUN apk add postgresql-dev +RUN apt update && apt install -y postgresql COPY ./requirements.txt /usr/src/app/ @@ -18,4 +17,4 @@ COPY . /usr/src/app ENV FLASK_ENV=production ENV APP_SETTINGS=common.config.ProductionConfig RUN pip3 install -e /usr/src/app -ENTRYPOINT gunicorn --workers=$WORKERS --bind=0.0.0.0:$PORT $MODULE_NAME.__main__:app \ No newline at end of file +ENTRYPOINT gunicorn --workers=$WORKERS --bind=0.0.0.0:$PORT $MODULE_NAME.__main__:app diff --git a/applications/workflows/tasks/extract-download/Dockerfile b/applications/workflows/tasks/extract-download/Dockerfile index 5a48923c2..f89460709 100644 --- a/applications/workflows/tasks/extract-download/Dockerfile +++ b/applications/workflows/tasks/extract-download/Dockerfile @@ -1,6 +1,6 @@ -FROM alpine -RUN apk update -RUN apk add file unzip wget +ARG CLOUDHARNESS_BASE +FROM $CLOUDHARNESS_BASE as base +RUN apt update && apt install -y file unzip wget ADD . / diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index 01bf9ba6d..5092e0813 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -50,28 +50,28 @@ steps: type: parallel stage: build steps: - cloudharness-frontend-build: + cloudharness-base: type: build stage: build - dockerfile: infrastructure/base-images/cloudharness-frontend-build/Dockerfile + dockerfile: infrastructure/base-images/cloudharness-base/Dockerfile registry: '${{CODEFRESH_REGISTRY}}' buildkit: true build_arguments: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/cloudharness-frontend-build - title: Cloudharness frontend build + image_name: cloudharness/cloudharness-base + title: Cloudharness base working_directory: ./. - tag: '${{CLOUDHARNESS_FRONTEND_BUILD_TAG}}' + tag: '${{CLOUDHARNESS_BASE_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}', - '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}') == true - forceNoCache: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}', - '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}') == false - test-e2e: + buildDoesNotExist: includes('${{CLOUDHARNESS_BASE_TAG_EXISTS}}', '{{CLOUDHARNESS_BASE_TAG_EXISTS}}') + == true + forceNoCache: includes('${{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}') + == false + accounts: type: build stage: build dockerfile: Dockerfile @@ -81,39 +81,39 @@ steps: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/test-e2e - title: Test e2e - working_directory: ./test/test-e2e - tag: '${{TEST_E2E_TAG}}' + image_name: cloudharness/accounts + title: Accounts + working_directory: ./applications/accounts + tag: '${{ACCOUNTS_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{TEST_E2E_TAG_EXISTS}}', '{{TEST_E2E_TAG_EXISTS}}') + buildDoesNotExist: includes('${{ACCOUNTS_TAG_EXISTS}}', '{{ACCOUNTS_TAG_EXISTS}}') == true - forceNoCache: includes('${{TEST_E2E_TAG_FORCE_BUILD}}', '{{TEST_E2E_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{ACCOUNTS_TAG_FORCE_BUILD}}', '{{ACCOUNTS_TAG_FORCE_BUILD}}') == false - cloudharness-base: + cloudharness-frontend-build: type: build stage: build - dockerfile: infrastructure/base-images/cloudharness-base/Dockerfile + dockerfile: infrastructure/base-images/cloudharness-frontend-build/Dockerfile registry: '${{CODEFRESH_REGISTRY}}' buildkit: true build_arguments: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/cloudharness-base - title: Cloudharness base + image_name: cloudharness/cloudharness-frontend-build + title: Cloudharness frontend build working_directory: ./. - tag: '${{CLOUDHARNESS_BASE_TAG}}' + tag: '${{CLOUDHARNESS_FRONTEND_BUILD_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{CLOUDHARNESS_BASE_TAG_EXISTS}}', '{{CLOUDHARNESS_BASE_TAG_EXISTS}}') - == true - forceNoCache: includes('${{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}') - == false - accounts: + buildDoesNotExist: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}', + '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}') == true + forceNoCache: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}', + '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}') == false + test-e2e: type: build stage: build dockerfile: Dockerfile @@ -123,18 +123,23 @@ steps: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/accounts - title: Accounts - working_directory: ./applications/accounts - tag: '${{ACCOUNTS_TAG}}' + image_name: cloudharness/test-e2e + title: Test e2e + working_directory: ./test/test-e2e + tag: '${{TEST_E2E_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{ACCOUNTS_TAG_EXISTS}}', '{{ACCOUNTS_TAG_EXISTS}}') + buildDoesNotExist: includes('${{TEST_E2E_TAG_EXISTS}}', '{{TEST_E2E_TAG_EXISTS}}') == true - forceNoCache: includes('${{ACCOUNTS_TAG_FORCE_BUILD}}', '{{ACCOUNTS_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{TEST_E2E_TAG_FORCE_BUILD}}', '{{TEST_E2E_TAG_FORCE_BUILD}}') == false - workflows-extract-download: + title: Build parallel step 1 + build_application_images_1: + type: parallel + stage: build + steps: + jupyterhub: type: build stage: build dockerfile: Dockerfile @@ -144,23 +149,19 @@ steps: - DOMAIN=${{DOMAIN}} - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/workflows-extract-download - title: Workflows extract download - working_directory: ./applications/workflows/tasks/extract-download - tag: '${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG}}' + - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} + image_name: cloudharness/jupyterhub + title: Jupyterhub + working_directory: ./applications/jupyterhub + tag: '${{JUPYTERHUB_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_EXISTS}}', - '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}', - '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}') == false - title: Build parallel step 1 - build_application_images_1: - type: parallel - stage: build - steps: - samples-secret: + buildDoesNotExist: includes('${{JUPYTERHUB_TAG_EXISTS}}', '{{JUPYTERHUB_TAG_EXISTS}}') + == true + forceNoCache: includes('${{JUPYTERHUB_TAG_FORCE_BUILD}}', '{{JUPYTERHUB_TAG_FORCE_BUILD}}') + == false + samples-sum: type: build stage: build dockerfile: Dockerfile @@ -171,18 +172,18 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/samples-secret - title: Samples secret - working_directory: ./applications/samples/tasks/secret - tag: '${{SAMPLES_SECRET_TAG}}' + image_name: cloudharness/samples-sum + title: Samples sum + working_directory: ./applications/samples/tasks/sum + tag: '${{SAMPLES_SUM_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{SAMPLES_SECRET_TAG_EXISTS}}', '{{SAMPLES_SECRET_TAG_EXISTS}}') + buildDoesNotExist: includes('${{SAMPLES_SUM_TAG_EXISTS}}', '{{SAMPLES_SUM_TAG_EXISTS}}') == true - forceNoCache: includes('${{SAMPLES_SECRET_TAG_FORCE_BUILD}}', '{{SAMPLES_SECRET_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{SAMPLES_SUM_TAG_FORCE_BUILD}}', '{{SAMPLES_SUM_TAG_FORCE_BUILD}}') == false - workflows-notify-queue: + samples-secret: type: build stage: build dockerfile: Dockerfile @@ -193,17 +194,17 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/workflows-notify-queue - title: Workflows notify queue - working_directory: ./applications/workflows/tasks/notify-queue - tag: '${{WORKFLOWS_NOTIFY_QUEUE_TAG}}' + image_name: cloudharness/samples-secret + title: Samples secret + working_directory: ./applications/samples/tasks/secret + tag: '${{SAMPLES_SECRET_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}', - '{{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}', - '{{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}') == false + buildDoesNotExist: includes('${{SAMPLES_SECRET_TAG_EXISTS}}', '{{SAMPLES_SECRET_TAG_EXISTS}}') + == true + forceNoCache: includes('${{SAMPLES_SECRET_TAG_FORCE_BUILD}}', '{{SAMPLES_SECRET_TAG_FORCE_BUILD}}') + == false cloudharness-flask: type: build stage: build @@ -226,7 +227,7 @@ steps: == true forceNoCache: includes('${{CLOUDHARNESS_FLASK_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_FLASK_TAG_FORCE_BUILD}}') == false - workflows-send-result-event: + workflows-notify-queue: type: build stage: build dockerfile: Dockerfile @@ -237,18 +238,18 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/workflows-send-result-event - title: Workflows send result event - working_directory: ./applications/workflows/tasks/send-result-event - tag: '${{WORKFLOWS_SEND_RESULT_EVENT_TAG}}' + image_name: cloudharness/workflows-notify-queue + title: Workflows notify queue + working_directory: ./applications/workflows/tasks/notify-queue + tag: '${{WORKFLOWS_NOTIFY_QUEUE_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}', - '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}', - '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}') == false - samples-print-file: + buildDoesNotExist: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}', + '{{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}') == true + forceNoCache: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}', + '{{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}') == false + workflows-extract-download: type: build stage: build dockerfile: Dockerfile @@ -259,17 +260,17 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/samples-print-file - title: Samples print file - working_directory: ./applications/samples/tasks/print-file - tag: '${{SAMPLES_PRINT_FILE_TAG}}' + image_name: cloudharness/workflows-extract-download + title: Workflows extract download + working_directory: ./applications/workflows/tasks/extract-download + tag: '${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{SAMPLES_PRINT_FILE_TAG_EXISTS}}', '{{SAMPLES_PRINT_FILE_TAG_EXISTS}}') - == true - forceNoCache: includes('${{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}', '{{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}') - == false + buildDoesNotExist: includes('${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_EXISTS}}', + '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_EXISTS}}') == true + forceNoCache: includes('${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}', + '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}') == false test-api: type: build stage: build @@ -292,7 +293,7 @@ steps: == true forceNoCache: includes('${{TEST_API_TAG_FORCE_BUILD}}', '{{TEST_API_TAG_FORCE_BUILD}}') == false - samples-sum: + samples-print-file: type: build stage: build dockerfile: Dockerfile @@ -303,18 +304,18 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/samples-sum - title: Samples sum - working_directory: ./applications/samples/tasks/sum - tag: '${{SAMPLES_SUM_TAG}}' + image_name: cloudharness/samples-print-file + title: Samples print file + working_directory: ./applications/samples/tasks/print-file + tag: '${{SAMPLES_PRINT_FILE_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{SAMPLES_SUM_TAG_EXISTS}}', '{{SAMPLES_SUM_TAG_EXISTS}}') + buildDoesNotExist: includes('${{SAMPLES_PRINT_FILE_TAG_EXISTS}}', '{{SAMPLES_PRINT_FILE_TAG_EXISTS}}') == true - forceNoCache: includes('${{SAMPLES_SUM_TAG_FORCE_BUILD}}', '{{SAMPLES_SUM_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}', '{{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}') == false - jupyterhub: + workflows-send-result-event: type: build stage: build dockerfile: Dockerfile @@ -325,23 +326,23 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/jupyterhub - title: Jupyterhub - working_directory: ./applications/jupyterhub - tag: '${{JUPYTERHUB_TAG}}' + image_name: cloudharness/workflows-send-result-event + title: Workflows send result event + working_directory: ./applications/workflows/tasks/send-result-event + tag: '${{WORKFLOWS_SEND_RESULT_EVENT_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{JUPYTERHUB_TAG_EXISTS}}', '{{JUPYTERHUB_TAG_EXISTS}}') - == true - forceNoCache: includes('${{JUPYTERHUB_TAG_FORCE_BUILD}}', '{{JUPYTERHUB_TAG_FORCE_BUILD}}') - == false + buildDoesNotExist: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}', + '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}') == true + forceNoCache: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}', + '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}') == false title: Build parallel step 2 build_application_images_2: type: parallel stage: build steps: - workflows: + common: type: build stage: build dockerfile: Dockerfile @@ -352,16 +353,16 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_FLASK=${{REGISTRY}}/cloudharness/cloudharness-flask:${{CLOUDHARNESS_FLASK_TAG}} - image_name: cloudharness/workflows - title: Workflows - working_directory: ./applications/workflows/server - tag: '${{WORKFLOWS_TAG}}' + image_name: cloudharness/common + title: Common + working_directory: ./applications/common/server + tag: '${{COMMON_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{WORKFLOWS_TAG_EXISTS}}', '{{WORKFLOWS_TAG_EXISTS}}') + buildDoesNotExist: includes('${{COMMON_TAG_EXISTS}}', '{{COMMON_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_TAG_FORCE_BUILD}}', '{{WORKFLOWS_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{COMMON_TAG_FORCE_BUILD}}', '{{COMMON_TAG_FORCE_BUILD}}') == false samples: type: build @@ -386,7 +387,7 @@ steps: == true forceNoCache: includes('${{SAMPLES_TAG_FORCE_BUILD}}', '{{SAMPLES_TAG_FORCE_BUILD}}') == false - common: + volumemanager: type: build stage: build dockerfile: Dockerfile @@ -397,18 +398,18 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_FLASK=${{REGISTRY}}/cloudharness/cloudharness-flask:${{CLOUDHARNESS_FLASK_TAG}} - image_name: cloudharness/common - title: Common - working_directory: ./applications/common/server - tag: '${{COMMON_TAG}}' + image_name: cloudharness/volumemanager + title: Volumemanager + working_directory: ./applications/volumemanager/server + tag: '${{VOLUMEMANAGER_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{COMMON_TAG_EXISTS}}', '{{COMMON_TAG_EXISTS}}') + buildDoesNotExist: includes('${{VOLUMEMANAGER_TAG_EXISTS}}', '{{VOLUMEMANAGER_TAG_EXISTS}}') == true - forceNoCache: includes('${{COMMON_TAG_FORCE_BUILD}}', '{{COMMON_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{VOLUMEMANAGER_TAG_FORCE_BUILD}}', '{{VOLUMEMANAGER_TAG_FORCE_BUILD}}') == false - volumemanager: + workflows: type: build stage: build dockerfile: Dockerfile @@ -419,16 +420,16 @@ steps: - NOCACHE=${{CF_BUILD_ID}} - REGISTRY=${{REGISTRY}}/cloudharness/ - CLOUDHARNESS_FLASK=${{REGISTRY}}/cloudharness/cloudharness-flask:${{CLOUDHARNESS_FLASK_TAG}} - image_name: cloudharness/volumemanager - title: Volumemanager - working_directory: ./applications/volumemanager/server - tag: '${{VOLUMEMANAGER_TAG}}' + image_name: cloudharness/workflows + title: Workflows + working_directory: ./applications/workflows/server + tag: '${{WORKFLOWS_TAG}}' when: condition: any: - buildDoesNotExist: includes('${{VOLUMEMANAGER_TAG_EXISTS}}', '{{VOLUMEMANAGER_TAG_EXISTS}}') + buildDoesNotExist: includes('${{WORKFLOWS_TAG_EXISTS}}', '{{WORKFLOWS_TAG_EXISTS}}') == true - forceNoCache: includes('${{VOLUMEMANAGER_TAG_FORCE_BUILD}}', '{{VOLUMEMANAGER_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{WORKFLOWS_TAG_FORCE_BUILD}}', '{{WORKFLOWS_TAG_FORCE_BUILD}}') == false title: Build parallel step 3 tests_unit: @@ -465,12 +466,12 @@ steps: - kubectl config use-context ${{CLUSTER_NAME}} - kubectl config set-context --current --namespace=test-${{NAMESPACE_BASENAME}} - kubectl rollout status deployment/workflows - - kubectl rollout status deployment/accounts - - kubectl rollout status deployment/samples - - kubectl rollout status deployment/samples-gk - kubectl rollout status deployment/common - kubectl rollout status deployment/volumemanager + - kubectl rollout status deployment/accounts - kubectl rollout status deployment/argo-gk + - kubectl rollout status deployment/samples + - kubectl rollout status deployment/samples-gk - sleep 60 tests_api: stage: qa diff --git a/docs/base-common-images.md b/docs/base-common-images.md index 29a5e2bfd..6d0ac6bf0 100644 --- a/docs/base-common-images.md +++ b/docs/base-common-images.md @@ -1,9 +1,10 @@ # Define and use base and common images ## Relevant files and directory structure - - `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks. - - `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks. - + +- `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks. +- `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks. + ## Base images and common images The main difference between the base images and common images is that base images are built in the root context, while @@ -25,6 +26,7 @@ harness: ``` 2. Refer to the base image with the uppercased-underscored name of the dependency as an argument + ```dockerfile ARG CLOUDHARNESS_BASE FROM $CLOUDHARNESS_BASE @@ -39,19 +41,18 @@ In workflow tasks, the build dependency must be specified in the main applicatio ## Default images CloudHarness defines the following base images: + - `cloudharness-base`: python-alpine with cloudharness common libraries preinstalled -- `cloudharness-base-debian`: python-debian with cloudharness common libraries preinstalled - `cloudharness-django`: cloudharness-base with cloudharness django fastapi libraries preinstalled -- `cloudharness-django-debian`: cloudharness-base-debian with cloudharness django fastapi libraries preinstalled - `cloudharness-fastapi`: cloudharness-base with fastapi libraries preinstalled -- `cloudharness-fastapi-debian`: cloudharness-base-debian with fastapi libraries preinstalled Also the following common images are defined: + - `cloudharness-flask`: common ground image to create Flask backends ## Override base and common images from CloudHarness -To override a base or common image just create the same directory path in your +To override a base or common image just create the same directory path in your solution. The overriding can be used to replace files used in the build process or the Dockerfile itself. For example, overriding `cloudharness-base` could be useful to change some behaviour in the CloudHarness @@ -59,4 +60,3 @@ libraries or to provide new libraries to share within all applications. To override cloudharness-base, create a directory `MY_SOLUTION/infrastructure/base-images/cloudharness-base` then run `harness-deployment cloudharness MY_SOLUTION` - diff --git a/infrastructure/base-images/cloudharness-base-debian/Dockerfile b/infrastructure/base-images/cloudharness-base-debian/Dockerfile deleted file mode 100644 index 6955614ab..000000000 --- a/infrastructure/base-images/cloudharness-base-debian/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -ARG PARENT=python:3.12 -FROM ${PARENT} - -RUN apt-get update && apt-get install -y nfs-common && rm -rf /var/lib/apt/lists/* - -RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ - pip install pytest --prefer-binary -COPY libraries/models/requirements.txt /libraries/models/requirements.txt -RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ - pip install -r /libraries/models/requirements.txt --prefer-binary -COPY libraries/cloudharness-common/requirements.txt /libraries/cloudharness-common/requirements.txt -RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ - pip install -r /libraries/cloudharness-common/requirements.txt --prefer-binary -COPY libraries/client/cloudharness_cli/requirements.txt /libraries/client/cloudharness_cli/requirements.txt -RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ - pip install -r /libraries/client/cloudharness_cli/requirements.txt --prefer-binary - -COPY libraries/models /libraries/models -RUN pip install -e /libraries/models --no-cache-dir - -COPY libraries/cloudharness-common /libraries/cloudharness-common -COPY libraries/client/cloudharness_cli /libraries/client/cloudharness_cli - -# -RUN pip install -e /libraries/models --no-cache-dir -RUN pip install -e /libraries/cloudharness-common --no-cache-dir -RUN pip install -e /libraries/client/cloudharness_cli --no-cache-dir - -ARG $DEBUG -RUN if [[ -z "$DEBUG" ]] ; then pip install debugpy --prefer-binary ; else echo "Debug not supported" ; fi - -WORKDIR / \ No newline at end of file diff --git a/infrastructure/base-images/cloudharness-base/Dockerfile b/infrastructure/base-images/cloudharness-base/Dockerfile index a4537877b..5e870d63e 100644 --- a/infrastructure/base-images/cloudharness-base/Dockerfile +++ b/infrastructure/base-images/cloudharness-base/Dockerfile @@ -1,13 +1,7 @@ -ARG PARENT=python:3.12-alpine +ARG PARENT=python:3.12 FROM ${PARENT} -RUN apk update -RUN apk upgrade - -# Add bash for k8s console -RUN apk add bash -# dev tools needed by some python libraries -RUN apk add gcc libc-dev g++ python3-dev libffi-dev openssl-dev rust musl-dev cargo +RUN apt-get update && apt-get install -y nfs-common && rm -rf /var/lib/apt/lists/* RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ pip install pytest --prefer-binary @@ -23,15 +17,14 @@ RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip & pip install -r /libraries/client/cloudharness_cli/requirements.txt --prefer-binary COPY libraries/models /libraries/models -RUN pip install -e /libraries/models +RUN pip install -e /libraries/models --no-cache-dir COPY libraries/cloudharness-common /libraries/cloudharness-common COPY libraries/client/cloudharness_cli /libraries/client/cloudharness_cli -# - -RUN pip install -e /libraries/cloudharness-common -RUN pip install -e /libraries/client/cloudharness_cli +RUN pip install -e /libraries/models --no-cache-dir +RUN pip install -e /libraries/cloudharness-common --no-cache-dir +RUN pip install -e /libraries/client/cloudharness_cli --no-cache-dir ARG $DEBUG RUN if [[ -z "$DEBUG" ]] ; then pip install debugpy --prefer-binary ; else echo "Debug not supported" ; fi diff --git a/infrastructure/common-images/cloudharness-django/Dockerfile b/infrastructure/common-images/cloudharness-django/Dockerfile index 941dad578..89a090891 100644 --- a/infrastructure/common-images/cloudharness-django/Dockerfile +++ b/infrastructure/common-images/cloudharness-django/Dockerfile @@ -1,5 +1,5 @@ -ARG CLOUDHARNESS_BASE_DEBIAN -FROM $CLOUDHARNESS_BASE_DEBIAN +ARG CLOUDHARNESS_BASE +FROM $CLOUDHARNESS_BASE ENV MODULE_NAME=backend ENV PORT=8080 diff --git a/infrastructure/common-images/cloudharness-django/README.md b/infrastructure/common-images/cloudharness-django/README.md index 1150631fb..e7abf0078 100644 --- a/infrastructure/common-images/cloudharness-django/README.md +++ b/infrastructure/common-images/cloudharness-django/README.md @@ -1,3 +1,3 @@ -# CloudHarness-Django Base Debian image +# CloudHarness-Django Base image Use this image to bring the package cloudharness-django into your image. diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py index 8f9d5c8d4..178e3b30b 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py @@ -99,4 +99,4 @@ if settings.ROOT_URLCONF: APP_URLCONF = settings.ROOT_URLCONF - ROOT_URLCONF = "cloudharness_django.urls" \ No newline at end of file + ROOT_URLCONF = "cloudharness_django.urls" diff --git a/infrastructure/common-images/cloudharness-fastapi/Dockerfile b/infrastructure/common-images/cloudharness-fastapi/Dockerfile index 257438faa..00de6ecc6 100644 --- a/infrastructure/common-images/cloudharness-fastapi/Dockerfile +++ b/infrastructure/common-images/cloudharness-fastapi/Dockerfile @@ -1,5 +1,5 @@ -ARG CLOUDHARNESS_BASE_DEBIAN -FROM $CLOUDHARNESS_BASE_DEBIAN +ARG CLOUDHARNESS_BASE +FROM $CLOUDHARNESS_BASE ENV MODULE_NAME=backend ENV PORT=8080 diff --git a/infrastructure/common-images/cloudharness-fastapi/README.md b/infrastructure/common-images/cloudharness-fastapi/README.md index 96949d87d..da4516e89 100644 --- a/infrastructure/common-images/cloudharness-fastapi/README.md +++ b/infrastructure/common-images/cloudharness-fastapi/README.md @@ -1,3 +1,3 @@ -# CloudHarness-FastAPI Base Debian image +# CloudHarness-FastAPI Base image Use this image for FastAPI based microservices. diff --git a/libraries/cloudharness-utils/cloudharness_utils/constants.py b/libraries/cloudharness-utils/cloudharness_utils/constants.py index 47771e6dd..479debca0 100644 --- a/libraries/cloudharness-utils/cloudharness_utils/constants.py +++ b/libraries/cloudharness-utils/cloudharness_utils/constants.py @@ -30,7 +30,7 @@ VALUES_MANUAL_PATH = 'values.yaml' VALUE_TEMPLATE_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/value-template.yaml' -CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10-alpine', 'cloudharness-base-debian': 'python:3.9.10'} +CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10'} CD_BUILD_STEP_PARALLEL = 'build_application_images' diff --git a/tools/deployment-cli-tools/ch_cli_tools/migration.py b/tools/deployment-cli-tools/ch_cli_tools/migration.py new file mode 100644 index 000000000..fa094f632 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/migration.py @@ -0,0 +1,151 @@ +""" +Utilities to perform a migration of the deployment to the latest supported version. +""" + +import os +import json + +from . import HERE +from cloudharness_utils.constants import APPS_PATH +from .utils import ( + get_sub_paths, + search_word_in_file, + search_word_in_folder, + search_word_by_pattern, +) + +TO_CHECK = ["deploy", "tasks", "Dockerfile"] + + +def read_file_and_replace(file, old, new, accept_all=False): + file_p = open(file, "r+") + lines = file_p.readlines() + file_p.seek(0) + for line in lines: + if old in line: + print(f"Found {old} in {file}") + + if not accept_all: + print(f"Would you like to replace:") + print(line) + print(f"with:") + print(line.replace(old, new)) + print("y/n") + + if accept_all: + file_p.write(line.replace(old, new)) + elif input() == "y": + file_p.write(line.replace(old, new)) + else: + file_p.write(line) + else: + file_p.write(line) + file_p.truncate() + file_p.close() + + +def perform_migration(base_root, accept_all=False): + all_files_detected = [] + + # Iterate the list of files and replace the old word with the new word + def replace_in_files(app_path, files, words): + for file in files: + file_path = os.path.join(app_path, file) + print(f">>> Running migration on {file_path} <<<") + all_files_detected.append(file_path) + for word_to_replace in words: + read_file_and_replace( + file_path, + word_to_replace["old"], + word_to_replace["new"], + accept_all, + ) + + # Iterate migrations, patterns and search the keyword using the pattern given. + # If the keyword is found then replace the list of words given from the old to the new + def run_migrations_with_pattern(app_path, with_pattern): + for single_migration in with_pattern: + for pattern in single_migration["patterns"]: + results = search_word_by_pattern( + app_path, pattern, single_migration["keyword"] + ) + replace_in_files( + app_path, + results, + single_migration["to_be_replaced"] + ) + + # Iterate the TO_CHECK array that is set to search inside the deploy and tasks folder + # plus the Dockerfile. If the keyword is found inside one of these folders or in the + # Dockerfile then replace the list of words given from the old to the new + def run_migrations_without_pattern(app_path, without_pattern): + for sub_path in TO_CHECK: + to_check = os.path.join(app_path, sub_path) + if os.path.isdir(to_check): + for migration_obj in without_pattern: + files = search_word_in_folder( + to_check, + migration_obj["keyword"] + ) + replace_in_files( + app_path, + files, + migration_obj["to_be_replaced"] + ) + elif os.path.isfile(to_check): + for migration_obj in without_pattern: + keyword = migration_obj["keyword"] + if len(search_word_in_file(to_check, keyword)) > 0: + print(f">>> Running migration on {to_check} <<<") + all_files_detected.append(to_check) + for word_to_replace in migration_obj["to_be_replaced"]: + read_file_and_replace( + to_check, + word_to_replace["old"], + word_to_replace["new"], + accept_all, + ) + + # Parse the migration and separate the ones with patterns and without + def extract_migrations(migration_obj): + migrations_with_pattern = [] + migrations_without_pattern = [] + for single_migration in migration_obj["deprecated"]: + if "patterns" in single_migration: + migrations_with_pattern.append(single_migration) + else: + migrations_without_pattern.append(single_migration) + return migrations_with_pattern, migrations_without_pattern + + # Iterate all the migration files and run the migrations + for file in os.listdir(os.path.join(HERE, "migrations")): + with open(os.path.join(HERE, "migrations", file), "r") as f: + migration_dict = json.load(f) + + sub_paths = [] + # Find all the apps in the base root + if isinstance(base_root, list): + app_base_path = [ + os.path.join(base_root_path, APPS_PATH) + for base_root_path in base_root + ] + for app_path in app_base_path: + sub_paths.extend(get_sub_paths(app_path)) + else: + app_base_path = os.path.join(base_root, APPS_PATH) + sub_paths = get_sub_paths(app_base_path) + + with_pattern, without_pattern = extract_migrations(migration_dict) + # Iterate all the application folders and run the migrations + for app_path in sub_paths: + run_migrations_with_pattern(app_path, with_pattern) + run_migrations_without_pattern(app_path, without_pattern) + print("=========================================") + print(">>> Cloud Harness migration completed <<<") + print("=========================================") + print(">>> Migration Summary <<<") + for file in all_files_detected: + print(f"{file}") + print("=========================================") + print(">>> End of Summary <<<") + print("=========================================") diff --git a/tools/deployment-cli-tools/ch_cli_tools/migrations/1_migration.json b/tools/deployment-cli-tools/ch_cli_tools/migrations/1_migration.json new file mode 100644 index 000000000..ec9dd3360 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/migrations/1_migration.json @@ -0,0 +1,47 @@ +{ + "deprecated": [ + { + "keyword": "CLOUDHARNESS_BASE_DEBIAN", + "to_be_replaced": [ + { + "old": "CLOUDHARNESS_BASE_DEBIAN", + "new": "CLOUDHARNESS_BASE" + }, + { + "old": "apk add", + "new": "apt-get install" + }, + { + "old": "apk update", + "new": "apt-get update" + }, + { + "old": "apk upgrade", + "new": "apt-get upgrade" + }, + { + "old": "apk add --no-cache", + "new": "apt-get install --no-cache" + }, + { + "old": "apk update --no-cache", + "new": "apt-get update --no-cache" + }, + { + "old": "apk upgrade --no-cache", + "new": "apt-get upgrade --no-cache" + } + ], + "patterns": ["**/Dockerfile", "**/*yaml"] + }, + { + "keyword": "cloudharness-base-debian", + "to_be_replaced": [ + { + "old": "cloudharness-base-debian", + "new": "cloudharness-base" + } + ] + } + ] +} \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 8f9f5e45f..c88bee158 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -471,6 +471,38 @@ def check_docker_manifest_exists(registry, image_name, tag, registry_secret=None return resp.status_code == 200 +def filter_empty_strings(value): + return value != "" + + +def search_word_in_file(filename, word): + if os.path.isdir(filename): + return [] + matches = [] + with open(filename) as f: + if word in f.read(): + matches.append(filename) + return list(filter(filter_empty_strings, matches)) + + +def search_word_in_folder(folder, word): + matches = [] + files = glob.glob(folder + '/**/*', recursive=True) + for file in files: + matches.extend(search_word_in_file(file, word)) + return list(filter(filter_empty_strings, matches)) + + +def search_word_by_pattern(folder, pattern, word): + if not folder.endswith('/'): + folder += '/' + matches = [] + files = glob.glob(folder + pattern, recursive=True) + for file in files: + matches.extend(search_word_in_file(file, word)) + return list(filter(filter_empty_strings, matches)) + + def get_git_commit_hash(path): # return the short git commit hash in that path # if the path is not a git repo, return None diff --git a/tools/deployment-cli-tools/harness-deployment b/tools/deployment-cli-tools/harness-deployment index f4c9fe3db..44445a068 100644 --- a/tools/deployment-cli-tools/harness-deployment +++ b/tools/deployment-cli-tools/harness-deployment @@ -11,6 +11,7 @@ from ch_cli_tools.skaffold import create_skaffold_configuration, create_vscode_d from ch_cli_tools.codefresh import create_codefresh_deployment_scripts, write_env_file from ch_cli_tools.preprocessing import preprocess_build_overrides from ch_cli_tools.utils import merge_app_directories +from ch_cli_tools.migration import perform_migration from cloudharness_utils.constants import DEPLOYMENT_PATH, COMPOSE_ENGINE, HELM_ENGINE HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/') @@ -63,6 +64,10 @@ if __name__ == "__main__": help=f'Do not generate ci/cd files') parser.add_argument('-we', '--write-env', dest='write_env', action="store_const", default=None, const=True, help=f'Write build env to .env file in {DEPLOYMENT_PATH}') + parser.add_argument('--migrate', dest='migration', action="store_true", + help='Perform a migration of the deployment to the latest supported version.') + parser.add_argument('--migrate-accept-all', dest='migration_accept_all', action="store_true", + help='Perform a migration of the deployment to the latest supported version without asking for confirmation. Use with caution.') parser.add_argument('--docker-compose', dest='docker_compose', action="store_true", help='Generate docker-compose.yaml and dedicated Skaffold configuration') @@ -77,13 +82,17 @@ if __name__ == "__main__": print('There are unknown args. Make sure to call the script with the accepted args. Try --help') print(f'unknown: {unknown}') else: - if args.merge: logging.warning( "Merge (-m, --merge) argument is deprecated. Directory merging is now set automatically") merge_app_directories(root_paths, destination=args.merge) root_paths = [args.merge] + if args.migration_accept_all: + perform_migration(root_paths, accept_all=True) + elif args.migration: + perform_migration(root_paths, accept_all=False) + chart_fn = create_helm_chart if not args.docker_compose else create_docker_compose_configuration helm_values = chart_fn( diff --git a/tools/deployment-cli-tools/harness-migrate b/tools/deployment-cli-tools/harness-migrate new file mode 100644 index 000000000..1b414a573 --- /dev/null +++ b/tools/deployment-cli-tools/harness-migrate @@ -0,0 +1,17 @@ +#!/usr/bin/env python +import os +import sys +from ch_cli_tools.migration import perform_migration + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description='Perform a migration of the deployment to the latest supported version.') + + parser.add_argument('--accept-all', dest='migration_accept_all', action="store_true", + help='Perform a migration of the deployment to the latest supported version without asking for confirmation. Use with caution.') + + args, unknown = parser.parse_known_args(sys.argv[1:]) + + perform_migration(os.getcwd(), accept_all=args.migration_accept_all) diff --git a/tools/deployment-cli-tools/setup.py b/tools/deployment-cli-tools/setup.py index 1b9b1bf2d..60a757f73 100644 --- a/tools/deployment-cli-tools/setup.py +++ b/tools/deployment-cli-tools/setup.py @@ -41,7 +41,7 @@ install_requires=REQUIREMENTS, packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), include_package_data=True, - scripts=['harness-deployment', 'harness-generate', 'harness-application'], + scripts=['harness-deployment', 'harness-generate', 'harness-application', 'harness-migrate'], long_description="""\ CloudHarness deploy library """ diff --git a/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile b/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile new file mode 100644 index 000000000..721ff5e5c --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apt install --no-cache bash diff --git a/tools/deployment-cli-tools/tests/resources/applications/migration_app/tasks/sub-task/Dockerfile b/tools/deployment-cli-tools/tests/resources/applications/migration_app/tasks/sub-task/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/migration_app/tasks/sub-task/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/Dockerfile b/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/tasks/sub-task/Dockerfile b/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/tasks/sub-task/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/tasks/sub-task/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/Dockerfile b/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/tasks/sub-task/Dockerfile b/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/tasks/sub-task/Dockerfile new file mode 100644 index 000000000..12e4f487d --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/migration/backup/migration_app/tasks/sub-task/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_BASE_DEBIAN +FROM $CLOUDHARNESS_BASE_DEBIAN + +RUN apk add --no-cache bash diff --git a/tools/deployment-cli-tools/tests/test_codefresh.py b/tools/deployment-cli-tools/tests/test_codefresh.py index 31dba2f49..92a7ce01b 100644 --- a/tools/deployment-cli-tools/tests/test_codefresh.py +++ b/tools/deployment-cli-tools/tests/test_codefresh.py @@ -59,7 +59,7 @@ def test_create_codefresh_configuration(): assert step_build_base["type"] == "parallel" steps = l1_steps[STEP_0]["steps"] - assert len(steps) == 8, "all images that do not depend on othe builds should be included in the first step" + assert len(steps) == 7, "all images that do not depend on othe builds should be included in the first step" assert "cloudharness-base" in steps, "cloudharness-base image should be included as dependency" assert "cloudharness-base-debian" not in steps, "cloudharness-base image should not be included" assert "cloudharness-frontend-build" in steps, "cloudharness-frontend-build image should be included as dependency" diff --git a/tools/deployment-cli-tools/tests/test_migration.py b/tools/deployment-cli-tools/tests/test_migration.py new file mode 100644 index 000000000..532ac5df6 --- /dev/null +++ b/tools/deployment-cli-tools/tests/test_migration.py @@ -0,0 +1,19 @@ +from ch_cli_tools.utils import * +from ch_cli_tools.migration import perform_migration + +HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/') + + +def test_migration_accept_all(): + found = search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN") + print(found) + assert len(found) == 2 + try: + perform_migration(os.path.join(HERE, './resources/migration'), accept_all=True) + found = search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN") + print(found) + assert len(found) == 0 + + finally: + os.system(f'cp -R {os.path.join(HERE, "resources/migration/backup/migration_app")} {os.path.join(HERE, "resources/migration/applications/")}') + assert len(search_word_in_folder(os.path.join(HERE, './resources/migration/applications'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 diff --git a/tools/deployment-cli-tools/tests/test_utils.py b/tools/deployment-cli-tools/tests/test_utils.py index 0c4060f36..aadc8764c 100644 --- a/tools/deployment-cli-tools/tests/test_utils.py +++ b/tools/deployment-cli-tools/tests/test_utils.py @@ -77,6 +77,14 @@ def test_check_docker_manifest_exists(): assert not check_docker_manifest_exists("gcr.io/metacellllc", "cloudharness/cloudharness-base", "RANDOM_TAG") +def test_search_word_in_file(): + assert len(search_word_in_file(os.path.join(HERE, './resources/applications/migration_app/Dockerfile'), "CLOUDHARNESS_BASE_DEBIAN")) == 1 + + +def test_search_word_in_folder(): + assert len(search_word_in_folder(os.path.join(HERE, './resources/applications/migration_app/'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 + + def test_find_dockerfile_paths(): myapp_path = os.path.join(HERE, "resources/applications/myapp")