diff --git a/README.md b/README.md index 44c494d..c9f3ede 100644 --- a/README.md +++ b/README.md @@ -10,16 +10,11 @@ Available actions are: 5. [generate_version](#generate_version) 6. [install_azcopy](#install_azcopy) 7. [login_to_aks](#login_to_aks) -8. [deploy_poetry_project_to_azfs](#deploy_poetry_project_to_azfs) -9. [deploy_dbt_project_to_azfs](#deploy_dbt_project_to_azfs) -10. [deploy_data_schemas_to_azfs](#deploy_data_schemas_to_azfs) -11. [run_azcopy](#run_azcopy) -12. [get_azure_share_sas](#get_azure_share_sas) -13. [setup_gh_app](#setup_gh_app) -14. [update_airflow_variables](#update_airflow_variables) -15. [contribute_changes](#contribute_changes) -16. [activate_workflow](#activate_workflow) -16. [setup_aws_ca](#setup_aws_ca) +8. [setup_gh_app](#setup_gh_app) +9. [update_airflow_variables](#update_airflow_variables) +10. [contribute_changes](#contribute_changes) +11. [activate_workflow](#activate_workflow) +12. [setup_aws_ca](#setup_aws_ca) ## semver_release @@ -240,10 +235,8 @@ Generates project version based on current git commit and git tags. 1) To use this action, your repository should contain [version tags](https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases). This action relies on git tags to be present in order to generate a version. -2) Generated version is will not be compatible with [PEP-440](https://peps.python.org/pep-0440/), so this versions -should not be used with python packages. Although, this action can be used with -[source code deployments](#deploy_poetry_project_to_azfs) of python applications. - +2) Generated version will not be compatible with [PEP-440](https://peps.python.org/pep-0440/), so these versions +should not be used with python packages. ### Inputs No inputs defined @@ -339,245 +332,6 @@ jobs: cluster_name: $AZURE_AKS_NAME ``` -# deploy_poetry_project_to_azfs -Copy python site-packages of current virtual environment and installs application into it. - -### Inputs -| Name | Description | Optional | Default value | -|-------------------|:-----------------------------------------------------------------|----------|---------------| -| project_version | Version of the project | False | | -| project_name | Name of the project | False | | -| project_directory | Directory name inside the project (if differs from project name) | True | "" | -| destination | Directory or SAS for upload | False | | -| python_version | Project python version | True | 3.9 | -| deployment_root | Root directory in the file share | False | | - -**NOTES**: -1) To use this action, your project should use poetry for virtual environment management. Ensure that you installed -the latest version of poetry and project dependencies (for instance, by [install_poetry](#install_poetry) action). - -### Outputs -No outputs defined - -### Usage -```yaml -name: Prepare python deployment - -on: - workflow_dispatch: - -jobs: - prepare_deployment: - name: Prepare python code for deployment - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Get project version - uses: SneaksAndData/github-actions/generate_version@v0.0.17 - id: version - - name: Generate SAS for upload - uses: SneaksAndData/github-actions/get_azure_share_sas@v0.0.17 - with: - directory_name: share-name/path/within/share - account_key: ${{ secrets.ACCOUNT_KEY }} - account_name: ${{ secrets.ACCOUNT_NAME }} - id: sas - - name: Prepare site-packages for deployment - uses: SneaksAndData/github-actions/deploy_poetry_project_to_azfs@v0.0.17 - with: - deployment_root: /python - project_version: ${{ steps.version.outputs.version }} - destination: ${{ steps.sas.outputs.authorized_destination }} - project_name: python_project -``` - -# deploy_dbt_project_to_azfs -Prepare DBT models for deployment to an Azure file share. - -### Inputs -| Name | Description | Optional | Default value | -|------------------|:----------------------------------------------|----------|---------------| -| project_version | Version of the project | False | | -| project_name | Name of the project | False | | -| destination | Directory or SAS for upload | False | | -| deployment_root | Root directory in the file share | False | | - -### Outputs -No outputs defined - -### Usage -```yaml -name: Prepare deployment - -on: - workflow_dispatch: - -jobs: - prepare_deployment: - name: Prepare dbt output for deployment - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Get project version - uses: SneaksAndData/github-actions/generate_version@v0.0.17 - id: version - - name: Generate SAS for upload - uses: SneaksAndData/github-actions/get_azure_share_sas@v0.0.17 - with: - directory_name: share-name/path/within/share - account_key: ${{ secrets.ACCOUNT_KEY }} - account_name: ${{ secrets.ACCOUNT_NAME }} - id: sas - - name: Prepare dbt for deployment - uses: SneaksAndData/github-actions/deploy_dbt_project_to_azfs@v0.0.17 - with: - deployment_root: /dbt - project_version: ${{ steps.version.outputs.version }} - destination: ${{ steps.sas.outputs.authorized_destination }} - project_name: dbt_project -``` - -# deploy_data_schemas_to_azfs -Prepare DBT schemas for deployment to an Azure file share. - -### Inputs -| Name | Description | Optional | Default value | -|------------------|:----------------------------------------------|----------|---------------| -| project_version | Version of the project | False | | -| project_name | Name of the project | False | | -| destination | Directory or SAS for upload | False | | -| deployment_root | Root directory in the file share | False | | - -### Outputs -No outputs defined - -### Usage -```yaml -name: Prepare deployment - -on: - workflow_dispatch: -jobs: - validate_commit: - name: Prepare schemas output for deployment - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Get project version - uses: SneaksAndData/github-actions/generate_version@v0.0.17 - id: version - - name: Generate SAS for upload - uses: SneaksAndData/github-actions/get_azure_share_sas@v0.0.17 - with: - directory_name: share-name/path/within/share - account_key: ${{ secrets.ACCOUNT_KEY }} - account_name: ${{ secrets.ACCOUNT_NAME }} - id: sas - - name: Prepare dbt for deployment - uses: SneaksAndData/github-actions/deploy_data_schemas_to_azfs@v0.0.17 - with: - deployment_root: /dbt - project_version: ${{ steps.version.outputs.version }} - destination: ${{ steps.sas.outputs.authorized_destination }} - project_name: dbt_project -``` - -# run_azcopy -Invoke [azcopy copy](https://learn.microsoft.com/en-us/azure/storage/common/storage-ref-azcopy-copy?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json) -command in pipeline. - -### Inputs -| Name | Description | Optional | Default value | -|--------------------|:-----------------------------------------------|----------|---------------| -| source | Source directory or SAS url to copy | False | | -| target | Target directory or SAS url | False | | -| mode | azcopy action mode (copy or sync) | True | copy | -| put_md5 | If `true` sets `--put-md5` parameter to azcopy | True | True | -| delete_destination | azcopy --delete-destination flag | True | False | - -### Outputs -No outputs defined - -### Usage -```yaml -name: Copy files - -on: - workflow_dispatch: - -jobs: - copy_files: - name: Copy files - steps: - - name: Generate SAS for upload - uses: SneaksAndData/github-actions/get_azure_share_sas@v0.0.17 - with: - directory_name: share-name/path/within/share - account_key: ${{ secrets.ACCOUNT_KEY }} - account_name: ${{ secrets.ACCOUNT_NAME }} - id: sas - - name: Copy data - uses: SneaksAndData/github-actions/run_azcopy@v0.0.17 - with: - source: source/directory/on/build/agent - target: ${{ steps.sas.outputs.authorized_destination }} -``` -## get_azure_share_sas - -### Description -Generates new temporary -[Shared Access Signature](https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview) -for a file share, attached to a storage account. - -### Inputs -| Name | Description | Optional | Default Value | -|-----------------|:-----------------------------------------------------------------|----------|---------------| -| directory_name | Path within file share | False | | -| account_key | Name of the storage account of the share | False | | -| account_name | Key of the storage account of the share | False | | -| expiration_date | Expiration date in format that can be used by the `date` command | True | +10 minutes | -| directory_type | Type of directory (blob or fileshare) | True | fileshare | - - -**NOTES**: -1) For the expiration date format see [man 1 date](https://man7.org/linux/man-pages/man1/date.1.html) - -### Outputs -| Name | Description | -|------------------------|-------------------------------------------------------------| -| authorized_destination | URL of the file share with attached shared access signature | - -### Usage -```yaml -name: Release a new version - -on: - workflow_dispatch: - -jobs: - create_release: - runs-on: ubuntu-latest - steps: - - name: Generate SAS for upload - uses: SneaksAndData/github-actions/get_azure_share_sas@v0.0.17 - with: - directory_name: share-name/path/within/share - account_key: ${{ secrets.ACCOUNT_KEY }} - account_name: ${{ secrets.ACCOUNT_NAME }} - id: sas - - name: Copy data - uses: SneaksAndData/github-actions/run_azcopy@v0.0.17 - with: - source_directory: source/directory/on/build/agent - target: ${{ steps.sas.outputs.authorized_destination }} -``` - ## setup_gh_app ### Description diff --git a/deploy_data_schemas_to_azfs/action.yaml b/deploy_data_schemas_to_azfs/action.yaml deleted file mode 100644 index 146f55d..0000000 --- a/deploy_data_schemas_to_azfs/action.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: Deploy schemas -description: Deploy schemas - -branding: - icon: 'battery-charging' - color: 'green' - -inputs: - project_version: - description: Version of project to deploy - required: true - - project_name: - description: Name of project to deploy - required: true - - destination: - required: true - description: Destination directory or SAS token - - deployment_root: - required: true - description: Root directory in the deployment file structure - -runs: - using: "composite" - steps: - - name: Prepare python deployment - shell: bash - run: $GITHUB_ACTION_PATH/deploy_data_schemas_to_azfs.sh - env: - DEPLOYMENT_ROOT: ${{ inputs.deployment_root }} - PROJECT_VERSION: ${{ inputs.project_version }} - PROJECT_NAME: ${{ inputs.project_name }} - DESTINATION: ${{ inputs.destination }} diff --git a/deploy_data_schemas_to_azfs/deploy_data_schemas_to_azfs.sh b/deploy_data_schemas_to_azfs/deploy_data_schemas_to_azfs.sh deleted file mode 100755 index 23536f0..0000000 --- a/deploy_data_schemas_to_azfs/deploy_data_schemas_to_azfs.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2022 Ecco Sneaks & Data -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -Eeuo pipefail - -SOURCE_DIRECTORY="./$DEPLOYMENT_ROOT/$PROJECT_NAME-schemas/$PROJECT_VERSION/" -mkdir -p "$SOURCE_DIRECTORY" -mv -v ./schemas/* "$SOURCE_DIRECTORY" - -./azcopy copy "./$SOURCE_DIRECTORY/*" "$DESTINATION" --recursive --overwrite true --put-md5 diff --git a/deploy_dbt_project_to_azfs/action.yaml b/deploy_dbt_project_to_azfs/action.yaml deleted file mode 100644 index 241b9d9..0000000 --- a/deploy_dbt_project_to_azfs/action.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: Deploy DBT -description: Deploy source code of DBT queries - -branding: - icon: 'battery-charging' - color: 'green' - -inputs: - project_version: - description: Version of project to deploy - required: true - - project_name: - description: Name of project to deploy - required: true - - destination: - required: true - description: Destination directory or SAS token - - deployment_root: - required: true - description: Root directory in the deployment file structure - -runs: - using: "composite" - steps: - - name: Prepare python deployment - shell: bash - run: $GITHUB_ACTION_PATH/deploy_dbt_project_to_azfs.sh - env: - DEPLOYMENT_ROOT: ${{ inputs.deployment_root }} - PROJECT_VERSION: ${{ inputs.project_version }} - PROJECT_NAME: ${{ inputs.project_name }} - DESTINATION: ${{ inputs.destination }} diff --git a/deploy_dbt_project_to_azfs/deploy_dbt_project_to_azfs.sh b/deploy_dbt_project_to_azfs/deploy_dbt_project_to_azfs.sh deleted file mode 100755 index 5cd9e48..0000000 --- a/deploy_dbt_project_to_azfs/deploy_dbt_project_to_azfs.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2022 Ecco Sneaks & Data -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -Eeuo pipefail - -SOURCE_DIRECTORY="./$DEPLOYMENT_ROOT/$PROJECT_NAME/$PROJECT_VERSION/" -mkdir -p "$SOURCE_DIRECTORY" -mv -v ./target/run/* "$SOURCE_DIRECTORY" - -./azcopy copy "./$SOURCE_DIRECTORY/*" "$DESTINATION" --recursive --overwrite true --put-md5 diff --git a/deploy_poetry_project_to_azfs/action.yaml b/deploy_poetry_project_to_azfs/action.yaml deleted file mode 100644 index 92b0843..0000000 --- a/deploy_poetry_project_to_azfs/action.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Deploy python code -description: Deploy source code of python application - -branding: - icon: 'battery-charging' - color: 'green' - -inputs: - project_version: - description: Version of project to deploy - required: true - - project_name: - description: Name of project to deploy - required: true - - destination: - required: true - description: Destination directory or SAS token - - project_directory: - description: Name of directory within the project to deploy - required: false - - python_version: - description: Python version - required: false - default: "3.9" - - deployment_root: - required: true - description: Root directory in the deployment file structure - -runs: - using: "composite" - steps: - - name: Prepare python deployment - shell: bash - run: $GITHUB_ACTION_PATH/deploy_poetry_project_to_azfs.sh - env: - DEPLOYMENT_ROOT: ${{ inputs.deployment_root }} - PROJECT_VERSION: ${{ inputs.project_version }} - PROJECT_NAME: ${{ inputs.project_name }} - PROJECT_DIRECTORY : ${{ inputs.project_directory }} - PYTHON_VERSION: ${{ inputs.python_version }} - DESTINATION: ${{ inputs.destination }} diff --git a/deploy_poetry_project_to_azfs/deploy_poetry_project_to_azfs.sh b/deploy_poetry_project_to_azfs/deploy_poetry_project_to_azfs.sh deleted file mode 100755 index fbe0d04..0000000 --- a/deploy_poetry_project_to_azfs/deploy_poetry_project_to_azfs.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2022 Ecco Sneaks & Data -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -Eeuo pipefail - -echo "Preparing to deploy $PROJECT_NAME $PROJECT_VERSION" -set -Eeuo pipefail -env_path=$(poetry env info | grep Path | head -n 1 | cut -d':' -f2 | xargs) - -if [ -z "$PROJECT_DIRECTORY" ]; then - PROJECT_DIRECTORY="${PROJECT_NAME//-/_}" -fi; - -SOURCE_DIRECTORY="./$DEPLOYMENT_ROOT/$PROJECT_NAME/$PROJECT_VERSION/" - -mkdir -p "$SOURCE_DIRECTORY/$PROJECT_DIRECTORY" -mv -v "$env_path"/lib/python"$PYTHON_VERSION"/site-packages/* "$SOURCE_DIRECTORY" -mv -v ./"$PROJECT_DIRECTORY"/* "$SOURCE_DIRECTORY/$PROJECT_DIRECTORY" - -./azcopy copy "./$SOURCE_DIRECTORY/*" "$DESTINATION" --recursive --overwrite true diff --git a/get_azure_share_sas/action.yaml b/get_azure_share_sas/action.yaml deleted file mode 100644 index 3bd4a01..0000000 --- a/get_azure_share_sas/action.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: Deploy to PVC -description: Deploy application Azure file share connected to an AKS Kubernetes cluster as a persistent volume claim - -branding: - icon: 'battery-charging' - color: 'green' - -inputs: - directory_name: - description: Relative path inside Azure file share - required: true - - account_key: - description: Storage account key - required: true - - account_name: - description: Storage account name - required: true - - expiration_date: - description: Expiration date in format that can be used by date command - required: false - default: "+10 minutes" - - directory_type: - description: Type of directory (blob or fileshare) - required: false - default: "fileshare" - -outputs: - authorized_destination: - description: "Generated url with sas for upload" - value: ${{ steps.sas.outputs.authorized_destination }} - -runs: - using: "composite" - steps: - - name: Get volume shared action signature - shell: bash - run: $GITHUB_ACTION_PATH/get_azure_share_sas.sh - env: - DIRECTORY_NAME: ${{ inputs.directory_name }} - ACCOUNT_KEY: ${{ inputs.account_key }} - ACCOUNT_NAME: ${{ inputs.account_name }} - EXPIRATION_DATE: ${{ inputs.expiration_date }} - DIRECTORY_TYPE: ${{ inputs.directory_type }} - id: sas diff --git a/get_azure_share_sas/get_azure_share_sas.sh b/get_azure_share_sas/get_azure_share_sas.sh deleted file mode 100755 index edc06ae..0000000 --- a/get_azure_share_sas/get_azure_share_sas.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2022 Ecco Sneaks & Data -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -Eeuo pipefail - - -if [[ "$(echo "$DIRECTORY_TYPE" | tr '[:upper:]' '[:lower:]')" == 'fileshare' ]]; then - destination="https://$ACCOUNT_NAME.file.core.windows.net/$DIRECTORY_NAME" - services="f" -elif [[ "$(echo "$DIRECTORY_TYPE" | tr '[:upper:]' '[:lower:]')" == 'blob' ]]; then - destination="https://$ACCOUNT_NAME.blob.core.windows.net/$DIRECTORY_NAME" - services="b" -else - echo "Directory type \"$DIRECTORY_TYPE\" not supported" - exit 1; -fi; - - -echo "Expiration date \"$EXPIRATION_DATE\"" -if [[ "$OSTYPE" =~ ^darwin ]]; then - end=$(date -v+10M "+%Y-%m-%dT%H:%MZ") -else - end=$(date -d "$EXPIRATION_DATE" '+%Y-%m-%dT%H:%MZ') -fi; - -echo "Generating SAS for $destination with expiration date $end" -sas=$( - az storage account generate-sas \ - --account-key "$ACCOUNT_KEY" \ - --account-name "$ACCOUNT_NAME" \ - --expiry "$end" \ - --https-only \ - --permissions acdlpruw \ - --resource-types sco \ - --services "$services" | cut -d'"' -f2 -) - -authorized_destination="$destination?$sas" -echo "::add-mask::$sas" -echo "::add-mask::$authorized_destination" -echo "authorized_destination=$authorized_destination" >> "$GITHUB_OUTPUT" diff --git a/run_azcopy/action.yaml b/run_azcopy/action.yaml deleted file mode 100644 index c366e56..0000000 --- a/run_azcopy/action.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: Run azcopy -description: Copy/Sync files with azcopy - -branding: - icon: 'battery-charging' - color: 'green' - -inputs: - source: - required: true - description: A directory to copy files from - - target: - required: true - description: Destination directory or SAS token - - mode: - required: false - default: "copy" - description: azcopy action mode (copy or sync) - - put_md5: - required: false - default: "true" - description: | - Create an MD5 hash of each file, and save the hash as the Content-MD5 property - of the destination blob or file. (By default the hash is NOT created.) Only available when uploading. - - delete_destination: - required: false - default: "false" - description: | - When mode="sync", if the 'delete_destination' flag is set to true, then sync will delete files and blobs - at the destination that aren't present at the source. - -runs: - using: "composite" - steps: - - name: Prepare python deployment - shell: bash - run: $GITHUB_ACTION_PATH/run_azcopy.sh - env: - SOURCE: ${{ inputs.source }} - TARGET: ${{ inputs.target }} - MODE: ${{ inputs.mode }} - PUT_MD5: ${{ inputs.put_md5 }} - DELETE_DESTINATION: ${{ inputs.delete_destination }} diff --git a/run_azcopy/run_azcopy.sh b/run_azcopy/run_azcopy.sh deleted file mode 100755 index 984c96a..0000000 --- a/run_azcopy/run_azcopy.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2022 Ecco Sneaks & Data -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -Eeuo pipefail - -echo "Copy from $SOURCE to $TARGET" -AZCOPY_OPTIONS=("--recursive") -if [[ "$(echo "$MODE" | tr '[:upper:]' '[:lower:]')" == 'copy' ]]; then - AZCOPY_OPTIONS+=("--overwrite" "true") -fi; -if [[ "$(echo "$PUT_MD5" | tr '[:upper:]' '[:lower:]')" == 'true' ]]; then - AZCOPY_OPTIONS+=("--put-md5") -fi; -if [[ "$(echo "$DELETE_DESTINATION" | tr '[:upper:]' '[:lower:]')" == 'true' ]]; then - AZCOPY_OPTIONS+=("--delete-destination" "true") -fi; -./azcopy "$MODE" "$SOURCE" "$TARGET" "${AZCOPY_OPTIONS[@]}"