diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml new file mode 100644 index 0000000..eff94bb --- /dev/null +++ b/.github/workflows/build-push-deploy.yml @@ -0,0 +1,170 @@ +on: + workflow_call: + inputs: + service-name: + description: "Name of the service to build. Used as the default image name and src dir unless 'image-name' or 'src-path' are used." + type: string + required: true + stage-name: + description: "The backend environment we are building for (API calls are pointed to). This should be one of (development, staging, production)." + type: string + required: true + deploy-namespace: + description: "The Kubernetes namespace to deploy the service to." + type: string + required: false + docker-build-args: + description: "Extra args passed to 'docker build'." + type: string + required: false + docker-image-ref: + description: "The version number or sha used in creating image tag." + type: string + required: false + default: "${{ github.sha }}" + dockerfiles: + description: "JSON list of dockerfiles to build, e.g. ['Dockerfile1', 'Dockerfile2']" + type: string + required: false + default: "['Dockerfile']" + docker-bake-target: + description: "The target to build with docker bake." + type: string + required: false + docker-bake-platforms: + description: "The platforms to build with docker bake." + type: string + required: false + migration-job-file: + description: "The file path to the migration k8s job YAML." + type: string + required: false + default: "deployment/migration-job.yaml" + + + +jobs: + # Looks for PR labels like "deploy-to-" so we can deploy to those envs + get-deploy-labels: + name: Get Deploy Envs + runs-on: mdb-dev + concurrency: + group: ${{ github.workflow_ref }} # workflow_ref contains the workflow name and branch ref + cancel-in-progress: true # Cancel any in-progress runs on this branch - this one is newer + outputs: + deploy-envs: ${{ steps.get-labels.outputs.deploy-envs }} + steps: + - id: get-labels + uses: mindsdb/github-actions/get-deploy-labels@main + + + # Build docker image(s) based on Dockerfile(s) and push to ECR + build: + runs-on: mdb-dev + needs: [get-deploy-labels] + if: ${{ !inputs.docker-bake && needs.get-deploy-labels.outputs.deploy-envs != '[]' }} + strategy: + matrix: + dockerfile: ${{fromJson(inputs.dockerfiles)}} + concurrency: + group: ${{ github.workflow_ref }} # workflow_ref contains the workflow name and branch ref + cancel-in-progress: true # Cancel any in-progress runs on this branch - this one is newer + env: + AWS_REGION: us-east-1 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.docker-image-ref }} + # Build via docker-bake if a bakefile is specified + - if: ${{ contains(matrix.dockerfile, '.hcl') }} + uses: mindsdb/github-actions/docker-bake@main + with: + git-sha: ${{ inputs.docker-image-ref }} + target: ${{ inputs.docker-bake-target }} + platforms: ${{ inputs.docker-bake-platforms }} + push-cache: false + # Otherwise build via regular docker + - if: ${{ !contains(matrix.dockerfile, '.hcl') }} + uses: mindsdb/github-actions/build-push-ecr@main + with: + module-name: ${{ inputs.service-name }} + build-for-environment: ${{ inputs.stage-name }} + image-ref: ${{ inputs.docker-image-ref }} + extra-build-args: "-f ${{ matrix.dockerfile }}" + + + migrate: + runs-on: mdb-dev + needs: [get-deploy-labels, build] + strategy: + matrix: + deploy-env: ${{fromJson(needs.get-deploy-labels.outputs.deploy-envs)}} + concurrency: + group: deploy-${{ matrix.deploy-env }} # All deployments for this env are grouped together + cancel-in-progress: false # Don't cancel in-progress deployments, it breaks helm + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.docker-image-ref }} + - name: Migrate + run: | + export NAMESPACE=${{inputs.deploy-namespace || matrix.deploy-env}} + export IMAGE_TAG=${{ inputs.stage-name }}-${{ inputs.docker-image-ref }} + export JOB_NAME=$(grep -E '^ *name:' ${{ inputs.migration-job-file }} | head -1 | awk '{print $2}') + + kubectl -n $NAMESPACE delete job --ignore-not-found $JOB_NAME + envsubst '${IMAGE_TAG} ${NAMESPACE}' < ${{ inputs.migration-job-file }} | kubectl apply -f - + + kubectl -n "$NAMESPACE" wait --for=condition=complete --timeout=1m "job/$JOB_NAME" + + # Deploy the built image to the specified environments + # Deploys to all environments at once + deploy: + runs-on: mdb-dev + needs: [ get-deploy-labels, build, migrate ] + strategy: + matrix: + deploy-env: ${{fromJson(needs.get-deploy-labels.outputs.deploy-envs)}} + concurrency: + group: deploy-${{ matrix.deploy-env }} # All deployments for this env are grouped together + cancel-in-progress: false # Don't cancel in-progress deployments, it breaks helm + environment: + # Assuming that ENV_URL is set in a github environment in the repo + # If not the link in the slack message will be borked, thats all + name: ${{ matrix.deploy-env }} + url: ${{ vars.ENV_URL }} + steps: + - uses: actions/checkout@v4 + - uses: mindsdb/github-actions/setup-env@main + - name: Notify of deployment starting + # This same message will be updated later with the deployment status + id: slack + uses: mindsdb/github-actions/slack-deploy-msg@main + with: + channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }} + status: "started" + color: "#0099CC" + env-name: ${{ matrix.deploy-env }} + env-url: ${{ vars.ENV_URL }} + slack-token: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }} + - uses: DevOps-Nirvana/aws-helm-multi-deploy-nodocker@v4 + # Do the actual deployment + with: + environment-slug: ${{matrix.deploy-env}} + k8s-namespace: ${{inputs.deploy-namespace || matrix.deploy-env}} + image-tag: ${{ inputs.stage-name }}-${{ github.sha }} + timeout: 600s + # We need to wait till deployment is finished here, since the calling workflow might test the deployment env once this job is done + wait: "true" + - name: Notify of deployment finish + # Update the slack message from before with the deployment status + uses: mindsdb/github-actions/slack-deploy-msg@main + if: always() + with: + channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }} + status: "${{ job.status == 'success' && 'finished' || 'failed' }}" + color: "${{ job.status == 'success' && '#00C851' || '#FF4444' }}" + env-name: ${{ matrix.deploy-env }} + env-url: ${{ vars.ENV_URL }} + slack-token: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }} + update-message-id: ${{ steps.slack.outputs.ts }} diff --git a/build-push-ecr/action.yml b/build-push-ecr/action.yml index eba4585..78f1faf 100644 --- a/build-push-ecr/action.yml +++ b/build-push-ecr/action.yml @@ -1,5 +1,10 @@ # Builds a docker image, then tags it with the github sha and pushes it to our Amazon ECR registry +outputs: + image: + description: "Full ECR image URI with tag (e.g. 123456.dkr.ecr.us-east-1.amazonaws.com/my-service:production-abc123)" + value: ${{ steps.push.outputs.image }} + inputs: module-name: description: "Name of the module to build. Used as the default image name and src dir unless 'image-name' or 'src-path' are used." @@ -30,28 +35,26 @@ runs: uses: aws-actions/amazon-ecr-login@v2 - shell: bash run: | - # Env var parsing - INPUT_SRC_PATH=${{ inputs.src-path }} SRC_PATH=${INPUT_SRC_PATH:-"./"} INPUT_IMAGE_REF=${{ inputs.image-ref }} IMAGE_REF=${INPUT_IMAGE_REF:-$CI_SHA} IMAGE_NAME=${{ inputs.module-name }} REPO_IMAGE=${{ steps.login-ecr.outputs.registry }}/$IMAGE_NAME - DOCKER_BUILDKIT=1 ENVIRONMENT=${{ inputs.build-for-environment }} - BRANCH_NAME=${{env.ENV_NAME}} IMAGE_TAG=$ENVIRONMENT-$IMAGE_REF - # Create repo if needed + # Create repo if needed (ignore error if it already exists) aws ecr create-repository --repository-name $IMAGE_NAME && \ - aws ecr set-repository-policy --repository-name $IMAGE_NAME --policy-text "$(cat ${{ github.action_path }}/shared-ecr-policy.json)" || \ - true # Just let this fail if the repo already exists + aws ecr set-repository-policy --repository-name $IMAGE_NAME --policy-text "$(cat ${{ github.action_path }}/shared-ecr-policy.json)" || true - docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist) + docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true cd $SRC_PATH BUILD_ARGS="--build-arg BUILD_FOR_ENVIRONMENT=$ENVIRONMENT --build-arg IMAGE_TAG=$IMAGE_TAG" - - # Finally, build our runner container docker buildx build ${{ inputs.extra-build-args }} $BUILD_ARGS -t $REPO_IMAGE:$IMAGE_TAG -t $REPO_IMAGE:latest --push . + + echo "BUILT_IMAGE=$REPO_IMAGE:$IMAGE_TAG" >> "$GITHUB_ENV" + - id: push + shell: bash + run: echo "image=$BUILT_IMAGE" >> "$GITHUB_OUTPUT"