dataflow-ops/.github/workflows/main_simple.yaml at main · anna-geller/dataflow-ops · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
name: (optional) Simple CD to S3 and Prefect Cloud w/o redeploying ECR image

on:
  workflow_dispatch:
#  push:
#    branches:
#      - main

env:
  PROJECT: dataflowops
  S3_BLOCK_NAME: prod
  PREFECT_VERSION: 2.*

jobs:
  set_description:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

  changes:
    name: Summary of flow and code dependency changes
    runs-on: ubuntu-latest
    outputs:
      prefect_flows: ${{ steps.filter.outputs.flows_files }}
      prefect_flows_changed: ${{ steps.filter.outputs.flows }}
      code_dependencies_changed: ${{ steps.filter.outputs.code }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          list-files: json
          filters: |
            flows:
              - added|modified: 'flows/*.py'
            code:
              - added|modified|deleted: 'dataflowops/**'
              - added|modified|deleted: 'requirements.txt'
              - added|modified|deleted: 'Dockerfile'
      - name: Generate Markdown Summary
        run: |
          echo Flows: ${{ steps.filter.outputs.flows_files }} >> $GITHUB_STEP_SUMMARY
          echo Code dependency changes: ${{ steps.filter.outputs.code_files }} >> $GITHUB_STEP_SUMMARY
  deploy:
    needs: changes
    if: ${{ needs.changes.outputs.prefect_flows_changed == 'true' }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        flows: ${{ fromJson(needs.changes.outputs.prefect_flows) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Python dependencies
        run: |
          pip install prefect==$PREFECT_VERSION
          pip install s3fs
          pip install .

      - name: Prefect Cloud login
        run: |
          prefect config set PREFECT_API_KEY=${{ secrets.PREFECT_API_KEY }}
          prefect config set PREFECT_API_URL=${{ secrets.PREFECT_API_URL }}

      - name: AWS credentials
        uses: aws-actions/configure-aws-credentials@v1-node16
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
          mask-aws-account-id: 'no'

      - name: Deploy flows to S3
        id: build
        run: |
          FLOW_NAME=$(basename ${{ matrix.flows }} .py)
          prefect deployment build ${{ matrix.flows }}:$FLOW_NAME --name $PROJECT -q $PROJECT -a -sb s3/$S3_BLOCK_NAME -v $GITHUB_SHA -o $FLOW_NAME.yaml
          echo "flow_manifest=$FLOW_NAME.yaml" >> $GITHUB_OUTPUT

      - name: Upload deployment manifest as artifact
        uses: actions/upload-artifact@v3
        with:
          name: Deployment YAML manifests
          path: ${{ steps.build.outputs.flow_manifest }}