spellbook/.github/workflows/commit_manifest.yml at main · duneanalytics/spellbook · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
name: Commit Manifest

on:
  workflow_dispatch:
  push:
    branches:
      - "main"

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  commit_manifest:
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        project: [ 'tokens', 'daily_spellbook', 'dex', 'solana', 'hourly_spellbook']
      fail-fast: false

    steps:
    - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
      with:
        python-version: "3.13"
    - name: Set up uv
      uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
      with:
        enable-cache: true
    - name: Checkout main branch
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
      with:
        ref: main

    - name: Sync Python dependencies
      run: uv sync --locked

    - name: Configure dbt
      env:
        DBT_PROFILES_YML_BASE64: ${{ secrets.DBT_PROFILES_YML_BASE64 }}
      run: |
        mkdir $HOME/.dbt
        echo $DBT_PROFILES_YML_BASE64 | base64 --decode > $HOME/.dbt/profiles.yml

    - name: Setup variables
      run: |
        echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
        echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
        echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{matrix.project}}" >> $GITHUB_ENV
        PROJECT_DIR=dbt_subprojects/${{ matrix.project }}
        echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV

    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
      with:
        role-to-assume: ${{ secrets.SPELLBOOK_CI_IAM_ROLE }}
        aws-region: eu-west-1

    - name: dbt dependencies
      working-directory: ${{env.PROJECT_DIR}}
      run: "uv run dbt deps"

    # Restore the previous run's partial-parse state so dbt only re-parses
    # files that changed since the last main commit. A cache miss (first run,
    # or after dbt invalidates the state) is fine — dbt just does a full parse.
    # dbt writes partial_parse.msgpack next to manifest.json in the target path
    # (--target-path . resolves to the project dir).
    - name: Restore dbt partial-parse cache
      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
      with:
        path: dbt_subprojects/${{ matrix.project }}/partial_parse.msgpack
        # Key is unique per commit so each run saves fresh state; restore-keys
        # falls back to the newest prior cache for this project (rolling cache).
        key: partial-parse-${{ matrix.project }}-${{ github.sha }}
        restore-keys: |
          partial-parse-${{ matrix.project }}-

    - name: dbt parse to create prod manifest from main
      run: "uv run dbt parse --target-path . $PROFILE --project-dir ${PROJECT_DIR}"

    # Persist the refreshed state for the next run. Placed right after parse so
    # a good parse is always cached, independent of the S3 upload steps below;
    # it only runs when parse succeeded, so we never cache a half-written file.
    - name: Save dbt partial-parse cache
      uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
      with:
        path: dbt_subprojects/${{ matrix.project }}/partial_parse.msgpack
        key: partial-parse-${{ matrix.project }}-${{ github.sha }}

    - name: copy old manifest locally
      run: "aws s3 cp $S3_LOCATION/manifest.json $S3_LOCATION/previous_manifest.json"

    - name: upload manifest
      run: "aws s3 cp $PROJECT_DIR/manifest.json $S3_LOCATION/manifest.json"