Staging to main

module-1/hamburg_weather/main.yml → .github/workflows/main.yml

@@ -6,62 +6,66 @@ on:
     branches:
       - staging
       - main
+
   push:
     branches:
       - staging
       - main
 
 jobs:
   deploy_data_environment:
-    # Only run when PR is merged or on direct push to environment branches
+    # Run on push OR merged PR
     if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true)
+
     runs-on: ubuntu-latest
 
     env:
-      # Read connection secret
       SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
       SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
       SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
 
     steps:
-      # Checkout step is necessary to access repository files
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # Determine environment based on target branch
+      # Determine deployment environment + branch
       - name: Set environment
         id: set_env
         run: |
           TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}"
-
-          if [[ $TARGET_BRANCH == "staging" ]]; then
+
+          echo "TARGET_BRANCH=$TARGET_BRANCH" >> $GITHUB_ENV
+
+          if [[ "$TARGET_BRANCH" == "staging" ]]; then
             echo "DEPLOY_ENV=staging" >> $GITHUB_ENV
             echo "Environment set to staging"
-          elif [[ $TARGET_BRANCH == "main" ]]; then
+
+          elif [[ "$TARGET_BRANCH" == "main" ]]; then
             echo "DEPLOY_ENV=prod" >> $GITHUB_ENV
             echo "Environment set to production"
+
           else
             echo "Unexpected branch: $TARGET_BRANCH"
             exit 1
           fi
 
-      # Install Snowflake CLI GitHub Action and point to config file
-      - name: Install SnowflakeCLI
+      - name: Install Snowflake CLI
         uses: snowflakedb/snowflake-cli-action@v1.5
         with:
           cli-version: "latest"
           default-config-file-path: "config.toml"
 
-      # Fetch latest changes from the repository to Snowflake
       - name: Fetch latest changes to Snowflake
-        run: snow git fetch course_repo.public.advanced_data_engineering_snowflake
+        run: |
+          snow git fetch course_repo.public.advanced_data_engineering_snowflake
 
-      # Deploy templates to the data environment -
       - name: Deploy templates to data environment
         run: |
-          # Export TARGET_BRANCH for use in this step
-          echo "Using branch: ${GITHUB_REF_NAME}"
-          snow git execute @advanced_data_engineering_snowflake/branches/FIX-ME/module-1/hamburg_weather/pipeline/data/ \
-          -D "env='${{  }}'" \
-          --database=COURSE_REPO \
-          --schema=PUBLIC
+          echo "Deploying branch: ${TARGET_BRANCH}"
+          echo "Deploy environment: ${DEPLOY_ENV}"
+
+          snow git execute \
+            @advanced_data_engineering_snowflake/branches/${TARGET_BRANCH}/module-1/hamburg_weather/pipeline/data/ \
+            -D "env='${DEPLOY_ENV}'" \
+            --database=COURSE_REPO \
+            --schema=PUBLIC

module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql

@@ -57,14 +57,15 @@ raw zone table build
 -- country table build
 
 -- todo: complete table build
-CREATE TABLE {{env}}_tasty_bytes.raw_pos.country
+CREATE OR ALTER TABLE {{env}}_tasty_bytes.raw_pos.country
 (
    country_id NUMBER(18,0),
    country VARCHAR(16777216),
    iso_currency VARCHAR(3),
    iso_country VARCHAR(2),
    city VARCHAR(16777216),
-   city_population VARCHAR(16777216)
+   city_population VARCHAR(16777216),
+   city_id NUMBER(38,0)
 );
 
 
@@ -293,17 +294,17 @@ USE WAREHOUSE demo_build_wh;
 
 
 -- country table load
--- COPY INTO {{env}}_tasty_bytes.raw_pos.country
--- (
---    country_id,
---    country,
---    iso_currency,
---    iso_country,
---    city_id,
---    city,
---    city_population
--- )
--- FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/;
+COPY INTO {{env}}_tasty_bytes.raw_pos.country
+(
+   country_id,
+   country,
+   iso_currency,
+   iso_country,
+   city_id,
+   city,
+   city_population
+)
+FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/;
 
 
 -- franchise table load