Staging branch

.github/workflows/main.yml

@@ -0,0 +1,96 @@
+name: Deploy Templatized Data Environment
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - staging
+      - main
+
+  push:
+    branches:
+      - staging
+      - main
+
+jobs:
+  deploy_data_environment:
+    # Run only for direct pushes or merged PRs
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true)
+    runs-on: ubuntu-latest
+
+    env:
+      # Tell Snowflake CLI where config.toml exists
+      SNOWFLAKE_HOME: ${{ github.workspace }}
+
+      # Snowflake connection credentials from GitHub secrets
+      SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+      SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+      SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+
+    steps:
+      # Checkout repository
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      # Setup Python for Snowflake CLI installation
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # Install Snowflake CLI
+      - name: Install Snowflake CLI
+        run: |
+          python -m pip install --upgrade pip
+          pip install snowflake-cli
+          snow --version
+
+      # Determine deployment environment
+      - name: Set deployment environment
+        shell: bash
+        run: |
+          TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}"
+
+          if [[ "$TARGET_BRANCH" == "staging" ]]; then
+            echo "DEPLOY_ENV=staging" >> $GITHUB_ENV
+            echo "Environment set to staging"
+
+          elif [[ "$TARGET_BRANCH" == "main" ]]; then
+            echo "DEPLOY_ENV=prod" >> $GITHUB_ENV
+            echo "Environment set to production"
+
+          else
+            echo "Unexpected branch: $TARGET_BRANCH"
+            exit 1
+          fi
+
+      # Verify that Snowflake CLI can see the config file
+      - name: Verify Snowflake configuration
+        run: |
+          echo "SNOWFLAKE_HOME=${SNOWFLAKE_HOME}"
+          ls -la ${SNOWFLAKE_HOME}
+          cat ${SNOWFLAKE_HOME}/config.toml | head -20
+
+      # Fix config.toml permissions
+      - name: Fix Snowflake config permissions
+        run: |
+          chmod 0600 ${SNOWFLAKE_HOME}/config.toml
+
+      # Fetch latest Git repository changes into Snowflake
+      - name: Fetch latest changes to Snowflake
+        run: |
+          snow git fetch course_repo.public.advanced_data_engineering_snowflake
+
+      # Execute the templated SQL files
+      - name: Deploy templates to data environment
+        shell: bash
+        run: |
+          echo "Using Git branch: ${GITHUB_REF_NAME}"
+          echo "Deploying environment: ${DEPLOY_ENV}"
+
+          snow git execute \
+            @advanced_data_engineering_snowflake/branches/${GITHUB_REF_NAME}/module-1/hamburg_weather/pipeline/data/ \
+            -D "env='${DEPLOY_ENV}'" \
+            --database COURSE_REPO \
+            --schema PUBLIC

Untitled.ipynb

@@ -0,0 +1,22 @@
+{
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Jupyter Notebook",
+      "name": "jupyter"
+    }
+  },
+  "nbformat_minor": 5,
+  "nbformat": 4,
+  "cells": [
+    {
+      "cell_type": "code",
+      "id": "29819d68-d8a2-4f82-b667-e7fbd790f073",
+      "metadata": {
+        "language": "python"
+      },
+      "source": "",
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}

module-1/hamburg_weather/notebooks/debugging.ipynb

@@ -1,50 +1,77 @@
 {
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3775908f-ca36-4846-8f38-5adca39217f2",
-   "metadata": {
-    "language": "sql",
-    "name": "cell1"
-   },
-   "outputs": [],
-   "source": [
-    "-- Set context\n",
-    "USE ROLE accountadmin;\n",
-    "USE DATABASE staging_tasty_bytes;\n",
-    "USE SCHEMA raw_pos;\n",
-    "USE WAREHOUSE compute_wh;"
-   ]
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "3775908f-ca36-4846-8f38-5adca39217f2",
+      "metadata": {
+        "language": "sql",
+        "name": "cell1",
+        "resultVariableName": "dataframe_1"
+      },
+      "outputs": [],
+      "source": "%%sql -r dataframe_1\n-- Set context\nUSE ROLE accountadmin;\nUSE DATABASE staging_tasty_bytes;\nUSE SCHEMA raw_pos;\nUSE WAREHOUSE compute_wh;"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "dfd78d46-46f3-40a7-be43-4ab79cf72a9f",
+      "metadata": {
+        "language": "sql",
+        "name": "cell2",
+        "resultVariableName": "dataframe_2"
+      },
+      "outputs": [],
+      "source": "%%sql -r dataframe_2\nSHOW TABLES IN staging_tasty_bytes.raw_pos;"
+    },
+    {
+      "id": "bd4d16a5-b00c-4f2e-ac19-b3cc64dc0c98",
+      "cell_type": "code",
+      "metadata": {
+        "resultVariableName": "dataframe_4",
+        "language": "sql"
+      },
+      "source": "CREATE OR ALTER TABLE STAGING_tasty_bytes.raw_pos.country\n(\n   country_id NUMBER(18,0),\n   country VARCHAR(16777216),\n   iso_currency VARCHAR(3),\n   iso_country VARCHAR(2),\n   city VARCHAR(16777216),\n   city_population VARCHAR(16777216),\n   city_id NUMBER(19,0)\n);",
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "id": "4b6c9d8d-6981-41df-870f-b493aa90403f",
+      "cell_type": "code",
+      "metadata": {
+        "resultVariableName": "dataframe_3",
+        "language": "sql"
+      },
+      "source": "-- country table load\nCOPY INTO STAGING_tasty_bytes.raw_pos.country\n(\n   country_id,\n   country,\n   iso_currency,\n   iso_country,\n   city_id,\n   city,\n   city_population\n)\nFROM @STAGING_tasty_bytes.public.s3load/raw_pos/country/;",
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "id": "d8221689-4b7f-42b1-b9f2-667d6426e234",
+      "cell_type": "code",
+      "metadata": {
+        "resultVariableName": "dataframe_5",
+        "language": "sql"
+      },
+      "source": "%%sql -r dataframe_5\n",
+      "outputs": [],
+      "execution_count": null
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Streamlit Notebook",
+      "name": "streamlit"
+    },
+    "lastEditStatus": {
+      "authorEmail": "gilberto.hernandez@snowflake.com",
+      "authorId": "6002244633419",
+      "authorName": "GILBERTO",
+      "lastEditTime": 1741667434593,
+      "notebookId": "sd4b3qffhw3yzmzwpcr2",
+      "sessionId": "2ad2d56d-f367-4eba-abad-28f4dc5b5f95"
+    }
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "dfd78d46-46f3-40a7-be43-4ab79cf72a9f",
-   "metadata": {
-    "language": "sql",
-    "name": "cell2"
-   },
-   "outputs": [],
-   "source": [
-    "SHOW TABLES IN staging_tasty_bytes.raw_pos;"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Streamlit Notebook",
-   "name": "streamlit"
-  },
-  "lastEditStatus": {
-   "authorEmail": "gilberto.hernandez@snowflake.com",
-   "authorId": "6002244633419",
-   "authorName": "GILBERTO",
-   "lastEditTime": 1741667434593,
-   "notebookId": "sd4b3qffhw3yzmzwpcr2",
-   "sessionId": "2ad2d56d-f367-4eba-abad-28f4dc5b5f95"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
+  "nbformat": 4,
+  "nbformat_minor": 5
+}

module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql

@@ -64,7 +64,8 @@ CREATE TABLE {{env}}_tasty_bytes.raw_pos.country
    iso_currency VARCHAR(3),
    iso_country VARCHAR(2),
    city VARCHAR(16777216),
-   city_population VARCHAR(16777216)
+   city_population VARCHAR(16777216),
+   city_id NUMBER(19,0)
 );
 
 
@@ -293,17 +294,17 @@ USE WAREHOUSE demo_build_wh;
 
 
 -- country table load
--- COPY INTO {{env}}_tasty_bytes.raw_pos.country
--- (
---    country_id,
---    country,
---    iso_currency,
---    iso_country,
---    city_id,
---    city,
---    city_population
--- )
--- FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/;
+COPY INTO {{env}}_tasty_bytes.raw_pos.country
+(
+   country_id,
+   country,
+   iso_currency,
+   iso_country,
+   city_id,
+   city,
+   city_population
+)
+FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/;
 
 
 -- franchise table load