From 00c0608b2e2e000a38af0e19febf018f9a6bc72f Mon Sep 17 00:00:00 2001 From: Varshini-Yenjarlapati Date: Wed, 25 Mar 2026 12:01:14 -0500 Subject: [PATCH 1/7] Add GitHub Actions workflow for data environment deployment --- .github/workflows/main.yml | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..e7413e0c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,67 @@ +name: Deploy Templatized Data Environment + +on: + pull_request: + types: [closed] + branches: + - staging + - main + push: + branches: + - staging + - main + +jobs: + deploy_data_environment: + # Only run when PR is merged or on direct push to environment branches + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) + runs-on: ubuntu-latest + + env: + # Read connection secret + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + + steps: + # Checkout step is necessary to access repository files + - name: Checkout repository + uses: actions/checkout@v4 + + # Determine environment based on target branch + - name: Set environment + id: set_env + run: | + TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}" + + if [[ $TARGET_BRANCH == "staging" ]]; then + echo "DEPLOY_ENV=staging" >> $GITHUB_ENV + echo "Environment set to staging" + elif [[ $TARGET_BRANCH == "main" ]]; then + echo "DEPLOY_ENV=prod" >> $GITHUB_ENV + echo "Environment set to production" + else + echo "Unexpected branch: $TARGET_BRANCH" + exit 1 + fi + + # Install Snowflake CLI GitHub Action and point to config file + - name: Install SnowflakeCLI + uses: snowflakedb/snowflake-cli-action@v1.5 + with: + cli-version: "latest" + default-config-file-path: "config.toml" + + # Fetch latest changes from the repository to Snowflake + - name: Fetch latest changes to Snowflake + run: snow git fetch course_repo.public.advanced_data_engineering_snowflake + + # Deploy templates to the data environment - + - name: Deploy templates to data environment + run: | + # Export TARGET_BRANCH for use in this step + echo "Using branch: ${GITHUB_REF_NAME}" + snow git execute @advanced_data_engineering_snowflake/branches/${GITHUB_REF_NAME}/module-1/hamburg_weather/pipeline/data/ \ + -D "env='${{ env.DEPLOY_ENV }}'" \ + --database=COURSE_REPO \ + --schema=PUBLIC From 81d2b13434dad646c7d4b3b1915da22bb92e74c6 Mon Sep 17 00:00:00 2001 From: Varshini-Yenjarlapati Date: Wed, 25 Mar 2026 12:12:39 -0500 Subject: [PATCH 2/7] Create main.yml --- .github/workflows/main.yml | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..e7413e0c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,67 @@ +name: Deploy Templatized Data Environment + +on: + pull_request: + types: [closed] + branches: + - staging + - main + push: + branches: + - staging + - main + +jobs: + deploy_data_environment: + # Only run when PR is merged or on direct push to environment branches + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) + runs-on: ubuntu-latest + + env: + # Read connection secret + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + + steps: + # Checkout step is necessary to access repository files + - name: Checkout repository + uses: actions/checkout@v4 + + # Determine environment based on target branch + - name: Set environment + id: set_env + run: | + TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}" + + if [[ $TARGET_BRANCH == "staging" ]]; then + echo "DEPLOY_ENV=staging" >> $GITHUB_ENV + echo "Environment set to staging" + elif [[ $TARGET_BRANCH == "main" ]]; then + echo "DEPLOY_ENV=prod" >> $GITHUB_ENV + echo "Environment set to production" + else + echo "Unexpected branch: $TARGET_BRANCH" + exit 1 + fi + + # Install Snowflake CLI GitHub Action and point to config file + - name: Install SnowflakeCLI + uses: snowflakedb/snowflake-cli-action@v1.5 + with: + cli-version: "latest" + default-config-file-path: "config.toml" + + # Fetch latest changes from the repository to Snowflake + - name: Fetch latest changes to Snowflake + run: snow git fetch course_repo.public.advanced_data_engineering_snowflake + + # Deploy templates to the data environment - + - name: Deploy templates to data environment + run: | + # Export TARGET_BRANCH for use in this step + echo "Using branch: ${GITHUB_REF_NAME}" + snow git execute @advanced_data_engineering_snowflake/branches/${GITHUB_REF_NAME}/module-1/hamburg_weather/pipeline/data/ \ + -D "env='${{ env.DEPLOY_ENV }}'" \ + --database=COURSE_REPO \ + --schema=PUBLIC From 324e7af6954d4e089a68fb44d7b6e3decb42f288 Mon Sep 17 00:00:00 2001 From: Varshini-Yenjarlapati Date: Wed, 25 Mar 2026 12:17:12 -0500 Subject: [PATCH 3/7] Create main.yml --- .github/workflows/main.yml | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..e7413e0c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,67 @@ +name: Deploy Templatized Data Environment + +on: + pull_request: + types: [closed] + branches: + - staging + - main + push: + branches: + - staging + - main + +jobs: + deploy_data_environment: + # Only run when PR is merged or on direct push to environment branches + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) + runs-on: ubuntu-latest + + env: + # Read connection secret + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + + steps: + # Checkout step is necessary to access repository files + - name: Checkout repository + uses: actions/checkout@v4 + + # Determine environment based on target branch + - name: Set environment + id: set_env + run: | + TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}" + + if [[ $TARGET_BRANCH == "staging" ]]; then + echo "DEPLOY_ENV=staging" >> $GITHUB_ENV + echo "Environment set to staging" + elif [[ $TARGET_BRANCH == "main" ]]; then + echo "DEPLOY_ENV=prod" >> $GITHUB_ENV + echo "Environment set to production" + else + echo "Unexpected branch: $TARGET_BRANCH" + exit 1 + fi + + # Install Snowflake CLI GitHub Action and point to config file + - name: Install SnowflakeCLI + uses: snowflakedb/snowflake-cli-action@v1.5 + with: + cli-version: "latest" + default-config-file-path: "config.toml" + + # Fetch latest changes from the repository to Snowflake + - name: Fetch latest changes to Snowflake + run: snow git fetch course_repo.public.advanced_data_engineering_snowflake + + # Deploy templates to the data environment - + - name: Deploy templates to data environment + run: | + # Export TARGET_BRANCH for use in this step + echo "Using branch: ${GITHUB_REF_NAME}" + snow git execute @advanced_data_engineering_snowflake/branches/${GITHUB_REF_NAME}/module-1/hamburg_weather/pipeline/data/ \ + -D "env='${{ env.DEPLOY_ENV }}'" \ + --database=COURSE_REPO \ + --schema=PUBLIC From df2bd3f3385d4af1c340809e8713399c98f1ae6c Mon Sep 17 00:00:00 2001 From: Varshini-Yenjarlapati Date: Wed, 25 Mar 2026 14:01:49 -0500 Subject: [PATCH 4/7] Delete .github directory --- .github/workflows/main.yml | 67 -------------------------------------- 1 file changed, 67 deletions(-) delete mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index e7413e0c..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Deploy Templatized Data Environment - -on: - pull_request: - types: [closed] - branches: - - staging - - main - push: - branches: - - staging - - main - -jobs: - deploy_data_environment: - # Only run when PR is merged or on direct push to environment branches - if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) - runs-on: ubuntu-latest - - env: - # Read connection secret - SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - - steps: - # Checkout step is necessary to access repository files - - name: Checkout repository - uses: actions/checkout@v4 - - # Determine environment based on target branch - - name: Set environment - id: set_env - run: | - TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}" - - if [[ $TARGET_BRANCH == "staging" ]]; then - echo "DEPLOY_ENV=staging" >> $GITHUB_ENV - echo "Environment set to staging" - elif [[ $TARGET_BRANCH == "main" ]]; then - echo "DEPLOY_ENV=prod" >> $GITHUB_ENV - echo "Environment set to production" - else - echo "Unexpected branch: $TARGET_BRANCH" - exit 1 - fi - - # Install Snowflake CLI GitHub Action and point to config file - - name: Install SnowflakeCLI - uses: snowflakedb/snowflake-cli-action@v1.5 - with: - cli-version: "latest" - default-config-file-path: "config.toml" - - # Fetch latest changes from the repository to Snowflake - - name: Fetch latest changes to Snowflake - run: snow git fetch course_repo.public.advanced_data_engineering_snowflake - - # Deploy templates to the data environment - - - name: Deploy templates to data environment - run: | - # Export TARGET_BRANCH for use in this step - echo "Using branch: ${GITHUB_REF_NAME}" - snow git execute @advanced_data_engineering_snowflake/branches/${GITHUB_REF_NAME}/module-1/hamburg_weather/pipeline/data/ \ - -D "env='${{ env.DEPLOY_ENV }}'" \ - --database=COURSE_REPO \ - --schema=PUBLIC From 556f6ea0ee82c11ccb431b1b7b2dd7f63c372e58 Mon Sep 17 00:00:00 2001 From: Varshini-Yenjarlapati Date: Wed, 25 Mar 2026 14:02:10 -0500 Subject: [PATCH 5/7] Delete .github directory --- .github/workflows/main.yml | 67 -------------------------------------- 1 file changed, 67 deletions(-) delete mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index e7413e0c..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Deploy Templatized Data Environment - -on: - pull_request: - types: [closed] - branches: - - staging - - main - push: - branches: - - staging - - main - -jobs: - deploy_data_environment: - # Only run when PR is merged or on direct push to environment branches - if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) - runs-on: ubuntu-latest - - env: - # Read connection secret - SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - - steps: - # Checkout step is necessary to access repository files - - name: Checkout repository - uses: actions/checkout@v4 - - # Determine environment based on target branch - - name: Set environment - id: set_env - run: | - TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}" - - if [[ $TARGET_BRANCH == "staging" ]]; then - echo "DEPLOY_ENV=staging" >> $GITHUB_ENV - echo "Environment set to staging" - elif [[ $TARGET_BRANCH == "main" ]]; then - echo "DEPLOY_ENV=prod" >> $GITHUB_ENV - echo "Environment set to production" - else - echo "Unexpected branch: $TARGET_BRANCH" - exit 1 - fi - - # Install Snowflake CLI GitHub Action and point to config file - - name: Install SnowflakeCLI - uses: snowflakedb/snowflake-cli-action@v1.5 - with: - cli-version: "latest" - default-config-file-path: "config.toml" - - # Fetch latest changes from the repository to Snowflake - - name: Fetch latest changes to Snowflake - run: snow git fetch course_repo.public.advanced_data_engineering_snowflake - - # Deploy templates to the data environment - - - name: Deploy templates to data environment - run: | - # Export TARGET_BRANCH for use in this step - echo "Using branch: ${GITHUB_REF_NAME}" - snow git execute @advanced_data_engineering_snowflake/branches/${GITHUB_REF_NAME}/module-1/hamburg_weather/pipeline/data/ \ - -D "env='${{ env.DEPLOY_ENV }}'" \ - --database=COURSE_REPO \ - --schema=PUBLIC From 05b1739a85525e598ce11b03a680f751545339fc Mon Sep 17 00:00:00 2001 From: Varshini-Yenjarlapati Date: Wed, 25 Mar 2026 14:07:04 -0500 Subject: [PATCH 6/7] Create main.yml --- .github/workflows/main.yml | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..e7413e0c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,67 @@ +name: Deploy Templatized Data Environment + +on: + pull_request: + types: [closed] + branches: + - staging + - main + push: + branches: + - staging + - main + +jobs: + deploy_data_environment: + # Only run when PR is merged or on direct push to environment branches + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) + runs-on: ubuntu-latest + + env: + # Read connection secret + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + + steps: + # Checkout step is necessary to access repository files + - name: Checkout repository + uses: actions/checkout@v4 + + # Determine environment based on target branch + - name: Set environment + id: set_env + run: | + TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}" + + if [[ $TARGET_BRANCH == "staging" ]]; then + echo "DEPLOY_ENV=staging" >> $GITHUB_ENV + echo "Environment set to staging" + elif [[ $TARGET_BRANCH == "main" ]]; then + echo "DEPLOY_ENV=prod" >> $GITHUB_ENV + echo "Environment set to production" + else + echo "Unexpected branch: $TARGET_BRANCH" + exit 1 + fi + + # Install Snowflake CLI GitHub Action and point to config file + - name: Install SnowflakeCLI + uses: snowflakedb/snowflake-cli-action@v1.5 + with: + cli-version: "latest" + default-config-file-path: "config.toml" + + # Fetch latest changes from the repository to Snowflake + - name: Fetch latest changes to Snowflake + run: snow git fetch course_repo.public.advanced_data_engineering_snowflake + + # Deploy templates to the data environment - + - name: Deploy templates to data environment + run: | + # Export TARGET_BRANCH for use in this step + echo "Using branch: ${GITHUB_REF_NAME}" + snow git execute @advanced_data_engineering_snowflake/branches/${GITHUB_REF_NAME}/module-1/hamburg_weather/pipeline/data/ \ + -D "env='${{ env.DEPLOY_ENV }}'" \ + --database=COURSE_REPO \ + --schema=PUBLIC From e3f29be14632d0ded3993422e1af6f48e7de0f7e Mon Sep 17 00:00:00 2001 From: Varshini-Yenjarlapati Date: Wed, 25 Mar 2026 14:20:48 -0500 Subject: [PATCH 7/7] Modify country table creation syntax and structure Changed table creation to use 'CREATE OR ALTER' syntax and added 'city_id' column. --- .../hamburg_weather/pipeline/data/load_tasty_bytes.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql b/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql index 4565f538..abad297b 100644 --- a/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql +++ b/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql @@ -57,14 +57,15 @@ raw zone table build -- country table build -- todo: complete table build -CREATE TABLE {{env}}_tasty_bytes.raw_pos.country +CREATE OR ALTER TABLE {{env}}_tasty_bytes.raw_pos.country ( country_id NUMBER(18,0), country VARCHAR(16777216), iso_currency VARCHAR(3), iso_country VARCHAR(2), city VARCHAR(16777216), - city_population VARCHAR(16777216) + city_population VARCHAR(16777216), + city_id NUMBER(19,0) ); @@ -338,4 +339,4 @@ FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/subset_order_header/; -- order_detail table load COPY INTO {{env}}_tasty_bytes.raw_pos.order_detail -FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/subset_order_detail/; \ No newline at end of file +FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/subset_order_detail/;