diff --git a/module-1/hamburg_weather/main.yml b/.github/workflows/main.yml similarity index 60% rename from module-1/hamburg_weather/main.yml rename to .github/workflows/main.yml index bc93eac2..e1c6fa34 100644 --- a/module-1/hamburg_weather/main.yml +++ b/.github/workflows/main.yml @@ -6,6 +6,7 @@ on: branches: - staging - main + push: branches: - staging @@ -13,55 +14,58 @@ on: jobs: deploy_data_environment: - # Only run when PR is merged or on direct push to environment branches + # Run on push OR merged PR if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) + runs-on: ubuntu-latest env: - # Read connection secret SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} steps: - # Checkout step is necessary to access repository files - name: Checkout repository uses: actions/checkout@v4 - # Determine environment based on target branch + # Determine deployment environment + branch - name: Set environment id: set_env run: | TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}" - - if [[ $TARGET_BRANCH == "staging" ]]; then + + echo "TARGET_BRANCH=$TARGET_BRANCH" >> $GITHUB_ENV + + if [[ "$TARGET_BRANCH" == "staging" ]]; then echo "DEPLOY_ENV=staging" >> $GITHUB_ENV echo "Environment set to staging" - elif [[ $TARGET_BRANCH == "main" ]]; then + + elif [[ "$TARGET_BRANCH" == "main" ]]; then echo "DEPLOY_ENV=prod" >> $GITHUB_ENV echo "Environment set to production" + else echo "Unexpected branch: $TARGET_BRANCH" exit 1 fi - # Install Snowflake CLI GitHub Action and point to config file - - name: Install SnowflakeCLI + - name: Install Snowflake CLI uses: snowflakedb/snowflake-cli-action@v1.5 with: cli-version: "latest" default-config-file-path: "config.toml" - # Fetch latest changes from the repository to Snowflake - name: Fetch latest changes to Snowflake - run: snow git fetch course_repo.public.advanced_data_engineering_snowflake + run: | + snow git fetch course_repo.public.advanced_data_engineering_snowflake - # Deploy templates to the data environment - - name: Deploy templates to data environment run: | - # Export TARGET_BRANCH for use in this step - echo "Using branch: ${GITHUB_REF_NAME}" - snow git execute @advanced_data_engineering_snowflake/branches/FIX-ME/module-1/hamburg_weather/pipeline/data/ \ - -D "env='${{ }}'" \ - --database=COURSE_REPO \ - --schema=PUBLIC \ No newline at end of file + echo "Deploying branch: ${TARGET_BRANCH}" + echo "Deploy environment: ${DEPLOY_ENV}" + + snow git execute \ + @advanced_data_engineering_snowflake/branches/${TARGET_BRANCH}/module-1/hamburg_weather/pipeline/data/ \ + -D "env='${DEPLOY_ENV}'" \ + --database=COURSE_REPO \ + --schema=PUBLIC diff --git a/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql b/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql index 4565f538..11da4457 100644 --- a/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql +++ b/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql @@ -57,14 +57,15 @@ raw zone table build -- country table build -- todo: complete table build -CREATE TABLE {{env}}_tasty_bytes.raw_pos.country +CREATE OR ALTER TABLE {{env}}_tasty_bytes.raw_pos.country ( country_id NUMBER(18,0), country VARCHAR(16777216), iso_currency VARCHAR(3), iso_country VARCHAR(2), city VARCHAR(16777216), - city_population VARCHAR(16777216) + city_population VARCHAR(16777216), + city_id NUMBER(38,0) ); @@ -293,17 +294,17 @@ USE WAREHOUSE demo_build_wh; -- country table load --- COPY INTO {{env}}_tasty_bytes.raw_pos.country --- ( --- country_id, --- country, --- iso_currency, --- iso_country, --- city_id, --- city, --- city_population --- ) --- FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/; +COPY INTO {{env}}_tasty_bytes.raw_pos.country +( + country_id, + country, + iso_currency, + iso_country, + city_id, + city, + city_population +) +FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/; -- franchise table load