Add new or updated feeds from Google Sheets/Form #550
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Add new or updated feeds from Google Sheets/Form | |
| on: | |
| workflow_dispatch: | |
| # schedule: | |
| # - cron: '55 3 * * *' # Run every night | |
| permissions: | |
| contents: write # Permissions to read/write content like code | |
| pull-requests: write # Permissions to create/merge pull requests | |
| env: | |
| DATE_FORMAT: "[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}" # this is the format we need to compare dates between the CSV and the local system. | |
| DATE_FORMAT_DESIRED: "MM/dd/yyyy" | |
| USERNAME: "github-actions[bot]" # GitHub username that will create the PR | |
| USERNAME_EMAIL: "41898282+github-actions[bot]@users.noreply.github.com" | |
| BASE: "main" | |
| REVIEWERS_JSON: "[\"emmambd\"]" # List of GitHub usernames of the reviewers, in a JSON array : ["username1", "username2"] | |
| jobs: | |
| add-new-updated-feeds: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup global variables | |
| id: global_vars | |
| run: | | |
| echo "TODAYS_DATE=$(date +%m/%d/%Y)" >> $GITHUB_ENV # Ex.: 07/27/2023 | |
| echo "TODAYS_DAY=$(date '+%d')" >> $GITHUB_ENV # Ex.: 27 | |
| echo "TODAYS_MONTH=$(date '+%m')" >> $GITHUB_ENV # Ex.: 07 | |
| echo "TODAYS_YEAR=$(date '+%Y')" >> $GITHUB_ENV # Ex.: 2023 | |
| echo "YESTERDAYS_DATE=$(date -d "yesterday" +"%m/%d/%Y")" >> $GITHUB_ENV # Ex.: 07/26/2023 | |
| echo "RUN_ID=$GITHUB_RUN_ID" >> $GITHUB_ENV | |
| - name: Create branch name | |
| id: create_branch_name | |
| run: | | |
| echo "BRANCH=${{ env.TODAYS_YEAR }}-${{ env.TODAYS_MONTH }}-${{ env.TODAYS_DAY }}-SOURCES" >> $GITHUB_OUTPUT # Branch name | |
| - name: Load secrets from 1Password | |
| id: onepw_secrets | |
| uses: 1password/load-secrets-action@v2.0.0 | |
| with: | |
| export-env: true # Export loaded secrets as environment variables | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| # The URL to obtain the csv file with the data of the google form. Taken from the "GiHub Actions — URLs" entry in 1password | |
| CSV_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/qkn5esttmtojawglm4l6t2bqaa/al2gjfhiuddabkp7o26hszuvia" | |
| SLACK_WEBHOOK_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Slack webhook URLs/rdpfgrmnbxqaelgi5oky3lryz4/internal-add-feeds" | |
| - name: Checkout repo | |
| id: checkout_repo | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.BASE }} | |
| fetch-depth: 0 | |
| - name: Create new branch | |
| shell: bash | |
| run: | | |
| git pull | |
| git checkout -b ${{ steps.create_branch_name.outputs.BRANCH }} | |
| git reset --hard ${{ env.BASE }} | |
| git clean -df | |
| - name: Download CSV and process each lines | |
| id: process-csv | |
| run: | | |
| cd ${{ github.workspace }}/scripts | |
| OUTPUT=$(swift process_csv_in_github_action.swift "${{ env.CSV_URL }}" "${{ env.YESTERDAYS_DATE }}" "${{ env.DATE_FORMAT }}" "${{ env.DATE_FORMAT_DESIRED }}") | |
| echo "PYTHON_SCRIPT_ARGS=${OUTPUT}" >> $GITHUB_OUTPUT | |
| # Count the number of feeds in the output | |
| COUNT=$(echo "$input_string" | grep -o '§' | wc -l) | |
| COUNT=$((COUNT + 1)) | |
| echo "NUMBER_OF_FEEDS=${COUNT}" >> $GITHUB_OUTPUT | |
| - name: Setup Python | |
| if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' # install the python version needed | |
| - name: Create + activate a Python virtual env & run script | |
| if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/tools | |
| PYTHONIOENCODING: "utf8" #ascii | |
| shell: bash | |
| run: | | |
| echo -e "\n\n" | |
| echo "BEGIN VIRTUAL ENV && SETUP PIP" | |
| python -m venv env | |
| source env/bin/activate | |
| pip install --upgrade pip | |
| pip install virtualenv --quiet | |
| pip install gtfs_kit --quiet | |
| pip install unidecode --quiet | |
| echo "DONE VIRTUAL ENV && SETUP PIP" | |
| echo -e "\n\n" | |
| echo "BEGIN SEND COMMAND TO PYTHON" | |
| sections=$(echo '${{ steps.process-csv.outputs.PYTHON_SCRIPT_ARGS }}' | sed 's/""/"/g' | sed "s/’/'/g" | sed 's/"comment": "}/"comment": ""}/g' | awk -F'§' '{for (i=1; i<=NF; i++) print $i}') | |
| for section in "${sections[@]}"; do | |
| # echo "processing line: ${section}" | |
| output=$(eval "python -c 'from tools.operations import *; ${section}'") | |
| # echo "$output" | |
| done | |
| echo "DONE SEND COMMAND TO PYTHON" | |
| echo -e "\n\n" | |
| git status | |
| - name: Commit, push, and create PR | |
| id: commit-push-create-pr | |
| if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| run: | | |
| BRANCH_NAME=${{ steps.create_branch_name.outputs.BRANCH }} | |
| # Fetch the latest changes from main | |
| echo "fetching from origin" | |
| git fetch origin main | |
| echo "Create or reset the branch to the current main" | |
| git checkout -B "$BRANCH_NAME" origin/main | |
| # Configure git | |
| git config --global user.name "${{ env.USERNAME }}" | |
| git config --global user.email "${{ env.USERNAME_EMAIL }}" | |
| echo "Add changes" | |
| git add . | |
| echo "----------" | |
| echo "Check commit" | |
| git status --porcelain | |
| echo "----------" | |
| echo "Count new files" | |
| NEW_FILE_COUNT=$(git status --porcelain | awk '$1 == "A" {count++} END {print count+0}') | |
| echo "----------" | |
| echo "Commit changes: $NEW_FILE_COUNT" | |
| git commit -m "Automated commit — New/Updated feed(s)" | |
| echo "----------" | |
| echo "Force push changes to the branch" | |
| git push -f origin "$BRANCH_NAME" || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) | |
| echo "----------" | |
| echo "Check if a PR already exists" | |
| EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') | |
| if [ -n "$EXISTING_PR" ]; then | |
| echo "Pull request already exists. Updating PR #$EXISTING_PR" | |
| PR_URL=$(gh pr view $EXISTING_PR --json url -q '.url') | |
| else | |
| echo "Creating new pull request" | |
| # Create pull request and capture the URL | |
| PR_URL=$(gh pr create --title "Automated Pull Request — New/Updated feed(s) [SOURCES]" \ | |
| --body "This pull request contains new or updated feed(s)" \ | |
| --base main \ | |
| --head "$BRANCH_NAME") | |
| fi | |
| # Set the PR URL as an output | |
| echo "pr-url=$PR_URL" >> $GITHUB_OUTPUT | |
| echo "new-file-count=$NEW_FILE_COUNT" >> $GITHUB_OUTPUT | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| shell: bash | |
| - name: Post notification in Slack channel of update | |
| if: success() && steps.commit-push-create-pr.outcome == 'success' | |
| uses: slackapi/slack-github-action@v1.26.0 | |
| with: | |
| payload: | | |
| { | |
| "channel": "C072DF86VQB", | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "✅ Added/updated ${{ steps.commit-push-create-pr.outputs.new-file-count }} feed(s)", | |
| "emoji": true | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "${{ steps.commit-push-create-pr.outputs.new-file-count }} feed(s) were added or updated in a new branch on the https://github.com/MobilityData/mobility-database-catalogs repo." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "New branch name: `${{ steps.create_branch_name.outputs.BRANCH }}`*Open the corresponding PR:* ${{ steps.commit-push-create-pr.outputs.pr-url }}\n\n*Open the new branch:* https://github.com/MobilityData/mobility-database-catalogs/tree/${{ steps.create_branch_name.outputs.BRANCH }}\n\n" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "See the source document here: https://docs.google.com/spreadsheets/d/1Q96KDppKsn2khdrkraZCQ7T_qRSfwj7WsvqXvuMt4Bc/edit#gid=2061813733" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| - name: Post notification in Slack in case of failure | |
| if: failure() | |
| uses: slackapi/slack-github-action@v1.26.0 | |
| with: | |
| payload: | | |
| { | |
| "channel": "C072DF86VQB", | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "❌ FAILED — Add/update ${{ steps.commit-push-create-pr.outputs.new-file-count }} feed(s)", | |
| "emoji": true | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "The GitHub action has failed on the https://github.com/MobilityData/mobility-database-catalogs repo." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "*Open the action run:* https://github.com/MobilityData/mobility-database-catalogs/actions/runs/${{ env.RUN_ID }}" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "See the source document here: https://docs.google.com/spreadsheets/d/1Q96KDppKsn2khdrkraZCQ7T_qRSfwj7WsvqXvuMt4Bc/edit#gid=2061813733" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |