test-doc-generator.yml

name: Test Doc Generator

on:
  workflow_dispatch:
    inputs:
      target_branch:
        description: 'Branch in appsmith-docs to create PR against'
        required: true
        default: 'docs-staging'
        type: string

jobs:
  generate_docs:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout appsmith-docs (target branch)
      uses: actions/checkout@v4
      with:
        token: ${{ secrets.test_REPO_ACCESS_TOKEN }}
        ref: ${{ github.event.inputs.target_branch }}
        fetch-depth: 0

    - name: Checkout integration-resources-test repo
      uses: actions/checkout@v4
      with:
        repository: harshilp24/integration-resources-test
        token: ${{ secrets.test_REPO_ACCESS_TOKEN }}
        ref: main
        path: integration-resources-test
        fetch-depth: 2

    - name: Get changed files in uqi_configs
      id: changed-files
      uses: tj-actions/changed-files@v46
      with:
        path: integration-resources-test/Generic UQI Creation/uqi_configs
        since_last_remote_commit: true

    - name: Save list of changed files
      run: |
        mkdir -p scripts
        echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' > files_to_process.txt
        if [ ! -s files_to_process.txt ]; then
          echo "No new or modified files to process. Exiting."
          exit 0
        fi
        echo "changes_found=true" >> $GITHUB_ENV

    - name: Process files with OpenAI
      if: env.changes_found == 'true'
      run: |
        mkdir -p generated_docs
        PROCESSED_COUNT=0

        while IFS= read -r FILE_PATH; do
          FILE_NAME=$(basename "$FILE_PATH")
          echo "⏳ Processing $FILE_NAME"

          FILE_URL="https://raw.githubusercontent.com/harshilp24/integration-resources-test/main/Generic%20UQI%20Creation/uqi_configs/$FILE_NAME"
          echo "Fetching content from: $FILE_URL"
          curl -fsSL --max-time 60 "$FILE_URL" -o input_file.json || {
            echo "❌ Failed to fetch $FILE_NAME"
            continue
          }

          SYSTEM_PROMPT=$(cat .github/prompts/extract_prompt.txt || echo "Extract important integration details.")
          USER_CONTENT=$(cat input_file.json)

          PAYLOAD=$(jq -n \
            --arg system "$SYSTEM_PROMPT" \
            --arg user "$USER_CONTENT" \
            '{
              model: "gpt-4-1106-preview",
              messages: [
                {"role": "system", "content": $system},
                {"role": "user", "content": $user}
              ],
              max_tokens: 2000,
              temperature: 0
            }')

          RESPONSE1=$(curl -s https://api.openai.com/v1/chat/completions \
            -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \
            -H "Content-Type: application/json" \
            -d "$PAYLOAD")

          if echo "$RESPONSE1" | jq -e '.error' > /dev/null; then
            echo "❌ OpenAI error on Prompt 1"
            echo "$RESPONSE1" | jq .
            continue
          fi
          echo "$RESPONSE1" | jq -r '.choices[0].message.content' > extracted_info.md

          SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt || echo "Generate reference documentation in markdown.")
          EXTRACTED_CONTENT=$(cat extracted_info.md)

          PAYLOAD=$(jq -n \
            --arg system "$SYSTEM_PROMPT" \
            --arg user "$EXTRACTED_CONTENT" \
            '{
              model: "gpt-4-1106-preview",
              messages: [
                {"role": "system", "content": $system},
                {"role": "user", "content": $user}
              ],
              max_tokens: 4000,
              temperature: 0.3
            }')

          RESPONSE2=$(curl -s https://api.openai.com/v1/chat/completions \
            -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \
            -H "Content-Type: application/json" \
            -d "$PAYLOAD")

          if echo "$RESPONSE2" | jq -e '.error' > /dev/null; then
            echo "❌ OpenAI error on Prompt 2"
            echo "$RESPONSE2" | jq .
            continue
          fi
          echo "$RESPONSE2" | jq -r '.choices[0].message.content' > generated_doc.md

          INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]')
          FINAL_PATH="website/docs/connect-data/reference/${INTEGRATION}.md"

          mkdir -p "$(dirname "$FINAL_PATH")"
          cp generated_doc.md "$FINAL_PATH"

          echo "$FILE_NAME" >> scripts/processed_files.txt
          PROCESSED_COUNT=$((PROCESSED_COUNT + 1))
          echo "✅ Finished $FILE_NAME"

        done < files_to_process.txt

        echo "processed_count=$PROCESSED_COUNT" >> $GITHUB_ENV
        if [ "$PROCESSED_COUNT" -gt 0 ]; then
          echo "content_generated=true" >> $GITHUB_ENV
        else
          echo "content_generated=false" >> $GITHUB_ENV
        fi
        rm -f input_file.json extracted_info.md generated_doc.md

    - name: Commit and open PR
      if: env.content_generated == 'true'
      uses: peter-evans/create-pull-request@v6
      with:
        token: ${{ secrets.test_REPO_ACCESS_TOKEN }}
        title: "docs: update integration docs for ${{ github.event.inputs.target_branch }}"
        commit-message: "docs: automated generation for ${{ github.event.inputs.target_branch }}\n\nProcessed changed files from integration-resources-test."
        branch: "docs-update/${{ github.event.inputs.target_branch }}-${{ github.run_id }}"
        base: ${{ github.event.inputs.target_branch }}
        add-paths: |
          website/docs/connect-data/reference/
          scripts/processed_files.txt
        body: |
          ✅ Auto-generated PR based on file changes in `integration-resources-test`.

          **Target Branch:** `${{ github.event.inputs.target_branch }}`
          **Source Repo:** `harshilp24/integration-resources-test`

          This PR includes:
          - New/updated docs from changed integration files.
          - Updated tracking list in `scripts/processed_files.txt`.