update-docs.yml

name: Update RPCN Connector Docs

on:
  workflow_dispatch:
  repository_dispatch:
    types: [generate-rpcn-docs]

jobs:
  generate-docs-and-submit-pull-request:
    runs-on: ubuntu-latest
    timeout-minutes: 30  # Prevent runaway workflows

    permissions:
      id-token: write
      contents: read
      issues: write  # For creating failure notification issues

    # Prevent concurrent runs to avoid conflicts on the same branch
    concurrency:
      group: update-rpcn-connector-docs
      cancel-in-progress: true  # Cancel in-progress runs when a new one starts

    env:
      NODE_VERSION: '24'
      DOCS_OVERRIDES: docs-data/overrides.json
      AUTO_BRANCH: auto-docs/update-rpcn-connector-docs

    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: ${{ vars.RP_AWS_CRED_REGION }}
          role-to-assume: arn:aws:iam::${{ secrets.RP_AWS_CRED_ACCOUNT_ID }}:role/${{ vars.RP_AWS_CRED_BASE_ROLE_NAME }}${{ github.event.repository.name }}

      - name: Retrieve GitHub bot token from AWS Secrets Manager
        uses: aws-actions/aws-secretsmanager-get-secrets@v2
        with:
          secret-ids: |
            ,sdlc/prod/github/actions_bot_token
          parse-json-secrets: true

      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          token: ${{ env.ACTIONS_BOT_TOKEN }}
          fetch-depth: 100  # Sufficient history for rebasing

      - name: Set up git identity
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

      - name: Check for existing PR branch and set up workspace
        id: setup
        run: |
          set -e  # Exit on error

          # Check if the auto-docs branch exists on remote
          if git ls-remote --heads origin "$AUTO_BRANCH" | grep -q "$AUTO_BRANCH"; then
            echo "branch_exists=true" >> "$GITHUB_OUTPUT"
            echo "✓ Found existing branch $AUTO_BRANCH"

            # Fetch and checkout the existing branch
            git fetch origin "$AUTO_BRANCH"
            git checkout "$AUTO_BRANCH"

            # Attempt to rebase on main to get latest changes
            echo "Attempting to rebase on main..."
            if git rebase origin/main; then
              echo "rebase_success=true" >> "$GITHUB_OUTPUT"
              echo "✓ Successfully rebased on main"
            else
              echo "rebase_success=false" >> "$GITHUB_OUTPUT"
              echo "✗ Rebase failed - conflicts detected"
              git rebase --abort

              # Try merge as fallback
              echo "Attempting merge as fallback..."
              if git merge origin/main -m "Merge main into $AUTO_BRANCH"; then
                echo "✓ Successfully merged main"
              else
                echo "✗ Merge also failed - branch has unresolvable conflicts"
                echo "::error::Cannot update branch $AUTO_BRANCH due to conflicts. Manual intervention required."

                # Save conflict details for issue creation
                {
                  echo "failure_type=merge_conflict"
                  echo "failure_message<<EOF"
                  echo "## Merge Conflict Detected"
                  echo ""
                  echo "The automated doc update workflow cannot merge \`main\` into \`$AUTO_BRANCH\` due to conflicts."
                  echo ""
                  echo "### Conflicting Files"
                  echo "\`\`\`"
                  git diff --name-only --diff-filter=U 2>/dev/null || echo "Unable to determine conflicting files"
                  echo "\`\`\`"
                  echo ""
                  echo "### Required Action"
                  echo "1. Checkout the \`$AUTO_BRANCH\` branch"
                  echo "2. Manually merge or rebase with \`main\`"
                  echo "3. Resolve conflicts"
                  echo "4. Re-run this workflow"
                  echo "EOF"
                } >> "$GITHUB_OUTPUT"

                exit 1
              fi
            fi

            # Check for manual commits (commits not from the bot)
            # Use grep to filter out bot commits instead of complex regex
            MANUAL_COMMITS=$(git log origin/main.."$AUTO_BRANCH" --format="%h %an: %s" | grep -v "github-actions\[bot\]" || true)

            if [ -n "$MANUAL_COMMITS" ]; then
              echo "has_manual_commits=true" >> "$GITHUB_OUTPUT"
              echo "⚠️  Found manual commits:"
              echo "$MANUAL_COMMITS"

              # Save the list for the PR body
              {
                echo "manual_commits<<EOF"
                echo "$MANUAL_COMMITS"
                echo "EOF"
              } >> "$GITHUB_OUTPUT"
            else
              echo "has_manual_commits=false" >> "$GITHUB_OUTPUT"
              echo "✓ No manual commits detected"
            fi
          else
            echo "branch_exists=false" >> "$GITHUB_OUTPUT"
            echo "has_manual_commits=false" >> "$GITHUB_OUTPUT"
            echo "rebase_success=true" >> "$GITHUB_OUTPUT"
            echo "✓ Branch $AUTO_BRANCH does not exist, will create new"

            # Create new branch from main
            git checkout -b "$AUTO_BRANCH"

            # Push the new branch to remote so it exists for peter-evans/create-pull-request
            git push -u origin "$AUTO_BRANCH"
            echo "✓ Pushed new branch to remote"
          fi

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install npm dependencies
        run: |
          npm ci  # Use ci for deterministic installs in CI
          echo "✓ npm dependencies installed"

      - name: Update doc-tools to latest version
        run: |
          # Ensure we use the latest doc-tools version after npm registry propagation
          # This prevents using stale package-lock.json versions that may have bugs
          npm update @redpanda-data/docs-extensions-and-macros
          echo "✓ Updated to latest doc-tools version"
          npm list @redpanda-data/docs-extensions-and-macros

      - name: Cache rpk connect components
        uses: actions/cache@v4
        with:
          path: |
            ~/.rpk
            ~/.cache/rpk
          key: rpk-connect-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            rpk-connect-${{ runner.os }}-

      - name: Install documentation tools
        run: |
          set -e
          echo "Installing doc-tools test dependencies..."
          npx --no-install doc-tools install-test-dependencies

          echo "Installing rpk connect..."
          rpk connect install

          echo "✓ All tools installed successfully"

      - name: Wait for npm registry propagation
        run: |
          # FIXME: This wait is required for npm registry propagation after package publish
          # Consider implementing polling/retry logic in doc-tools instead of fixed wait
          # See: [link to relevant issue/doc]

          WAIT_TIME=300  # Reduced from 500s to 5 minutes
          echo "⏳ Waiting ${WAIT_TIME}s for registry propagation..."
          echo "::notice::Sleeping for registry sync - consider implementing retry logic"
          sleep $WAIT_TIME
          echo "✓ Wait complete"

      - name: Generate RPCN Connector docs
        id: generate
        env:
          GITHUB_TOKEN: ${{ env.ACTIONS_BOT_TOKEN }}
        run: |
          set -e

          echo "Starting RPCN connector docs generation..."
          if ! npx --no-install doc-tools generate rpcn-connector-docs \
            --fetch-connectors \
            --overrides "$DOCS_OVERRIDES" \
            --draft-missing \
            --update-whats-new \
            --include-bloblang > full_output.txt 2>&1; then

            echo "::error::Doc generation failed"
            cat full_output.txt

            # Save failure details for issue creation
            {
              echo "failure_type=generation_failed"
              echo "failure_message<<EOF"
              echo "## Documentation Generation Failed"
              echo ""
              echo "The \`doc-tools generate rpcn-connector-docs\` command failed during execution."
              echo ""
              echo "### Error Output"
              echo "\`\`\`"
              tail -50 full_output.txt 2>/dev/null || echo "Unable to read output"
              echo "\`\`\`"
              echo ""
              echo "### Required Action"
              echo "1. Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for full details"
              echo "2. Verify \`doc-tools\` and \`rpk connect\` are properly installed"
              echo "3. Check if there are issues with the connector data source"
              echo "4. Review the \`$DOCS_OVERRIDES\` file for syntax errors"
              echo "EOF"
            } >> "$GITHUB_OUTPUT"

            exit 1
          fi

          echo "✓ Doc generation completed"

          # Extract PR summary (between markers)
          if grep -q "PR_SUMMARY_START" full_output.txt; then
            sed -n '/<!-- PR_SUMMARY_START -->/,/<!-- PR_SUMMARY_END -->/p' full_output.txt > pr_summary.txt
            echo "✓ PR summary extracted"
          else
            echo "ℹ️  No PR summary found in output"
            touch pr_summary.txt
          fi

          # Build the PR body
          {
            echo "delta_report<<EOF"

            # Include PR summary at the top if it exists
            if [ -s pr_summary.txt ]; then
              cat pr_summary.txt
              echo ""
              echo "---"
              echo ""
            fi

            echo "## 📚 Review Guide"
            echo ""
            echo "For help reviewing this content, see our [Review Guide](https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Review+autogenerated+reference+docs+for+Redpanda+Connect)."
            echo ""
            echo "## 🤖 Generation Details"
            echo ""
            echo "- **Workflow Run:** [\`${{ github.run_id }}\`](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})"
            echo "- **Triggered By:** \`${{ github.event_name }}\`"
            echo "- **Generated At:** \`$(date -u +'%Y-%m-%d %H:%M:%S UTC')\`"
            echo "- **Node Version:** \`${{ env.NODE_VERSION }}\`"
            echo ""
            echo "<details><summary>📋 Generation Output (last 200 lines)</summary>"
            echo ""
            echo '```'
            # Truncate to last 200 lines to avoid PR body size limits (65536 chars)
            TOTAL_LINES=$(wc -l < full_output.txt)
            if [ "$TOTAL_LINES" -gt 200 ]; then
              echo "... (output truncated, showing last 200 of $TOTAL_LINES lines)"
              echo ""
              tail -200 full_output.txt
            else
              cat full_output.txt
            fi
            echo '```'
            echo ""
            echo "_Full logs available in [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})_"
            echo ""
            echo "</details>"
            echo "EOF"
          } >> "$GITHUB_OUTPUT"

      - name: Check for changes
        id: check_changes
        run: |
          if git diff --quiet && git diff --cached --quiet; then
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
            echo "No changes detected"
          else
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
            echo "✓ Changes detected"

            # Show summary of changes
            echo "Changed files:"
            git diff --name-status
          fi

      - name: Create or Update Pull Request
        if: steps.check_changes.outputs.has_changes == 'true'
        id: create_pr
        uses: peter-evans/create-pull-request@v6
        with:
          token: ${{ env.ACTIONS_BOT_TOKEN }}
          base: main
          branch: ${{ env.AUTO_BRANCH }}
          title: 'auto-docs: Update RPCN connector docs'
          commit-message: |
            docs: Update RPCN connector docs

            Auto-generated by workflow run ${{ github.run_id }}
          body: |
            ${{ steps.generate.outputs.delta_report }}

            ${{ steps.setup.outputs.has_manual_commits == 'true' && format('---\n\n## Manual Commits Detected\n\nThis PR contains manual commits that have been preserved:\n\n```\n{0}\n```\n\n**Note:** The automation has added new changes on top of these manual edits. Please review carefully to ensure nothing was overwritten.\n', steps.setup.outputs.manual_commits) || '' }}

            ---

            _This PR was automatically generated. Please review the changes and merge when ready._
          labels: |
            auto-docs
            documentation
          assignees: ${{ github.event.client_payload.assignees || '' }}
          delete-branch: false

      - name: PR Result
        if: always()
        run: |
          if [ "${{ steps.check_changes.outputs.has_changes }}" != "true" ]; then
            echo "No changes detected - PR not created/updated"
            echo "Branch is up to date with generated docs"
          elif [ "${{ steps.create_pr.outputs.pull-request-number }}" != "" ]; then
            echo "✓ PR #${{ steps.create_pr.outputs.pull-request-number }} created/updated successfully"
            echo "PR URL: ${{ steps.create_pr.outputs.pull-request-url }}"
            echo "::notice title=PR Created::PR #${{ steps.create_pr.outputs.pull-request-number }} - ${{ steps.create_pr.outputs.pull-request-url }}"
          else
            echo "No PR created - branch may be up to date with main"
          fi

      - name: Create failure notification issue
        if: failure()
        id: create_issue
        uses: actions/github-script@v7
        with:
          github-token: ${{ env.ACTIONS_BOT_TOKEN }}
          script: |
            const failureType = ${{ toJSON(steps.setup.outputs.failure_type || steps.generate.outputs.failure_type || 'unknown') }};
            const failureMessage = ${{ toJSON(steps.setup.outputs.failure_message || steps.generate.outputs.failure_message || '') }}.trim();

            // Default message if no specific failure was captured
            let issueBody = failureMessage || `## Workflow Failure

            The automated RPCN connector docs update workflow failed.

            ### Details
            - **Workflow Run:** [\`${{ github.run_id }}\`](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
            - **Branch:** \`${{ env.AUTO_BRANCH }}\`
            - **Triggered By:** \`${{ github.event_name }}\`
            - **Failed At:** \`${new Date().toISOString()}\`

            ### Required Action
            Check the workflow logs for details and resolve the issue manually.`;

            // Add common footer
            issueBody += `

            ---

            _This issue was automatically created by the [Update RPCN Connector Docs workflow](${{ github.server_url }}/${{ github.repository }}/actions/workflows/update-rpcn-connector-docs.yml)._`;

            // Check if there's already an open issue for this workflow
            const existingIssues = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: 'open',
              labels: 'auto-docs-failure',
              per_page: 1
            });

            let issueNumber;

            if (existingIssues.data.length > 0) {
              // Update existing issue
              issueNumber = existingIssues.data[0].number;
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: issueNumber,
                body: `## 🔄 Workflow Failed Again\n\n${issueBody}`
              });
              console.log(`Updated existing issue #${issueNumber}`);
            } else {
              // Create new issue
              const issue = await github.rest.issues.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                title: `Auto-docs workflow failure: ${failureType.replace('_', ' ')}`,
                body: issueBody,
                labels: ['auto-docs-failure', 'bug', 'documentation']
              });
              issueNumber = issue.data.number;
              console.log(`Created new issue #${issueNumber}`);
            }

            // Set output for later steps
            core.setOutput('issue_number', issueNumber);

      - name: Notify on failure
        if: failure()
        run: |
          if [ -n "${{ steps.create_issue.outputs.issue_number }}" ]; then
            echo "::error::Workflow failed - Issue #${{ steps.create_issue.outputs.issue_number }} created/updated"
            echo "Issue: ${{ github.server_url }}/${{ github.repository }}/issues/${{ steps.create_issue.outputs.issue_number }}"
          else
            echo "::error::Workflow failed - check logs for details"
          fi
          echo "Branch: ${{ env.AUTO_BRANCH }}"