Test Copilot CLI #245
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Copilot CLI | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - action.yml | |
| - .github/workflows/test-copilot.yml | |
| pull_request: | |
| paths: | |
| - action.yml | |
| - .github/workflows/test-copilot.yml | |
| schedule: | |
| # Nightly drift check — the action installs @github/copilot@prerelease at | |
| # runtime, so this is the only thing that catches a breaking CLI release. | |
| - cron: '0 7 * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| copilot-requests: write | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Basic smoke test — defaults only | |
| basic: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| id: copilot | |
| with: | |
| prompt: "Say 'hello world' and nothing else." | |
| max-turns: 1 | |
| - name: Verify exit code output | |
| run: | | |
| echo "Exit code: ${{ steps.copilot.outputs.exit-code }}" | |
| test "${{ steps.copilot.outputs.exit-code }}" = "0" | |
| # Test autopilot + max-turns limit with tool use | |
| autopilot: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Create a file called test.txt with the content 'autopilot works'. Then read it back to confirm." | |
| max-turns: 3 | |
| - name: Verify file was created | |
| run: | | |
| test -f test.txt | |
| grep -q "autopilot works" test.txt | |
| # Test silent mode — no usage stats in output | |
| silent: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say 'silent mode works' and nothing else." | |
| silent: true | |
| max-turns: 1 | |
| # Test model selection + reasoning effort | |
| model: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "What model are you? Reply with just your model name." | |
| model: claude-sonnet-4 | |
| reasoning-effort: low | |
| max-turns: 1 | |
| # Test JSON output format | |
| json-output: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say 'json works'." | |
| output-format: json | |
| max-turns: 1 | |
| # Test tool deny list — shell should be blocked | |
| denied-tools: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Try to run 'echo hello' in a shell. If you can't, just say 'shell denied'." | |
| denied-tools: "shell" | |
| max-turns: 2 | |
| # Test GitHub MCP toolsets | |
| github-mcp: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Use the GitHub MCP to get info about the repo austenstone/copilot-cli. Tell me the description and star count." | |
| add-github-mcp-toolsets: "repos" | |
| max-turns: 3 | |
| # Test disable builtin MCPs | |
| no-builtin-mcps: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "List your available MCP servers. If none, say 'no MCP servers available'." | |
| disable-builtin-mcps: true | |
| max-turns: 1 | |
| # Test URL allow/deny | |
| url-controls: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Fetch https://api.github.com/zen and tell me what it says." | |
| allowed-urls: "api.github.com" | |
| denied-urls: "evil.com" | |
| max-turns: 2 | |
| # Test share session to file + verify output | |
| share: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| id: copilot | |
| with: | |
| prompt: "Say 'session sharing works'." | |
| share: true | |
| max-turns: 1 | |
| - name: Verify session file exists | |
| run: | | |
| echo "Session path: ${{ steps.copilot.outputs.session-path }}" | |
| test -f "${{ steps.copilot.outputs.session-path }}" | |
| cat "${{ steps.copilot.outputs.session-path }}" | |
| # Test experimental flag | |
| experimental: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say 'experimental mode active'." | |
| experimental: true | |
| max-turns: 1 | |
| # Test additional directories | |
| additional-dirs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - run: | | |
| mkdir -p /tmp/copilot-test | |
| echo "found me" > /tmp/copilot-test/secret.txt | |
| - uses: ./ | |
| with: | |
| prompt: "Read the file /tmp/copilot-test/secret.txt and tell me its contents." | |
| additional-directories: "/tmp/copilot-test" | |
| max-turns: 2 | |
| # Test custom agent | |
| agent: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "What files are in this repo?" | |
| agent: explore | |
| max-turns: 2 | |
| # Test fail-on-error catches non-zero exit | |
| fail-on-error: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| id: copilot | |
| continue-on-error: true | |
| with: | |
| prompt: "Exit with error code 1 by running: exit 1" | |
| fail-on-error: true | |
| max-turns: 2 | |
| - name: Verify exit code was captured | |
| run: | | |
| echo "Exit code: ${{ steps.copilot.outputs.exit-code }}" | |
| echo "Step outcome: ${{ steps.copilot.outcome }}" | |
| # Test copilot-config is applied | |
| config: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say 'config test passed'." | |
| copilot-config: | | |
| { | |
| "banner": "never", | |
| "render_markdown": false, | |
| "theme": "dark", | |
| "trusted_folders": [] | |
| } | |
| max-turns: 1 | |
| # Test long context window selection | |
| context: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say hello and exit." | |
| context: long_context | |
| max-turns: 1 | |
| # Test blanket URL allow | |
| allow-all-urls: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say hello and exit." | |
| allow-all-urls: true | |
| max-turns: 1 | |
| # Test blanket path allow | |
| allow-all-paths: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say hello and exit." | |
| allow-all-paths: true | |
| max-turns: 1 | |
| # Test forwarding environment variables to the CLI | |
| secret-env-vars: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| env: | |
| DUMMY_SECRET: hunter2 | |
| with: | |
| prompt: "Print nothing sensitive and exit." | |
| secret-env-vars: DUMMY_SECRET | |
| max-turns: 1 | |
| # Test tool exclusion list | |
| excluded-tools: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say hello." | |
| excluded-tools: "shell" | |
| max-turns: 1 | |
| # Test tool allow list | |
| available-tools: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say hello." | |
| available-tools: "view" | |
| max-turns: 1 | |
| # Test persistent memory | |
| enable-memory: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say hello and exit." | |
| enable-memory: true | |
| max-turns: 1 | |
| # Test plan mode — the action skips --autopilot when mode is set | |
| mode-plan: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say hello and exit." | |
| mode: plan | |
| max-turns: 1 | |
| # Test file attachments | |
| attachments: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - run: echo hi > note.txt | |
| - uses: ./ | |
| with: | |
| prompt: "Summarize the attached file." | |
| attachments: note.txt | |
| max-turns: 1 | |
| # Test disabling the temp directory | |
| disallow-temp-dir: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./ | |
| with: | |
| prompt: "Say hello and exit." | |
| disallow-temp-dir: true | |
| max-turns: 1 |