Skip to content

Commit cad63bd

Browse files
data-douserdata-douser
authored
Fixes and integration tests for MCP-provided prompts and VSIX bundle (#71)
1 parent 532791a commit cad63bd

28 files changed

+56215
-2691
lines changed

.github/workflows/build-extension.yml renamed to .github/workflows/build-and-test-extension.yml

Lines changed: 30 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,73 +1,85 @@
1-
name: Build Extension - CodeQL Development MCP Server
1+
name: Build and Test Extension - CodeQL Development MCP Server
22

33
on:
44
pull_request:
55
branches: ['main']
66
paths:
7-
- '.github/workflows/build-extension.yml'
7+
- '.github/workflows/build-and-test-extension.yml'
88
- '.node-version'
99
- 'extensions/vscode/**'
1010
- 'server/dist/**'
1111
- 'server/ql/*/tools/src/**'
12+
- 'server/src/**'
1213
push:
1314
branches: ['main']
1415
paths:
15-
- '.github/workflows/build-extension.yml'
16+
- '.github/workflows/build-and-test-extension.yml'
1617
- '.node-version'
1718
- 'extensions/vscode/**'
1819
- 'server/dist/**'
1920
- 'server/ql/*/tools/src/**'
21+
- 'server/src/**'
2022
workflow_dispatch:
2123

2224
permissions:
2325
contents: read
2426

2527
jobs:
26-
build-extension:
27-
name: Build Extension
28+
build-and-test-extension:
29+
name: Build and Test Extension
2830
runs-on: ubuntu-latest
2931

3032
steps:
31-
- name: Build Extension - Checkout repository
33+
- name: Checkout repository
3234
uses: actions/checkout@v6
3335

34-
- name: Build Extension - Setup Node.js environment
36+
- name: Setup Node.js environment
3537
uses: actions/setup-node@v6
3638
with:
3739
cache: 'npm'
3840
node-version-file: '.node-version'
3941

40-
- name: Build Extension - Install dependencies
42+
- name: Install dependencies
4143
run: npm ci --include=optional
4244

43-
- name: Build Extension - Build server (dependency)
45+
- name: Build server (dependency)
4446
run: npm run build -w server
4547

46-
- name: Build Extension - Run extension tests with coverage
48+
- name: Run extension unit tests with coverage
4749
working-directory: extensions/vscode
4850
run: npm run test:coverage
4951

50-
- name: Build Extension - Bundle extension and server
52+
- name: Bundle extension and server
5153
working-directory: extensions/vscode
5254
run: |
5355
npm run clean
5456
npm run lint
5557
npm run bundle
5658
npm run bundle:server
5759
58-
- name: Build Extension - Verify VSIX packaging
60+
- name: Setup CodeQL environment for integration tests
61+
uses: ./.github/actions/setup-codeql-environment
62+
with:
63+
add-to-path: 'true'
64+
install-language-runtimes: 'false'
65+
66+
- name: Run Extension Host integration tests
67+
working-directory: extensions/vscode
68+
run: xvfb-run -a npm run test:integration
69+
70+
- name: Verify VSIX packaging
5971
working-directory: extensions/vscode
6072
run: npx @vscode/vsce package --no-dependencies --out codeql-development-mcp-server.vsix
6173

62-
- name: Build Extension - Verify VSIX contents
74+
- name: Verify VSIX contents
6375
working-directory: extensions/vscode
6476
run: |
6577
echo "## VSIX Contents" >> $GITHUB_STEP_SUMMARY
6678
echo '```' >> $GITHUB_STEP_SUMMARY
6779
npx @vscode/vsce ls --no-dependencies --tree 2>&1 | head -50 >> $GITHUB_STEP_SUMMARY
6880
echo '```' >> $GITHUB_STEP_SUMMARY
6981
70-
- name: Build Extension - Check for uncommitted changes
82+
- name: Check for uncommitted changes
7183
run: |
7284
if [ -n "$(git status --porcelain)" ]; then
7385
echo "❌ Uncommitted changes detected after build:"
@@ -78,12 +90,13 @@ jobs:
7890
echo "✅ No uncommitted changes after build"
7991
fi
8092
81-
- name: Build Extension - Summary
93+
- name: Summary
8294
run: |
83-
echo "## Build Extension Summary" >> $GITHUB_STEP_SUMMARY
95+
echo "## Build and Test Extension Summary" >> $GITHUB_STEP_SUMMARY
8496
echo "✅ ESLint checks completed" >> $GITHUB_STEP_SUMMARY
85-
echo "✅ All tests passed with coverage" >> $GITHUB_STEP_SUMMARY
97+
echo "✅ Unit tests passed with coverage" >> $GITHUB_STEP_SUMMARY
8698
echo "✅ Extension bundled successfully" >> $GITHUB_STEP_SUMMARY
8799
echo "✅ Server bundled into extension" >> $GITHUB_STEP_SUMMARY
100+
echo "✅ Extension Host integration tests passed" >> $GITHUB_STEP_SUMMARY
88101
echo "✅ VSIX packaging verified" >> $GITHUB_STEP_SUMMARY
89102
echo "✅ No uncommitted changes detected" >> $GITHUB_STEP_SUMMARY

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ codeql-development-mcp-server.code-workspace
1919

2020
## Globs to ignore
2121
*.log
22+
# Allow *.log files committed as test fixtures
23+
!extensions/vscode/test/fixtures/**/*.log
2224
*.pid
2325
*.swp
2426
*.swo

‎docs/getting-started.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ See the [VS Code Extension guide](./vscode/extension.md) for details.
1818

1919
The `.vsix` can be downloaded from
2020
[GitHub Releases](https://github.com/advanced-security/codeql-development-mcp-server/releases)
21-
or built from source (`npm run package:vscode` at the repository root).
21+
or built from source (`npm run package:vsix` at the repository root).
2222

2323
### From npm
2424

‎docs/testing.md‎

Lines changed: 47 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,40 @@
44
55
## Overview
66

7-
Every release of the CodeQL Development MCP Server is validated through three progressively broader layers of testing. Each layer builds on the previous one, moving from fast unit checks to real-world agentic validation.
7+
Every release of the CodeQL Development MCP Server is validated through four progressively broader layers of testing. Each layer builds on the previous one, moving from fast unit checks to real-world agentic validation.
88

9-
| Layer | Scope | Framework | Location |
10-
| ---------------------- | ----------------------------------------------- | ------------------------------ | ----------------------------- |
11-
| 1 — Unit tests | TypeScript source code | Vitest | `server/test/**/*.test.ts` |
12-
| 2 — Integration tests | Individual MCP tools against a running server | Custom MCP client | `client/integration-tests/**` |
13-
| 3 — Agentic validation | End-to-end CodeQL workflows driven by AI agents | GitHub Copilot agents + skills | `.github/{agents,skills}/**` |
9+
| Layer | Scope | Framework | Location |
10+
| -------------------------------- | ----------------------------------------------- | ------------------------------ | ------------------------------------------- |
11+
| 1a — Server unit tests | MCP server TypeScript source code | Vitest | `server/test/**/*.test.ts` |
12+
| 1b — Extension unit tests | VS Code extension TypeScript source code | Vitest | `extensions/vscode/test/**/*.test.ts` |
13+
| 2a — MCP tool integration tests | Individual MCP tools against a running server | Custom MCP client | `client/integration-tests/**` |
14+
| 2b — Extension integration tests | Extension behaviour inside a VS Code host | @vscode/test-cli + Mocha | `extensions/vscode/test/suite/**/*.test.ts` |
15+
| 3 — Agentic validation | End-to-end CodeQL workflows driven by AI agents | GitHub Copilot agents + skills | `.github/{agents,skills}/**` |
1416

1517
## Layer 1 — Unit Tests
1618

19+
### 1a — Server unit tests
20+
1721
Unit tests verify the TypeScript implementation of the MCP server.
1822

1923
- **Framework**: [Vitest](https://vitest.dev/) with code coverage via `@vitest/coverage-v8`.
2024
- **Location**: `server/test/**/*.test.ts`, mirroring `server/src/**/*.ts`.
2125
- **Run command**: `npm run test -w server` (or `npm run test:server` from the repo root).
2226
- **Conventions**: Tests follow the Arrange-Act-Assert (AAA) pattern, use descriptive names, and mock external dependencies where necessary.
2327

28+
### 1b — Extension unit tests
29+
30+
Unit tests verify the VS Code extension's TypeScript code outside of the Extension Host (no VS Code API dependency).
31+
32+
- **Framework**: [Vitest](https://vitest.dev/) with code coverage via `@vitest/coverage-v8`.
33+
- **Location**: `extensions/vscode/test/**/*.test.ts` (excluding `test/suite/`).
34+
- **Run command**: `npm run test:coverage -w extensions/vscode` (or `npm run test:vscode` from the repo root, which also runs integration tests).
35+
- **Conventions**: Same AAA pattern and naming conventions as the server unit tests.
36+
2437
## Layer 2 — Integration Tests
2538

39+
### 2a — MCP tool integration tests
40+
2641
Integration tests exercise individual MCP tools against a live server instance using the custom MCP client.
2742

2843
- **Client**: `client/src/ql-mcp-client.js` — starts the MCP server, invokes tools, and validates results.
@@ -33,6 +48,22 @@ Integration tests exercise individual MCP tools against a live server instance u
3348
- No mocks — tests use real CodeQL databases and queries bundled under `server/ql/`.
3449
- The `before/monitoring-state.json` file supplies tool arguments. For file-based tests, the integration-test runner diffs filesystem state from `before/` to `after/`; for monitoring-based tests, `after/` artifacts are generally not diffed and are only interpreted for specific validations (for example, `codeql_query_run` interpreted output).
3550

51+
### 2b — Extension integration tests
52+
53+
Extension integration tests run inside a real VS Code Extension Development Host with the full VS Code API. They validate activation, MCP server registration, CodeQL extension bridging, and workspace-aware scenarios.
54+
55+
- **Framework**: [@vscode/test-cli](https://github.com/nicolo-ribaudo/vscode-test-cli) + [Mocha](https://mochajs.org/) (TDD UI).
56+
- **Location**: `extensions/vscode/test/suite/**/*.integration.test.ts` (compiled to `dist/test/suite/*.test.cjs` by esbuild).
57+
- **Configuration**: `extensions/vscode/.vscode-test.mjs` — defines three test profiles:
58+
- `noWorkspace` — no workspace folder open
59+
- `singleFolder` — single-folder workspace fixture
60+
- `multiRoot` — multi-root workspace fixture
61+
- **Run commands**:
62+
- `npm run test:integration -w extensions/vscode` — run all profiles
63+
- `npm run test:integration:label -w extensions/vscode -- noWorkspace` — run a single profile
64+
- **Prerequisites**: `npm run bundle -w extensions/vscode` must be run first to compile the extension and test suite.
65+
- **CI**: Runs under `xvfb-run` on Linux for headless display support (see `.github/workflows/build-and-test-extension.yml`).
66+
3667
## Layer 3 — Agentic Validation
3768

3869
Agentic validation uses GitHub Copilot agents and skills to solve CodeQL query development challenges, exercising MCP tools in realistic end-to-end scenarios.
@@ -49,14 +80,20 @@ Agentic validation uses GitHub Copilot agents and skills to solve CodeQL query d
4980
From the repository root:
5081

5182
```bash
52-
# Build everything and run all layers (unit + integration)
83+
# Build everything and run all layers (1a + 1b + 2a + 2b)
5384
npm run build-and-test
5485

55-
# Run only unit tests
86+
# Run only server unit tests (1a)
5687
npm run test:server
5788

58-
# Run only integration tests
89+
# Run extension unit tests + integration tests (1b + 2b)
90+
npm run test:vscode
91+
92+
# Run only MCP tool integration tests (2a)
5993
npm run test:client
94+
95+
# Run only extension integration tests (2b)
96+
npm run test:integration -w extensions/vscode
6097
```
6198

62-
> **Note**: Layer 3 (agentic validation) runs as part of the CI/CD pipeline via GitHub Actions workflows and is not included in the local `build-and-test` command.
99+
> **Note**: `npm run test:vscode` runs `npm test -w extensions/vscode`, which executes both Vitest unit tests (`test:coverage`) and Extension Host integration tests (`test:integration`) in sequence. Layer 3 (agentic validation) runs as part of the CI/CD pipeline via GitHub Actions workflows and is not included in the local `build-and-test` command.

‎docs/vscode/extension.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ Or in VS Code: **Extensions** sidebar → `⋯` menu → **Install from VSIX…*
3939
From the repository root:
4040

4141
```bash
42-
npm run package:vscode
42+
npm run package:vsix
4343
code --install-extension extensions/vscode/codeql-development-mcp-server.vsix
4444
```
4545

‎extensions/vscode/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,6 @@ dist/
33
server/
44
coverage/
55
*.vsix
6+
7+
# @vscode/test-cli downloads and runtime data
8+
.vscode-test/

‎extensions/vscode/.vscode-test.mjs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/**
2+
* Configuration for @vscode/test-cli.
3+
*
4+
* Defines integration test profiles that run inside a VS Code Extension
5+
* Development Host with the REAL VS Code API. The compiled Mocha test
6+
* suite (dist/test/suite/*.cjs) is discovered by the runner in index.ts.
7+
*
8+
* Run all profiles: npx vscode-test
9+
* Run one profile: npx vscode-test --label noWorkspace
10+
*
11+
* Prerequisites:
12+
* npm run bundle # builds extension + test suite
13+
* npm run bundle:server # bundles MCP server for e2e tests
14+
*/
15+
16+
import { defineConfig } from '@vscode/test-cli';
17+
18+
export default defineConfig([
19+
{
20+
label: 'noWorkspace',
21+
files: 'dist/test/suite/*.test.cjs',
22+
version: 'stable',
23+
mocha: {
24+
ui: 'tdd',
25+
color: true,
26+
timeout: 60_000,
27+
},
28+
},
29+
{
30+
label: 'singleFolder',
31+
files: 'dist/test/suite/*.test.cjs',
32+
version: 'stable',
33+
workspaceFolder: './test/fixtures/single-folder-workspace',
34+
mocha: {
35+
ui: 'tdd',
36+
color: true,
37+
timeout: 60_000,
38+
},
39+
},
40+
{
41+
label: 'multiRoot',
42+
files: 'dist/test/suite/*.test.cjs',
43+
version: 'stable',
44+
workspaceFolder: './test/fixtures/multi-root-workspace/test.code-workspace',
45+
mocha: {
46+
ui: 'tdd',
47+
color: true,
48+
timeout: 60_000,
49+
},
50+
},
51+
]);

‎extensions/vscode/.vscodeignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Source and config (not needed at runtime)
22
.vscode/**
33
.vscode-test/**
4+
.vscode-test.mjs
45
src/**
56
test/**
67
scripts/**

‎extensions/vscode/package.json‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "vscode-codeql-development-mcp-server",
33
"displayName": "CodeQL Development MCP Server",
4-
"description": "Automatically installs, configures, and manages the CodeQL Development MCP Server in VS Code.",
4+
"description": "LLM-assisted development of CodeQL queries, libraries, and tests via #ql-mcp prompts, resources, and tools.",
55
"version": "2.24.1",
66
"publisher": "advanced-security",
77
"license": "SEE LICENSE IN LICENSE",
@@ -65,7 +65,9 @@
6565
},
6666
"codeql-mcp.serverArgs": {
6767
"type": "array",
68-
"items": { "type": "string" },
68+
"items": {
69+
"type": "string"
70+
},
6971
"default": [],
7072
"description": "Custom arguments for the MCP server command. When empty, the bundled server entry point is used automatically. Set to e.g. ['/path/to/server/dist/codeql-development-mcp-server.js'] for local development."
7173
},
@@ -84,19 +86,25 @@
8486
},
8587
"codeql-mcp.additionalDatabaseDirs": {
8688
"type": "array",
87-
"items": { "type": "string" },
89+
"items": {
90+
"type": "string"
91+
},
8892
"default": [],
8993
"description": "Additional directories to search for CodeQL databases. Appended to CODEQL_DATABASES_BASE_DIRS alongside the vscode-codeql storage paths."
9094
},
9195
"codeql-mcp.additionalMrvaRunResultsDirs": {
9296
"type": "array",
93-
"items": { "type": "string" },
97+
"items": {
98+
"type": "string"
99+
},
94100
"default": [],
95101
"description": "Additional directories containing MRVA run result subdirectories. Appended to CODEQL_MRVA_RUN_RESULTS_DIRS alongside the vscode-codeql variant analysis storage path."
96102
},
97103
"codeql-mcp.additionalQueryRunResultsDirs": {
98104
"type": "array",
99-
"items": { "type": "string" },
105+
"items": {
106+
"type": "string"
107+
},
100108
"default": [],
101109
"description": "Additional directories containing query run result subdirectories. Appended to CODEQL_QUERY_RUN_RESULTS_DIRS alongside the vscode-codeql query storage path."
102110
}
@@ -133,8 +141,10 @@
133141
"lint": "eslint src/ test/",
134142
"lint:fix": "eslint src/ test/ --fix",
135143
"package": "vsce package --no-dependencies --out codeql-development-mcp-server.vsix",
136-
"test": "vitest --run",
144+
"test": "npm run test:coverage && npm run test:integration",
137145
"test:coverage": "vitest --run --coverage",
146+
"test:integration": "vscode-test",
147+
"test:integration:label": "vscode-test --label",
138148
"test:watch": "vitest --watch",
139149
"vscode:prepublish": "npm run clean && npm run lint && npm run bundle && npm run bundle:server",
140150
"watch": "node esbuild.config.js --watch"
@@ -145,6 +155,8 @@
145155
"@types/node": "^25.3.0",
146156
"@types/vscode": "^1.109.0",
147157
"@vitest/coverage-v8": "^4.0.18",
158+
"@vscode/test-cli": "^0.0.12",
159+
"@vscode/test-electron": "^2.5.2",
148160
"@vscode/vsce": "^3.7.1",
149161
"esbuild": "^0.27.3",
150162
"eslint": "^10.0.0",

0 commit comments

Comments
 (0)