Skip to content

Commit 14f9735

Browse files
mikekistlerhalter73
mikekistler
and
halter73
authored
Add client conformance tests (#1102)
Co-authored-by: Stephen Halter <halter73@gmail.com>
1 parent f3fb617 commit 14f9735

8 files changed

Lines changed: 343 additions & 40 deletions

File tree

‎ModelContextProtocol.slnx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
<Folder Name="/tests/">
7171
<Project Path="tests/ModelContextProtocol.Analyzers.Tests/ModelContextProtocol.Analyzers.Tests.csproj" />
7272
<Project Path="tests/ModelContextProtocol.AspNetCore.Tests/ModelContextProtocol.AspNetCore.Tests.csproj" />
73+
<Project Path="tests/ModelContextProtocol.ConformanceClient/ModelContextProtocol.ConformanceClient.csproj" />
7374
<Project Path="tests/ModelContextProtocol.ConformanceServer/ModelContextProtocol.ConformanceServer.csproj" />
7475
<Project Path="tests/ModelContextProtocol.TestOAuthServer/ModelContextProtocol.TestOAuthServer.csproj" />
7576
<Project Path="tests/ModelContextProtocol.Tests/ModelContextProtocol.Tests.csproj" />

‎tests/Common/Utils/NodeHelpers.cs‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
using System.Diagnostics;
2+
using System.Runtime.InteropServices;
3+
4+
namespace ModelContextProtocol.Tests.Utils;
5+
6+
/// <summary>
7+
/// Helper utilities for Node.js and npm operations.
8+
/// </summary>
9+
public static class NodeHelpers
10+
{
11+
/// <summary>
12+
/// Creates a ProcessStartInfo configured to run npx with the specified arguments.
13+
/// </summary>
14+
/// <param name="arguments">The arguments to pass to npx.</param>
15+
/// <returns>A configured ProcessStartInfo for running npx.</returns>
16+
public static ProcessStartInfo NpxStartInfo(string arguments)
17+
{
18+
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
19+
{
20+
// On Windows, npx is a PowerShell script, so we need to use cmd.exe to invoke it
21+
return new ProcessStartInfo
22+
{
23+
FileName = "cmd.exe",
24+
Arguments = $"/c npx {arguments}",
25+
RedirectStandardOutput = true,
26+
RedirectStandardError = true,
27+
UseShellExecute = false,
28+
CreateNoWindow = true
29+
};
30+
}
31+
else
32+
{
33+
// On Unix-like systems, npx is typically a shell script that can be executed directly
34+
return new ProcessStartInfo
35+
{
36+
FileName = "npx",
37+
Arguments = arguments,
38+
RedirectStandardOutput = true,
39+
RedirectStandardError = true,
40+
UseShellExecute = false,
41+
CreateNoWindow = true
42+
};
43+
}
44+
}
45+
46+
/// <summary>
47+
/// Checks if Node.js and npx are installed and available on the system.
48+
/// </summary>
49+
/// <returns>True if npx is available, false otherwise.</returns>
50+
public static bool IsNpxInstalled()
51+
{
52+
try
53+
{
54+
var startInfo = NpxStartInfo("--version");
55+
56+
using var process = Process.Start(startInfo);
57+
if (process == null)
58+
{
59+
return false;
60+
}
61+
62+
process.WaitForExit(5000);
63+
return process.ExitCode == 0;
64+
}
65+
catch
66+
{
67+
return false;
68+
}
69+
}
70+
}

‎tests/ModelContextProtocol.AspNetCore.Tests/ClientConformanceTests.cs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
using System.Diagnostics;
2+
using System.Text;
3+
using ModelContextProtocol.Tests.Utils;
4+
5+
namespace ModelContextProtocol.ConformanceTests;
6+
7+
/// <summary>
8+
/// Runs the official MCP conformance tests against the ConformanceClient.
9+
/// This test runs the Node.js-based conformance test suite for the client
10+
/// and reports the results.
11+
/// </summary>
12+
public class ClientConformanceTests //: IAsyncLifetime
13+
{
14+
private readonly ITestOutputHelper _output;
15+
16+
// Public static property required for SkipUnless attribute
17+
public static bool IsNpxInstalled => NodeHelpers.IsNpxInstalled();
18+
19+
public ClientConformanceTests(ITestOutputHelper output)
20+
{
21+
_output = output;
22+
}
23+
24+
[Theory(Skip = "npx is not installed. Skipping client conformance tests.", SkipUnless = nameof(IsNpxInstalled))]
25+
[InlineData("initialize")]
26+
[InlineData("tools_call")]
27+
[InlineData("auth/metadata-default")]
28+
[InlineData("auth/metadata-var1")]
29+
[InlineData("auth/metadata-var2")]
30+
[InlineData("auth/metadata-var3")]
31+
[InlineData("auth/basic-cimd")]
32+
// [InlineData("auth/2025-03-26-oauth-metadata-backcompat")]
33+
// [InlineData("auth/2025-03-26-oauth-endpoint-fallback")]
34+
[InlineData("auth/scope-from-www-authenticate")]
35+
[InlineData("auth/scope-from-scopes-supported")]
36+
[InlineData("auth/scope-omitted-when-undefined")]
37+
[InlineData("auth/scope-step-up")]
38+
public async Task RunConformanceTest(string scenario)
39+
{
40+
// Run the conformance test suite
41+
var result = await RunClientConformanceScenario(scenario);
42+
43+
// Report the results
44+
Assert.True(result.Success,
45+
$"Conformance test failed.\n\nStdout:\n{result.Output}\n\nStderr:\n{result.Error}");
46+
}
47+
48+
private async Task<(bool Success, string Output, string Error)> RunClientConformanceScenario(string scenario)
49+
{
50+
// Construct an absolute path to the conformance client executable
51+
var exeSuffix = OperatingSystem.IsWindows() ? ".exe" : "";
52+
var conformanceClientPath = Path.GetFullPath($"./ModelContextProtocol.ConformanceClient{exeSuffix}");
53+
// Replace AspNetCore.Tests with ConformanceClient in the path
54+
conformanceClientPath = conformanceClientPath.Replace("AspNetCore.Tests", "ConformanceClient");
55+
56+
if (!File.Exists(conformanceClientPath))
57+
{
58+
throw new FileNotFoundException(
59+
$"ConformanceClient executable not found at: {conformanceClientPath}");
60+
}
61+
62+
var startInfo = NodeHelpers.NpxStartInfo($"-y @modelcontextprotocol/conformance client --scenario {scenario} --command \"{conformanceClientPath} {scenario}\"");
63+
64+
var outputBuilder = new StringBuilder();
65+
var errorBuilder = new StringBuilder();
66+
67+
var process = new Process { StartInfo = startInfo };
68+
69+
process.OutputDataReceived += (sender, e) =>
70+
{
71+
if (e.Data != null)
72+
{
73+
_output.WriteLine(e.Data);
74+
outputBuilder.AppendLine(e.Data);
75+
}
76+
};
77+
78+
process.ErrorDataReceived += (sender, e) =>
79+
{
80+
if (e.Data != null)
81+
{
82+
_output.WriteLine(e.Data);
83+
errorBuilder.AppendLine(e.Data);
84+
}
85+
};
86+
87+
process.Start();
88+
process.BeginOutputReadLine();
89+
process.BeginErrorReadLine();
90+
91+
await process.WaitForExitAsync();
92+
93+
return (
94+
Success: process.ExitCode == 0,
95+
Output: outputBuilder.ToString(),
96+
Error: errorBuilder.ToString()
97+
);
98+
}
99+
}

‎tests/ModelContextProtocol.AspNetCore.Tests/ModelContextProtocol.AspNetCore.Tests.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
<ProjectReference Include="..\..\src\ModelContextProtocol.AspNetCore\ModelContextProtocol.AspNetCore.csproj" />
5858
<ProjectReference Include="..\ModelContextProtocol.TestSseServer\ModelContextProtocol.TestSseServer.csproj" />
5959
<ProjectReference Include="..\ModelContextProtocol.TestOAuthServer\ModelContextProtocol.TestOAuthServer.csproj" />
60+
<ProjectReference Include="..\ModelContextProtocol.ConformanceClient\ModelContextProtocol.ConformanceClient.csproj" />
6061
<ProjectReference Include="..\ModelContextProtocol.ConformanceServer\ModelContextProtocol.ConformanceServer.csproj" />
6162
</ItemGroup>
6263

‎tests/ModelContextProtocol.AspNetCore.Tests/ServerConformanceTests.cs‎

Lines changed: 2 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ public async ValueTask DisposeAsync()
9898
public async Task RunConformanceTests()
9999
{
100100
// Check if Node.js is installed
101-
Assert.SkipWhen(!IsNodeInstalled(), "Node.js is not installed. Skipping conformance tests.");
101+
Assert.SkipWhen(!NodeHelpers.IsNpxInstalled(), "Node.js is not installed. Skipping conformance tests.");
102102

103103
// Run the conformance test suite
104104
var result = await RunNpxConformanceTests();
@@ -117,15 +117,7 @@ private void StartConformanceServer()
117117

118118
private async Task<(bool Success, string Output, string Error)> RunNpxConformanceTests()
119119
{
120-
var startInfo = new ProcessStartInfo
121-
{
122-
FileName = "npx",
123-
Arguments = $"-y @modelcontextprotocol/conformance server --url {_serverUrl}",
124-
RedirectStandardOutput = true,
125-
RedirectStandardError = true,
126-
UseShellExecute = false,
127-
CreateNoWindow = true
128-
};
120+
var startInfo = NodeHelpers.NpxStartInfo($"-y @modelcontextprotocol/conformance server --url {_serverUrl}");
129121

130122
var outputBuilder = new StringBuilder();
131123
var errorBuilder = new StringBuilder();
@@ -162,33 +154,4 @@ private void StartConformanceServer()
162154
Error: errorBuilder.ToString()
163155
);
164156
}
165-
166-
private static bool IsNodeInstalled()
167-
{
168-
try
169-
{
170-
var startInfo = new ProcessStartInfo
171-
{
172-
FileName = "npx", // Check specifically for npx because windows seems unable to find it
173-
Arguments = "--version",
174-
RedirectStandardOutput = true,
175-
RedirectStandardError = true,
176-
UseShellExecute = false,
177-
CreateNoWindow = true
178-
};
179-
180-
using var process = Process.Start(startInfo);
181-
if (process == null)
182-
{
183-
return false;
184-
}
185-
186-
process.WaitForExit(5000);
187-
return process.ExitCode == 0;
188-
}
189-
catch
190-
{
191-
return false;
192-
}
193-
}
194157
}

‎tests/ModelContextProtocol.ConformanceClient/ModelContextProtocol.ConformanceClient.csproj‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFrameworks>net10.0;net9.0;net8.0</TargetFrameworks>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
<OutputType>Exe</OutputType>
8+
</PropertyGroup>
9+
10+
<PropertyGroup Condition="'$(TargetFramework)' == 'net9.0'">
11+
<!-- For better test coverage, only disable reflection in one of the targets -->
12+
<JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
13+
</PropertyGroup>
14+
15+
<ItemGroup>
16+
<ProjectReference Include="../../src/ModelContextProtocol.Core/ModelContextProtocol.Core.csproj" />
17+
</ItemGroup>
18+
19+
<ItemGroup>
20+
<PackageReference Include="Microsoft.Extensions.Logging.Console" />
21+
</ItemGroup>
22+
23+
</Project>

0 commit comments

Comments
 (0)