Skip to content

Commit d96eb70

Browse files
mdaiglemdaigle
authored
Test | Split TvpTest part1 (#3956)
* Split baseline files. * Fix split test compilation. * Remove original TestMain and combined output files. * Fix compilation. Add flaky category. * Add tests to single collection to force them to run serially. * Set culture. * Remove retry on flaky manual tests. * Mark flaky tests. Fix json streaming tests. * Avoid extra brackets.
1 parent f322be6 commit d96eb70

38 files changed

Lines changed: 4378 additions & 3539 deletions

‎eng/pipelines/common/templates/steps/build-and-run-tests-netcore-step.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,5 +121,4 @@ steps:
121121
--blame-hang
122122
--blame-hang-dump-type full
123123
--blame-hang-timeout 10m
124-
retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}
125124
continueOnError: true

‎eng/pipelines/common/templates/steps/build-and-run-tests-netfx-step.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,5 +121,4 @@ steps:
121121
--blame-hang
122122
--blame-hang-dump-type full
123123
--blame-hang-timeout 10m
124-
retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}
125124
continueOnError: true

‎eng/pipelines/common/templates/steps/run-all-tests-step.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,6 @@ steps:
226226
-p:Filter="category=flaky"
227227
-p:CollectCodeCoverage=false
228228
continueOnError: true
229-
retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}
230229

231230
- ${{ else }}: # Linux or macOS
232231
- ${{if eq(parameters.referenceType, 'Project')}}:
@@ -349,4 +348,3 @@ steps:
349348
verbosityRestore: Detailed
350349
verbosityPack: Detailed
351350
continueOnError: true
352-
retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}

‎src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj‎

Lines changed: 104 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,12 @@
199199
<Compile Include="SQL\ParameterTest\StePermutationSet.cs" />
200200
<Compile Include="SQL\ParameterTest\SteTypeBoundaries.cs" />
201201
<Compile Include="SQL\ParameterTest\StreamInputParam.cs" />
202+
<Compile Include="SQL\ParameterTest\StreamInputParameterTests.cs" />
203+
<Compile Include="SQL\ParameterTest\TvpColumnBoundariesTests.cs" />
204+
<Compile Include="SQL\ParameterTest\TvpQueryHintsTests.cs" />
205+
<Compile Include="SQL\ParameterTest\SqlVariantParameterTests.cs" />
206+
<Compile Include="SQL\ParameterTest\DateTimeVariantTests.cs" />
207+
<Compile Include="SQL\ParameterTest\OutputParameterTests.cs" />
202208
<Compile Include="SQL\ParameterTest\TvpTest.cs" />
203209
<Compile Include="SQL\SplitPacketTest\SplitPacketTest.cs" />
204210
<Compile Include="SQL\SqlCommand\SqlCommandCancelTest.cs" />
@@ -237,14 +243,104 @@
237243
<!-- @TODO: Move these into some folder or a resx or something -->
238244
<Content Include="DDDataTypesTest_Data.xml"
239245
CopyToOutputDirectory="PreserveNewest" />
240-
<Content Include="SqlParameterTest_DebugMode.bsl"
241-
CopyToOutputDirectory="PreserveNewest" />
242-
<Content Include="SqlParameterTest_DebugMode_Azure.bsl"
243-
CopyToOutputDirectory="PreserveNewest" />
244-
<Content Include="SqlParameterTest_ReleaseMode.bsl"
245-
CopyToOutputDirectory="PreserveNewest" />
246-
<Content Include="SqlParameterTest_ReleaseMode_Azure.bsl"
247-
CopyToOutputDirectory="PreserveNewest" />
246+
247+
<!-- Split baseline files for individual test groups -->
248+
<Content Include="SQL\ParameterTest\StreamInputParameter_DebugMode.bsl">
249+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
250+
<Link>StreamInputParameter_DebugMode.bsl</Link>
251+
</Content>
252+
<Content Include="SQL\ParameterTest\StreamInputParameter_DebugMode_Azure.bsl">
253+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
254+
<Link>StreamInputParameter_DebugMode_Azure.bsl</Link>
255+
</Content>
256+
<Content Include="SQL\ParameterTest\StreamInputParameter_ReleaseMode.bsl">
257+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
258+
<Link>StreamInputParameter_ReleaseMode.bsl</Link>
259+
</Content>
260+
<Content Include="SQL\ParameterTest\StreamInputParameter_ReleaseMode_Azure.bsl">
261+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
262+
<Link>StreamInputParameter_ReleaseMode_Azure.bsl</Link>
263+
</Content>
264+
<Content Include="SQL\ParameterTest\TvpColumnBoundaries_DebugMode.bsl">
265+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
266+
<Link>TvpColumnBoundaries_DebugMode.bsl</Link>
267+
</Content>
268+
<Content Include="SQL\ParameterTest\TvpColumnBoundaries_DebugMode_Azure.bsl">
269+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
270+
<Link>TvpColumnBoundaries_DebugMode_Azure.bsl</Link>
271+
</Content>
272+
<Content Include="SQL\ParameterTest\TvpColumnBoundaries_ReleaseMode.bsl">
273+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
274+
<Link>TvpColumnBoundaries_ReleaseMode.bsl</Link>
275+
</Content>
276+
<Content Include="SQL\ParameterTest\TvpColumnBoundaries_ReleaseMode_Azure.bsl">
277+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
278+
<Link>TvpColumnBoundaries_ReleaseMode_Azure.bsl</Link>
279+
</Content>
280+
<Content Include="SQL\ParameterTest\TvpQueryHints_DebugMode.bsl">
281+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
282+
<Link>TvpQueryHints_DebugMode.bsl</Link>
283+
</Content>
284+
<Content Include="SQL\ParameterTest\TvpQueryHints_DebugMode_Azure.bsl">
285+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
286+
<Link>TvpQueryHints_DebugMode_Azure.bsl</Link>
287+
</Content>
288+
<Content Include="SQL\ParameterTest\TvpQueryHints_ReleaseMode.bsl">
289+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
290+
<Link>TvpQueryHints_ReleaseMode.bsl</Link>
291+
</Content>
292+
<Content Include="SQL\ParameterTest\TvpQueryHints_ReleaseMode_Azure.bsl">
293+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
294+
<Link>TvpQueryHints_ReleaseMode_Azure.bsl</Link>
295+
</Content>
296+
<Content Include="SQL\ParameterTest\SqlVariantParameter_DebugMode.bsl">
297+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
298+
<Link>SqlVariantParameter_DebugMode.bsl</Link>
299+
</Content>
300+
<Content Include="SQL\ParameterTest\SqlVariantParameter_DebugMode_Azure.bsl">
301+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
302+
<Link>SqlVariantParameter_DebugMode_Azure.bsl</Link>
303+
</Content>
304+
<Content Include="SQL\ParameterTest\SqlVariantParameter_ReleaseMode.bsl">
305+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
306+
<Link>SqlVariantParameter_ReleaseMode.bsl</Link>
307+
</Content>
308+
<Content Include="SQL\ParameterTest\SqlVariantParameter_ReleaseMode_Azure.bsl">
309+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
310+
<Link>SqlVariantParameter_ReleaseMode_Azure.bsl</Link>
311+
</Content>
312+
<Content Include="SQL\ParameterTest\DateTimeVariant_DebugMode.bsl">
313+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
314+
<Link>DateTimeVariant_DebugMode.bsl</Link>
315+
</Content>
316+
<Content Include="SQL\ParameterTest\DateTimeVariant_DebugMode_Azure.bsl">
317+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
318+
<Link>DateTimeVariant_DebugMode_Azure.bsl</Link>
319+
</Content>
320+
<Content Include="SQL\ParameterTest\DateTimeVariant_ReleaseMode.bsl">
321+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
322+
<Link>DateTimeVariant_ReleaseMode.bsl</Link>
323+
</Content>
324+
<Content Include="SQL\ParameterTest\DateTimeVariant_ReleaseMode_Azure.bsl">
325+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
326+
<Link>DateTimeVariant_ReleaseMode_Azure.bsl</Link>
327+
</Content>
328+
<Content Include="SQL\ParameterTest\OutputParameter_DebugMode.bsl">
329+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
330+
<Link>OutputParameter_DebugMode.bsl</Link>
331+
</Content>
332+
<Content Include="SQL\ParameterTest\OutputParameter_DebugMode_Azure.bsl">
333+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
334+
<Link>OutputParameter_DebugMode_Azure.bsl</Link>
335+
</Content>
336+
<Content Include="SQL\ParameterTest\OutputParameter_ReleaseMode.bsl">
337+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
338+
<Link>OutputParameter_ReleaseMode.bsl</Link>
339+
</Content>
340+
<Content Include="SQL\ParameterTest\OutputParameter_ReleaseMode_Azure.bsl">
341+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
342+
<Link>OutputParameter_ReleaseMode_Azure.bsl</Link>
343+
</Content>
248344
</ItemGroup>
249345

250346
<!-- Items to include in AE test set -->

‎src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,10 @@ private void CompareJsonFiles()
6969
}
7070
}
7171

72-
private void PrintJsonDataToFile(SqlConnection connection)
72+
private void PrintJsonDataToFile(SqlConnection connection, string tableName)
7373
{
7474
DeleteFile(_outputFile);
75-
using (SqlCommand command = new SqlCommand("SELECT [data] FROM [jsonTab]", connection))
75+
using (SqlCommand command = new SqlCommand($"SELECT [data] FROM {tableName}", connection))
7676
{
7777
using (SqlDataReader reader = command.ExecuteReader(CommandBehavior.SequentialAccess))
7878
{
@@ -99,10 +99,10 @@ private void PrintJsonDataToFile(SqlConnection connection)
9999
}
100100
}
101101

102-
private async Task PrintJsonDataToFileAsync(SqlConnection connection)
102+
private async Task PrintJsonDataToFileAsync(SqlConnection connection, string tableName)
103103
{
104104
DeleteFile(_outputFile);
105-
using (SqlCommand command = new SqlCommand("SELECT [data] FROM [jsonTab]", connection))
105+
using (SqlCommand command = new SqlCommand($"SELECT [data] FROM {tableName}", connection))
106106
{
107107
using (SqlDataReader reader = await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess))
108108
{
@@ -129,9 +129,9 @@ private async Task PrintJsonDataToFileAsync(SqlConnection connection)
129129
}
130130
}
131131

132-
private void StreamJsonFileToServer(SqlConnection connection)
132+
private void StreamJsonFileToServer(SqlConnection connection, string tableName)
133133
{
134-
using (SqlCommand cmd = new SqlCommand("INSERT INTO [jsonTab] (data) VALUES (@jsondata)", connection))
134+
using (SqlCommand cmd = new SqlCommand($"INSERT INTO {tableName} (data) VALUES (@jsondata)", connection))
135135
{
136136
using (StreamReader jsonFile = File.OpenText(_jsonFile))
137137
{
@@ -141,9 +141,9 @@ private void StreamJsonFileToServer(SqlConnection connection)
141141
}
142142
}
143143

144-
private async Task StreamJsonFileToServerAsync(SqlConnection connection)
144+
private async Task StreamJsonFileToServerAsync(SqlConnection connection, string tableName)
145145
{
146-
using (SqlCommand cmd = new SqlCommand("INSERT INTO [jsonTab] (data) VALUES (@jsondata)", connection))
146+
using (SqlCommand cmd = new SqlCommand($"INSERT INTO {tableName} (data) VALUES (@jsondata)", connection))
147147
{
148148
using (StreamReader jsonFile = File.OpenText(_jsonFile))
149149
{
@@ -168,9 +168,10 @@ public void TestJsonStreaming()
168168
using (SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString))
169169
{
170170
connection.Open();
171-
DataTestUtility.CreateTable(connection, "jsonTab", "(data json)");
172-
StreamJsonFileToServer(connection);
173-
PrintJsonDataToFile(connection);
171+
var tableName = DataTestUtility.GetLongName("jsonTab");
172+
DataTestUtility.CreateTable(connection, tableName, "(data json)");
173+
StreamJsonFileToServer(connection, tableName);
174+
PrintJsonDataToFile(connection, tableName);
174175
CompareJsonFiles();
175176
DeleteFile(_jsonFile);
176177
DeleteFile(_outputFile);
@@ -184,9 +185,10 @@ public async Task TestJsonStreamingAsync()
184185
using (SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString))
185186
{
186187
await connection.OpenAsync();
187-
DataTestUtility.CreateTable(connection, "jsonTab", "(data json)");
188-
await StreamJsonFileToServerAsync(connection);
189-
await PrintJsonDataToFileAsync(connection);
188+
var tableName = DataTestUtility.GetLongName("jsonTab");
189+
DataTestUtility.CreateTable(connection, tableName, "(data json)");
190+
await StreamJsonFileToServerAsync(connection, tableName);
191+
await PrintJsonDataToFileAsync(connection, tableName);
190192
CompareJsonFiles();
191193
DeleteFile(_jsonFile);
192194
DeleteFile(_outputFile);

‎src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/DateTimeVariantTests.cs‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.Globalization;
7+
using System.IO;
8+
using System.Text;
9+
using System.Threading;
10+
using Xunit;
11+
12+
namespace Microsoft.Data.SqlClient.ManualTesting.Tests
13+
{
14+
/// <summary>
15+
/// Tests for DateTime variant parameters with different date/time types.
16+
/// These tests run independently with their own baseline comparison.
17+
/// </summary>
18+
[Collection("ParameterBaselineTests")]
19+
public class DateTimeVariantTests
20+
{
21+
private readonly string _connStr;
22+
23+
public DateTimeVariantTests()
24+
{
25+
_connStr = DataTestUtility.TCPConnectionString;
26+
}
27+
28+
[Trait("Category", "flaky")]
29+
[ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))]
30+
public void DateTimeVariantParameterTest()
31+
{
32+
Assert.True(RunTestAndCompareWithBaseline());
33+
}
34+
35+
private bool RunTestAndCompareWithBaseline()
36+
{
37+
CultureInfo previousCulture = Thread.CurrentThread.CurrentCulture;
38+
Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");
39+
try
40+
{
41+
string outputPath = "DateTimeVariant.out";
42+
string baselinePath;
43+
#if DEBUG
44+
if (DataTestUtility.IsNotAzureServer() || DataTestUtility.IsManagedInstance)
45+
{
46+
baselinePath = "DateTimeVariant_DebugMode.bsl";
47+
}
48+
else
49+
{
50+
baselinePath = "DateTimeVariant_DebugMode_Azure.bsl";
51+
}
52+
#else
53+
if (DataTestUtility.IsNotAzureServer() || DataTestUtility.IsManagedInstance)
54+
{
55+
baselinePath = "DateTimeVariant_ReleaseMode.bsl";
56+
}
57+
else
58+
{
59+
baselinePath = "DateTimeVariant_ReleaseMode_Azure.bsl";
60+
}
61+
#endif
62+
63+
var fstream = new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.Read);
64+
var swriter = new StreamWriter(fstream, Encoding.UTF8);
65+
var twriter = new TvpTest.CarriageReturnLineFeedReplacer(swriter);
66+
Console.SetOut(twriter);
67+
68+
// Run Test
69+
DateTimeVariantTest.TestAllDateTimeWithDataTypeAndVariant(_connStr);
70+
71+
Console.Out.Flush();
72+
Console.Out.Dispose();
73+
74+
// Recover the standard output stream
75+
StreamWriter standardOutput = new(Console.OpenStandardOutput());
76+
standardOutput.AutoFlush = true;
77+
Console.SetOut(standardOutput);
78+
79+
// Compare output file
80+
var comparisonResult = FindDiffFromBaseline(baselinePath, outputPath);
81+
82+
if (string.IsNullOrEmpty(comparisonResult))
83+
{
84+
return true;
85+
}
86+
87+
Console.WriteLine("DateTimeVariantParameterTest Failed!");
88+
Console.WriteLine("Please compare baseline: {0} with output: {1}", Path.GetFullPath(baselinePath), Path.GetFullPath(outputPath));
89+
Console.WriteLine("Comparison Results:");
90+
Console.WriteLine(comparisonResult);
91+
return false;
92+
}
93+
finally
94+
{
95+
Thread.CurrentThread.CurrentCulture = previousCulture;
96+
}
97+
}
98+
99+
private static string FindDiffFromBaseline(string baselinePath, string outputPath)
100+
{
101+
var expectedLines = File.ReadAllLines(baselinePath);
102+
var outputLines = File.ReadAllLines(outputPath);
103+
104+
var comparisonSb = new StringBuilder();
105+
106+
var expectedLength = expectedLines.Length;
107+
var outputLength = outputLines.Length;
108+
var findDiffLength = Math.Min(expectedLength, outputLength);
109+
110+
for (var lineNo = 0; lineNo < findDiffLength; lineNo++)
111+
{
112+
if (!expectedLines[lineNo].Equals(outputLines[lineNo]))
113+
{
114+
comparisonSb.AppendFormat("** DIFF at line {0} \n", lineNo);
115+
comparisonSb.AppendFormat("A : {0} \n", outputLines[lineNo]);
116+
comparisonSb.AppendFormat("E : {0} \n", expectedLines[lineNo]);
117+
}
118+
}
119+
120+
var startIndex = findDiffLength - 1;
121+
if (startIndex < 0)
122+
{
123+
startIndex = 0;
124+
}
125+
126+
if (findDiffLength < expectedLength)
127+
{
128+
comparisonSb.AppendFormat("** MISSING \n");
129+
for (var lineNo = startIndex; lineNo < expectedLength; lineNo++)
130+
{
131+
comparisonSb.AppendFormat("{0} : {1}", lineNo, expectedLines[lineNo]);
132+
}
133+
}
134+
if (findDiffLength < outputLength)
135+
{
136+
comparisonSb.AppendFormat("** EXTRA \n");
137+
for (var lineNo = startIndex; lineNo < outputLength; lineNo++)
138+
{
139+
comparisonSb.AppendFormat("{0} : {1}", lineNo, outputLines[lineNo]);
140+
}
141+
}
142+
143+
return comparisonSb.ToString();
144+
}
145+
}
146+
}

0 commit comments

Comments
 (0)