Skip to content

Commit 3b00897

Browse files
committed
perf: reduce search batch test from 10k to 100 documents
Reduce the document count in the batch-size search test (test 3) from 10,000 to 100 documents and re-enable it. Test 3 was previously gated off because 10k documents made the search1 job a ~27-minute long-pole. At 100 documents (batch size 20, preserving the 5:1 ratio), the test exercises the same batch-write code paths without the excessive runtime. Changes: - bigDF: createTestData(10000) -> createTestData(100) - batchSize: 2000 -> 20 - assertSize: 10000 -> 100 - testsToRun: Set(1, 2) -> Set(1, 2, 3) [re-enabled]
1 parent 895752c commit 3b00897

1 file changed

Lines changed: 4 additions & 4 deletions

File tree

cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/split1/SearchWriterSuitePart1.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ class SearchWriterSuiteUtilities extends TestBase with AzureSearchKey
240240

241241
lazy val df4: DataFrame = createTestData(4)
242242
lazy val df10: DataFrame = createTestData(10)
243-
lazy val bigDF: DataFrame = createTestData(10000)
243+
lazy val bigDF: DataFrame = createTestData(100)
244244

245245
lazy val ad: AddDocuments = {
246246
new AddDocuments()
@@ -297,7 +297,7 @@ class SearchWriterSuitePart1 extends SearchWriterSuiteUtilities
297297
}
298298

299299
test("Run azure-search tests with waits") {
300-
val testsToRun = Set(1, 2) //, 3)
300+
val testsToRun = Set(1, 2, 3)
301301

302302
def dependsOn(testNumber: Int, f: => Unit): Unit = {
303303
if (testsToRun(testNumber)) {
@@ -326,11 +326,11 @@ class SearchWriterSuitePart1 extends SearchWriterSuiteUtilities
326326

327327
//push docs with custom batch size
328328
lazy val in3 = generateIndexName()
329-
dependsOn(3, writeHelper(bigDF, in3, isVectorField=false, Map("batchSize" -> "2000")))
329+
dependsOn(3, writeHelper(bigDF, in3, isVectorField=false, Map("batchSize" -> "20")))
330330

331331
dependsOn(1, retryWithBackoff(assertSize(in1, 4)))
332332
dependsOn(2, retryWithBackoff(assertSize(in2, 10)))
333-
dependsOn(3, retryWithBackoff(assertSize(in3, 10000)))
333+
dependsOn(3, retryWithBackoff(assertSize(in3, 100)))
334334

335335
}
336336

0 commit comments

Comments
 (0)