Skip to content

Commit 22b2f38

Browse files
Amar3ttoAmar3tto
committed
Refactor LoadTestBase
1 parent 4d975e5 commit 22b2f38

39 files changed

Lines changed: 291 additions & 42 deletions

‎.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ jobs:
6464
matrix:
6565
job_name: ["beam_PostCommit_Java_IO_Performance_Tests"]
6666
job_phrase: ["Run Java PostCommit IO Performance Tests"]
67-
test_case: ["GCSPerformanceTest", "BigTablePerformanceTest", "BigQueryStorageApiStreamingPerformanceTest"]
67+
test_case: ["IcebergPerformanceTest"]
6868
steps:
6969
- uses: actions/checkout@v7
7070
- name: Setup repository

‎it/build.gradle‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,8 @@ tasks.register('BigTablePerformanceTest') {
3434

3535
tasks.register('BigQueryStorageApiStreamingPerformanceTest') {
3636
dependsOn(":it:google-cloud-platform:BigQueryStorageApiStreamingPerformanceTest")
37-
}
37+
}
38+
39+
tasks.register('IcebergPerformanceTest') {
40+
dependsOn(":it:iceberg:IcebergPerformanceTest")
41+
}

‎it/common/build.gradle‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,25 @@ ext.summary = "Code used by all integration test utilities."
2828
dependencies {
2929
implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
3030
implementation project(path: ":sdks:java:core", configuration: "shadow")
31+
implementation project(path: ":runners:google-cloud-dataflow-java")
32+
implementation project(path: ":sdks:java:testing:test-utils")
33+
implementation 'com.google.cloud:google-cloud-bigquery'
34+
implementation 'com.google.cloud:google-cloud-monitoring'
35+
provided 'com.google.api.grpc:proto-google-cloud-monitoring-v3'
36+
implementation library.java.gax
3137
implementation library.java.google_api_services_dataflow
3238
implementation library.java.google_auth_library_credentials
3339
implementation library.java.google_auth_library_oauth2_http
40+
implementation library.java.google_api_services_bigquery
3441
implementation library.java.vendored_guava_32_1_2_jre
3542
implementation library.java.slf4j_api
3643
implementation library.java.commons_lang3
3744
implementation library.java.failsafe
3845
implementation library.java.google_code_gson
3946
implementation library.java.google_http_client
4047
implementation library.java.guava
48+
implementation library.java.protobuf_java_util
49+
implementation library.java.protobuf_java
4150

4251
// TODO: excluding Guava until Truth updates it to >32.1.x
4352
testImplementation(library.java.truth) {
@@ -46,4 +55,5 @@ dependencies {
4655
testImplementation library.java.junit
4756
testImplementation library.java.mockito_inline
4857
testRuntimeOnly library.java.slf4j_simple
58+
testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration")
4959
}

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManager.java renamed to it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManager.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
package org.apache.beam.it.gcp.bigquery;
18+
package org.apache.beam.it.common.bigquery;
1919

2020
import com.google.api.gax.paging.Page;
2121
import com.google.auth.Credentials;

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerException.java renamed to it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManagerException.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
package org.apache.beam.it.gcp.bigquery;
18+
package org.apache.beam.it.common.bigquery;
1919

2020
/** Custom exception for {@link BigQueryResourceManager} implementations. */
2121
public class BigQueryResourceManagerException extends RuntimeException {

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtils.java renamed to it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManagerUtils.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
package org.apache.beam.it.gcp.bigquery;
18+
package org.apache.beam.it.common.bigquery;
1919

2020
import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId;
2121

‎it/common/src/main/java/org/apache/beam/it/common/bigquery/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
/** Package for managing BigQuery common resources within integration tests. */
20+
package org.apache.beam.it.common.bigquery;

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncher.java renamed to it/common/src/main/java/org/apache/beam/it/common/dataflow/AbstractPipelineLauncher.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
package org.apache.beam.it.gcp.dataflow;
18+
package org.apache.beam.it.common.dataflow;
1919

2020
import static org.apache.beam.it.common.PipelineLauncher.JobState.ACTIVE_STATES;
2121
import static org.apache.beam.it.common.PipelineLauncher.JobState.FAILED;

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java renamed to it/common/src/main/java/org/apache/beam/it/common/dataflow/DefaultPipelineLauncher.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
package org.apache.beam.it.gcp.dataflow;
18+
package org.apache.beam.it.common.dataflow;
1919

2020
import static org.apache.beam.it.common.logging.LogStrings.formatForLogging;
2121
import static org.apache.beam.sdk.testing.TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS;
@@ -41,7 +41,7 @@
4141
import java.util.stream.StreamSupport;
4242
import org.apache.beam.it.common.PipelineLauncher;
4343
import org.apache.beam.it.common.utils.PipelineUtils;
44-
import org.apache.beam.it.gcp.IOLoadTestBase;
44+
import org.apache.beam.it.common.IOLoadTestBase;
4545
import org.apache.beam.runners.dataflow.DataflowPipelineJob;
4646
import org.apache.beam.sdk.PipelineResult;
4747
import org.apache.beam.sdk.metrics.DistributionResult;

‎it/common/src/main/java/org/apache/beam/it/common/dataflow/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
/** Package for managing Dataflow jobs from integration tests. */
20+
package org.apache.beam.it.common.dataflow;

0 commit comments

Comments
 (0)