Skip to content

Commit eccd9b7

Browse files
Amar3ttoAmar3tto
authored
Merge pull request #39270 from apache/icerbergio-benchmark
IcerbergIO Load test and benchmarking
2 parents c54595b + ab142e9 commit eccd9b7

68 files changed

Lines changed: 789 additions & 180 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ jobs:
6464
matrix:
6565
job_name: ["beam_PostCommit_Java_IO_Performance_Tests"]
6666
job_phrase: ["Run Java PostCommit IO Performance Tests"]
67-
test_case: ["GCSPerformanceTest", "BigTablePerformanceTest", "BigQueryStorageApiStreamingPerformanceTest"]
67+
test_case: ["IcebergPerformanceTest", "GCSPerformanceTest", "BigTablePerformanceTest", "BigQueryStorageApiStreamingPerformanceTest"]
6868
steps:
6969
- uses: actions/checkout@v7
7070
with:

‎.test-infra/tools/refresh_looker_metrics.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
("33", ["21", "70", "116", "69", "115"]), # BigTableIO_Write
3535
("34", ["22", "56", "96", "55", "95"]), # TextIO_Read
3636
("35", ["23", "64", "110", "63", "109"]), # TextIO_Write
37+
("113", ["386", "388", "390", "392", "394"]), # IcebergIO_Read
38+
("114", ["387", "389", "391", "393", "395"]), # IcebergIO_Write
3739
("75", ["258", "259", "260", "261", "262"]), # TensorFlow MNIST
3840
("76", ["233", "234", "235", "236", "237"]), # PyTorch BERT base uncased
3941
("77", ["238", "239", "240", "241", "242"]), # PyTorch BERT large uncased

‎it/build.gradle‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,8 @@ tasks.register('BigTablePerformanceTest') {
3434

3535
tasks.register('BigQueryStorageApiStreamingPerformanceTest') {
3636
dependsOn(":it:google-cloud-platform:BigQueryStorageApiStreamingPerformanceTest")
37-
}
37+
}
38+
39+
tasks.register('IcebergPerformanceTest') {
40+
dependsOn(":it:iceberg:IcebergPerformanceTest")
41+
}

‎it/clickhouse/build.gradle‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@ ext.summary = "Integration test utilities for ClickHouse."
2929

3030
def clickhouse_java_client_version = "0.9.6"
3131

32+
configurations.configureEach {
33+
resolutionStrategy.capabilitiesResolution.withCapability("org.lz4:lz4-java") {
34+
select("at.yawk.lz4:lz4-java:1.10.2")
35+
}
36+
}
37+
3238
dependencies {
3339
implementation project(path: ":it:common")
3440
implementation project(path: ":it:testcontainers")
@@ -52,4 +58,4 @@ dependencies {
5258

5359
permitUnusedDeclared "com.clickhouse:client-v2:$clickhouse_java_client_version:all"
5460

55-
}
61+
}

‎it/common/build.gradle‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,19 @@ ext.summary = "Code used by all integration test utilities."
2828
dependencies {
2929
implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
3030
implementation project(path: ":sdks:java:core", configuration: "shadow")
31+
implementation project(path: ":runners:google-cloud-dataflow-java")
32+
implementation 'com.google.cloud:google-cloud-bigquery'
33+
implementation 'com.google.cloud:google-cloud-monitoring'
34+
implementation 'com.google.cloud:google-cloud-storage'
35+
implementation 'org.apache.hadoop:hadoop-common:3.3.6'
36+
implementation 'org.apache.avro:avro:1.11.1'
37+
implementation 'org.apache.parquet:parquet-avro:1.15.2'
38+
implementation 'org.apache.parquet:parquet-common:1.15.2'
39+
implementation 'org.apache.parquet:parquet-hadoop:1.15.2'
40+
provided 'com.google.api.grpc:proto-google-cloud-monitoring-v3'
41+
implementation library.java.jackson_core
42+
implementation library.java.jackson_databind
43+
implementation library.java.gax
3144
implementation library.java.google_api_services_dataflow
3245
implementation library.java.google_auth_library_credentials
3346
implementation library.java.google_auth_library_oauth2_http
@@ -36,14 +49,20 @@ dependencies {
3649
implementation library.java.commons_lang3
3750
implementation library.java.failsafe
3851
implementation library.java.google_code_gson
52+
implementation library.java.google_cloud_core
53+
implementation library.java.google_api_client
3954
implementation library.java.google_http_client
4055
implementation library.java.guava
41-
56+
implementation library.java.protobuf_java_util
57+
implementation library.java.protobuf_java
58+
implementation library.java.junit
59+
testImplementation library.java.mockito_inline
60+
testRuntimeOnly library.java.slf4j_simple
4261
// TODO: excluding Guava until Truth updates it to >32.1.x
4362
testImplementation(library.java.truth) {
44-
exclude group: 'com.google.guava', module: 'guava'
63+
exclude group: 'com.google.guava', module: 'guava'
4564
}
46-
testImplementation library.java.junit
47-
testImplementation library.java.mockito_inline
48-
testRuntimeOnly library.java.slf4j_simple
65+
testImplementation project(path: ":runners:direct-java", configuration: "shadow")
66+
testImplementation project(path: ":sdks:java:testing:test-utils")
67+
testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration")
4968
}

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/Artifact.java renamed to it/common/src/main/java/org/apache/beam/it/common/artifacts/Artifact.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
package org.apache.beam.it.gcp.artifacts;
18+
package org.apache.beam.it.common.artifacts;
1919

2020
/**
2121
* Represents a single artifact.

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/ArtifactClient.java renamed to it/common/src/main/java/org/apache/beam/it/common/artifacts/ArtifactClient.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
package org.apache.beam.it.gcp.artifacts;
18+
package org.apache.beam.it.common.artifacts;
1919

2020
import java.io.IOException;
2121
import java.nio.file.Path;

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/GcsArtifact.java renamed to it/common/src/main/java/org/apache/beam/it/common/artifacts/GcsArtifact.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
package org.apache.beam.it.gcp.artifacts;
18+
package org.apache.beam.it.common.artifacts;
1919

2020
import com.google.cloud.storage.Blob;
2121

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/package-info.java renamed to it/common/src/main/java/org/apache/beam/it/common/artifacts/package-info.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@
1717
*/
1818

1919
/** Package for working with test artifacts. */
20-
package org.apache.beam.it.gcp.artifacts;
20+
package org.apache.beam.it.common.artifacts;

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/ArtifactUtils.java renamed to it/common/src/main/java/org/apache/beam/it/common/artifacts/utils/ArtifactUtils.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
package org.apache.beam.it.gcp.artifacts.utils;
18+
package org.apache.beam.it.common.artifacts.utils;
1919

2020
import static java.util.Arrays.stream;
2121
import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;

0 commit comments

Comments
 (0)