Skip to content

Commit 6ee638c

Browse files
committed
Add support for overriding the number of retries.
1 parent 728fb52 commit 6ee638c

File tree

3 files changed

+72
-0
lines changed

3 files changed

+72
-0
lines changed

bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/TemplateUtils.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ public static CloudBigtableScanConfiguration buildExportConfig(ExportOptions opt
106106
BigtableOptionsFactory.BIGTABLE_READ_RPC_ATTEMPT_TIMEOUT_MS_KEY,
107107
options.getBigtableReadRpcAttemptTimeoutMs());
108108
}
109+
if (options.getBigtableMaxAttempts() != null) {
110+
configBuilder.withConfiguration(
111+
BigtableOptionsFactory.MAX_SCAN_TIMEOUT_RETRIES,
112+
ValueProvider.NestedValueProvider.of(
113+
options.getBigtableMaxAttempts(), String::valueOf)));
114+
}
109115
return configBuilder.build();
110116
}
111117
}

bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/sequencefiles/ExportJob.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,12 @@ public interface ExportOptions extends GcpOptions, GcsOptions {
153153
@SuppressWarnings("unused")
154154
void setBigtableFilter(ValueProvider<String> filter);
155155

156+
@Description("The maximum number of retry attempts for the Bigtable client.")
157+
ValueProvider<Integer> getBigtableMaxAttempts();
158+
159+
@SuppressWarnings("unused")
160+
void setBigtableMaxAttempts(ValueProvider<Integer> maxAttempts);
161+
156162
@Description("The destination directory")
157163
ValueProvider<String> getDestinationPath();
158164

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.beam;
17+
18+
import static org.junit.Assert.assertEquals;
19+
import static org.junit.Assert.assertNull;
20+
21+
import com.google.cloud.bigtable.beam.sequencefiles.ExportJob.ExportOptions;
22+
import com.google.cloud.bigtable.hbase.BigtableOptionsFactory;
23+
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
24+
import org.apache.hadoop.conf.Configuration;
25+
import org.junit.Test;
26+
import org.junit.runner.RunWith;
27+
import org.junit.runners.JUnit4;
28+
import org.mockito.Mockito;
29+
30+
@RunWith(JUnit4.class)
31+
public class TemplateUtilsTest {
32+
33+
@Test
34+
public void testBuildExportConfig_propagatesMaxAttempts() {
35+
ExportOptions options = Mockito.mock(ExportOptions.class);
36+
Mockito.when(options.getBigtableProject()).thenReturn(StaticValueProvider.of("my-project"));
37+
Mockito.when(options.getBigtableInstanceId()).thenReturn(StaticValueProvider.of("my-instance"));
38+
Mockito.when(options.getBigtableTableId()).thenReturn(StaticValueProvider.of("my-table"));
39+
Mockito.when(options.getBigtableMaxAttempts()).thenReturn(StaticValueProvider.of(15));
40+
Mockito.when(options.getBigtableAppProfileId()).thenReturn(StaticValueProvider.of("my-app-profile"));
41+
42+
CloudBigtableScanConfiguration config = TemplateUtils.buildExportConfig(options);
43+
Configuration hbaseConfig = config.toHBaseConfig();
44+
assertEquals("15", hbaseConfig.get(BigtableOptionsFactory.MAX_SCAN_TIMEOUT_RETRIES));
45+
}
46+
47+
@Test
48+
public void testBuildExportConfig_defaultMaxAttempts() {
49+
ExportOptions options = Mockito.mock(ExportOptions.class);
50+
Mockito.when(options.getBigtableProject()).thenReturn(StaticValueProvider.of("my-project"));
51+
Mockito.when(options.getBigtableInstanceId()).thenReturn(StaticValueProvider.of("my-instance"));
52+
Mockito.when(options.getBigtableTableId()).thenReturn(StaticValueProvider.of("my-table"));
53+
Mockito.when(options.getBigtableAppProfileId()).thenReturn(StaticValueProvider.of((String) null));
54+
Mockito.when(options.getBigtableMaxAttempts()).thenReturn(null);
55+
56+
CloudBigtableScanConfiguration config = TemplateUtils.buildExportConfig(options);
57+
Configuration hbaseConfig = config.toHBaseConfig();
58+
assertNull(hbaseConfig.get(BigtableOptionsFactory.MAX_SCAN_TIMEOUT_RETRIES));
59+
}
60+
}

0 commit comments

Comments
 (0)