Skip to content
This repository was archived by the owner on May 8, 2026. It is now read-only.

Commit 526bf8b

Browse files
mutianfmutianf
committed
add the utils
Change-Id: I643390642ecbf3fd6867d6defd33e3de12d0c40a
1 parent aa007ce commit 526bf8b

2 files changed

Lines changed: 284 additions & 0 deletions

File tree

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/LargeRowPaginationUtil.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.data.v2.stub.readrows;
17+
18+
import com.google.api.gax.rpc.ApiException;
19+
import com.google.api.gax.rpc.StatusCode.Code;
20+
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
21+
import com.google.cloud.bigtable.data.v2.models.Filters;
22+
import com.google.cloud.bigtable.data.v2.models.Row;
23+
import com.google.cloud.bigtable.data.v2.models.RowCell;
24+
import com.google.cloud.bigtable.data.v2.models.TableId;
25+
import com.google.common.collect.Lists;
26+
import com.google.protobuf.ByteString;
27+
import java.util.List;
28+
import java.util.logging.Level;
29+
import java.util.logging.Logger;
30+
import javax.annotation.Nullable;
31+
32+
/** Public utility class to read a large row by paginating over cells. */
33+
public final class LargeRowPaginationUtil {
34+
private static final Logger LOGGER = Logger.getLogger(LargeRowPaginationUtil.class.getName());
35+
36+
private LargeRowPaginationUtil() {}
37+
38+
/**
39+
* Reads a large row by paginating over cells. 1. Reads the total count of cells without fetching
40+
* values using strip filter. 2. Reads cells in chunks using limit and offset filters. 3. Divides
41+
* the chunk size by half if a failure occurs with FAILED_PRECONDITION.
42+
*/
43+
public static Row readLargeRow(
44+
BigtableDataClient client,
45+
String tableId,
46+
ByteString rowKey,
47+
@Nullable Filters.Filter rowFilter) {
48+
// Step 1: Count the number of cells with a strip value filter
49+
Filters.ChainFilter countFilter =
50+
Filters.FILTERS.chain().filter(Filters.FILTERS.value().strip());
51+
if (rowFilter != null) {
52+
countFilter.filter(rowFilter);
53+
}
54+
Row countRow = client.readRow(TableId.of(tableId), rowKey, countFilter);
55+
if (countRow == null) {
56+
return null; // row not found
57+
}
58+
int totalCells = countRow.getCells().size();
59+
60+
List<RowCell> resultCells = Lists.newArrayList();
61+
int offset = 0;
62+
int limit = totalCells; // start with trying to read all cells
63+
64+
while (offset < totalCells) {
65+
try {
66+
Filters.ChainFilter chain = Filters.FILTERS.chain();
67+
if (rowFilter != null) {
68+
chain.filter(rowFilter);
69+
}
70+
if (offset > 0) {
71+
chain.filter(Filters.FILTERS.offset().cellsPerRow(offset));
72+
}
73+
chain.filter(Filters.FILTERS.limit().cellsPerRow(limit));
74+
75+
Row partialRow = client.readRow(TableId.of(tableId), rowKey, chain);
76+
if (partialRow == null) {
77+
break;
78+
}
79+
resultCells.addAll(partialRow.getCells());
80+
offset += partialRow.getCells().size();
81+
} catch (ApiException e) {
82+
if (e.getStatusCode().getCode() != Code.FAILED_PRECONDITION) {
83+
throw e;
84+
}
85+
limit = limit / 2;
86+
if (limit == 0) {
87+
throw new RuntimeException("Cannot divide limit further. Cell might be too large.");
88+
}
89+
LOGGER.log(
90+
Level.FINE,
91+
"Failed to read chunk with limit {0} at offset {1}. Dividing limit by half.",
92+
new Object[] {limit, offset});
93+
}
94+
}
95+
return Row.create(rowKey, resultCells);
96+
}
97+
}

‎google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/LargeRowPaginationUtilIT.java‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.data.v2.it;
17+
18+
import static com.google.common.truth.Truth.assertThat;
19+
import static com.google.common.truth.TruthJUnit.assume;
20+
21+
import com.google.api.gax.grpc.GrpcStatusCode;
22+
import com.google.api.gax.rpc.ApiException;
23+
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
24+
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
25+
import com.google.cloud.bigtable.admin.v2.models.Table;
26+
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
27+
import com.google.cloud.bigtable.data.v2.models.Filters;
28+
import com.google.cloud.bigtable.data.v2.models.Row;
29+
import com.google.cloud.bigtable.data.v2.models.RowMutation;
30+
import com.google.cloud.bigtable.data.v2.models.TableId;
31+
import com.google.cloud.bigtable.data.v2.stub.readrows.LargeRowPaginationUtil;
32+
import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv;
33+
import com.google.cloud.bigtable.test_helpers.env.PrefixGenerator;
34+
import com.google.cloud.bigtable.test_helpers.env.TestEnvRule;
35+
import com.google.protobuf.ByteString;
36+
import io.grpc.Status;
37+
import java.util.Random;
38+
import java.util.concurrent.TimeUnit;
39+
import org.junit.After;
40+
import org.junit.Before;
41+
import org.junit.ClassRule;
42+
import org.junit.Test;
43+
import org.junit.runner.RunWith;
44+
import org.junit.runners.JUnit4;
45+
46+
@RunWith(JUnit4.class)
47+
public class LargeRowPaginationUtilIT {
48+
49+
@ClassRule public static final TestEnvRule testEnvRule = new TestEnvRule();
50+
51+
private BigtableTableAdminClient tableAdminClient;
52+
private BigtableDataClient dataClient;
53+
private Table table;
54+
private final String familyId1 = "cf1";
55+
private final String familyId2 = "cf2";
56+
private ByteString rowKey;
57+
private ByteString largeValue;
58+
59+
@Before
60+
public void setup() throws Exception {
61+
tableAdminClient = testEnvRule.env().getTableAdminClient();
62+
dataClient = testEnvRule.env().getDataClient();
63+
64+
// Skip population for emulator as it doesn't support the expected failure
65+
if (testEnvRule.env() instanceof EmulatorEnv) {
66+
return;
67+
}
68+
69+
String tableId = PrefixGenerator.newPrefix("LargeRowPagination");
70+
table =
71+
tableAdminClient.createTable(
72+
CreateTableRequest.of(tableId).addFamily(familyId1).addFamily(familyId2));
73+
74+
rowKey = ByteString.copyFromUtf8("large-row-key");
75+
byte[] largeValueBytes = new byte[100 * 1024 * 1024];
76+
new Random().nextBytes(largeValueBytes);
77+
largeValue = ByteString.copyFrom(largeValueBytes);
78+
79+
// Populate cf1: 1 qualifier, 2 versions
80+
for (int i = 0; i < 2; i++) {
81+
dataClient
82+
.mutateRowAsync(
83+
RowMutation.create(TableId.of(table.getId()), rowKey)
84+
.setCell(familyId1, ByteString.copyFromUtf8("q1"), largeValue))
85+
.get(10, TimeUnit.MINUTES);
86+
}
87+
88+
// Populate cf2: 2 qualifiers, 2 versions each
89+
for (int q = 1; q <= 2; q++) {
90+
for (int v = 0; v < 2; v++) {
91+
dataClient
92+
.mutateRowAsync(
93+
RowMutation.create(TableId.of(table.getId()), rowKey)
94+
.setCell(familyId2, ByteString.copyFromUtf8("q" + q), largeValue))
95+
.get(10, TimeUnit.MINUTES);
96+
}
97+
}
98+
}
99+
100+
@After
101+
public void tearDown() {
102+
if (table != null) {
103+
tableAdminClient.deleteTable(table.getId());
104+
}
105+
}
106+
107+
@Test
108+
public void testReadLargeRow() throws Exception {
109+
assume()
110+
.withMessage("Large row read errors are not supported by emulator")
111+
.that(testEnvRule.env())
112+
.isNotInstanceOf(EmulatorEnv.class);
113+
114+
// Verify it fails with standard read
115+
try {
116+
dataClient.readRow(TableId.of(table.getId()), rowKey);
117+
org.junit.Assert.fail("Should have failed with FAILED_PRECONDITION");
118+
} catch (ApiException e) {
119+
assertThat(e.getStatusCode()).isEqualTo(GrpcStatusCode.of(Status.Code.FAILED_PRECONDITION));
120+
}
121+
122+
// Read without filter: 2 (cf1) + 4 (cf2) = 6 cells
123+
Row row = LargeRowPaginationUtil.readLargeRow(dataClient, table.getId(), rowKey, null);
124+
assertThat(row).isNotNull();
125+
assertThat(row.getKey()).isEqualTo(rowKey);
126+
assertThat(row.getCells()).hasSize(6);
127+
128+
// Verify cell content
129+
// cf1:q1 (2 versions)
130+
assertThat(row.getCells().get(0).getFamily()).isEqualTo(familyId1);
131+
assertThat(row.getCells().get(0).getQualifier().toStringUtf8()).isEqualTo("q1");
132+
assertThat(row.getCells().get(0).getValue()).isEqualTo(largeValue);
133+
134+
assertThat(row.getCells().get(1).getFamily()).isEqualTo(familyId1);
135+
assertThat(row.getCells().get(1).getQualifier().toStringUtf8()).isEqualTo("q1");
136+
assertThat(row.getCells().get(1).getValue()).isEqualTo(largeValue);
137+
138+
// cf2:q1 (2 versions)
139+
assertThat(row.getCells().get(2).getFamily()).isEqualTo(familyId2);
140+
assertThat(row.getCells().get(2).getQualifier().toStringUtf8()).isEqualTo("q1");
141+
assertThat(row.getCells().get(2).getValue()).isEqualTo(largeValue);
142+
143+
assertThat(row.getCells().get(3).getFamily()).isEqualTo(familyId2);
144+
assertThat(row.getCells().get(3).getQualifier().toStringUtf8()).isEqualTo("q1");
145+
assertThat(row.getCells().get(3).getValue()).isEqualTo(largeValue);
146+
147+
// cf2:q2 (2 versions)
148+
assertThat(row.getCells().get(4).getFamily()).isEqualTo(familyId2);
149+
assertThat(row.getCells().get(4).getQualifier().toStringUtf8()).isEqualTo("q2");
150+
assertThat(row.getCells().get(4).getValue()).isEqualTo(largeValue);
151+
152+
assertThat(row.getCells().get(5).getFamily()).isEqualTo(familyId2);
153+
assertThat(row.getCells().get(5).getQualifier().toStringUtf8()).isEqualTo("q2");
154+
assertThat(row.getCells().get(5).getValue()).isEqualTo(largeValue);
155+
}
156+
157+
@Test
158+
public void testReadLargeRowWithFilter() throws Exception {
159+
assume()
160+
.withMessage("Large row read errors are not supported by emulator")
161+
.that(testEnvRule.env())
162+
.isNotInstanceOf(EmulatorEnv.class);
163+
164+
// Filter for most recent version: 1 (cf1) + 2 (cf2) = 3 cells
165+
Filters.Filter filter = Filters.FILTERS.limit().cellsPerColumn(1);
166+
Row row = LargeRowPaginationUtil.readLargeRow(dataClient, table.getId(), rowKey, filter);
167+
168+
assertThat(row).isNotNull();
169+
assertThat(row.getKey()).isEqualTo(rowKey);
170+
assertThat(row.getCells()).hasSize(3);
171+
172+
// cf1:q1
173+
assertThat(row.getCells().get(0).getFamily()).isEqualTo(familyId1);
174+
assertThat(row.getCells().get(0).getQualifier().toStringUtf8()).isEqualTo("q1");
175+
assertThat(row.getCells().get(0).getValue()).isEqualTo(largeValue);
176+
177+
// cf2:q1
178+
assertThat(row.getCells().get(1).getFamily()).isEqualTo(familyId2);
179+
assertThat(row.getCells().get(1).getQualifier().toStringUtf8()).isEqualTo("q1");
180+
assertThat(row.getCells().get(1).getValue()).isEqualTo(largeValue);
181+
182+
// cf2:q2
183+
assertThat(row.getCells().get(2).getFamily()).isEqualTo(familyId2);
184+
assertThat(row.getCells().get(2).getQualifier().toStringUtf8()).isEqualTo("q2");
185+
assertThat(row.getCells().get(2).getValue()).isEqualTo(largeValue);
186+
}
187+
}

0 commit comments

Comments
 (0)