Skip to content

Commit 1b7c2e0

Browse files
dlarocquedlarocque
authored
feat(firestore): Add support for 16MB documents (#13478)
1 parent 38e272e commit 1b7c2e0

1 file changed

Lines changed: 233 additions & 0 deletions

File tree

‎java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.firestore.it;
18+
19+
import static com.google.cloud.firestore.LocalFirestoreHelper.autoId;
20+
import static com.google.cloud.firestore.it.ITQueryTest.map;
21+
import static org.junit.Assert.assertEquals;
22+
import static org.junit.Assert.assertNotNull;
23+
import static org.junit.Assert.assertTrue;
24+
import static org.junit.Assert.fail;
25+
import static org.junit.Assume.assumeTrue;
26+
27+
import com.google.api.core.ApiFuture;
28+
import com.google.api.core.ApiFutures;
29+
import com.google.cloud.firestore.CollectionReference;
30+
import com.google.cloud.firestore.DocumentReference;
31+
import com.google.cloud.firestore.DocumentSnapshot;
32+
import com.google.cloud.firestore.FieldPath;
33+
import com.google.cloud.firestore.FieldValue;
34+
import com.google.cloud.firestore.ListenerRegistration;
35+
import com.google.cloud.firestore.QuerySnapshot;
36+
import com.google.cloud.firestore.WriteResult;
37+
import java.util.Arrays;
38+
import java.util.concurrent.CompletableFuture;
39+
import java.util.concurrent.TimeUnit;
40+
import org.junit.After;
41+
import org.junit.Before;
42+
import org.junit.Test;
43+
import org.junit.runner.RunWith;
44+
import org.junit.runners.JUnit4;
45+
46+
@RunWith(JUnit4.class)
47+
public class ITLargeDocumentTest extends ITBaseTest {
48+
49+
private String collectionName;
50+
private String unicodePayload;
51+
private String asciiPayload;
52+
53+
private DocumentReference docRef;
54+
private DocumentReference docA;
55+
private DocumentReference docB;
56+
57+
static boolean runLargeDocTests() {
58+
String propertyName = "FIRESTORE_RUN_LARGE_DOC_TESTS";
59+
String runLargeTests = System.getProperty(propertyName);
60+
if (runLargeTests == null) {
61+
runLargeTests = System.getenv(propertyName);
62+
}
63+
return "YES".equalsIgnoreCase(runLargeTests) || "true".equalsIgnoreCase(runLargeTests);
64+
}
65+
66+
private static String generateUnicodeString(int targetUtf8Bytes) {
67+
StringBuilder sb = new StringBuilder();
68+
String emoji = "🚀"; // 4 bytes in UTF-8
69+
int bytes = 0;
70+
while (bytes < targetUtf8Bytes) {
71+
if (bytes % 2 == 0 && bytes + 4 <= targetUtf8Bytes) {
72+
sb.append(emoji);
73+
bytes += 4;
74+
} else {
75+
sb.append('a');
76+
bytes += 1;
77+
}
78+
}
79+
return sb.toString();
80+
}
81+
82+
private static String generateAsciiString(int sizeInBytes) {
83+
char[] chars = new char[sizeInBytes];
84+
Arrays.fill(chars, 'a');
85+
return new String(chars);
86+
}
87+
88+
@Before
89+
@Override
90+
public void before() throws Exception {
91+
// Check preconditions before setting up
92+
assumeTrue(runLargeDocTests());
93+
assumeTrue("NIGHTLY".equalsIgnoreCase(getTargetBackend()));
94+
assumeTrue(getFirestoreEdition() == FirestoreEdition.ENTERPRISE);
95+
96+
// Call base class before() to initialize firestore
97+
super.before();
98+
99+
collectionName = "large_doc_tests_" + autoId();
100+
CollectionReference colRef = firestore.collection(collectionName);
101+
docRef = colRef.document("doc_15_9MB_unicode");
102+
docA = colRef.document("doc_a");
103+
docB = colRef.document("doc_b");
104+
105+
int targetBytes = (int) Math.floor(15.9 * 1024 * 1024);
106+
unicodePayload = generateUnicodeString(targetBytes);
107+
asciiPayload = generateAsciiString(targetBytes);
108+
109+
// Write documents in parallel
110+
ApiFuture<WriteResult> f1 = docRef.set(map("chunk", unicodePayload));
111+
ApiFuture<WriteResult> f2 = docA.set(map("chunk", asciiPayload));
112+
ApiFuture<WriteResult> f3 = docB.set(map("chunk", asciiPayload));
113+
114+
ApiFutures.allAsList(Arrays.asList(f1, f2, f3)).get(90, TimeUnit.SECONDS);
115+
}
116+
117+
@After
118+
@Override
119+
public void after() throws Exception {
120+
if (firestore != null) {
121+
if (collectionName != null) {
122+
try {
123+
// Delete documents in parallel
124+
ApiFuture<WriteResult> d1 = docRef.delete();
125+
ApiFuture<WriteResult> d2 = docA.delete();
126+
ApiFuture<WriteResult> d3 = docB.delete();
127+
ApiFutures.allAsList(Arrays.asList(d1, d2, d3)).get(30, TimeUnit.SECONDS);
128+
} catch (Exception e) {
129+
// Suppress errors during cleanup to not mask test failures
130+
}
131+
}
132+
super.after();
133+
}
134+
}
135+
136+
@Test
137+
public void testReadLargeUnicodeDocument() throws Exception {
138+
DocumentSnapshot snapshot = docRef.get().get();
139+
assertTrue(snapshot.exists());
140+
String chunk = snapshot.getString("chunk");
141+
assertNotNull(chunk);
142+
assertEquals(unicodePayload.length(), chunk.length());
143+
assertEquals(unicodePayload, chunk);
144+
}
145+
146+
@Test
147+
public void testQueryMultipleLargeDocuments() throws Exception {
148+
CollectionReference colRef = firestore.collection(collectionName);
149+
QuerySnapshot querySnapshot =
150+
colRef.whereIn(FieldPath.documentId(), Arrays.asList("doc_a", "doc_b")).get().get();
151+
assertEquals(2, querySnapshot.size());
152+
153+
DocumentSnapshot snapshotA = querySnapshot.getDocuments().get(0);
154+
DocumentSnapshot snapshotB = querySnapshot.getDocuments().get(1);
155+
assertEquals(asciiPayload, snapshotA.getString("chunk"));
156+
assertEquals(asciiPayload, snapshotB.getString("chunk"));
157+
}
158+
159+
@Test
160+
public void testWatchStreamInitialization() throws Exception {
161+
CompletableFuture<DocumentSnapshot> snapshotFuture = new CompletableFuture<>();
162+
ListenerRegistration registration =
163+
docRef.addSnapshotListener(
164+
(snapshot, error) -> {
165+
if (error != null) {
166+
snapshotFuture.completeExceptionally(error);
167+
} else if (snapshot != null && snapshot.exists()) {
168+
snapshotFuture.complete(snapshot);
169+
}
170+
});
171+
172+
try {
173+
DocumentSnapshot snapshot = snapshotFuture.get(60, TimeUnit.SECONDS);
174+
assertTrue(snapshot.exists());
175+
assertEquals(unicodePayload, snapshot.getString("chunk"));
176+
} finally {
177+
registration.remove();
178+
}
179+
}
180+
181+
@Test
182+
public void testTransactionReadModifyWrite() throws Exception {
183+
firestore
184+
.runTransaction(
185+
transaction -> {
186+
DocumentSnapshot snapshot = transaction.get(docRef).get();
187+
assertTrue(snapshot.exists());
188+
transaction.update(
189+
docRef, map("transaction_timestamp", FieldValue.serverTimestamp()));
190+
return null;
191+
})
192+
.get(60, TimeUnit.SECONDS);
193+
}
194+
195+
@Test
196+
public void testPaginateLargeDocuments() throws Exception {
197+
CollectionReference colRef = firestore.collection(collectionName);
198+
com.google.cloud.firestore.Query q =
199+
colRef
200+
.whereIn(FieldPath.documentId(), Arrays.asList("doc_a", "doc_b"))
201+
.orderBy(FieldPath.documentId());
202+
203+
QuerySnapshot firstPage = q.limit(1).get().get();
204+
assertEquals(1, firstPage.size());
205+
DocumentSnapshot doc1 = firstPage.getDocuments().get(0);
206+
assertEquals("doc_a", doc1.getId());
207+
assertEquals(asciiPayload, doc1.getString("chunk"));
208+
209+
QuerySnapshot secondPage = q.startAfter(doc1).limit(1).get().get();
210+
assertEquals(1, secondPage.size());
211+
DocumentSnapshot doc2 = secondPage.getDocuments().get(0);
212+
assertEquals("doc_b", doc2.getId());
213+
assertEquals(asciiPayload, doc2.getString("chunk"));
214+
}
215+
216+
@Test
217+
public void testOversizedPayloadRejection() {
218+
DocumentReference oversizedDoc =
219+
firestore.collection(collectionName).document("temp_oversized_doc");
220+
int targetBytes = 16 * 1024 * 1024 + 102400;
221+
char[] chars = new char[targetBytes];
222+
Arrays.fill(chars, 'a');
223+
String largePayload = new String(chars);
224+
225+
try {
226+
oversizedDoc.set(map("chunk", largePayload)).get(60, TimeUnit.SECONDS);
227+
fail("Setting a document exceeding the 16MB limit should fail.");
228+
} catch (Exception e) {
229+
Throwable cause = e.getCause();
230+
assertTrue(cause instanceof com.google.api.gax.rpc.InvalidArgumentException);
231+
}
232+
}
233+
}

0 commit comments

Comments
 (0)