Skip to content

Commit ef20f5e

Browse files
dlarocquedlarocque
committed
feat(firestore): Add support for 16MB documents
1 parent 79e26b8 commit ef20f5e

1 file changed

Lines changed: 230 additions & 0 deletions

File tree

‎java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java‎

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.firestore.it;
18+
19+
import static com.google.cloud.firestore.LocalFirestoreHelper.autoId;
20+
import static com.google.cloud.firestore.it.ITQueryTest.map;
21+
import static com.google.common.truth.Truth.assertThat;
22+
import static org.junit.Assert.assertEquals;
23+
import static org.junit.Assert.assertNotNull;
24+
import static org.junit.Assert.assertTrue;
25+
import static org.junit.Assert.fail;
26+
import static org.junit.Assume.assumeTrue;
27+
28+
import com.google.api.core.ApiFuture;
29+
import com.google.api.core.ApiFutures;
30+
import com.google.cloud.firestore.CollectionReference;
31+
import com.google.cloud.firestore.DocumentReference;
32+
import com.google.cloud.firestore.DocumentSnapshot;
33+
import com.google.cloud.firestore.FieldPath;
34+
import com.google.cloud.firestore.FieldValue;
35+
import com.google.cloud.firestore.FirestoreException;
36+
import com.google.cloud.firestore.ListenerRegistration;
37+
import com.google.cloud.firestore.QuerySnapshot;
38+
import com.google.cloud.firestore.WriteResult;
39+
import io.grpc.Status;
40+
import java.util.Arrays;
41+
import java.util.HashMap;
42+
import java.util.Map;
43+
import java.util.concurrent.CompletableFuture;
44+
import java.util.concurrent.TimeUnit;
45+
import org.junit.After;
46+
import org.junit.Before;
47+
import org.junit.Test;
48+
import org.junit.runner.RunWith;
49+
import org.junit.runners.JUnit4;
50+
51+
@RunWith(JUnit4.class)
52+
public class ITLargeDocumentTest extends ITBaseTest {
53+
54+
private String collectionName;
55+
private String unicodePayload;
56+
private String asciiPayload;
57+
58+
private DocumentReference docRef;
59+
private DocumentReference docA;
60+
private DocumentReference docB;
61+
62+
static boolean runLargeDocTests() {
63+
String propertyName = "FIRESTORE_RUN_LARGE_DOC_TESTS";
64+
String runLargeTests = System.getProperty(propertyName);
65+
if (runLargeTests == null) {
66+
runLargeTests = System.getenv(propertyName);
67+
}
68+
return "YES".equalsIgnoreCase(runLargeTests) || "true".equalsIgnoreCase(runLargeTests);
69+
}
70+
71+
private static String generateUnicodeString(int targetUtf8Bytes) {
72+
StringBuilder sb = new StringBuilder();
73+
String emoji = "🚀"; // 4 bytes in UTF-8
74+
int bytes = 0;
75+
while (bytes < targetUtf8Bytes) {
76+
if (bytes % 2 == 0 && bytes + 4 <= targetUtf8Bytes) {
77+
sb.append(emoji);
78+
bytes += 4;
79+
} else {
80+
sb.append('a');
81+
bytes += 1;
82+
}
83+
}
84+
return sb.toString();
85+
}
86+
87+
private static String generateAsciiString(int sizeInBytes) {
88+
char[] chars = new char[sizeInBytes];
89+
Arrays.fill(chars, 'a');
90+
return new String(chars);
91+
}
92+
93+
@Before
94+
@Override
95+
public void before() throws Exception {
96+
// Check preconditions before setting up
97+
assumeTrue(runLargeDocTests());
98+
assumeTrue("NIGHTLY".equalsIgnoreCase(getTargetBackend()));
99+
assumeTrue(getFirestoreEdition() == FirestoreEdition.ENTERPRISE);
100+
101+
// Call base class before() to initialize firestore
102+
super.before();
103+
104+
collectionName = "large_doc_tests_" + autoId();
105+
CollectionReference colRef = firestore.collection(collectionName);
106+
docRef = colRef.document("doc_15_9MB_unicode");
107+
docA = colRef.document("doc_a");
108+
docB = colRef.document("doc_b");
109+
110+
int targetBytes = (int) Math.floor(15.9 * 1024 * 1024);
111+
unicodePayload = generateUnicodeString(targetBytes);
112+
asciiPayload = generateAsciiString(targetBytes);
113+
114+
// Write documents in parallel
115+
ApiFuture<WriteResult> f1 = docRef.set(map("chunk", unicodePayload));
116+
ApiFuture<WriteResult> f2 = docA.set(map("chunk", asciiPayload));
117+
ApiFuture<WriteResult> f3 = docB.set(map("chunk", asciiPayload));
118+
119+
ApiFutures.allAsList(Arrays.asList(f1, f2, f3)).get(90, TimeUnit.SECONDS);
120+
}
121+
122+
@After
123+
@Override
124+
public void after() throws Exception {
125+
if (firestore != null && collectionName != null) {
126+
try {
127+
// Delete documents in parallel
128+
ApiFuture<WriteResult> d1 = docRef.delete();
129+
ApiFuture<WriteResult> d2 = docA.delete();
130+
ApiFuture<WriteResult> d3 = docB.delete();
131+
ApiFutures.allAsList(Arrays.asList(d1, d2, d3)).get(30, TimeUnit.SECONDS);
132+
} catch (Exception e) {
133+
// Suppress errors during cleanup to not mask test failures
134+
}
135+
}
136+
super.after();
137+
}
138+
139+
@Test
140+
public void testReadLargeUnicodeDocument() throws Exception {
141+
DocumentSnapshot snapshot = docRef.get().get();
142+
assertTrue(snapshot.exists());
143+
String chunk = snapshot.getString("chunk");
144+
assertNotNull(chunk);
145+
assertEquals(unicodePayload.length(), chunk.length());
146+
assertEquals(unicodePayload, chunk);
147+
}
148+
149+
@Test
150+
public void testQueryMultipleLargeDocuments() throws Exception {
151+
CollectionReference colRef = firestore.collection(collectionName);
152+
QuerySnapshot querySnapshot =
153+
colRef.whereIn(FieldPath.documentId(), Arrays.asList("doc_a", "doc_b")).get().get();
154+
assertEquals(2, querySnapshot.size());
155+
156+
DocumentSnapshot snapshotA = querySnapshot.getDocuments().get(0);
157+
DocumentSnapshot snapshotB = querySnapshot.getDocuments().get(1);
158+
assertEquals(asciiPayload, snapshotA.getString("chunk"));
159+
assertEquals(asciiPayload, snapshotB.getString("chunk"));
160+
}
161+
162+
@Test
163+
public void testWatchStreamInitialization() throws Exception {
164+
CompletableFuture<DocumentSnapshot> snapshotFuture = new CompletableFuture<>();
165+
ListenerRegistration registration =
166+
docRef.addSnapshotListener(
167+
(snapshot, error) -> {
168+
if (error != null) {
169+
snapshotFuture.completeExceptionally(error);
170+
} else if (snapshot != null && snapshot.exists()) {
171+
snapshotFuture.complete(snapshot);
172+
}
173+
});
174+
175+
try {
176+
DocumentSnapshot snapshot = snapshotFuture.get(60, TimeUnit.SECONDS);
177+
assertTrue(snapshot.exists());
178+
assertEquals(unicodePayload, snapshot.getString("chunk"));
179+
} finally {
180+
registration.remove();
181+
}
182+
}
183+
184+
@Test
185+
public void testTransactionReadModifyWrite() throws Exception {
186+
firestore
187+
.runTransaction(
188+
transaction -> {
189+
DocumentSnapshot snapshot = transaction.get(docRef).get();
190+
assertTrue(snapshot.exists());
191+
transaction.update(docRef, map("transaction_timestamp", FieldValue.serverTimestamp()));
192+
return null;
193+
})
194+
.get(60, TimeUnit.SECONDS);
195+
}
196+
197+
@Test
198+
public void testPaginateLargeDocuments() throws Exception {
199+
CollectionReference colRef = firestore.collection(collectionName);
200+
com.google.cloud.firestore.Query q =
201+
colRef.whereIn(FieldPath.documentId(), Arrays.asList("doc_a", "doc_b")).orderBy(FieldPath.documentId());
202+
203+
QuerySnapshot firstPage = q.limit(1).get().get();
204+
assertEquals(1, firstPage.size());
205+
DocumentSnapshot doc1 = firstPage.getDocuments().get(0);
206+
assertEquals(asciiPayload, doc1.getString("chunk"));
207+
208+
QuerySnapshot secondPage = q.startAfter(doc1).limit(1).get().get();
209+
assertEquals(1, secondPage.size());
210+
DocumentSnapshot doc2 = secondPage.getDocuments().get(0);
211+
assertEquals(asciiPayload, doc2.getString("chunk"));
212+
}
213+
214+
@Test
215+
public void testOversizedPayloadRejection() {
216+
DocumentReference oversizedDoc = firestore.collection(collectionName).document("temp_oversized_doc");
217+
int targetBytes = 16 * 1024 * 1024 + 102400;
218+
String largePayload = generateAsciiString(targetBytes);
219+
Map<String, Object> data = new HashMap<>();
220+
data.put("chunk", largePayload);
221+
222+
try {
223+
oversizedDoc.set(data).get(60, TimeUnit.SECONDS);
224+
fail("Setting a document exceeding the 16MB limit should fail.");
225+
} catch (Exception e) {
226+
Throwable cause = e.getCause();
227+
assertTrue(cause instanceof com.google.api.gax.rpc.InvalidArgumentException);
228+
}
229+
}
230+
}

0 commit comments

Comments
 (0)