Skip to content

Commit d913901

Browse files
committed
test(jsonata): add regression test for large-dataset Frame.lookup() StackOverflowError
Reproduces a production crash: 5,000 LDAP-like records with a 15-field flat-lookup expression (ternary, $string, $join on a multi-value array) run on the -Xss512k JVM. The expression mirrors a real-world actual flow. Without the 4 MB eval thread this crashes; with it the test passes.
1 parent d0b5a34 commit d913901

1 file changed

Lines changed: 68 additions & 0 deletions

File tree

plugin-transform-json/src/test/java/io/kestra/plugin/transform/jsonata/TransformItemsTest.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import java.net.URI;
2020
import java.nio.file.Files;
2121
import java.nio.file.Path;
22+
import java.util.ArrayList;
23+
import java.util.HashMap;
2224
import java.util.List;
2325
import java.util.Map;
2426

@@ -155,6 +157,72 @@ void shouldReuseEvalThreadAcrossRecords() throws Exception {
155157
Assertions.assertEquals(0, liveAfter, "jsonata-eval thread should be terminated after run()");
156158
}
157159

160+
@Test
161+
void shouldHandleLargeDatasetWithFlatFieldLookupOnConstrainedStack() throws Exception {
162+
// Regression test for Pylon #1703 (T-Systems): TransformItems crashed the Windows worker with
163+
// StackOverflowError when processing a large LDAP dataset (~200k records, ~30 attributes each).
164+
// The crash was in Jsonata$Frame.lookup() scope-chain recursion — unrelated to user-defined
165+
// function depth, so lowering maxDepth had no effect. The fix is the 4 MB eval thread.
166+
// This test JVM runs at -Xss512k (build.gradle) to simulate the constrained Windows stack.
167+
RunContext runContext = runContextFactory.of();
168+
final Path outputFilePath = runContext.workingDir().createTempFile(".ion");
169+
170+
int recordCount = 5_000;
171+
List<Map<String, Object>> records = new ArrayList<>(recordCount);
172+
for (int i = 0; i < recordCount; i++) {
173+
Map<String, Object> attributes = new HashMap<>();
174+
attributes.put("mail", List.of("user" + i + "@example.com"));
175+
attributes.put("cn", List.of("User " + i));
176+
attributes.put("displayName", List.of("Display User " + i));
177+
attributes.put("givenName", List.of("First" + i));
178+
attributes.put("sn", List.of("Last" + i));
179+
attributes.put("uid", List.of("uid" + i));
180+
attributes.put("employeenumber", List.of("EMP" + i));
181+
attributes.put("tCID", List.of("CID" + i));
182+
attributes.put("tWrID", List.of("WR" + i));
183+
attributes.put("tMainWrID", List.of("MWR" + i));
184+
attributes.put("tisActive", List.of("TRUE"));
185+
attributes.put("tStatusOfEmployment", List.of("active"));
186+
attributes.put("preferredLanguage", List.of("en"));
187+
// Multi-value field — mirrors the isMemberOf array the customer used $join() on
188+
attributes.put("isMemberOf", List.of("cn=group1,ou=groups", "cn=group2,ou=groups", "cn=group3,ou=groups"));
189+
records.add(Map.of("dn", "uid=user" + i + ",ou=Account,o=DTAG", "attributes", attributes));
190+
}
191+
192+
try (Writer writer = new OutputStreamWriter(Files.newOutputStream(outputFilePath))) {
193+
FileSerde.writeAll(writer, Flux.fromIterable(records)).block();
194+
writer.flush();
195+
}
196+
URI uri = runContext.storage().putFile(outputFilePath.toFile());
197+
198+
TransformItems task = TransformItems.builder()
199+
.from(Property.ofValue(uri.toString()))
200+
.expression(Property.ofValue("""
201+
{
202+
"DN": dn ? $string(dn) : null,
203+
"MAIL": attributes.mail[0] ? $string(attributes.mail[0]) : null,
204+
"CN": attributes.cn[0] ? $string(attributes.cn[0]) : null,
205+
"DISPLAY_NAME": attributes.displayName[0] ? $string(attributes.displayName[0]) : null,
206+
"GIVEN_NAME": attributes.givenName[0] ? $string(attributes.givenName[0]) : null,
207+
"SN": attributes.sn[0] ? $string(attributes.sn[0]) : null,
208+
"UID": attributes.uid[0] ? $string(attributes.uid[0]) : null,
209+
"EMPLOYEENUMBER": attributes.employeenumber[0] ? $string(attributes.employeenumber[0]) : null,
210+
"TCID": attributes.tCID[0] ? $string(attributes.tCID[0]) : null,
211+
"TWRID": attributes.tWrID[$ != attributes.tMainWrID[0]][0] ? $string(attributes.tWrID[$ != attributes.tMainWrID[0]][0]) : (attributes.tWrID[0] ? $string(attributes.tWrID[0]) : null),
212+
"TMAINWRID": attributes.tMainWrID[0] ? $string(attributes.tMainWrID[0]) : null,
213+
"TIS_ACTIVE": attributes.tisActive[0] ? $string(attributes.tisActive[0]) : null,
214+
"TSTATUS_OF_EMPLOYMENT": attributes.tStatusOfEmployment[0] ? $string(attributes.tStatusOfEmployment[0]) : null,
215+
"PREFERREDLANGUAGE": attributes.preferredLanguage[0] ? $string(attributes.preferredLanguage[0]) : null,
216+
"ISMEMBEROF": attributes.isMemberOf ? $join(attributes.isMemberOf, "|") : null
217+
}
218+
"""))
219+
.build();
220+
221+
TransformItems.Output output = task.run(runContext);
222+
223+
Assertions.assertEquals(recordCount, output.getProcessedItemsTotal());
224+
}
225+
158226
@Test
159227
void shouldTransformJsonInputWithDefaultIonMapper() throws Exception {
160228
// Given

0 commit comments

Comments
 (0)