|
19 | 19 | import java.net.URI; |
20 | 20 | import java.nio.file.Files; |
21 | 21 | import java.nio.file.Path; |
| 22 | +import java.util.ArrayList; |
| 23 | +import java.util.HashMap; |
22 | 24 | import java.util.List; |
23 | 25 | import java.util.Map; |
24 | 26 |
|
@@ -155,6 +157,72 @@ void shouldReuseEvalThreadAcrossRecords() throws Exception { |
155 | 157 | Assertions.assertEquals(0, liveAfter, "jsonata-eval thread should be terminated after run()"); |
156 | 158 | } |
157 | 159 |
|
| 160 | + @Test |
| 161 | + void shouldHandleLargeDatasetWithFlatFieldLookupOnConstrainedStack() throws Exception { |
| 162 | + // Regression test for Pylon #1703 (T-Systems): TransformItems crashed the Windows worker with |
| 163 | + // StackOverflowError when processing a large LDAP dataset (~200k records, ~30 attributes each). |
| 164 | + // The crash was in Jsonata$Frame.lookup() scope-chain recursion — unrelated to user-defined |
| 165 | + // function depth, so lowering maxDepth had no effect. The fix is the 4 MB eval thread. |
| 166 | + // This test JVM runs at -Xss512k (build.gradle) to simulate the constrained Windows stack. |
| 167 | + RunContext runContext = runContextFactory.of(); |
| 168 | + final Path outputFilePath = runContext.workingDir().createTempFile(".ion"); |
| 169 | + |
| 170 | + int recordCount = 5_000; |
| 171 | + List<Map<String, Object>> records = new ArrayList<>(recordCount); |
| 172 | + for (int i = 0; i < recordCount; i++) { |
| 173 | + Map<String, Object> attributes = new HashMap<>(); |
| 174 | + attributes.put("mail", List.of("user" + i + "@example.com")); |
| 175 | + attributes.put("cn", List.of("User " + i)); |
| 176 | + attributes.put("displayName", List.of("Display User " + i)); |
| 177 | + attributes.put("givenName", List.of("First" + i)); |
| 178 | + attributes.put("sn", List.of("Last" + i)); |
| 179 | + attributes.put("uid", List.of("uid" + i)); |
| 180 | + attributes.put("employeenumber", List.of("EMP" + i)); |
| 181 | + attributes.put("tCID", List.of("CID" + i)); |
| 182 | + attributes.put("tWrID", List.of("WR" + i)); |
| 183 | + attributes.put("tMainWrID", List.of("MWR" + i)); |
| 184 | + attributes.put("tisActive", List.of("TRUE")); |
| 185 | + attributes.put("tStatusOfEmployment", List.of("active")); |
| 186 | + attributes.put("preferredLanguage", List.of("en")); |
| 187 | + // Multi-value field — mirrors the isMemberOf array the customer used $join() on |
| 188 | + attributes.put("isMemberOf", List.of("cn=group1,ou=groups", "cn=group2,ou=groups", "cn=group3,ou=groups")); |
| 189 | + records.add(Map.of("dn", "uid=user" + i + ",ou=Account,o=DTAG", "attributes", attributes)); |
| 190 | + } |
| 191 | + |
| 192 | + try (Writer writer = new OutputStreamWriter(Files.newOutputStream(outputFilePath))) { |
| 193 | + FileSerde.writeAll(writer, Flux.fromIterable(records)).block(); |
| 194 | + writer.flush(); |
| 195 | + } |
| 196 | + URI uri = runContext.storage().putFile(outputFilePath.toFile()); |
| 197 | + |
| 198 | + TransformItems task = TransformItems.builder() |
| 199 | + .from(Property.ofValue(uri.toString())) |
| 200 | + .expression(Property.ofValue(""" |
| 201 | + { |
| 202 | + "DN": dn ? $string(dn) : null, |
| 203 | + "MAIL": attributes.mail[0] ? $string(attributes.mail[0]) : null, |
| 204 | + "CN": attributes.cn[0] ? $string(attributes.cn[0]) : null, |
| 205 | + "DISPLAY_NAME": attributes.displayName[0] ? $string(attributes.displayName[0]) : null, |
| 206 | + "GIVEN_NAME": attributes.givenName[0] ? $string(attributes.givenName[0]) : null, |
| 207 | + "SN": attributes.sn[0] ? $string(attributes.sn[0]) : null, |
| 208 | + "UID": attributes.uid[0] ? $string(attributes.uid[0]) : null, |
| 209 | + "EMPLOYEENUMBER": attributes.employeenumber[0] ? $string(attributes.employeenumber[0]) : null, |
| 210 | + "TCID": attributes.tCID[0] ? $string(attributes.tCID[0]) : null, |
| 211 | + "TWRID": attributes.tWrID[$ != attributes.tMainWrID[0]][0] ? $string(attributes.tWrID[$ != attributes.tMainWrID[0]][0]) : (attributes.tWrID[0] ? $string(attributes.tWrID[0]) : null), |
| 212 | + "TMAINWRID": attributes.tMainWrID[0] ? $string(attributes.tMainWrID[0]) : null, |
| 213 | + "TIS_ACTIVE": attributes.tisActive[0] ? $string(attributes.tisActive[0]) : null, |
| 214 | + "TSTATUS_OF_EMPLOYMENT": attributes.tStatusOfEmployment[0] ? $string(attributes.tStatusOfEmployment[0]) : null, |
| 215 | + "PREFERREDLANGUAGE": attributes.preferredLanguage[0] ? $string(attributes.preferredLanguage[0]) : null, |
| 216 | + "ISMEMBEROF": attributes.isMemberOf ? $join(attributes.isMemberOf, "|") : null |
| 217 | + } |
| 218 | + """)) |
| 219 | + .build(); |
| 220 | + |
| 221 | + TransformItems.Output output = task.run(runContext); |
| 222 | + |
| 223 | + Assertions.assertEquals(recordCount, output.getProcessedItemsTotal()); |
| 224 | + } |
| 225 | + |
158 | 226 | @Test |
159 | 227 | void shouldTransformJsonInputWithDefaultIonMapper() throws Exception { |
160 | 228 | // Given |
|
0 commit comments