Skip to content

feat(generic): Ignore large payload support for generic importers #1453

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ameya9 wants to merge 4 commits into from
+109 −46
Closed

feat(generic): Ignore large payload support for generic importers #1453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9c7e3e2
feat: Ingore large payload support for generic importers
ameya9 Apr 25, 2025
c40b71a
feat: Ingore large payload support for generic importers
ameya9 Apr 25, 2025
d199df6
feat: Ingore large payload support for generic importers
ameya9 Apr 25, 2025
ad82eed
feat: Ingore large payload support for generic importers
ameya9 Apr 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion extensions/data-transfer/portability-data-transfer-generic/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -753,4 +753,27 @@ Content-Type: application/json
}
```
The worker will pause the job and won't retry in such scenario.
Some exporters also support the ability to retry transfers after user frees up space in the destination.
Some exporters also support the ability to retry transfers after user frees up space in the destination.

### Destination Not Accepting Large Payloads

If the destination is unable to accept large payloads due to API gateway limitations or other reasons,
the POST APIs should also throw a `413 Payload Too Large` error.

`error` field should strictly be set to `too_large_payload` to indicate that the destination cannot handle large payloads.

Example error response:

```
HTTP/1.1 413 Payload Too Large
Content-Type: application/json
{
"error": "too_large_payload",
"error_description": "Destination not accepting payloads above XMB"
}
```

Generic Importers will throw an IOException with a message starting with:
`GenericImporter payload too large: `
Idempotent executors can ignore such IOExceptions if they want. This allows us to ignore large payload errors gracefully.
If Idempotent executors are configured correctly, the worker will ignore such imports and continue with other imports.
19 changes: 14 additions & 5 deletions ...r-generic/src/main/java/org/datatransferproject/datatransfer/generic/GenericImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@

public class GenericImporter<C extends ContainerResource, R>
implements Importer<TokensAndUrlAuthData, C> {

public static final String PAYLOAD_TOO_LARGE = "GenericImporter payload too large: ";

@JsonIgnoreProperties(ignoreUnknown = true)
static class ErrorResponse {
private final String error;
Expand Down Expand Up @@ -139,20 +140,28 @@ boolean parseResponse(Response response) throws IOException, InvalidTokenExcepti
response.code(), new String(body, StandardCharsets.UTF_8)),
e);
}

if (response.code() == 401 && error.getError().equals("invalid_token")) {
throw new InvalidTokenException(error.toString(), null);
} if (response.code() == 413 && error.getError().equals("destination_full")) {
}

if (response.code() == 413 && error.getError().equals("destination_full")) {
throw new DestinationMemoryFullException(
String.format("Generic importer failed with code (%s)", response.code()),
new RuntimeException("destination_full"));
} else {
throw new IOException(format("Error (%d) %s", response.code(), error.toString()));
}

if (response.code() == 413 && error.getError().equals("too_large_payload")) {
throw new IOException(format("%s (%d) %s", PAYLOAD_TOO_LARGE, response.code(), error));
}

throw new IOException(format("Error (%d) %s", response.code(), error.toString()));

}
if (response.code() < 200 || response.code() >= 300) {
throw new IOException(format("Unexpected response code (%d)", response.code()));
}

return true;
}

Expand Down
111 changes: 71 additions & 40 deletions ...neric/src/test/java/org/datatransferproject/datatransfer/generic/GenericImporterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,11 @@ public void testGenericImporter() throws Exception {
getImporter(
importerClass,
container ->
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
webServer.setDispatcher(getDispatcher());

importer.importItem(
Expand Down Expand Up @@ -199,11 +199,11 @@ public void testGenericImporterTokenRefresh() throws Exception {
getImporter(
importerClass,
container ->
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
webServer.setDispatcher(getDispatcher());

importer.importItem(
Expand Down Expand Up @@ -243,11 +243,11 @@ public void testGenericImporterBadRequest() throws Exception {
getImporter(
importerClass,
container ->
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
webServer.enqueue(
new MockResponse().setResponseCode(400).setBody("{\"error\":\"bad_request\"}"));

Expand All @@ -272,11 +272,11 @@ public void testGenericImporterUnexpectedResponse() throws Exception {
getImporter(
importerClass,
container ->
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
webServer.enqueue(new MockResponse().setResponseCode(400).setBody("notjson"));

importer.importItem(
Expand All @@ -300,11 +300,11 @@ public void testGenericImporterUnexpectedResponseCode() throws Exception {
getImporter(
importerClass,
container ->
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
webServer.enqueue(new MockResponse().setResponseCode(111));

importer.importItem(
Expand All @@ -325,30 +325,61 @@ public void testGenericImporterUnexpectedResponseCode() throws Exception {
public void testGenericImporterDestinationFull() throws Exception {
InMemoryIdempotentImportExecutor executor = new InMemoryIdempotentImportExecutor(monitor);
GenericImporter<IdOnlyContainerResource, String> importer =
getImporter(
importerClass,
container ->
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
webServer.enqueue(new MockResponse().setResponseCode(413).setBody("{\"error\":\"destination_full\"}"));

assertThrows(DestinationMemoryFullException.class, () -> {
importer.importItem(
getImporter(
importerClass,
container ->
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
webServer.enqueue(
new MockResponse().setResponseCode(413).setBody("{\"error\":\"destination_full\"}"));

assertThrows(
DestinationMemoryFullException.class,
() -> {
importer.importItem(
UUID.randomUUID(),
executor,
new TokensAndUrlAuthData(
"accessToken", "refreshToken", webServer.url("/refresh").toString()),
"accessToken", "refreshToken", webServer.url("/refresh").toString()),
new IdOnlyContainerResource("itemId"));
});

});

Collection<ErrorDetail> errors = executor.getErrors();
assertEquals(1, errors.size());
ErrorDetail error = errors.iterator().next();
assertEquals("itemId", error.title());
assertContains("Generic importer failed with code (413)", error.exception());
}

@Test
public void testGenericImporterIgnoreLargePayload() throws Exception {
InMemoryIdempotentImportExecutor executor = new InMemoryIdempotentImportExecutor(monitor);
GenericImporter<IdOnlyContainerResource, String> importer =
getImporter(
importerClass,
container ->
List.of(
new ImportableData<>(
new GenericPayload<>(container.getId(), "schemasource"),
container.getId(),
container.getId())));
webServer.enqueue(
new MockResponse().setResponseCode(413).setBody("{\"error\":\"too_large_payload\"}"));

importer.importItem(
UUID.randomUUID(),
executor,
new TokensAndUrlAuthData(
"accessToken", "refreshToken", webServer.url("/refresh").toString()),
new IdOnlyContainerResource("itemId"));

Collection<ErrorDetail> errors = executor.getErrors();
assertEquals(1, errors.size());
ErrorDetail error = errors.iterator().next();
assertEquals("itemId", error.title());
assertContains(GenericImporter.PAYLOAD_TOO_LARGE, error.exception());
}
}