Skip to content

Commit 47fca57

Browse files
kylstevensonkylstevenson
authored
Add Support for Multifile Distribution Downloads (#167)
* Implement multifile downloads Update method signature for distribution files to match existing methods Skip checksum validation for multifile downloads where no checksum is provided * Use original structured name if no identifier (file name is the same as distribution name)
1 parent f5760dd commit 47fca57

17 files changed

Lines changed: 1056 additions & 87 deletions

‎README.md‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,13 +148,24 @@ Map<String, DatasetSeries> members = fusion.listDatasetMembers("my-catalog", "my
148148
```java
149149
Map<String, Distribution> distributions = fusion.listDistributions("my-catalog", "my-dataset", "my-series-member");
150150
```
151-
6. Download as a file
151+
6. List the files available in a distribution
152+
```java
153+
Map<String, DistributionFile> files = fusion.listDistributionFiles("my-catalog", "my-dataset", "my-series-member", "csv", 0);
154+
```
155+
7. Download all files in a distribution
152156
```java
153157
fusion.download("my-catalog", "my-dataset", "my-series-member", "csv", "/downloads/distributions");
154158
```
155-
7. Download as a stream
159+
8. Download specific files in a distribution
160+
```java
161+
List<String> fileNames = Arrays.asList("file1", "file2");
162+
fusion.download("my-catalog", "my-dataset", "my-series-member", "csv", "/downloads/distributions", fileNames);
163+
```
164+
9. Download as a stream (returns a Map of file names to InputStreams)
156165
```java
157-
InputStream is = fusion.downloadStream("my-catalog", "my-dataset", "my-series-member", "csv");
166+
Map<String, InputStream> streams = fusion.downloadStream("my-catalog", "my-dataset", "my-series-member", "csv");
167+
// Access individual file streams
168+
InputStream fileStream = streams.get("file1");
158169
```
159170

160171
#### Logging

‎src/main/java/io/github/jpmorganchase/fusion/Fusion.java‎

Lines changed: 295 additions & 25 deletions
Large diffs are not rendered by default.

‎src/main/java/io/github/jpmorganchase/fusion/api/FusionAPIManager.java‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,13 +138,68 @@ public String callAPIToDelete(String apiPath) {
138138
public void callAPIFileDownload(
139139
String apiPath, String fileName, String catalog, String dataset, Map<String, String> headers)
140140
throws APICallException, FileDownloadException {
141-
downloader.callAPIFileDownload(apiPath, fileName, catalog, dataset, headers);
141+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, false);
142+
}
143+
144+
@Override
145+
public void callAPIFileDownload(
146+
String apiPath,
147+
String fileName,
148+
String catalog,
149+
String dataset,
150+
Map<String, String> headers,
151+
boolean skipChecksumValidationIfMissing)
152+
throws APICallException, FileDownloadException {
153+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
154+
}
155+
156+
@Override
157+
public void callAPIFileDownload(
158+
String apiPath,
159+
String fileName,
160+
String catalog,
161+
String dataset,
162+
Map<String, String> headers,
163+
String fileIdentifier,
164+
boolean skipChecksumValidationIfMissing)
165+
throws APICallException, FileDownloadException {
166+
downloader.callAPIFileDownload(
167+
apiPath, fileName, catalog, dataset, headers, fileIdentifier, skipChecksumValidationIfMissing);
142168
}
143169

144170
@Override
145171
public InputStream callAPIFileDownload(String apiPath, String catalog, String dataset, Map<String, String> headers)
146172
throws APICallException, FileDownloadException {
147-
return downloader.callAPIFileDownload(APIManager.encodeUrl(apiPath), catalog, dataset, headers);
173+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, false);
174+
}
175+
176+
@Override
177+
public InputStream callAPIFileDownload(
178+
String apiPath,
179+
String catalog,
180+
String dataset,
181+
Map<String, String> headers,
182+
boolean skipChecksumValidationIfMissing)
183+
throws APICallException, FileDownloadException {
184+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
185+
}
186+
187+
@Override
188+
public InputStream callAPIFileDownload(
189+
String apiPath,
190+
String catalog,
191+
String dataset,
192+
Map<String, String> headers,
193+
String fileIdentifier,
194+
boolean skipChecksumValidationIfMissing)
195+
throws APICallException, FileDownloadException {
196+
return downloader.callAPIFileDownload(
197+
APIManager.encodeUrl(apiPath),
198+
catalog,
199+
dataset,
200+
headers,
201+
fileIdentifier,
202+
skipChecksumValidationIfMissing);
148203
}
149204

150205
@Override

‎src/main/java/io/github/jpmorganchase/fusion/api/exception/APICallException.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public String getMessage() {
6161

6262
private String getBadRequestMessage() {
6363
if (UNKNOWN.equalsIgnoreCase(responseDetail)) {
64-
return "Bad Request. Please verify the correct data has been provided.";
64+
return "Bad Request. Please verify the correct data has been provided.";
6565
}
6666
return responseDetail;
6767
}

‎src/main/java/io/github/jpmorganchase/fusion/api/operations/APIDownloadOperations.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,54 @@ public interface APIDownloadOperations {
99

1010
default void callAPIFileDownload(
1111
String apiPath, String fileName, String catalog, String dataset, Map<String, String> headers)
12+
throws APICallException, FileDownloadException {
13+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, false);
14+
}
15+
16+
default void callAPIFileDownload(
17+
String apiPath,
18+
String fileName,
19+
String catalog,
20+
String dataset,
21+
Map<String, String> headers,
22+
boolean skipChecksumValidationIfMissing)
23+
throws APICallException, FileDownloadException {
24+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
25+
}
26+
27+
default void callAPIFileDownload(
28+
String apiPath,
29+
String fileName,
30+
String catalog,
31+
String dataset,
32+
Map<String, String> headers,
33+
String fileIdentifier,
34+
boolean skipChecksumValidationIfMissing)
1235
throws APICallException, FileDownloadException {}
1336

1437
default InputStream callAPIFileDownload(String apiPath, String catalog, String dataset, Map<String, String> headers)
1538
throws APICallException, FileDownloadException {
39+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, false);
40+
}
41+
42+
default InputStream callAPIFileDownload(
43+
String apiPath,
44+
String catalog,
45+
String dataset,
46+
Map<String, String> headers,
47+
boolean skipChecksumValidationIfMissing)
48+
throws APICallException, FileDownloadException {
49+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
50+
}
51+
52+
default InputStream callAPIFileDownload(
53+
String apiPath,
54+
String catalog,
55+
String dataset,
56+
Map<String, String> headers,
57+
String fileIdentifier,
58+
boolean skipChecksumValidationIfMissing)
59+
throws APICallException, FileDownloadException {
1660
return null;
1761
}
1862
}

‎src/main/java/io/github/jpmorganchase/fusion/api/operations/FusionAPIDownloadOperations.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,40 @@ public void callAPIFileDownload(
5959
String apiPath, String filePath, String catalog, String dataset, Map<String, String> headers)
6060
throws APICallException, FileDownloadException {
6161

62+
callAPIFileDownload(apiPath, filePath, catalog, dataset, headers, null, false);
63+
}
64+
65+
@Override
66+
public void callAPIFileDownload(
67+
String apiPath,
68+
String filePath,
69+
String catalog,
70+
String dataset,
71+
Map<String, String> headers,
72+
boolean skipChecksumValidationIfMissing)
73+
throws APICallException, FileDownloadException {
74+
75+
callAPIFileDownload(apiPath, filePath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
76+
}
77+
78+
@Override
79+
public void callAPIFileDownload(
80+
String apiPath,
81+
String filePath,
82+
String catalog,
83+
String dataset,
84+
Map<String, String> headers,
85+
String fileIdentifier,
86+
boolean skipChecksumValidationIfMissing)
87+
throws APICallException, FileDownloadException {
88+
6289
DownloadRequest dr = DownloadRequest.builder()
6390
.apiPath(apiPath)
6491
.filePath(filePath)
6592
.catalog(catalog)
6693
.dataset(dataset)
94+
.fileIdentifier(fileIdentifier)
95+
.skipChecksumValidationIfMissing(skipChecksumValidationIfMissing)
6796
.headers(headers)
6897
.build();
6998

@@ -85,11 +114,38 @@ public void callAPIFileDownload(
85114
public InputStream callAPIFileDownload(String apiPath, String catalog, String dataset, Map<String, String> headers)
86115
throws APICallException, FileDownloadException {
87116

117+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, false);
118+
}
119+
120+
@Override
121+
public InputStream callAPIFileDownload(
122+
String apiPath,
123+
String catalog,
124+
String dataset,
125+
Map<String, String> headers,
126+
boolean skipChecksumValidationIfMissing)
127+
throws APICallException, FileDownloadException {
128+
129+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
130+
}
131+
132+
@Override
133+
public InputStream callAPIFileDownload(
134+
String apiPath,
135+
String catalog,
136+
String dataset,
137+
Map<String, String> headers,
138+
String fileIdentifier,
139+
boolean skipChecksumValidationIfMissing)
140+
throws APICallException, FileDownloadException {
141+
88142
DownloadRequest dr = DownloadRequest.builder()
89143
.apiPath(apiPath)
90144
.catalog(catalog)
91145
.dataset(dataset)
146+
.fileIdentifier(fileIdentifier)
92147
.isDownloadToStream(true)
148+
.skipChecksumValidationIfMissing(skipChecksumValidationIfMissing)
93149
.headers(headers)
94150
.build();
95151

‎src/main/java/io/github/jpmorganchase/fusion/api/request/DownloadRequest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ public class DownloadRequest {
1818
private String catalog;
1919
private String dataset;
2020
private String filePath;
21+
private String fileIdentifier;
2122
private boolean isDownloadToStream;
2223
private Map<String, String> headers;
24+
private boolean skipChecksumValidationIfMissing;
2325
}

‎src/main/java/io/github/jpmorganchase/fusion/api/request/PartFetcher.java‎

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import static io.github.jpmorganchase.fusion.api.tools.ResponseChecker.checkResponseStatus;
44

55
import io.github.jpmorganchase.fusion.FusionConfiguration;
6+
import io.github.jpmorganchase.fusion.api.exception.FileDownloadException;
67
import io.github.jpmorganchase.fusion.api.response.GetPartResponse;
78
import io.github.jpmorganchase.fusion.api.response.Head;
89
import io.github.jpmorganchase.fusion.api.stream.IntegrityCheckingInputStream;
@@ -13,6 +14,7 @@
1314
import java.io.InputStream;
1415
import java.util.HashMap;
1516
import java.util.Map;
17+
import java.util.Objects;
1618
import java.util.Optional;
1719
import lombok.Builder;
1820
import lombok.Getter;
@@ -26,7 +28,6 @@
2628
public class PartFetcher {
2729

2830
private static final String HEAD_PATH = "%s/operationType/download";
29-
private static final String HEAD_PATH_FOR_PART = "%s?downloadPartNumber=%d";
3031

3132
Client client;
3233
FusionTokenProvider credentials;
@@ -48,16 +49,28 @@ public GetPartResponse fetch(PartRequest pr) {
4849
checkResponseStatus(response);
4950

5051
Head head = getHead(response, pr);
51-
InputStream inputStream = getIntegrityCheckingInputStream(response, head);
52+
InputStream inputStream = getIntegrityCheckingInputStream(response, head, pr);
5253

5354
return GetPartResponse.builder().content(inputStream).head(head).build();
5455
}
5556

56-
private IntegrityCheckingInputStream getIntegrityCheckingInputStream(
57-
HttpResponse<InputStream> response, Head head) {
57+
private InputStream getIntegrityCheckingInputStream(HttpResponse<InputStream> response, Head head, PartRequest pr) {
58+
String checksum = head.getChecksum();
59+
boolean checksumMissing = Objects.isNull(checksum) || checksum.isEmpty();
60+
DownloadRequest downloadRequest = pr.getDownloadRequest();
61+
62+
if (downloadRequest.isSkipChecksumValidationIfMissing()) {
63+
log.debug("Skipping checksum validation for download request {}", downloadRequest);
64+
return response.getBody();
65+
} else if (checksumMissing) {
66+
throw new FileDownloadException(
67+
"Checksum validation failed: checksum is missing for download request: " + downloadRequest);
68+
}
69+
70+
log.debug("Verifying checksum for download request {} with checksum: {}", downloadRequest, checksum);
5871
return IntegrityCheckingInputStream.builder()
5972
.part(response.getBody())
60-
.checksum(head.getChecksum())
73+
.checksum(checksum)
6174
.partChecker(
6275
PartChecker.builder().digestAlgo(getDigestAlgo(head)).build())
6376
.build();
@@ -89,28 +102,20 @@ private Map<String, String> getSecurityHeaders(PartRequest pr) {
89102
}
90103

91104
private String getPath(PartRequest pr) {
92-
String path;
93-
94-
if (pr.isHeadRequest()) {
95-
path = getHeadPath(pr);
96-
} else if (pr.isSinglePartDownloadRequest()) {
97-
path = getSinglePartPath(pr);
98-
} else {
99-
path = getMultiPartPath(pr);
105+
String path = appendFileQueryParam(pr.getDownloadRequest().getApiPath(), pr.getDownloadRequest());
106+
107+
if (!pr.isHeadRequest() && !pr.isSinglePartDownloadRequest()) {
108+
String separator = path.contains("?") ? "&" : "?";
109+
return path + separator + "downloadPartNumber=" + pr.getPartNo();
100110
}
101111

102112
return path;
103113
}
104114

105-
private String getSinglePartPath(PartRequest pr) {
106-
return pr.getDownloadRequest().getApiPath();
107-
}
108-
109-
private String getMultiPartPath(PartRequest pr) {
110-
return String.format(HEAD_PATH_FOR_PART, getHeadPath(pr), pr.getPartNo());
111-
}
112-
113-
private String getHeadPath(PartRequest pr) {
114-
return String.format(HEAD_PATH, pr.getDownloadRequest().getApiPath());
115+
private String appendFileQueryParam(String path, DownloadRequest dr) {
116+
if (dr.getFileIdentifier() != null && !dr.getFileIdentifier().isEmpty()) {
117+
return path + "?file=" + dr.getFileIdentifier();
118+
}
119+
return path;
115120
}
116121
}

‎src/main/java/io/github/jpmorganchase/fusion/digest/PartChecker.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ public void update(int bytesRead) throws IOException {
3737
}
3838

3939
public void verify(String expectedChecksum) throws IOException {
40+
if (Objects.isNull(expectedChecksum) || expectedChecksum.isEmpty()) {
41+
return;
42+
}
43+
4044
String calculatedChecksum = digestProvider.getDigest();
4145

4246
if (!Objects.equals(expectedChecksum, calculatedChecksum)) {

0 commit comments

Comments
 (0)