Skip to content

Commit e74b4ed

Browse files
kylstevensonkylstevenson
committed
Implement multifile downloads
Update method signature for distribution files to match existing methods Skip checksum validation for multifile downloads where no checksum is provided
1 parent f5760dd commit e74b4ed

14 files changed

Lines changed: 1050 additions & 60 deletions

File tree

‎README.md‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,13 +148,24 @@ Map<String, DatasetSeries> members = fusion.listDatasetMembers("my-catalog", "my
148148
```java
149149
Map<String, Distribution> distributions = fusion.listDistributions("my-catalog", "my-dataset", "my-series-member");
150150
```
151-
6. Download as a file
151+
6. List the files available in a distribution
152+
```java
153+
Map<String, DistributionFile> files = fusion.listDistributionFiles("my-catalog", "my-dataset", "my-series-member", "csv", 0);
154+
```
155+
7. Download all files in a distribution
152156
```java
153157
fusion.download("my-catalog", "my-dataset", "my-series-member", "csv", "/downloads/distributions");
154158
```
155-
7. Download as a stream
159+
8. Download specific files in a distribution
160+
```java
161+
List<String> fileNames = Arrays.asList("file1", "file2");
162+
fusion.download("my-catalog", "my-dataset", "my-series-member", "csv", "/downloads/distributions", fileNames);
163+
```
164+
9. Download as a stream (returns a Map of file names to InputStreams)
156165
```java
157-
InputStream is = fusion.downloadStream("my-catalog", "my-dataset", "my-series-member", "csv");
166+
Map<String, InputStream> streams = fusion.downloadStream("my-catalog", "my-dataset", "my-series-member", "csv");
167+
// Access individual file streams
168+
InputStream fileStream = streams.get("file1");
158169
```
159170

160171
#### Logging

‎src/main/java/io/github/jpmorganchase/fusion/Fusion.java‎

Lines changed: 293 additions & 25 deletions
Large diffs are not rendered by default.

‎src/main/java/io/github/jpmorganchase/fusion/api/FusionAPIManager.java‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,13 +138,68 @@ public String callAPIToDelete(String apiPath) {
138138
public void callAPIFileDownload(
139139
String apiPath, String fileName, String catalog, String dataset, Map<String, String> headers)
140140
throws APICallException, FileDownloadException {
141-
downloader.callAPIFileDownload(apiPath, fileName, catalog, dataset, headers);
141+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, false);
142+
}
143+
144+
@Override
145+
public void callAPIFileDownload(
146+
String apiPath,
147+
String fileName,
148+
String catalog,
149+
String dataset,
150+
Map<String, String> headers,
151+
boolean skipChecksumValidationIfMissing)
152+
throws APICallException, FileDownloadException {
153+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
154+
}
155+
156+
@Override
157+
public void callAPIFileDownload(
158+
String apiPath,
159+
String fileName,
160+
String catalog,
161+
String dataset,
162+
Map<String, String> headers,
163+
String fileIdentifier,
164+
boolean skipChecksumValidationIfMissing)
165+
throws APICallException, FileDownloadException {
166+
downloader.callAPIFileDownload(
167+
apiPath, fileName, catalog, dataset, headers, fileIdentifier, skipChecksumValidationIfMissing);
142168
}
143169

144170
@Override
145171
public InputStream callAPIFileDownload(String apiPath, String catalog, String dataset, Map<String, String> headers)
146172
throws APICallException, FileDownloadException {
147-
return downloader.callAPIFileDownload(APIManager.encodeUrl(apiPath), catalog, dataset, headers);
173+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, false);
174+
}
175+
176+
@Override
177+
public InputStream callAPIFileDownload(
178+
String apiPath,
179+
String catalog,
180+
String dataset,
181+
Map<String, String> headers,
182+
boolean skipChecksumValidationIfMissing)
183+
throws APICallException, FileDownloadException {
184+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
185+
}
186+
187+
@Override
188+
public InputStream callAPIFileDownload(
189+
String apiPath,
190+
String catalog,
191+
String dataset,
192+
Map<String, String> headers,
193+
String fileIdentifier,
194+
boolean skipChecksumValidationIfMissing)
195+
throws APICallException, FileDownloadException {
196+
return downloader.callAPIFileDownload(
197+
APIManager.encodeUrl(apiPath),
198+
catalog,
199+
dataset,
200+
headers,
201+
fileIdentifier,
202+
skipChecksumValidationIfMissing);
148203
}
149204

150205
@Override

‎src/main/java/io/github/jpmorganchase/fusion/api/operations/APIDownloadOperations.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,54 @@ public interface APIDownloadOperations {
99

1010
default void callAPIFileDownload(
1111
String apiPath, String fileName, String catalog, String dataset, Map<String, String> headers)
12+
throws APICallException, FileDownloadException {
13+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, false);
14+
}
15+
16+
default void callAPIFileDownload(
17+
String apiPath,
18+
String fileName,
19+
String catalog,
20+
String dataset,
21+
Map<String, String> headers,
22+
boolean skipChecksumValidationIfMissing)
23+
throws APICallException, FileDownloadException {
24+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
25+
}
26+
27+
default void callAPIFileDownload(
28+
String apiPath,
29+
String fileName,
30+
String catalog,
31+
String dataset,
32+
Map<String, String> headers,
33+
String fileIdentifier,
34+
boolean skipChecksumValidationIfMissing)
1235
throws APICallException, FileDownloadException {}
1336

1437
default InputStream callAPIFileDownload(String apiPath, String catalog, String dataset, Map<String, String> headers)
1538
throws APICallException, FileDownloadException {
39+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, false);
40+
}
41+
42+
default InputStream callAPIFileDownload(
43+
String apiPath,
44+
String catalog,
45+
String dataset,
46+
Map<String, String> headers,
47+
boolean skipChecksumValidationIfMissing)
48+
throws APICallException, FileDownloadException {
49+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
50+
}
51+
52+
default InputStream callAPIFileDownload(
53+
String apiPath,
54+
String catalog,
55+
String dataset,
56+
Map<String, String> headers,
57+
String fileIdentifier,
58+
boolean skipChecksumValidationIfMissing)
59+
throws APICallException, FileDownloadException {
1660
return null;
1761
}
1862
}

‎src/main/java/io/github/jpmorganchase/fusion/api/operations/FusionAPIDownloadOperations.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,40 @@ public void callAPIFileDownload(
5959
String apiPath, String filePath, String catalog, String dataset, Map<String, String> headers)
6060
throws APICallException, FileDownloadException {
6161

62+
callAPIFileDownload(apiPath, filePath, catalog, dataset, headers, null, false);
63+
}
64+
65+
@Override
66+
public void callAPIFileDownload(
67+
String apiPath,
68+
String filePath,
69+
String catalog,
70+
String dataset,
71+
Map<String, String> headers,
72+
boolean skipChecksumValidationIfMissing)
73+
throws APICallException, FileDownloadException {
74+
75+
callAPIFileDownload(apiPath, filePath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
76+
}
77+
78+
@Override
79+
public void callAPIFileDownload(
80+
String apiPath,
81+
String filePath,
82+
String catalog,
83+
String dataset,
84+
Map<String, String> headers,
85+
String fileIdentifier,
86+
boolean skipChecksumValidationIfMissing)
87+
throws APICallException, FileDownloadException {
88+
6289
DownloadRequest dr = DownloadRequest.builder()
6390
.apiPath(apiPath)
6491
.filePath(filePath)
6592
.catalog(catalog)
6693
.dataset(dataset)
94+
.fileIdentifier(fileIdentifier)
95+
.skipChecksumValidationIfMissing(skipChecksumValidationIfMissing)
6796
.headers(headers)
6897
.build();
6998

@@ -85,11 +114,38 @@ public void callAPIFileDownload(
85114
public InputStream callAPIFileDownload(String apiPath, String catalog, String dataset, Map<String, String> headers)
86115
throws APICallException, FileDownloadException {
87116

117+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, false);
118+
}
119+
120+
@Override
121+
public InputStream callAPIFileDownload(
122+
String apiPath,
123+
String catalog,
124+
String dataset,
125+
Map<String, String> headers,
126+
boolean skipChecksumValidationIfMissing)
127+
throws APICallException, FileDownloadException {
128+
129+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
130+
}
131+
132+
@Override
133+
public InputStream callAPIFileDownload(
134+
String apiPath,
135+
String catalog,
136+
String dataset,
137+
Map<String, String> headers,
138+
String fileIdentifier,
139+
boolean skipChecksumValidationIfMissing)
140+
throws APICallException, FileDownloadException {
141+
88142
DownloadRequest dr = DownloadRequest.builder()
89143
.apiPath(apiPath)
90144
.catalog(catalog)
91145
.dataset(dataset)
146+
.fileIdentifier(fileIdentifier)
92147
.isDownloadToStream(true)
148+
.skipChecksumValidationIfMissing(skipChecksumValidationIfMissing)
93149
.headers(headers)
94150
.build();
95151

‎src/main/java/io/github/jpmorganchase/fusion/api/request/DownloadRequest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ public class DownloadRequest {
1818
private String catalog;
1919
private String dataset;
2020
private String filePath;
21+
private String fileIdentifier;
2122
private boolean isDownloadToStream;
2223
private Map<String, String> headers;
24+
private boolean skipChecksumValidationIfMissing;
2325
}

‎src/main/java/io/github/jpmorganchase/fusion/api/request/PartFetcher.java‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import static io.github.jpmorganchase.fusion.api.tools.ResponseChecker.checkResponseStatus;
44

55
import io.github.jpmorganchase.fusion.FusionConfiguration;
6+
import io.github.jpmorganchase.fusion.api.exception.FileDownloadException;
67
import io.github.jpmorganchase.fusion.api.response.GetPartResponse;
78
import io.github.jpmorganchase.fusion.api.response.Head;
89
import io.github.jpmorganchase.fusion.api.stream.IntegrityCheckingInputStream;
@@ -13,6 +14,7 @@
1314
import java.io.InputStream;
1415
import java.util.HashMap;
1516
import java.util.Map;
17+
import java.util.Objects;
1618
import java.util.Optional;
1719
import lombok.Builder;
1820
import lombok.Getter;
@@ -26,7 +28,6 @@
2628
public class PartFetcher {
2729

2830
private static final String HEAD_PATH = "%s/operationType/download";
29-
private static final String HEAD_PATH_FOR_PART = "%s?downloadPartNumber=%d";
3031

3132
Client client;
3233
FusionTokenProvider credentials;
@@ -48,16 +49,30 @@ public GetPartResponse fetch(PartRequest pr) {
4849
checkResponseStatus(response);
4950

5051
Head head = getHead(response, pr);
51-
InputStream inputStream = getIntegrityCheckingInputStream(response, head);
52+
InputStream inputStream = getIntegrityCheckingInputStream(response, head, pr);
5253

5354
return GetPartResponse.builder().content(inputStream).head(head).build();
5455
}
5556

56-
private IntegrityCheckingInputStream getIntegrityCheckingInputStream(
57-
HttpResponse<InputStream> response, Head head) {
57+
private InputStream getIntegrityCheckingInputStream(HttpResponse<InputStream> response, Head head, PartRequest pr) {
58+
String checksum = head.getChecksum();
59+
boolean checksumMissing = Objects.isNull(checksum) || checksum.isEmpty();
60+
DownloadRequest downloadRequest = pr.getDownloadRequest();
61+
62+
if (checksumMissing) {
63+
if (downloadRequest.isSkipChecksumValidationIfMissing()) {
64+
log.debug("Skipping checksum validation for download request {}", downloadRequest);
65+
return response.getBody();
66+
} else {
67+
throw new FileDownloadException(
68+
"Checksum validation failed: checksum is missing for download request: " + downloadRequest);
69+
}
70+
}
71+
72+
log.debug("Verifying checksum for download request {} with checksum: {}", downloadRequest, checksum);
5873
return IntegrityCheckingInputStream.builder()
5974
.part(response.getBody())
60-
.checksum(head.getChecksum())
75+
.checksum(checksum)
6176
.partChecker(
6277
PartChecker.builder().digestAlgo(getDigestAlgo(head)).build())
6378
.build();
@@ -103,14 +118,24 @@ private String getPath(PartRequest pr) {
103118
}
104119

105120
private String getSinglePartPath(PartRequest pr) {
106-
return pr.getDownloadRequest().getApiPath();
121+
return appendFileQueryParam(pr.getDownloadRequest().getApiPath(), pr.getDownloadRequest());
107122
}
108123

109124
private String getMultiPartPath(PartRequest pr) {
110-
return String.format(HEAD_PATH_FOR_PART, getHeadPath(pr), pr.getPartNo());
125+
String headPath = getHeadPath(pr);
126+
String separator = headPath.contains("?") ? "&" : "?";
127+
return headPath + separator + "downloadPartNumber=" + pr.getPartNo();
111128
}
112129

113130
private String getHeadPath(PartRequest pr) {
114-
return String.format(HEAD_PATH, pr.getDownloadRequest().getApiPath());
131+
String basePath = String.format(HEAD_PATH, pr.getDownloadRequest().getApiPath());
132+
return appendFileQueryParam(basePath, pr.getDownloadRequest());
133+
}
134+
135+
private String appendFileQueryParam(String path, DownloadRequest dr) {
136+
if (dr.getFileIdentifier() != null && !dr.getFileIdentifier().isEmpty()) {
137+
return path + "?file=" + dr.getFileIdentifier();
138+
}
139+
return path;
115140
}
116141
}

‎src/main/java/io/github/jpmorganchase/fusion/digest/PartChecker.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ public void update(int bytesRead) throws IOException {
3737
}
3838

3939
public void verify(String expectedChecksum) throws IOException {
40+
if (Objects.isNull(expectedChecksum) || expectedChecksum.isEmpty()) {
41+
return;
42+
}
43+
4044
String calculatedChecksum = digestProvider.getDigest();
4145

4246
if (!Objects.equals(expectedChecksum, calculatedChecksum)) {

0 commit comments

Comments
 (0)