Skip to content

Commit e1c0453

Browse files
kylstevensonkylstevenson
committed
Skip checksum validation for multifile downloads where no checksum is provided
1 parent 0bb9b16 commit e1c0453

7 files changed

Lines changed: 139 additions & 34 deletions

File tree

‎src/main/java/io/github/jpmorganchase/fusion/Fusion.java‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -635,22 +635,22 @@ public void download(
635635
String safeFileName = identifier.replaceAll("[^a-zA-Z0-9_.\\-]", "_");
636636
String fullPath = path + "/" + safeFileName + extension;
637637

638-
String url = null;
638+
String url = String.format(
639+
"%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s",
640+
this.rootURL, catalogName, dataset, seriesMember, distribution);
641+
642+
String encodedIdentifier = null;
639643
try {
640644
// URL encode the identifier, handling special characters like '.' and '+'
641-
String encodedIdentifier = URLEncoder.encode(identifier, "UTF-8")
645+
encodedIdentifier = URLEncoder.encode(identifier, "UTF-8")
642646
.replace(".", "%2E")
643647
.replace("+", "%2B");
644-
645-
url = String.format(
646-
"%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s?file=%s",
647-
this.rootURL, catalogName, dataset, seriesMember, distribution, encodedIdentifier);
648-
649648
} catch (UnsupportedEncodingException e) {
650649
throw new RuntimeException(e);
651650
}
652651

653-
this.api.callAPIFileDownload(url, fullPath, catalogName, dataset, headers, skipChecksumValidationIfMissing);
652+
this.api.callAPIFileDownload(
653+
url, fullPath, catalogName, dataset, headers, encodedIdentifier, skipChecksumValidationIfMissing);
654654
}
655655
}
656656

@@ -811,22 +811,22 @@ public Map<String, InputStream> downloadStream(
811811
continue;
812812
}
813813

814-
String url = null;
814+
String url = String.format(
815+
"%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s",
816+
this.rootURL, catalogName, dataset, seriesMember, distribution);
817+
818+
String encodedIdentifier = null;
815819
try {
816820
// URL encode the identifier, handling special characters like '.' and '+'
817-
String encodedIdentifier = URLEncoder.encode(identifier, "UTF-8")
821+
encodedIdentifier = URLEncoder.encode(identifier, "UTF-8")
818822
.replace(".", "%2E")
819823
.replace("+", "%2B");
820-
821-
url = String.format(
822-
"%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s/files/operationType/download?file=%s",
823-
this.rootURL, catalogName, dataset, seriesMember, distribution, encodedIdentifier);
824824
} catch (UnsupportedEncodingException e) {
825825
throw new RuntimeException(e);
826826
}
827827

828-
InputStream stream =
829-
this.api.callAPIFileDownload(url, catalogName, dataset, headers, skipChecksumValidationIfMissing);
828+
InputStream stream = this.api.callAPIFileDownload(
829+
url, catalogName, dataset, headers, encodedIdentifier, skipChecksumValidationIfMissing);
830830
result.put(identifier, stream);
831831
}
832832

‎src/main/java/io/github/jpmorganchase/fusion/api/FusionAPIManager.java‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ public String callAPIToDelete(String apiPath) {
138138
public void callAPIFileDownload(
139139
String apiPath, String fileName, String catalog, String dataset, Map<String, String> headers)
140140
throws APICallException, FileDownloadException {
141-
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, false);
141+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, false);
142142
}
143143

144144
@Override
@@ -150,13 +150,38 @@ public void callAPIFileDownload(
150150
Map<String, String> headers,
151151
boolean skipChecksumValidationIfMissing)
152152
throws APICallException, FileDownloadException {
153-
downloader.callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, skipChecksumValidationIfMissing);
153+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
154+
}
155+
156+
@Override
157+
public void callAPIFileDownload(
158+
String apiPath,
159+
String fileName,
160+
String catalog,
161+
String dataset,
162+
Map<String, String> headers,
163+
String fileIdentifier,
164+
boolean skipChecksumValidationIfMissing)
165+
throws APICallException, FileDownloadException {
166+
downloader.callAPIFileDownload(
167+
apiPath, fileName, catalog, dataset, headers, fileIdentifier, skipChecksumValidationIfMissing);
154168
}
155169

156170
@Override
157171
public InputStream callAPIFileDownload(String apiPath, String catalog, String dataset, Map<String, String> headers)
158172
throws APICallException, FileDownloadException {
159-
return callAPIFileDownload(apiPath, catalog, dataset, headers, false);
173+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, false);
174+
}
175+
176+
@Override
177+
public InputStream callAPIFileDownload(
178+
String apiPath,
179+
String catalog,
180+
String dataset,
181+
Map<String, String> headers,
182+
boolean skipChecksumValidationIfMissing)
183+
throws APICallException, FileDownloadException {
184+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
160185
}
161186

162187
@Override
@@ -165,10 +190,16 @@ public InputStream callAPIFileDownload(
165190
String catalog,
166191
String dataset,
167192
Map<String, String> headers,
193+
String fileIdentifier,
168194
boolean skipChecksumValidationIfMissing)
169195
throws APICallException, FileDownloadException {
170196
return downloader.callAPIFileDownload(
171-
APIManager.encodeUrl(apiPath), catalog, dataset, headers, skipChecksumValidationIfMissing);
197+
APIManager.encodeUrl(apiPath),
198+
catalog,
199+
dataset,
200+
headers,
201+
fileIdentifier,
202+
skipChecksumValidationIfMissing);
172203
}
173204

174205
@Override

‎src/main/java/io/github/jpmorganchase/fusion/api/operations/APIDownloadOperations.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public interface APIDownloadOperations {
1010
default void callAPIFileDownload(
1111
String apiPath, String fileName, String catalog, String dataset, Map<String, String> headers)
1212
throws APICallException, FileDownloadException {
13-
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, false);
13+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, false);
1414
}
1515

1616
default void callAPIFileDownload(
@@ -20,18 +20,41 @@ default void callAPIFileDownload(
2020
String dataset,
2121
Map<String, String> headers,
2222
boolean skipChecksumValidationIfMissing)
23+
throws APICallException, FileDownloadException {
24+
callAPIFileDownload(apiPath, fileName, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
25+
}
26+
27+
default void callAPIFileDownload(
28+
String apiPath,
29+
String fileName,
30+
String catalog,
31+
String dataset,
32+
Map<String, String> headers,
33+
String fileIdentifier,
34+
boolean skipChecksumValidationIfMissing)
2335
throws APICallException, FileDownloadException {}
2436

2537
default InputStream callAPIFileDownload(String apiPath, String catalog, String dataset, Map<String, String> headers)
2638
throws APICallException, FileDownloadException {
27-
return callAPIFileDownload(apiPath, catalog, dataset, headers, false);
39+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, false);
40+
}
41+
42+
default InputStream callAPIFileDownload(
43+
String apiPath,
44+
String catalog,
45+
String dataset,
46+
Map<String, String> headers,
47+
boolean skipChecksumValidationIfMissing)
48+
throws APICallException, FileDownloadException {
49+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
2850
}
2951

3052
default InputStream callAPIFileDownload(
3153
String apiPath,
3254
String catalog,
3355
String dataset,
3456
Map<String, String> headers,
57+
String fileIdentifier,
3558
boolean skipChecksumValidationIfMissing)
3659
throws APICallException, FileDownloadException {
3760
return null;

‎src/main/java/io/github/jpmorganchase/fusion/api/operations/FusionAPIDownloadOperations.java‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public void callAPIFileDownload(
5959
String apiPath, String filePath, String catalog, String dataset, Map<String, String> headers)
6060
throws APICallException, FileDownloadException {
6161

62-
callAPIFileDownload(apiPath, filePath, catalog, dataset, headers, false);
62+
callAPIFileDownload(apiPath, filePath, catalog, dataset, headers, null, false);
6363
}
6464

6565
@Override
@@ -72,11 +72,26 @@ public void callAPIFileDownload(
7272
boolean skipChecksumValidationIfMissing)
7373
throws APICallException, FileDownloadException {
7474

75+
callAPIFileDownload(apiPath, filePath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
76+
}
77+
78+
@Override
79+
public void callAPIFileDownload(
80+
String apiPath,
81+
String filePath,
82+
String catalog,
83+
String dataset,
84+
Map<String, String> headers,
85+
String fileIdentifier,
86+
boolean skipChecksumValidationIfMissing)
87+
throws APICallException, FileDownloadException {
88+
7589
DownloadRequest dr = DownloadRequest.builder()
7690
.apiPath(apiPath)
7791
.filePath(filePath)
7892
.catalog(catalog)
7993
.dataset(dataset)
94+
.fileIdentifier(fileIdentifier)
8095
.skipChecksumValidationIfMissing(skipChecksumValidationIfMissing)
8196
.headers(headers)
8297
.build();
@@ -99,7 +114,19 @@ public void callAPIFileDownload(
99114
public InputStream callAPIFileDownload(String apiPath, String catalog, String dataset, Map<String, String> headers)
100115
throws APICallException, FileDownloadException {
101116

102-
return callAPIFileDownload(apiPath, catalog, dataset, headers, false);
117+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, false);
118+
}
119+
120+
@Override
121+
public InputStream callAPIFileDownload(
122+
String apiPath,
123+
String catalog,
124+
String dataset,
125+
Map<String, String> headers,
126+
boolean skipChecksumValidationIfMissing)
127+
throws APICallException, FileDownloadException {
128+
129+
return callAPIFileDownload(apiPath, catalog, dataset, headers, null, skipChecksumValidationIfMissing);
103130
}
104131

105132
@Override
@@ -108,13 +135,15 @@ public InputStream callAPIFileDownload(
108135
String catalog,
109136
String dataset,
110137
Map<String, String> headers,
138+
String fileIdentifier,
111139
boolean skipChecksumValidationIfMissing)
112140
throws APICallException, FileDownloadException {
113141

114142
DownloadRequest dr = DownloadRequest.builder()
115143
.apiPath(apiPath)
116144
.catalog(catalog)
117145
.dataset(dataset)
146+
.fileIdentifier(fileIdentifier)
118147
.isDownloadToStream(true)
119148
.skipChecksumValidationIfMissing(skipChecksumValidationIfMissing)
120149
.headers(headers)

‎src/main/java/io/github/jpmorganchase/fusion/api/request/DownloadRequest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ public class DownloadRequest {
1818
private String catalog;
1919
private String dataset;
2020
private String filePath;
21+
private String fileIdentifier;
2122
private boolean isDownloadToStream;
2223
private Map<String, String> headers;
2324
private boolean skipChecksumValidationIfMissing;

‎src/main/java/io/github/jpmorganchase/fusion/api/request/PartFetcher.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,14 +112,22 @@ private String getPath(PartRequest pr) {
112112
}
113113

114114
private String getSinglePartPath(PartRequest pr) {
115-
return pr.getDownloadRequest().getApiPath();
115+
return appendFileQueryParam(pr.getDownloadRequest().getApiPath(), pr.getDownloadRequest());
116116
}
117117

118118
private String getMultiPartPath(PartRequest pr) {
119119
return String.format(HEAD_PATH_FOR_PART, getHeadPath(pr), pr.getPartNo());
120120
}
121121

122122
private String getHeadPath(PartRequest pr) {
123-
return String.format(HEAD_PATH, pr.getDownloadRequest().getApiPath());
123+
String basePath = String.format(HEAD_PATH, pr.getDownloadRequest().getApiPath());
124+
return appendFileQueryParam(basePath, pr.getDownloadRequest());
125+
}
126+
127+
private String appendFileQueryParam(String path, DownloadRequest dr) {
128+
if (dr.getFileIdentifier() != null && !dr.getFileIdentifier().isEmpty()) {
129+
return path + "?file=" + dr.getFileIdentifier();
130+
}
131+
return path;
124132
}
125133
}

0 commit comments

Comments
 (0)