|
1 | 1 | import { assert } from "tsafe/assert"; |
2 | 2 | import { Evt } from "evt"; |
| 3 | +import { Zip, ZipPassThrough } from "fflate/browser"; |
3 | 4 | import type { Thunks } from "core/bootstrap"; |
4 | 5 | import { name, actions } from "./state"; |
5 | 6 | import { protectedSelectors } from "./selectors"; |
@@ -31,7 +32,7 @@ export declare namespace ExplorersCreateParams { |
31 | 32 | const privateThunks = { |
32 | 33 | createOperation: |
33 | 34 | (params: { |
34 | | - operation: "create" | "delete" | "modifyPolicy"; |
| 35 | + operation: "create" | "delete" | "modifyPolicy" | "downloading"; |
35 | 36 | objects: S3Object[]; |
36 | 37 | directoryPath: string; |
37 | 38 | }) => |
@@ -178,6 +179,184 @@ const privateThunks = { |
178 | 179 | isBucketPolicyAvailable |
179 | 180 | }) |
180 | 181 | ); |
| 182 | + }, |
| 183 | + downloadObjectsAsZip: |
| 184 | + (params: { s3Objects: S3Object[] }) => |
| 185 | + async (...args) => { |
| 186 | + const [dispatch, getState] = args; |
| 187 | + |
| 188 | + const { directoryPath } = getState()[name]; |
| 189 | + assert(directoryPath !== undefined); |
| 190 | + |
| 191 | + const { s3Objects } = params; |
| 192 | + |
| 193 | + const operationId = await dispatch( |
| 194 | + privateThunks.createOperation({ |
| 195 | + operation: "downloading", |
| 196 | + objects: s3Objects, |
| 197 | + directoryPath |
| 198 | + }) |
| 199 | + ); |
| 200 | + |
| 201 | + const s3Client = await dispatch( |
| 202 | + s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() |
| 203 | + ).then(r => { |
| 204 | + assert(r !== undefined); |
| 205 | + return r.s3Client; |
| 206 | + }); |
| 207 | + |
| 208 | + const { readable, writable } = new TransformStream(); |
| 209 | + const writer = writable.getWriter(); |
| 210 | + |
| 211 | + { |
| 212 | + const zip = new Zip((err, chunk, final) => { |
| 213 | + if (err) { |
| 214 | + writer.abort(err); |
| 215 | + throw err; |
| 216 | + } |
| 217 | + |
| 218 | + writer.write(chunk); |
| 219 | + |
| 220 | + if (final) { |
| 221 | + writer.close(); |
| 222 | + } |
| 223 | + }); |
| 224 | + |
| 225 | + const { crawl } = crawlFactory({ |
| 226 | + list: async ({ directoryPath }) => { |
| 227 | + const { objects } = await s3Client.listObjects({ |
| 228 | + path: directoryPath |
| 229 | + }); |
| 230 | + |
| 231 | + return objects.reduce<{ |
| 232 | + fileBasenames: string[]; |
| 233 | + directoryBasenames: string[]; |
| 234 | + }>( |
| 235 | + (acc, { kind, basename }) => { |
| 236 | + switch (kind) { |
| 237 | + case "directory": |
| 238 | + acc.directoryBasenames.push(basename); |
| 239 | + break; |
| 240 | + case "file": |
| 241 | + if (basename !== ".keep") { |
| 242 | + acc.fileBasenames.push(basename); |
| 243 | + } |
| 244 | + break; |
| 245 | + } |
| 246 | + return acc; |
| 247 | + }, |
| 248 | + { |
| 249 | + fileBasenames: [], |
| 250 | + directoryBasenames: [] |
| 251 | + } |
| 252 | + ); |
| 253 | + } |
| 254 | + }); |
| 255 | + |
| 256 | + const createZipEntryFromUrl = async ({ |
| 257 | + zipPath, |
| 258 | + url, |
| 259 | + modifiedDate |
| 260 | + }: { |
| 261 | + zipPath: string; |
| 262 | + url: string; |
| 263 | + modifiedDate?: string | number | Date; |
| 264 | + }) => { |
| 265 | + const res = await fetch(url); |
| 266 | + if (!res.ok || !res.body) return; |
| 267 | + |
| 268 | + const entry = new ZipPassThrough(zipPath); |
| 269 | + if (modifiedDate) entry.mtime = modifiedDate; |
| 270 | + |
| 271 | + zip.add(entry); |
| 272 | + |
| 273 | + const reader = res.body.getReader(); |
| 274 | + while (true) { |
| 275 | + const { done, value } = await reader.read(); |
| 276 | + if (done) break; |
| 277 | + entry.push(value); |
| 278 | + } |
| 279 | + |
| 280 | + entry.push(new Uint8Array(0), true); |
| 281 | + }; |
| 282 | + |
| 283 | + const downloadTasks: Promise<void>[] = []; |
| 284 | + |
| 285 | + for (const object of s3Objects) { |
| 286 | + const basePath = pathJoin(directoryPath, object.basename); |
| 287 | + |
| 288 | + switch (object.kind) { |
| 289 | + case "directory": { |
| 290 | + const { filePaths, directoryPaths } = await crawl({ |
| 291 | + directoryPath: basePath |
| 292 | + }); |
| 293 | + |
| 294 | + directoryPaths.forEach(path => { |
| 295 | + const zipEntry = new ZipPassThrough( |
| 296 | + `${pathJoin(object.basename, path)}/` |
| 297 | + ); |
| 298 | + zip.add(zipEntry); |
| 299 | + zipEntry.push(new Uint8Array(0), true); |
| 300 | + }); |
| 301 | + |
| 302 | + for (const relativeFilePath of filePaths) { |
| 303 | + const absolutePath = pathJoin(basePath, relativeFilePath); |
| 304 | + const zipEntryPath = pathJoin( |
| 305 | + object.basename, |
| 306 | + relativeFilePath |
| 307 | + ); |
| 308 | + |
| 309 | + const url = await s3Client.getFileDownloadUrl({ |
| 310 | + path: absolutePath, |
| 311 | + validityDurationSecond: 300 |
| 312 | + }); |
| 313 | + |
| 314 | + downloadTasks.push( |
| 315 | + createZipEntryFromUrl({ |
| 316 | + zipPath: zipEntryPath, |
| 317 | + url, |
| 318 | + modifiedDate: undefined |
| 319 | + }) |
| 320 | + ); |
| 321 | + } |
| 322 | + break; |
| 323 | + } |
| 324 | + |
| 325 | + case "file": { |
| 326 | + const url = await s3Client.getFileDownloadUrl({ |
| 327 | + path: basePath, |
| 328 | + validityDurationSecond: 300 |
| 329 | + }); |
| 330 | + |
| 331 | + downloadTasks.push( |
| 332 | + createZipEntryFromUrl({ |
| 333 | + zipPath: object.basename, |
| 334 | + url, |
| 335 | + modifiedDate: object.lastModified |
| 336 | + }) |
| 337 | + ); |
| 338 | + break; |
| 339 | + } |
| 340 | + } |
| 341 | + } |
| 342 | + |
| 343 | + await Promise.all(downloadTasks); |
| 344 | + zip.end(); |
| 345 | + } |
| 346 | + |
| 347 | + dispatch( |
| 348 | + actions.operationCompleted({ |
| 349 | + objects: s3Objects, |
| 350 | + operationId |
| 351 | + }) |
| 352 | + ); |
| 353 | + return { |
| 354 | + stream: readable, |
| 355 | + zipFileName: |
| 356 | + s3Objects.length === 1 |
| 357 | + ? `${s3Objects[0].basename}.zip` |
| 358 | + : `onyxia-download-${new Date().toISOString()}.zip` |
| 359 | + }; |
181 | 360 | } |
182 | 361 | } satisfies Thunks; |
183 | 362 |
|
@@ -858,5 +1037,30 @@ export const thunks = { |
858 | 1037 | ); |
859 | 1038 |
|
860 | 1039 | dispatch(actions.requestSignedUrlCompleted({ url })); |
| 1040 | + }, |
| 1041 | + getDownloadUrl: |
| 1042 | + (params: { s3Objects: S3Object[] }) => |
| 1043 | + async (...args): Promise<{ url: string; filename: string }> => { |
| 1044 | + const { s3Objects } = params; |
| 1045 | + const [dispatch] = args; |
| 1046 | + |
| 1047 | + if (s3Objects.length === 1 && s3Objects[0].kind === "file") { |
| 1048 | + const basename = s3Objects[0].basename; |
| 1049 | + const url = await dispatch( |
| 1050 | + thunks.getFileDownloadUrl({ |
| 1051 | + basename |
| 1052 | + }) |
| 1053 | + ); |
| 1054 | + return { url, filename: basename }; |
| 1055 | + } |
| 1056 | + |
| 1057 | + const { stream, zipFileName } = await dispatch( |
| 1058 | + privateThunks.downloadObjectsAsZip({ s3Objects }) |
| 1059 | + ); |
| 1060 | + |
| 1061 | + const blob = await new Response(stream).blob(); |
| 1062 | + const blobUrl = URL.createObjectURL(blob); |
| 1063 | + |
| 1064 | + return { url: blobUrl, filename: zipFileName }; |
861 | 1065 | } |
862 | 1066 | } satisfies Thunks; |
0 commit comments