Skip to content

Commit 48aa447

Browse files
ylliethyllieth
authored
Merge pull request #36 from yllieth/add-source_job_awaiting
[master] follow api changes
2 parents ff4b57f + 0d1300a commit 48aa447

9 files changed

Lines changed: 233 additions & 169 deletions

File tree

‎bower.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "predicsis_ml_sdk-javascript",
3-
"version": "0.7.3",
3+
"version": "0.7.4",
44
"authors": [
55
"Yllieth <sylvnimes@hotmail.com>",
66
"Dehau"
@@ -20,7 +20,7 @@
2020
"angular": "1.3.17",
2121
"restangular": "1.5.1",
2222
"rehttp": "~1.1.5",
23-
"swissknife": "~0.1.3"
23+
"bluebird": "^3.4.1"
2424
},
2525
"main": "dist/predicsis-jsSDK.js",
2626
"ignore": [

‎dist/predicsis-jsSDK.js‎

Lines changed: 114 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -1204,7 +1204,7 @@ angular
12041204

12051205
//resolve promise if status is completed (and stop interval loop)
12061206
clearInterval(intervalId);
1207-
deferred.resolve(jobId);
1207+
deferred.resolve(job.next_job_id ? self.listen(job.next_job_id) : jobId);
12081208

12091209
} else {
12101210

@@ -2895,7 +2895,7 @@ angular
28952895
*/
28962896
angular
28972897
.module('predicsis.jsSDK.models')
2898-
.service('Sources', function($q, Restangular) {
2898+
.service('Sources', function($q, Restangular, Jobs) {
28992899
'use strict';
29002900

29012901
function source(id) { return Restangular.one('sources', id); }
@@ -2931,8 +2931,11 @@ angular
29312931
* @param {Object} params See above example.
29322932
* @return {Promise} New source
29332933
*/
2934-
this.create = function(source, dataStore) {
2935-
return sources().post({source: source, data_store: dataStore });
2934+
this.create = function(_source, dataStore) {
2935+
return Jobs.wrapAsyncPromise(sources().post({source: _source, data_store: dataStore }))
2936+
.then(function(result) {
2937+
return source(result.id).get();
2938+
});
29362939
};
29372940

29382941
/**
@@ -3079,7 +3082,7 @@ angular
30793082
* @return {Promise} An object containing a part_url field (PUT part presigned url)
30803083
*/
30813084
this.getPartUrl = function(id, partNumber, path) {
3082-
return upload(id).get({ part_number: partNumber, path: path });
3085+
return upload(id).get({ part_number: partNumber, path: path, hideErrors: true });
30833086
};
30843087

30853088
/**
@@ -3446,8 +3449,6 @@ angular
34463449
.service('uploadHelper', function($rootScope, $injector) {
34473450
'use strict';
34483451

3449-
var tasks = swissknife.tasks;
3450-
var collection = swissknife.collection;
34513452
var HTTP = { CREATED: 201, OK: 200, NOT_FOUND: 404, BAD_REQUEST: 400, FORBIDDEN: 403 };
34523453
var Uploads = $injector.get('Uploads');
34533454

@@ -3472,47 +3473,73 @@ angular
34723473
this.removeEventListener = this.off;
34733474
}
34743475

3476+
function wait(ms) {
3477+
return function (value) {
3478+
return new Promise(function(resolve) {
3479+
setTimeout(function () { resolve(value) }, ms);
3480+
});
3481+
};
3482+
}
3483+
3484+
function retry(options) {
3485+
options = options || {};
3486+
options.delay = options.delay || function (cpt) { return 0; }
3487+
options.isRetryable = options.isRetryable || function (err) { return true; }
3488+
options.maxRetry = options.maxRetry || 5;
3489+
3490+
var events = new EventEmitter();
3491+
3492+
function tryTask(retryCpt) {
3493+
retryCpt = retryCpt || 1;
3494+
var result = options.task(options.ctx);
3495+
events.emit('try', { promise: result, retries: retryCpt });
3496+
return result.catch(function (err) {
3497+
if (options.isRetryable(err) && retryCpt < options.maxRetry) {
3498+
return wait(retryCpt ? options.delay(retryCpt) : 0)()
3499+
.then(function () {
3500+
return tryTask(retryCpt + 1);
3501+
});
3502+
} else {
3503+
throw err;
3504+
}
3505+
});
3506+
}
3507+
return angular.extend(tryTask(), { events: events });
3508+
}
3509+
34753510
function chunks(file, options) {
3511+
var chunks = [];
34763512
var CHUNK_SIZE = options.chunkSize;
34773513
var offset = options.fileOffset || 0;
3478-
var done = false;
3479-
var index = 0;
3480-
return {
3481-
next: function() {
3482-
if (done) {
3483-
return { done: true };
3484-
}
3485-
var chunk = file.slice(offset, offset + CHUNK_SIZE);
3486-
done = chunk.size < CHUNK_SIZE;
3487-
offset += CHUNK_SIZE;
3488-
index++;
3489-
return { done: false, value: { chunk: chunk, index: index } };
3490-
}
3491-
};
3514+
var index = parseInt(offset / CHUNK_SIZE, 10);
3515+
while (offset < file.size) {
3516+
index++;
3517+
var chunk = file.slice(offset, offset + CHUNK_SIZE);
3518+
chunks.push({ chunk: chunk, index: index });
3519+
offset += CHUNK_SIZE;
3520+
}
3521+
return chunks;
34923522
}
34933523

3494-
function uploadChunk(chunk, index, id, path) {
3495-
var events = new EventEmitter();
3524+
function uploadChunk(chunk, index, id, path, events) {
34963525
var cancel = function() {};
34973526
var isCancelled = false;
3498-
var promise = tasks.retry({
3499-
task: tasks.chain([
3500-
function getUploadUrl() {
3501-
return Uploads.getPartUrl(id, index, path);
3502-
},
3503-
function upload(authorization) {
3504-
var result = rehttp.request({ url: authorization.part_url, method: 'PUT', body: chunk });
3505-
cancel = function() { result.cancel(); };
3506-
result.events.on('uploadProgress', function(progress) { events.emit('progress', progress); });
3507-
return result;
3508-
},
3509-
function checkUploadStatus(res) {
3510-
if (res.status !== HTTP.OK) {
3511-
throw res;
3512-
}
3513-
return res;
3514-
}
3515-
]),
3527+
var promise = retry({
3528+
task: function () {
3529+
return Uploads.getPartUrl(id, index, path)
3530+
.then(function upload(authorization) {
3531+
var result = rehttp.request({ url: authorization.part_url, method: 'PUT', body: chunk });
3532+
cancel = function() { result.cancel(); };
3533+
result.events.on('uploadProgress', function(progress) { events.emit('progress', { progress: progress, index: index }); });
3534+
return result;
3535+
})
3536+
.then(function checkUploadStatus(res) {
3537+
if (res.status !== HTTP.OK) {
3538+
throw res;
3539+
}
3540+
return res;
3541+
});
3542+
},
35163543
isRetryable: function(err) {
35173544
// AWS S3 could return 400 after network issues => retyable
35183545
if ([HTTP.NOT_FOUND, HTTP.FORBIDDEN].indexOf(err.status) > -1) {
@@ -3523,7 +3550,9 @@ angular
35233550
delay: function(cpt) { return cpt * 10000; },
35243551
maxRetry: 5
35253552
});
3526-
return Object.assign(promise, { events: events, cancel: function() { isCancelled = true; cancel(); } });
3553+
var promiseMeta = { events: events, cancel: function() { isCancelled = true; cancel(); } };
3554+
events.emit('start', { index: index, cancel: promiseMeta.cancel });
3555+
return angular.extend(promise, promiseMeta);
35273556
}
35283557

35293558
function upload(file, options) {
@@ -3534,48 +3563,52 @@ angular
35343563
var chunksProgress = [fileOffset];
35353564
var chunksCancel = [];
35363565
var events = new EventEmitter();
3537-
var promise = tasks.chain([
3538-
function initializeUpload() {
3539-
if (uploadId) {
3540-
return { path: uploadPath, id: uploadId };
3541-
} else {
3542-
return Uploads.initiate()
3543-
.then(function(ctx) {
3544-
if (ctx.type === 's3') {
3545-
ctx.path = ctx.key;
3546-
} else if (ctx.type === 'swift') {
3547-
ctx.path = ctx.object;
3548-
}
3549-
return ctx;
3550-
});
3551-
}
3552-
},
3553-
function uploadChunks(ctx) {
3566+
var promise = Promise.resolve(function initializeUpload() {
3567+
if (uploadId) {
3568+
return { path: uploadPath, id: uploadId };
3569+
} else {
3570+
return Uploads.initiate()
3571+
.then(function(ctx) {
3572+
if (ctx.type === 's3') {
3573+
ctx.path = ctx.key;
3574+
} else if (ctx.type === 'swift') {
3575+
ctx.path = ctx.object;
3576+
}
3577+
return ctx;
3578+
});
3579+
}
3580+
}());
3581+
promise = promise
3582+
.then(function uploadChunks(ctx) {
35543583
var uploadId = ctx.id;
35553584
var uploadPath = ctx.path;
3556-
var result = collection
3557-
.map(
3558-
chunks(file, { chunkSize: chunkSize, fileOffset: fileOffset }),
3559-
function(v) { return uploadChunk(v.chunk, v.index, uploadId, uploadPath); }
3560-
);
3561-
result.events.on('start', function(ctx) {
3585+
var container = ctx.container;
3586+
var type = ctx.type;
3587+
var uploadChunksEvents = new EventEmitter();
3588+
uploadChunksEvents.on('start', function(ctx) {
35623589
chunksProgress[ctx.index] = 0;
3563-
if (ctx.promise && ctx.promise.cancel) {
3564-
chunksCancel[ctx.index] = function() { ctx.promise.cancel(); };
3590+
if (ctx.cancel) {
3591+
chunksCancel[ctx.index] = function() { ctx.cancel(); };
35653592
}
3566-
ctx.promise.events.on('progress', function(progress) {
3567-
chunksProgress[ctx.index] = progress.loaded;
3593+
uploadChunksEvents.on('progress', function(ctx) {
3594+
chunksProgress[ctx.index] = ctx.progress.loaded;
35683595
var progression = chunksProgress.reduce(function(m, v) { return m + v; }, 0) / file.size;
35693596
events.emit('progress', progression * 100);
35703597
});
35713598
});
3572-
result.events.on('end', function() { fileOffset += chunkSize;});
3573-
result.events.on('end', function(ctx) { delete chunksCancel[ctx.index]; });
3574-
return result.all()
3575-
.then(function() { return { uploadId: uploadId, uploadPath: uploadPath, container: container }; });
3576-
},
3577-
function completeUpload(ctx) {
3578-
return tasks.retry({
3599+
uploadChunksEvents.on('end', function() { fileOffset += chunkSize;});
3600+
uploadChunksEvents.on('end', function(ctx) { delete chunksCancel[ctx.index]; });
3601+
var result = Promise
3602+
.map(
3603+
chunks(file, { chunkSize: chunkSize, fileOffset: fileOffset }),
3604+
function(v) { return uploadChunk(v.chunk, v.index, uploadId, uploadPath, uploadChunksEvents); },
3605+
{ concurrency : 3 }
3606+
);
3607+
return result
3608+
.then(function() { return { uploadId: uploadId, uploadPath: uploadPath, container: container, type: type }; });
3609+
})
3610+
.then(function completeUpload(ctx) {
3611+
return retry({
35793612
task: function() {
35803613
return Uploads.complete(ctx.uploadId, ctx.uploadPath)
35813614
.then(function() { return { uploadId: ctx.uploadId, uploadPath: ctx.uploadPath, type: ctx.type, container: ctx.container }; });
@@ -3590,14 +3623,13 @@ angular
35903623
delay: function(cpt) { return cpt * 10000; },
35913624
maxRetry: 5
35923625
});
3593-
}
3594-
])()
3626+
})
35953627
.catch(function(err) {
35963628
throw { err: err, uploadId: uploadId, uploadPath: uploadPath, fileOffset: fileOffset };
35973629
});
3598-
return Object.assign(promise, { events: events, cancel: function() {
3599-
chunksCancel.forEach(function(cancel) { cancel(); });
3600-
} });
3630+
return angular.extend(promise, { events: events, cancel: function() {
3631+
chunksCancel.forEach(function(cancel) { cancel(); });
3632+
} });
36013633
}
36023634

36033635
var concurrentUploads = {};
@@ -3689,7 +3721,7 @@ angular
36893721
});
36903722
uploadRes.then(function(ctx) {
36913723
delete concurrentUploads[uploadId];
3692-
$rootScope.$broadcast('jsSDK.upload.uploaded', { id: uploadId, path: ctx.uploadPath, fileName: file.name, type: ctx.type, container: ctx.container });
3724+
$rootScope.$broadcast('jsSDK.upload.uploaded', { id: uploadId, path: ctx.uploadPath, fileName: file.name, fileSize: file.size, type: ctx.type, container: ctx.container });
36933725
});
36943726
uploadRes.catch(function(err) {
36953727
//delete concurrentUploads[uploadId];

‎dist/predicsis-jsSDK.min.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dist/predicsis-jsSDK.min.js.map‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)