Skip to content

Commit 2e2970f

Browse files
yangzong18yangzong18
committed
add resuamable upload and download
1 parent 572d0f8 commit 2e2970f

3 files changed

Lines changed: 441 additions & 0 deletions

File tree

‎src/OSS/Core/OssUtil.php‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -531,4 +531,35 @@ public static function decodeKey($key, $encoding)
531531
throw new OssException("Unrecognized encoding type: " . $encoding);
532532
}
533533
}
534+
535+
/**
536+
* getUploadCpFilePath return the file path of the checkpoint
537+
* @param string $filePath
538+
* @param $bucketName
539+
* @param $objectKey
540+
* @return string
541+
*/
542+
public static function getCpFilePath($bucketName,$objectKey,$versionId=""){
543+
$dest = sprintf("oss://%s/%s", $bucketName, $objectKey);
544+
$cpFileName = self::getCpFileName("", $dest, $versionId);
545+
return sys_get_temp_dir(). DIRECTORY_SEPARATOR . $cpFileName;
546+
}
547+
548+
/**
549+
* getCpFileName return the name of the checkpoint file
550+
* @param string $src
551+
* @param string $dest
552+
* @param string $versionId
553+
* @return string
554+
*/
555+
public static function getCpFileName($src,$dest,$versionId){
556+
$srcCheckSum = md5($src);
557+
$destCheckSum = md5($dest);
558+
if ($versionId == ''){
559+
return sprintf("%s-%s.cp", $srcCheckSum, $destCheckSum);
560+
}
561+
$versionCheckSum = md5($versionId);
562+
return sprintf("%s-%s-%s.cp", $srcCheckSum, $destCheckSum,$versionCheckSum);
563+
}
564+
534565
}

‎src/OSS/OssClient.php‎

Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2718,6 +2718,275 @@ public function multiuploadFile($bucket, $object, $file, $options = null)
27182718
return $this->completeMultipartUpload($bucket, $object, $uploadId, $uploadParts, $cmp_options);
27192719
}
27202720

2721+
2722+
/**
2723+
* A higher level API for uploading a file with multipart upload and resumeable upload. It consists of initialization, parts upload and completion.
2724+
*
2725+
* @param string $bucket bucket name
2726+
* @param string $object object name
2727+
* @param string $file The local file to upload
2728+
* @param array $options Key-Value array
2729+
* @return null
2730+
* @throws OssException
2731+
*/
2732+
public function resumableUpload($bucket, $object, $file, $options = null)
2733+
{
2734+
$cpFilePath = OssUtil::getCpFilePath($bucket,$object);
2735+
$resumable = false;
2736+
if (file_exists($cpFilePath)){
2737+
$content = file_get_contents($cpFilePath);
2738+
$uploadInfo = json_decode($content,true);
2739+
$uploadId = $uploadInfo['uploadId'];
2740+
$parts = $uploadInfo['parts'];
2741+
$object = $uploadInfo['object'];
2742+
$partSize = $uploadInfo['partSize'];
2743+
$totalSize = $uploadInfo['totalSize'];
2744+
if ($this->isValidUpload($file,$uploadInfo)){
2745+
$resumable = true;
2746+
}
2747+
}
2748+
$this->precheckCommon($bucket, $object, $options);
2749+
if (isset($options[self::OSS_LENGTH])) {
2750+
$options[self::OSS_CONTENT_LENGTH] = $options[self::OSS_LENGTH];
2751+
unset($options[self::OSS_LENGTH]);
2752+
}
2753+
if (empty($file)) {
2754+
throw new OssException("parameter invalid, file is empty");
2755+
}
2756+
$uploadFile = OssUtil::encodePath($file);
2757+
if (!isset($options[self::OSS_CONTENT_TYPE])) {
2758+
$options[self::OSS_CONTENT_TYPE] = $this->getMimeType($object, $uploadFile);
2759+
}
2760+
$position = isset($options[self::OSS_SEEK_TO]) ? (integer)$options[self::OSS_SEEK_TO] : 0;
2761+
2762+
if ($resumable){
2763+
$num = count($parts);
2764+
$upload_position = $position + $options[self::OSS_PART_SIZE]*$num;
2765+
$upload_file_size = $totalSize -= $upload_position;
2766+
}else{
2767+
$upload_position = $position;
2768+
if (isset($options[self::OSS_CONTENT_LENGTH])) {
2769+
$upload_file_size = (integer)$options[self::OSS_CONTENT_LENGTH];
2770+
} else {
2771+
$upload_file_size = sprintf('%u',filesize($uploadFile));
2772+
if ($upload_file_size !== false) {
2773+
$upload_file_size -= $upload_position;
2774+
}
2775+
}
2776+
}
2777+
if ($upload_position === false || !isset($upload_file_size) || $upload_file_size === false || $upload_file_size < 0) {
2778+
throw new OssException('The size of `fileUpload` cannot be determined in ' . __FUNCTION__ . '().');
2779+
}
2780+
if ($resumable){
2781+
$options[self::OSS_PART_SIZE] = $partSize;
2782+
}else{
2783+
// Computes the part size and assign it to options.
2784+
if (isset($options[self::OSS_PART_SIZE])) {
2785+
$options[self::OSS_PART_SIZE] = $this->computePartSize($options[self::OSS_PART_SIZE]);
2786+
} else {
2787+
$options[self::OSS_PART_SIZE] = self::OSS_MID_PART_SIZE;
2788+
}
2789+
}
2790+
$is_check_md5 = $this->isCheckMD5($options);
2791+
// if the file size is less than part size, use simple file upload.
2792+
if ($upload_file_size < $options[self::OSS_PART_SIZE] && !isset($options[self::OSS_UPLOAD_ID])) {
2793+
return $this->uploadFile($bucket, $object, $uploadFile, $options);
2794+
}
2795+
if (!$resumable){
2796+
// Using multipart upload, initialize if no OSS_UPLOAD_ID is specified in options.
2797+
if (isset($options[self::OSS_UPLOAD_ID])) {
2798+
$uploadId = $options[self::OSS_UPLOAD_ID];
2799+
} else {
2800+
// initialize
2801+
$uploadId = $this->initiateMultipartUpload($bucket, $object, $options);
2802+
}
2803+
}
2804+
$upload_position = isset($options[self::OSS_SEEK_TO]) ? (integer)$options[self::OSS_SEEK_TO] : 0;
2805+
if (!$resumable){
2806+
$uploadInfo = array(
2807+
'file'=>$uploadFile,
2808+
'fileStat'=>array(
2809+
'size'=>$upload_file_size,
2810+
'lastModified'=>filemtime($uploadFile),
2811+
),
2812+
'uploadId' =>$uploadId,
2813+
'object'=>$object,
2814+
'partSize'=>$options[self::OSS_PART_SIZE],
2815+
'totalSize'=>$upload_file_size,
2816+
);
2817+
}
2818+
// generates the parts information and upload them one by one
2819+
$pieces = $this->generateMultiuploadParts($upload_file_size, (integer)$options[self::OSS_PART_SIZE]);
2820+
if($resumable){
2821+
$num = count($parts);
2822+
$todo_pieces = array_slice($pieces,$num,-1,true);
2823+
}else{
2824+
$todo_pieces = $pieces;
2825+
}
2826+
$response_upload_part = array();
2827+
foreach ($todo_pieces as $i => $piece) {
2828+
$from_pos = $upload_position + (integer)$piece[self::OSS_SEEK_TO];
2829+
$to_pos = (integer)$piece[self::OSS_LENGTH] + $from_pos - 1;
2830+
$up_options = array(
2831+
self::OSS_FILE_UPLOAD => $uploadFile,
2832+
self::OSS_PART_NUM => ($i + 1),
2833+
self::OSS_SEEK_TO => $from_pos,
2834+
self::OSS_LENGTH => $to_pos - $from_pos + 1,
2835+
self::OSS_CHECK_MD5 => $is_check_md5,
2836+
);
2837+
if ($is_check_md5) {
2838+
$content_md5 = OssUtil::getMd5SumForFile($uploadFile, $from_pos, $to_pos);
2839+
$up_options[self::OSS_CONTENT_MD5] = $content_md5;
2840+
}
2841+
$response_upload_part[] = $this->uploadPart($bucket, $object, $uploadId, $up_options);
2842+
if ($resumable){
2843+
$uploadInfo['parts'] = array_merge($parts,$response_upload_part);
2844+
}else{
2845+
$uploadInfo['parts'] = $response_upload_part;
2846+
}
2847+
file_put_contents($cpFilePath,json_encode($uploadInfo));
2848+
if (isset($options['uploadPartHooker']) && $i+1 == $options['uploadPartHooker']){
2849+
throw new OssException('ErrorHooker in ' . __FUNCTION__ . '().');
2850+
}
2851+
}
2852+
$uploadParts = array();
2853+
foreach ($uploadInfo['parts'] as $i => $etag) {
2854+
$uploadParts[] = array(
2855+
'PartNumber' => ($i + 1),
2856+
'ETag' => $etag,
2857+
);
2858+
}
2859+
//build complete options
2860+
$cmp_options = null;
2861+
if (isset($options[self::OSS_HEADERS]) && isset($options[self::OSS_HEADERS][self::OSS_REQUEST_PAYER])) {
2862+
$cmp_options = array(
2863+
OssClient::OSS_HEADERS => array(
2864+
OssClient::OSS_REQUEST_PAYER => $options[self::OSS_HEADERS][self::OSS_REQUEST_PAYER],
2865+
));
2866+
}
2867+
$result = $this->completeMultipartUpload($bucket, $object, $uploadId, $uploadParts, $cmp_options);
2868+
unlink($cpFilePath);
2869+
return $result;
2870+
}
2871+
2872+
/**
2873+
* A higher level API for download a file with get object.
2874+
*
2875+
* @param string $bucket bucket name
2876+
* @param string $object object name
2877+
* @param array $options Key-Value array
2878+
* @return null
2879+
* @throws OssException
2880+
*/
2881+
public function resumableDownload($bucket, $object, $options = null)
2882+
{
2883+
if (!isset($options[self::OSS_FILE_DOWNLOAD])){
2884+
throw new OssException('options file download can not be empty! ');
2885+
}
2886+
if (isset($options[self::OSS_VERSION_ID])){
2887+
$cpFilePath = OssUtil::getCpFilePath($bucket,$object,$options[self::OSS_VERSION_ID]);
2888+
}else{
2889+
$cpFilePath = OssUtil::getCpFilePath($bucket,$object);
2890+
}
2891+
$resumable = false;
2892+
$objectMeta = $this->getObjectMeta($bucket, $object);
2893+
$size = $objectMeta['content-length'];
2894+
if (file_exists($cpFilePath)){
2895+
$content = file_get_contents($cpFilePath);
2896+
$downloadInfo = json_decode($content,true);
2897+
$num = $downloadInfo['parts'];
2898+
$pieces = $downloadInfo['pieces'];
2899+
$object = $downloadInfo['object'];
2900+
$partSize = $downloadInfo['partSize'];
2901+
if ($this->isValidDownload($objectMeta,$downloadInfo)){
2902+
$resumable = true;
2903+
}else{
2904+
unlink($options[self::OSS_FILE_DOWNLOAD]);
2905+
}
2906+
}
2907+
if ($resumable){
2908+
$options[self::OSS_PART_SIZE] = $partSize;
2909+
}else{
2910+
// Computes the part size and assign it to options.
2911+
if (isset($options[self::OSS_PART_SIZE])) {
2912+
$options[self::OSS_PART_SIZE] = $this->computePartSize($options[self::OSS_PART_SIZE]);
2913+
} else {
2914+
$options[self::OSS_PART_SIZE] = self::OSS_MID_PART_SIZE;
2915+
}
2916+
}
2917+
// if the file size is less than part size, use simple file download.
2918+
if ($size < $options[self::OSS_PART_SIZE]) {
2919+
return $this->getObject($bucket, $object, $options);
2920+
}
2921+
if (!$resumable){
2922+
// Computes the part size and assign it to options.
2923+
if (isset($options[self::OSS_PART_SIZE])) {
2924+
$options[self::OSS_PART_SIZE] = $this->computePartSize($options[self::OSS_PART_SIZE]);
2925+
} else {
2926+
$options[self::OSS_PART_SIZE] = self::OSS_MID_PART_SIZE;
2927+
}
2928+
$todo_pieces = $pieces = $this->generateMultiuploadParts($size, $options[self::OSS_PART_SIZE]);
2929+
$downloadPosition = 0;
2930+
$downloadArray = array(
2931+
"object" => $object,
2932+
"partSize" => $options[self::OSS_PART_SIZE],
2933+
'fileSize' => $size,
2934+
"pieces" => $pieces,
2935+
'fileStat'=>array(
2936+
'size'=>$size,
2937+
'lastModified'=>$objectMeta['last-modified'],
2938+
),
2939+
);
2940+
}else{
2941+
$todo_pieces = array_slice($pieces,$num,-1,true);
2942+
}
2943+
foreach ($todo_pieces as $i => $piece) {
2944+
$fromPos = $downloadPosition + (integer)$piece[self::OSS_SEEK_TO];
2945+
$toPos = (integer)$piece[self::OSS_LENGTH] + $fromPos - 1;
2946+
$downOptions = array(
2947+
OssClient::OSS_RANGE => $fromPos.'-'.$toPos,
2948+
);
2949+
$downloadArray['parts'] = $i+1;
2950+
$content = $this->getObject($bucket,$object,$downOptions);
2951+
file_put_contents($options[self::OSS_FILE_DOWNLOAD], $content,FILE_APPEND );
2952+
file_put_contents($cpFilePath, json_encode($downloadArray));
2953+
if ($i+1 == count($pieces)){
2954+
unlink($cpFilePath);
2955+
}
2956+
if (isset($options['uploadPartHooker']) && $i+1 == $options['uploadPartHooker']){
2957+
throw new OssException('ErrorHooker in ' . __FUNCTION__ . '().');
2958+
}
2959+
}
2960+
}
2961+
2962+
2963+
/**
2964+
* Valid this upload file
2965+
* @param string $file
2966+
* @param array $uploadInfo
2967+
* @return bool
2968+
*/
2969+
private function isValidUpload($file,$uploadInfo){
2970+
$fileInfo = $uploadInfo['fileStat'];
2971+
if ($fileInfo['size'] != sprintf('%u',filesize($file)) || filemtime($file) != $fileInfo['lastModified']){
2972+
return false;
2973+
}
2974+
return true;
2975+
}
2976+
2977+
/**
2978+
* Valid this download file
2979+
* @param array $meta
2980+
* @param array $downloadInfo
2981+
* @return bool
2982+
*/
2983+
private function isValidDownload($meta,$downloadInfo){
2984+
if ($downloadInfo['size'] != $meta['content-length'] || $downloadInfo['lastModified'] != $meta['last-modified']){
2985+
return false;
2986+
}
2987+
return true;
2988+
}
2989+
27212990
/**
27222991
* Uploads the local directory to the specified bucket into specified folder (prefix)
27232992
*

0 commit comments

Comments
 (0)