Skip to content

Commit 0152bc3

Browse files
lewzylulewzylu
lewzylu
and
lewzylu
authored
support rangDownload/progress (#178)
Co-authored-by: lewzylu <lewzylu@tencent.com>
1 parent a56d033 commit 0152bc3

File tree

5 files changed

+161
-15
lines changed

5 files changed

+161
-15
lines changed

‎src/Qcloud/Cos/Client.php

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ public function __construct($cosConfig) {
106106
$this->cosConfig['retry'] = isset($cosConfig['retry']) ? $cosConfig['retry'] : 1;
107107
$this->cosConfig['userAgent'] = isset($cosConfig['userAgent']) ? $cosConfig['userAgent'] : 'cos-php-sdk-v5.'. Client::VERSION;
108108
$this->cosConfig['pathStyle'] = isset($cosConfig['pathStyle']) ? $cosConfig['pathStyle'] : false;
109-
109+
110110

111111
$service = Service::getService();
112112
$handler = HandlerStack::create();
@@ -135,7 +135,6 @@ public function __construct($cosConfig) {
135135
'commandToRequestTransformer'], [$this, 'responseToResultTransformer'],
136136
null);
137137
}
138-
139138
public function commandToRequestTransformer(CommandInterface $command)
140139
{
141140
$this->action = $command->GetName();
@@ -169,12 +168,13 @@ public function __destruct() {
169168
}
170169

171170
public function __call($method, array $args) {
172-
for ($i = 1; $i <= $this->cosConfig['retry']; $i++) {
171+
for ($i = 0; $i <= $this->cosConfig['retry']; $i++) {
173172
try {
174-
return parent::__call(ucfirst($method), $args);
175-
} catch (CommandException $e) {
173+
$rt = parent::__call(ucfirst($method), $args);
174+
return $rt;
175+
} catch (\Exception $e) {
176176
if ($i != $this->cosConfig['retry']) {
177-
sleep(1 << ($i-1));
177+
sleep(1 << ($i));
178178
continue;
179179
}
180180
$previous = $e->getPrevious();
@@ -213,6 +213,7 @@ public function getObjectUrl($bucket, $key, $expires = "+30 minutes", array $arg
213213

214214
public function upload($bucket, $key, $body, $options = array()) {
215215
$body = Psr7\stream_for($body);
216+
$options['Retry'] = $this->cosConfig['retry'];
216217
$options['PartSize'] = isset($options['PartSize']) ? $options['PartSize'] : MultipartUpload::DEFAULT_PART_SIZE;
217218
if ($body->getSize() < $options['PartSize']) {
218219
$rt = $this->putObject(array(
@@ -232,6 +233,38 @@ public function upload($bucket, $key, $body, $options = array()) {
232233
return $rt;
233234
}
234235

236+
public function download($bucket, $key, $saveAs, $options = array()) {
237+
$options['PartSize'] = isset($options['PartSize']) ? $options['PartSize'] : RangeDownload::DEFAULT_PART_SIZE;
238+
$contentLength = 0;
239+
$versionId = isset($options['VersionId']) ? $options['VersionId'] : "";
240+
try {
241+
$rt = $this->headObject(array(
242+
'Bucket'=>$bucket,
243+
'Key'=>$key,
244+
'VersionId'=>$versionId,
245+
)
246+
);
247+
$contentLength = $rt['ContentLength'];
248+
} catch (\Exception $e) {
249+
throw ($e);
250+
}
251+
if ($contentLength < $options['PartSize']) {
252+
$rt = $this->getObject(array(
253+
'Bucket' => $bucket,
254+
'Key' => $key,
255+
'SaveAs' => $saveAs,
256+
) + $options);
257+
} else {
258+
$rangeDownload = new RangeDownload($this, $contentLength, $saveAs, array(
259+
'Bucket' => $bucket,
260+
'Key' => $key,
261+
) + $options);
262+
263+
$rt = $rangeDownload->performDownloading();
264+
}
265+
return $rt;
266+
}
267+
235268
public function resumeUpload($bucket, $key, $body, $uploadId, $options = array()) {
236269
$body = Psr7\stream_for($body);
237270
$options['PartSize'] = isset($options['PartSize']) ? $options['PartSize'] : MultipartUpload::DEFAULT_PART_SIZE;

‎src/Qcloud/Cos/CommandToRequestTransformer.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ public function md5Transformer( CommandInterface $command, $request ) {
122122
if ( isset( $operation['parameters']['ContentMD5'] ) &&
123123
isset( $command['ContentMD5'] ) ) {
124124
$value = $command['ContentMD5'];
125-
if ( $value === true ) {
125+
if ( $value != false ) {
126126
$request = $this->addMd5( $request );
127127
}
128128
}

‎src/Qcloud/Cos/MultipartUpload.php

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
use Qcloud\Cos\Exception\CosException;
66
use GuzzleHttp\Pool;
77

8+
use function GuzzleHttp\Promise\rejection_for;
9+
810
class MultipartUpload {
911
const MIN_PART_SIZE = 1048576;
1012
const MAX_PART_SIZE = 5368709120;
@@ -16,6 +18,9 @@ class MultipartUpload {
1618
private $partSize;
1719
private $parts;
1820
private $body;
21+
private $progress;
22+
private $totolSize;
23+
private $uploadedSize;
1924

2025
public function __construct($client, $body, $options = array()) {
2126
$minPartSize = $options['PartSize'];
@@ -25,8 +30,13 @@ public function __construct($client, $body, $options = array()) {
2530
$this->options = $options;
2631
$this->partSize = $this->calculatePartSize($minPartSize);
2732
$this->concurrency = isset($options['Concurrency']) ? $options['Concurrency'] : 10;
33+
$this->progress = isset($options['Progress']) ? $options['Progress'] : function($totolSize, $uploadedSize) {};
2834
$this->parts = [];
2935
$this->partNumberList = [];
36+
$this->uploadedSize = 0;
37+
$this->totolSize = $this->body->getSize();
38+
$this->needMd5 = isset($options['ContentMD5']) ? $options['ContentMD5'] : true;
39+
$this->retry = isset($options['Retry']) ? $options['Retry'] : 3;
3040
}
3141
public function performUploading() {
3242
$uploadId= $this->initiateMultipartUpload();
@@ -48,45 +58,60 @@ public function uploadParts($uploadId) {
4858
$uploadRequests = function ($uploadId) {
4959
$partNumber = 1;
5060
$index = 1;
61+
$offset = 0;
62+
$partSize = 0;
5163
for ( ; ; $partNumber ++) {
5264
if ($this->body->eof()) {
5365
break;
5466
}
5567
$body = $this->body->read($this->partSize);
68+
$partSize = $this->partSize;
69+
if ($offset + $this->partSize >= $this->totolSize) {
70+
$partSize = $this->totolSize - $offset;
71+
}
72+
$offset += $partSize;
5673
if (empty($body)) {
5774
break;
5875
}
5976
if (isset($this->parts[$partNumber])) {
6077
continue;
6178
}
62-
$this->partNumberList[$index] = $partNumber;
79+
$this->partNumberList[$index]['PartNumber'] = $partNumber;
80+
$this->partNumberList[$index]['PartSize'] = $partSize;
6381
$params = array(
6482
'Bucket' => $this->options['Bucket'],
6583
'Key' => $this->options['Key'],
6684
'UploadId' => $uploadId,
6785
'PartNumber' => $partNumber,
68-
'Body' => $body
86+
'Body' => $body,
87+
'ContentMD5' => $this->needMd5
6988
);
70-
if(!isset($this->parts[$partNumber])) {
89+
if ($this->needMd5 == false) {
90+
unset($params["ContentMD5"]);
91+
}
92+
if (!isset($this->parts[$partNumber])) {
7193
$command = $this->client->getCommand('uploadPart', $params);
7294
$request = $this->client->commandToRequestTransformer($command);
7395
$index ++;
7496
yield $request;
7597
}
7698
}
77-
};
99+
};
78100
$pool = new Pool($this->client->httpClient, $uploadRequests($uploadId), [
79101
'concurrency' => $this->concurrency,
80102
'fulfilled' => function ($response, $index) {
81103
$index = $index + 1;
82-
$partNumber = $this->partNumberList[$index];
104+
$partNumber = $this->partNumberList[$index]['PartNumber'];
105+
$partSize = $this->partNumberList[$index]['PartSize'];
83106
$etag = $response->getHeaders()["ETag"][0];
84107
$part = array('PartNumber' => $partNumber, 'ETag' => $etag);
85108
$this->parts[$partNumber] = $part;
109+
$this->uploadedSize += $partSize;
110+
call_user_func_array($this->progress, [$this->totolSize, $this->uploadedSize]);
86111
},
87112

88113
'rejected' => function ($reason, $index) {
89-
throw($reason);
114+
printf("part [%d] upload failed, reason: %s\n", $index, $reason);
90115
}
91116
]);
92117
$promise = $pool->promise();

‎src/Qcloud/Cos/RangeDownload.php

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
<?php
2+
3+
namespace Qcloud\Cos;
4+
5+
use Exception;
6+
use Qcloud\Cos\Exception\CosException;
7+
use GuzzleHttp\Pool;
8+
9+
class RangeDownload {
10+
const DEFAULT_PART_SIZE = 52428800;
11+
12+
private $client;
13+
private $options;
14+
private $partSize;
15+
private $parts;
16+
private $body;
17+
private $progress;
18+
private $totolSize;
19+
private $uploadedSize;
20+
21+
public function __construct($client, $contentLength, $saveAs, $options = array()) {
22+
$this->client = $client;
23+
$this->options = $options;
24+
$this->partSize = isset($options['PartSize']) ? $options['PartSize'] : self::DEFAULT_PART_SIZE;
25+
$this->concurrency = isset($options['Concurrency']) ? $options['Concurrency'] : 10;
26+
$this->progress = isset($options['Progress']) ? $options['Progress'] : function($totolSize, $downloadedSize) {};
27+
$this->parts = [];
28+
$this->partNumberList = [];
29+
$this->downloadedSize = 0;
30+
$this->totolSize = $contentLength;
31+
$this->saveAs = $saveAs;
32+
}
33+
public function performdownloading() {
34+
$this->fp = fopen($this->saveAs, "wb");
35+
$rt = $this->donwloadParts();
36+
fclose($this->fp);
37+
return $rt;
38+
}
39+
public function donwloadParts() {
40+
$uploadRequests = function () {
41+
$index = 1;
42+
$partSize = 0;
43+
for ($offset = 0; $offset < $this->totolSize; ) {
44+
$partSize = $this->partSize;
45+
if ($offset + $this->partSize >= $this->totolSize) {
46+
$partSize = $this->totolSize - $offset;
47+
}
48+
$this->parts[$index]['PartSize'] = $partSize;
49+
$this->parts[$index]['Offset'] = $offset;
50+
$params = array(
51+
'Bucket' => $this->options['Bucket'],
52+
'Key' => $this->options['Key'],
53+
'Range' => sprintf("bytes=%d-%d", $offset, $offset + $partSize)
54+
);
55+
$command = $this->client->getCommand('getObject', $params);
56+
$request = $this->client->commandToRequestTransformer($command);
57+
yield $request;
58+
$offset += $partSize;
59+
$index += 1;
60+
}
61+
};
62+
$pool = new Pool($this->client->httpClient, $uploadRequests(), [
63+
'concurrency' => $this->concurrency,
64+
'fulfilled' => function ($response, $index) {
65+
$index = $index + 1;
66+
67+
$stream = $response->getBody();
68+
$offset = $this->parts[$index]['Offset'];
69+
$partsize = 8192;
70+
while (!$stream->eof()) {
71+
$output = $stream->read($partsize);
72+
fseek($this->fp, $offset);
73+
$writeLen = fwrite($this->fp, $output);
74+
$offset += $writeLen;
75+
}
76+
$partSize = $this->parts[$index]['PartSize'];
77+
$this->downloadedSize += $partSize;
78+
call_user_func_array($this->progress, [$this->totolSize, $this->downloadedSize]);
79+
},
80+
'rejected' => function ($reason, $index) {
81+
throw($reason);
82+
}
83+
]);
84+
$promise = $pool->promise();
85+
$promise->wait();
86+
}
87+
88+
}

‎src/Qcloud/Cos/Tests/COSTest.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ class COSTest extends \PHPUnit\Framework\TestCase
1010
private $cosClient;
1111
private $bucket;
1212
private $region;
13-
protected function setUp()
13+
protected function setUp(): void
1414
{
1515
$this->bucket = getenv('COS_BUCKET');
1616
$this->region = getenv('COS_REGION');
@@ -25,7 +25,7 @@ protected function setUp()
2525
}
2626
}
2727

28-
protected function tearDown() {
28+
protected function tearDown(): void {
2929
}
3030

3131
function generateRandomString($length = 10) {

0 commit comments

Comments
 (0)