From 0c672d84e8855b39d0604cb4fd360b7257e8a0c3 Mon Sep 17 00:00:00 2001 From: LiHS Date: Wed, 6 Jul 2022 18:20:12 +0800 Subject: [PATCH] add timeout per request for upload --- .github/workflows/test-ci.yml | 3 ++ src/Qiniu/Http/Client.php | 68 ++++++++++++++++++++++------ src/Qiniu/Http/Request.php | 10 +++- src/Qiniu/Http/RequestOptions.php | 62 +++++++++++++++++++++++++ src/Qiniu/Storage/FormUploader.php | 32 ++++++++++--- src/Qiniu/Storage/ResumeUploader.php | 39 ++++++++++------ src/Qiniu/Storage/UploadManager.php | 34 +++++++++++--- tests/Qiniu/Tests/HttpTest.php | 45 ++++++++++++++++-- tests/mock-server/timeout.php | 3 ++ 9 files changed, 251 insertions(+), 45 deletions(-) create mode 100644 src/Qiniu/Http/RequestOptions.php create mode 100644 tests/mock-server/timeout.php diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index a3f76f19..9943cfac 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -29,10 +29,13 @@ jobs: - name: Run cases run: | + nohup php -S localhost:9000 -t ./tests/mock-server/ > phpd.log 2>&1 & + export PHP_SERVER_PID=$! ./vendor/bin/phpcs --standard=PSR2 src ./vendor/bin/phpcs --standard=PSR2 examples ./vendor/bin/phpcs --standard=PSR2 tests ./vendor/bin/phpunit --coverage-clover=coverage.xml + kill $PHP_SERVER_PID env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} diff --git a/src/Qiniu/Http/Client.php b/src/Qiniu/Http/Client.php index 3735aab4..acc69b1e 100644 --- a/src/Qiniu/Http/Client.php +++ b/src/Qiniu/Http/Client.php @@ -2,35 +2,70 @@ namespace Qiniu\Http; use Qiniu\Config; -use Qiniu\Http\Request; -use Qiniu\Http\Response; final class Client { - public static function get($url, array $headers = array()) + /** + * @param $url + * @param array $headers + * @param RequestOptions $opt + * @return Response + */ + public static function get($url, array $headers = array(), $opt = null) { - $request = new Request('GET', $url, $headers); + $request = new Request('GET', $url, $headers, null, $opt); return self::sendRequest($request); } - public static function delete($url, array $headers = array()) + /** + * @param $url + * @param array $headers + * @param array $opt detail see {@see Request::$opt} + * @return Response + */ + public static function delete($url, array $headers = array(), $opt = null) { - $request = new Request('DELETE', $url, $headers); + $request = new Request('DELETE', $url, $headers, null, $opt); return self::sendRequest($request); } - public static function post($url, $body, array $headers = array()) + /** + * @param $url + * @param $body + * @param array $headers + * @param RequestOptions $opt + * @return Response + */ + public static function post($url, $body, array $headers = array(), $opt = null) { - $request = new Request('POST', $url, $headers, $body); + $request = new Request('POST', $url, $headers, $body, $opt); return self::sendRequest($request); } - public static function PUT($url, $body, array $headers = array()) + /** + * @param $url + * @param $body + * @param array $headers + * @param RequestOptions $opt + * @return Response + */ + public static function PUT($url, $body, array $headers = array(), $opt = null) { - $request = new Request('PUT', $url, $headers, $body); + $request = new Request('PUT', $url, $headers, $body, $opt); return self::sendRequest($request); } + /** + * @param $url + * @param array $fields + * @param string $name + * @param string $fileName + * @param $fileBody + * @param null $mimeType + * @param array $headers + * @param RequestOptions $opt + * @return Response + */ public static function multipartPost( $url, $fields, @@ -38,7 +73,8 @@ public static function multipartPost( $fileName, $fileBody, $mimeType = null, - array $headers = array() + $headers = array(), + $opt = null ) { $data = array(); $mimeBoundary = md5(microtime()); @@ -62,10 +98,9 @@ public static function multipartPost( array_push($data, ''); $body = implode("\r\n", $data); - // var_dump($data);exit; $contentType = 'multipart/form-data; boundary=' . $mimeBoundary; $headers['Content-Type'] = $contentType; - $request = new Request('POST', $url, $headers, $body); + $request = new Request('POST', $url, $headers, $body, $opt); return self::sendRequest($request); } @@ -84,6 +119,10 @@ private static function userAgent() return $ua; } + /** + * @param Request $request + * @return Response + */ public static function sendRequest($request) { $t1 = microtime(true); @@ -98,6 +137,9 @@ public static function sendRequest($request) CURLOPT_CUSTOMREQUEST => $request->method, CURLOPT_URL => $request->url, ); + foreach ($request->opt->getCurlOpt() as $k => $v) { + $options[$k] = $v; + } // Handle open_basedir & safe mode if (!ini_get('safe_mode') && !ini_get('open_basedir')) { $options[CURLOPT_FOLLOWLOCATION] = true; diff --git a/src/Qiniu/Http/Request.php b/src/Qiniu/Http/Request.php index 43b0bfdb..2c7814b5 100644 --- a/src/Qiniu/Http/Request.php +++ b/src/Qiniu/Http/Request.php @@ -7,12 +7,20 @@ final class Request public $headers; public $body; public $method; + /** + * @var RequestOptions + */ + public $opt; - public function __construct($method, $url, array $headers = array(), $body = null) + public function __construct($method, $url, array $headers = array(), $body = null, $opt = null) { $this->method = strtoupper($method); $this->url = $url; $this->headers = $headers; $this->body = $body; + if ($opt === null) { + $opt = new RequestOptions(); + } + $this->opt = $opt; } } diff --git a/src/Qiniu/Http/RequestOptions.php b/src/Qiniu/Http/RequestOptions.php new file mode 100644 index 00000000..0a31c83f --- /dev/null +++ b/src/Qiniu/Http/RequestOptions.php @@ -0,0 +1,62 @@ +connection_timeout = $connection_timeout; + $this->connection_timeout_ms = $connection_timeout_ms; + $this->timeout = $timeout; + $this->timeout_ms = $timeout_ms; + } + + public function getCurlOpt() + { + $result = array(); + if ($this->connection_timeout != null) { + $result[CURLOPT_CONNECTTIMEOUT] = $this->connection_timeout; + } + if ($this->connection_timeout_ms != null) { + $result[CURLOPT_CONNECTTIMEOUT_MS] = $this->connection_timeout_ms; + } + if ($this->timeout != null) { + $result[CURLOPT_TIMEOUT] = $this->timeout; + } + if ($this->timeout_ms != null) { + $result[CURLOPT_TIMEOUT_MS] = $this->timeout_ms; + } + return $result; + } +} diff --git a/src/Qiniu/Storage/FormUploader.php b/src/Qiniu/Storage/FormUploader.php index 453cc389..d907e16f 100644 --- a/src/Qiniu/Storage/FormUploader.php +++ b/src/Qiniu/Storage/FormUploader.php @@ -5,6 +5,7 @@ use Qiniu\Config; use Qiniu\Http\Error; use Qiniu\Http\Client; +use Qiniu\Http\RequestOptions; final class FormUploader { @@ -17,10 +18,10 @@ final class FormUploader * @param string $data 上传二进制流 * @param Config $config 上传配置 * @param string $params 自定义变量，规格参考 - * https://developer.qiniu.com/kodo/manual/1235/vars#xvar + * {@link https://developer.qiniu.com/kodo/manual/1235/vars#xvar} * @param string $mime 上传数据的mimeType - * * @param string $fname + * @param RequestOptions $reqOpt * * @return array 包含已上传文件的信息，类似： * [ @@ -35,8 +36,12 @@ public static function put( $config, $params, $mime, - $fname + $fname, + $reqOpt = null ) { + if ($reqOpt == null) { + $reqOpt = new RequestOptions(); + } $fields = array('token' => $upToken); if ($key === null) { } else { @@ -63,7 +68,17 @@ public static function put( return array(null, $err); } - $response = Client::multipartPost($upHost, $fields, 'file', $fname, $data, $mime); + + $response = Client::multipartPost( + $upHost, + $fields, + 'file', + $fname, + $data, + $mime, + array(), + $reqOpt + ); if (!$response->ok()) { return array(null, new Error($upHost, $response)); } @@ -93,9 +108,12 @@ public static function putFile( $filePath, $config, $params, - $mime + $mime, + $reqOpt = null ) { - + if ($reqOpt == null) { + $reqOpt = new RequestOptions(); + } $fields = array('token' => $upToken, 'file' => self::createFile($filePath, $mime)); if ($key !== null) { @@ -123,7 +141,7 @@ public static function putFile( return array(null, $err); } - $response = Client::post($upHost, $fields, $headers); + $response = Client::post($upHost, $fields, $headers, $reqOpt); if (!$response->ok()) { return array(null, new Error($upHost, $response)); } diff --git a/src/Qiniu/Storage/ResumeUploader.php b/src/Qiniu/Storage/ResumeUploader.php index eedc9afe..a983d38d 100644 --- a/src/Qiniu/Storage/ResumeUploader.php +++ b/src/Qiniu/Storage/ResumeUploader.php @@ -6,6 +6,7 @@ use Qiniu\Http\Client; use Qiniu\Http\Error; use Qiniu\Enum\SplitUploadVersion; +use Qiniu\Http\RequestOptions; /** * 断点续上传类, 该类主要实现了断点续上传中的分块上传, @@ -31,6 +32,10 @@ final class ResumeUploader private $resumeRecordFile; private $version; private $partSize; + /** + * @var RequestOptions + */ + private $reqOpt; /** * 上传二进制流到七牛 @@ -44,9 +49,11 @@ final class ResumeUploader * @param Config $config * @param string $resumeRecordFile 断点续传的已上传的部分信息记录文件 * @param string $version 分片上传版本 目前支持v1/v2版本 默认v1 - * @param string $partSize 分片上传v2字段 默认大小为4MB 分片大小范围为1 MB - 1 GB - * + * @param int $partSize 分片上传v2字段 默认大小为4MB 分片大小范围为1 MB - 1 GB + * @param RequestOptions $reqOpt 分片上传v2字段 默认大小为4MB 分片大小范围为1 MB - 1 GB * @link http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + * + * @throws \Exception */ public function __construct( $upToken, @@ -58,7 +65,8 @@ public function __construct( $config, $resumeRecordFile = null, $version = 'v1', - $partSize = config::BLOCK_SIZE + $partSize = config::BLOCK_SIZE, + $reqOpt = null ) { $this->upToken = $upToken; @@ -68,11 +76,16 @@ public function __construct( $this->params = $params; $this->mime = $mime; $this->contexts = array(); - $this->finishedEtags = array("etags"=>array(), "uploadId"=>"", "expiredAt"=>0, "uploaded"=>0); + $this->finishedEtags = array("etags" => array(), "uploadId" => "", "expiredAt" => 0, "uploaded" => 0); $this->config = $config; $this->resumeRecordFile = $resumeRecordFile ? $resumeRecordFile : null; $this->partSize = $partSize ? $partSize : config::BLOCK_SIZE; + if ($reqOpt === null) { + $reqOpt = new RequestOptions(); + } + $this->reqOpt = $reqOpt; + try { $this->version = SplitUploadVersion::from($version ? $version : 'v1'); } catch (\Exception $e) { @@ -100,7 +113,7 @@ public function upload($fname) $uploaded = 0; if ($this->version == SplitUploadVersion::V2) { $partNumber = 1; - $encodedObjectName = $this->key? \Qiniu\base64_urlSafeEncode($this->key) : '~'; + $encodedObjectName = $this->key ? \Qiniu\base64_urlSafeEncode($this->key) : '~'; }; // get upload record from resumeRecordFile if ($this->resumeRecordFile != null) { @@ -316,7 +329,7 @@ private function post($url, $data) { $this->currentUrl = $url; $headers = array('Authorization' => 'UpToken ' . $this->upToken); - return Client::post($url, $data, $headers); + return Client::post($url, $data, $headers, $this->reqOpt); } private function blockSize($uploaded) @@ -339,7 +352,7 @@ private function makeInitReq($encodedObjectName) */ private function initReq($encodedObjectName) { - $url = $this->host.'/buckets/'.$this->bucket.'/objects/'.$encodedObjectName.'/uploads'; + $url = $this->host . '/buckets/' . $this->bucket . '/objects/' . $encodedObjectName . '/uploads'; $headers = array( 'Authorization' => 'UpToken ' . $this->upToken, 'Content-Type' => 'application/json' @@ -358,8 +371,8 @@ private function uploadPart($block, $partNumber, $uploadId, $encodedObjectName, 'Content-Type' => 'application/octet-stream', 'Content-MD5' => $md5 ); - $url = $this->host.'/buckets/'.$this->bucket.'/objects/'.$encodedObjectName. - '/uploads/'.$uploadId.'/'.$partNumber; + $url = $this->host . '/buckets/' . $this->bucket . '/objects/' . $encodedObjectName . + '/uploads/' . $uploadId . '/' . $partNumber; $response = $this->put($url, $block, $headers); return $response; } @@ -367,7 +380,7 @@ private function uploadPart($block, $partNumber, $uploadId, $encodedObjectName, private function completeParts($fname, $uploadId, $encodedObjectName) { $headers = array( - 'Authorization' => 'UpToken '.$this->upToken, + 'Authorization' => 'UpToken ' . $this->upToken, 'Content-Type' => 'application/json' ); $etags = $this->finishedEtags['etags']; @@ -397,7 +410,7 @@ private function completeParts($fname, $uploadId, $encodedObjectName) 'parts' => $sortedEtags ); $jsonBody = json_encode($body); - $url = $this->host.'/buckets/'.$this->bucket.'/objects/'.$encodedObjectName.'/uploads/'.$uploadId; + $url = $this->host . '/buckets/' . $this->bucket . '/objects/' . $encodedObjectName . '/uploads/' . $uploadId; $response = $this->postWithHeaders($url, $jsonBody, $headers); if ($response->needRetry()) { $response = $this->postWithHeaders($url, $jsonBody, $headers); @@ -411,12 +424,12 @@ private function completeParts($fname, $uploadId, $encodedObjectName) private function put($url, $data, $headers) { $this->currentUrl = $url; - return Client::put($url, $data, $headers); + return Client::put($url, $data, $headers, $this->reqOpt); } private function postWithHeaders($url, $data, $headers) { $this->currentUrl = $url; - return Client::post($url, $data, $headers); + return Client::post($url, $data, $headers, $this->reqOpt); } } diff --git a/src/Qiniu/Storage/UploadManager.php b/src/Qiniu/Storage/UploadManager.php index abadeb69..6a3e46f4 100644 --- a/src/Qiniu/Storage/UploadManager.php +++ b/src/Qiniu/Storage/UploadManager.php @@ -3,6 +3,7 @@ use Qiniu\Config; use Qiniu\Http\HttpClient; +use Qiniu\Http\RequestOptions; use Qiniu\Storage\ResumeUploader; use Qiniu\Storage\FormUploader; @@ -14,13 +15,27 @@ final class UploadManager { private $config; + /** + * @var RequestOptions + */ + private $reqOpt; - public function __construct(Config $config = null) + /** + * @param Config|null $config + * @param RequestOptions|null $reqOpt + */ + public function __construct(Config $config = null, RequestOptions $reqOpt = null) { if ($config === null) { $config = new Config(); } $this->config = $config; + + if ($reqOpt === null) { + $reqOpt = new RequestOptions(); + } + + $this->reqOpt = $reqOpt; } /** @@ -46,8 +61,10 @@ public function put( $data, $params = null, $mime = 'application/octet-stream', - $fname = "default_filename" + $fname = "default_filename", + $reqOpt = null ) { + $reqOpt = $reqOpt === null ? $this->reqOpt : $reqOpt; $params = self::trimParams($params); return FormUploader::put( @@ -57,7 +74,8 @@ public function put( $this->config, $params, $mime, - $fname + $fname, + $reqOpt ); } @@ -92,8 +110,10 @@ public function putFile( $checkCrc = false, $resumeRecordFile = null, $version = 'v1', - $partSize = config::BLOCK_SIZE + $partSize = config::BLOCK_SIZE, + $reqOpt = null ) { + $reqOpt = $reqOpt === null ? $this->reqOpt : $reqOpt; $file = fopen($filePath, 'rb'); if ($file === false) { @@ -115,7 +135,8 @@ public function putFile( $this->config, $params, $mime, - basename($filePath) + basename($filePath), + $reqOpt ); } @@ -129,7 +150,8 @@ public function putFile( $this->config, $resumeRecordFile, $version, - $partSize + $partSize, + $reqOpt ); $ret = $up->upload(basename($filePath)); fclose($file); diff --git a/tests/Qiniu/Tests/HttpTest.php b/tests/Qiniu/Tests/HttpTest.php index 558bc1f1..abbce295 100755 --- a/tests/Qiniu/Tests/HttpTest.php +++ b/tests/Qiniu/Tests/HttpTest.php @@ -2,13 +2,14 @@ namespace Qiniu\Tests; use Qiniu\Http\Client; +use Qiniu\Http\RequestOptions; class HttpTest extends \PHPUnit_Framework_TestCase { public function testGet() { $response = Client::get('qiniu.com'); - $this->assertEquals($response->statusCode, 200); + $this->assertEquals(200, $response->statusCode); $this->assertNotNull($response->body); $this->assertNull($response->error); } @@ -23,10 +24,18 @@ public function testGetQiniu() $this->assertNotNull($response->error); } + public function testGetTimeout() + { + $reqOpt = new RequestOptions(); + $reqOpt->timeout = 1; + $response = Client::get('localhost:9000/timeout.php', array(), $reqOpt); + $this->assertEquals(-1, $response->statusCode); + } + public function testDelete() { $response = Client::delete('uc.qbox.me/bucketTagging', array()); - $this->assertEquals($response->statusCode, 401); + $this->assertEquals(401, $response->statusCode); $this->assertNotNull($response->body); $this->assertNotNull($response->error); } @@ -41,10 +50,19 @@ public function testDeleteQiniu() $this->assertNotNull($response->error); } + public function testDeleteTimeout() + { + $reqOpt = new RequestOptions(); + $reqOpt->timeout = 1; + $response = Client::delete('localhost:9000/timeout.php', array(), $reqOpt); + $this->assertEquals(-1, $response->statusCode); + } + + public function testPost() { $response = Client::post('qiniu.com', null); - $this->assertEquals($response->statusCode, 200); + $this->assertEquals(200, $response->statusCode); $this->assertNotNull($response->body); $this->assertNull($response->error); } @@ -52,17 +70,25 @@ public function testPost() public function testPostQiniu() { $response = Client::post('upload.qiniu.com', null); - $this->assertEquals($response->statusCode, 400); + $this->assertEquals(400, $response->statusCode); $this->assertNotNull($response->body); $this->assertNotNull($response->xReqId()); $this->assertNotNull($response->xLog()); $this->assertNotNull($response->error); } + public function testPostTimeout() + { + $reqOpt = new RequestOptions(); + $reqOpt->timeout = 1; + $response = Client::post('localhost:9000/timeout.php', null, array(), $reqOpt); + $this->assertEquals(-1, $response->statusCode); + } + public function testPut() { $response = Client::PUT('uc.qbox.me/bucketTagging', null); - $this->assertEquals($response->statusCode, 401); + $this->assertEquals(401, $response->statusCode); $this->assertNotNull($response->body); $this->assertNotNull($response->error); } @@ -76,4 +102,13 @@ public function testPutQiniu() $this->assertNotNull($response->xLog()); $this->assertNotNull($response->error); } + + + public function testPutTimeout() + { + $reqOpt = new RequestOptions(); + $reqOpt->timeout = 1; + $response = Client::put('localhost:9000/timeout.php', null, array(), $reqOpt); + $this->assertEquals(-1, $response->statusCode); + } } diff --git a/tests/mock-server/timeout.php b/tests/mock-server/timeout.php new file mode 100644 index 00000000..0d4e991d --- /dev/null +++ b/tests/mock-server/timeout.php @@ -0,0 +1,3 @@ +