diff --git a/Core/src/Upload/MultipartUploader.php b/Core/src/Upload/MultipartUploader.php index 3affaf71b1b3..2b0b52e717bb 100644 --- a/Core/src/Upload/MultipartUploader.php +++ b/Core/src/Upload/MultipartUploader.php @@ -98,6 +98,9 @@ private function prepareRequest() $headers['Content-Length'] = $size; } + $customHeaders = $this->requestOptions['restOptions']['headers'] ?? []; + $headers = array_merge($headers, $customHeaders); + return new Request( 'POST', $this->uri, diff --git a/Core/src/Upload/ResumableUploader.php b/Core/src/Upload/ResumableUploader.php index ecea6703f59f..d7d1f7924d66 100644 --- a/Core/src/Upload/ResumableUploader.php +++ b/Core/src/Upload/ResumableUploader.php @@ -170,6 +170,13 @@ public function upload() 'Content-Range' => "bytes $rangeStart-$rangeEnd/$size", ]; + if ($size !== '*' && ($rangeEnd + 1) == (int) $size) { + $customHeaders = $this->requestOptions['restOptions']['headers'] ?? []; + if (isset($customHeaders['X-Goog-Hash'])) { + $headers['X-Goog-Hash'] = $customHeaders['X-Goog-Hash']; + } + } + $request = new Request( 'PUT', $resumeUri, diff --git a/Core/src/Upload/StreamableUploader.php b/Core/src/Upload/StreamableUploader.php index 123b2b27c13c..cd5ef54caa1b 100644 --- a/Core/src/Upload/StreamableUploader.php +++ b/Core/src/Upload/StreamableUploader.php @@ -43,10 +43,12 @@ public function upload($writeSize = null) return []; } + $isFinalRequest = ($writeSize === null); + // find or create the resumeUri $resumeUri = $this->getResumeUri(); - if ($writeSize) { + if ($writeSize !== null) { $rangeEnd = $this->rangeStart + $writeSize - 1; $data = $this->data->read($writeSize); } else { @@ -62,6 +64,13 @@ public function upload($writeSize = null) 'Content-Range' => "bytes {$this->rangeStart}-$rangeEnd/*" ]; + if ($isFinalRequest) { + $customHeaders = $this->requestOptions['restOptions']['headers'] ?? []; + if (isset($customHeaders['X-Goog-Hash'])) { + $headers['X-Goog-Hash'] = $customHeaders['X-Goog-Hash']; + } + } + $request = new Request( 'PUT', $resumeUri, diff --git a/Core/tests/Unit/Upload/MultipartUploaderTest.php b/Core/tests/Unit/Upload/MultipartUploaderTest.php index 05ad6906d7af..91c30a1d654f 100644 --- a/Core/tests/Unit/Upload/MultipartUploaderTest.php +++ b/Core/tests/Unit/Upload/MultipartUploaderTest.php @@ -85,6 +85,48 @@ public function testUploadsAsyncData() $actualPromise->wait() ); } + + public function testUploadsWithCustomHeaders() + { + $customHeaders = [ + 'X-Goog-Custom-Header' => 'custom-value', + 'User-Agent' => 'custom-ua' + ]; + + $requestWrapper = $this->prophesize(RequestWrapper::class); + $stream = Utils::streamFor('abcd'); + $successBody = '{"canI":"kickIt"}'; + $response = new Response(200, [], $successBody); + + $requestWrapper->send( + Argument::that(function (RequestInterface $request) use ($customHeaders) { + foreach ($customHeaders as $key => $value) { + if ($request->getHeaderLine($key) !== $value) { + return false; + } + } + + $contentType = $request->getHeaderLine('Content-Type'); + return str_contains($contentType, 'multipart/related') + && str_contains($contentType, 'boundary=boundary'); + }), + Argument::type('array') + )->willReturn($response); + + $uploader = new MultipartUploader( + $requestWrapper->reveal(), + $stream, + 'http://www.example.com', + [ + 'restOptions' => [ + 'headers' => $customHeaders + ] + ] + ); + + $this->assertEquals(json_decode($successBody, true), $uploader->upload()); + } + /** * @dataProvider streamSizes */ diff --git a/Core/tests/Unit/Upload/ResumableUploaderTest.php b/Core/tests/Unit/Upload/ResumableUploaderTest.php index 0e0255e66451..2ab311d396b0 100644 --- a/Core/tests/Unit/Upload/ResumableUploaderTest.php +++ b/Core/tests/Unit/Upload/ResumableUploaderTest.php @@ -238,6 +238,119 @@ public function testRetryOptionsPassing() $this->assertTrue($retryListenerCalled); } + public function testUploadSendsGoogHashOnFinalChunk() + { + $hashValue = 'crc32c=abc123'; + $resumeUri = 'http://some-resume-uri.example.com'; + + $this->requestWrapper->send( + Argument::which('getMethod', 'POST'), + Argument::type('array') + )->willReturn(new Response(200, ['Location' => $resumeUri])); + + $this->requestWrapper->send( + Argument::that(function (RequestInterface $request) use ($hashValue) { + return $request->getHeaderLine('X-Goog-Hash') === $hashValue; + }), + Argument::type('array') + )->willReturn(new Response(200, [], $this->successBody)); + + $uploader = new ResumableUploader( + $this->requestWrapper->reveal(), + $this->stream, + 'http://www.example.com', + [ + 'restOptions' => [ + 'headers' => ['X-Goog-Hash' => $hashValue] + ] + ] + ); + + $this->assertEquals(json_decode($this->successBody, true), $uploader->upload()); + } + + public function testUploadDoesNotSendGoogHashOnIntermediateChunk() + { + $hashValue = 'crc32c=abc123'; + $resumeUri = 'http://some-resume-uri.example.com'; + + $this->requestWrapper->send( + Argument::which('getMethod', 'POST'), + Argument::type('array') + )->willReturn(new Response(200, ['Location' => $resumeUri])); + + $this->requestWrapper->send( + Argument::that(function (RequestInterface $request) { + return $request->getHeaderLine('Content-Range') === 'bytes 0-1/4' + && !$request->hasHeader('X-Goog-Hash'); + }), + Argument::type('array') + )->willReturn(new Response(308, ['Range' => 'bytes 0-1'])); + + $this->requestWrapper->send( + Argument::that(function (RequestInterface $request) { + return $request->getHeaderLine('Content-Range') === 'bytes 2-3/4'; + }), + Argument::type('array') + )->willReturn(new Response(200, [], $this->successBody)); + + $uploader = new ResumableUploader( + $this->requestWrapper->reveal(), + $this->stream, // size 4 + 'http://www.example.com', + [ + 'chunkSize' => 2, // Force multi-chunk upload + 'restOptions' => [ + 'headers' => ['X-Goog-Hash' => $hashValue] + ] + ] + ); + + $this->assertEquals(json_decode($this->successBody, true), $uploader->upload()); + } + + public function testGoogHashOnlyOnFinalChunkOfMultiChunkUpload() + { + $hashValue = 'crc32c=abc123'; + $resumeUri = 'http://some-resume-uri.example.com'; + $this->stream = Utils::streamFor('01234567'); // 8 bytes total + + $this->requestWrapper->send( + Argument::which('getMethod', 'POST'), + Argument::type('array') + )->willReturn(new Response(200, ['Location' => $resumeUri])); + + $this->requestWrapper->send( + Argument::that(function (RequestInterface $request) { + return $request->getHeaderLine('Content-Range') === 'bytes 0-3/8' + && !$request->hasHeader('X-Goog-Hash'); + }), + Argument::type('array') + )->willReturn(new Response(308, ['Range' => 'bytes 0-3'])); + + $this->requestWrapper->send( + Argument::that(function (RequestInterface $request) use ($hashValue) { + return $request->getHeaderLine('Content-Range') === 'bytes 4-7/8' + && $request->getHeaderLine('X-Goog-Hash') === $hashValue; + }), + Argument::type('array') + )->willReturn(new Response(200, [], $this->successBody)); + + $uploader = new ResumableUploader( + $this->requestWrapper->reveal(), + $this->stream, + 'http://www.example.com', + [ + 'chunkSize' => 4, + 'restOptions' => [ + 'headers' => ['X-Goog-Hash' => $hashValue] + ] + ] + ); + + $this->assertEquals(json_decode($this->successBody, true), $uploader->upload()); + } + public function testThrowsExceptionWhenAttemptsAsyncUpload() { $this->expectException(GoogleException::class); diff --git a/Core/tests/Unit/Upload/StreamableUploaderTest.php b/Core/tests/Unit/Upload/StreamableUploaderTest.php index 15fa377e4482..2dfeb0422db2 100644 --- a/Core/tests/Unit/Upload/StreamableUploaderTest.php +++ b/Core/tests/Unit/Upload/StreamableUploaderTest.php @@ -174,6 +174,115 @@ public function testLastChunkSendsCorrectHeaders() $uploader->upload(); } + public function testUploadWithCustomGoogHashHeader() + { + $hashValue = 'md5=abc123'; + $resumeUri = 'http://some-resume-uri.example.com'; + + $resumeResponse = new Response(200, ['Location' => $resumeUri]); + $this->requestWrapper->send( + Argument::type(RequestInterface::class), + Argument::any() + )->willReturn($resumeResponse)->shouldBeCalled(); + + $uploadResponse = new Response(200, ['Location' => $resumeUri], $this->successBody); + $this->requestWrapper->send( + Argument::that(function ($request) use ($resumeUri, $hashValue) { + return (string) $request->getUri() === $resumeUri + && $request->getHeaderLine('X-Goog-Hash') === $hashValue; + }), + Argument::any() + )->willReturn($uploadResponse)->shouldBeCalled(); + + $uploader = new StreamableUploader( + $this->requestWrapper->reveal(), + $this->stream, + 'http://www.example.com', + [ + 'restOptions' => [ + 'headers' => [ + 'X-Goog-Hash' => $hashValue, + 'Other-Header' => 'should-be-ignored' + ] + ] + ] + ); + + $this->stream->write("some data"); + + $this->assertEquals(json_decode($this->successBody, true), $uploader->upload()); + } + + public function testUploadDoesNotSendGoogHashWhenConditionNotMet() + { + $hashValue = 'md5=abc123'; + $resumeUri = 'http://some-resume-uri.example.com'; + + $resumeResponse = new Response(200, ['Location' => $resumeUri]); + $this->requestWrapper->send( + Argument::which('getMethod', 'POST'), + Argument::type('array') + )->willReturn($resumeResponse); + + $uploadResponse = new Response(200, [], $this->successBody); + $this->requestWrapper->send( + Argument::that(function (RequestInterface $request) { + return !$request->hasHeader('X-Goog-Hash'); + }), + Argument::type('array') + )->willReturn($uploadResponse); + + $uploader = new StreamableUploader( + $this->requestWrapper->reveal(), + $this->stream, + 'http://www.example.com', + [ + 'restOptions' => [ + 'headers' => ['X-Goog-Hash' => $hashValue] + ] + ] + ); + + $this->stream->write("0123456789ABCDEF"); + + $this->assertEquals(json_decode($this->successBody, true), $uploader->upload(16)); + } + + public function testUploadSendsGoogHashOnFinalStep() + { + $hashValue = 'md5=finalHash'; + $resumeUri = 'http://some-resume-uri.example.com'; + + $resumeResponse = new Response(200, ['Location' => $resumeUri]); + $this->requestWrapper->send( + Argument::which('getMethod', 'POST'), + Argument::type('array') + )->willReturn($resumeResponse)->shouldBeCalled(); + + $uploadResponse = new Response(200, [], $this->successBody); + $this->requestWrapper->send( + Argument::that(function (RequestInterface $request) use ($hashValue) { + return $request->getHeaderLine('X-Goog-Hash') === $hashValue; + }), + Argument::type('array') + )->willReturn($uploadResponse)->shouldBeCalled(); + + $uploader = new StreamableUploader( + $this->requestWrapper->reveal(), + $this->stream, + 'http://www.example.com', + [ + 'restOptions' => [ + 'headers' => ['X-Goog-Hash' => $hashValue] + ] + ] + ); + + $this->stream->write("final data"); + + $this->assertEquals(json_decode($this->successBody, true), $uploader->upload()); + } + public function testThrowsExceptionWhenAttemptsAsyncUpload() { $this->expectException(GoogleException::class); diff --git a/Storage/src/Connection/Rest.php b/Storage/src/Connection/Rest.php index 4766c0ac2ee8..ac18b5f8a399 100644 --- a/Storage/src/Connection/Rest.php +++ b/Storage/src/Connection/Rest.php @@ -500,12 +500,30 @@ private function resolveUploadOptions(array $args) } $validate = $this->chooseValidationMethod($args); - if ($validate === 'md5') { - $args['metadata']['md5Hash'] = base64_encode(Utils::hash($args['data'], 'md5', true)); - } elseif ($validate === 'crc32') { - $args['metadata']['crc32c'] = $this->crcFromStream($args['data']); + $md5Hash = null; + $crc32c = null; + + if ($validate !== false) { + $md5Hash = base64_encode(Utils::hash($args['data'], 'md5', true)); + $crc32c = $this->crcFromStream($args['data']); + + if ($validate === 'md5') { + $args['metadata']['md5Hash'] = $md5Hash; + } elseif ($validate === 'crc32') { + $args['metadata']['crc32c'] = $crc32c; + } } + // Prepare the X-Goog-Hash header string + $xGoogHash = []; + if ($crc32c) { + $xGoogHash[] = 'crc32c=' . $crc32c; + } + if ($md5Hash) { + $xGoogHash[] = 'md5=' . $md5Hash; + } + $xGoogHashHeader = implode(',', $xGoogHash); + $args['metadata']['name'] = $args['name']; if (isset($args['retention'])) { // during object creation retention properties go into metadata @@ -532,6 +550,22 @@ private function resolveUploadOptions(array $args) $args['uploaderOptions'] = array_intersect_key($args, array_flip($uploaderOptionKeys)); $args = array_diff_key($args, array_flip($uploaderOptionKeys)); + $args['uploaderOptions']['restOptions'] ??= []; + $args['uploaderOptions']['restOptions']['headers'] ??= []; + + // Add the X-Goog-Hash header only if there are hashes to include + if (!empty($xGoogHashHeader)) { + $args['uploaderOptions']['restOptions']['headers']['X-Goog-Hash'] = $xGoogHashHeader; + } + + if (!empty($args['headers'])) { + $args['uploaderOptions']['restOptions']['headers'] = array_merge( + $args['uploaderOptions']['restOptions']['headers'], + $args['headers'] + ); + } + unset($args['headers']); + // Passing on custom retry function to $args['uploaderOptions'] $retryFunc = $this->getRestRetryFunction( 'objects', @@ -714,7 +748,10 @@ private function buildDownloadObjectParams(array $args) private function chooseValidationMethod(array $args) { // If the user provided a hash, skip hashing. - if (isset($args['metadata']['md5Hash']) || isset($args['metadata']['crc32c'])) { + if (isset($args['metadata']['md5Hash']) + || isset($args['metadata']['crc32c']) + || isset($args['headers']['X-Goog-Hash']) + ) { return false; } diff --git a/Storage/tests/Unit/BucketTest.php b/Storage/tests/Unit/BucketTest.php index a01a46b452f9..2584fff032cc 100644 --- a/Storage/tests/Unit/BucketTest.php +++ b/Storage/tests/Unit/BucketTest.php @@ -183,6 +183,68 @@ public function testGetResumableUploaderWithStringWithNoName() $bucket->getResumableUploader('some more data'); } + /** + * Verifies that a resumable upload triggered through a Bucket + * only sends the X-Goog-Hash on the final chunk. + */ + public function testUploadResumableFinalChunkHashes() + { + $data = 'chunk1chunk2'; // 12 bytes + $name = 'test-resumable.txt'; + $resumeUri = 'http://example.com/resumable/123'; + $hash = 'crc32c=mb+64g==,md5=ecA+ttxFBv4gFVBh52kiWA=='; + + $rw = $this->prophesize(RequestWrapper::class); + + // Handshake call (POST) + $rw->send(Argument::that(function ($request) { + return $request->getMethod() === 'POST'; + }), Argument::any())->willReturn(new \GuzzleHttp\Psr7\Response(200, ['Location' => $resumeUri])); + + // Intermediate chunk (PUT) - Should NOT have X-Goog-Hash + $rw->send(Argument::that(function ($request) { + return $request->getMethod() === 'PUT' + && $request->getHeaderLine('Content-Range') === 'bytes 0-5/12' + && !$request->hasHeader('X-Goog-Hash'); + }), Argument::any())->willReturn(new \GuzzleHttp\Psr7\Response(308, ['Range' => 'bytes=0-5'])); + + // FINAL chunk (PUT) - MUST HAVE X-Goog-Hash + $rw->send(Argument::that(function ($request) use ($hash) { + return $request->getMethod() === 'PUT' + && $request->getHeaderLine('Content-Range') === 'bytes 6-11/12' + && $request->getHeaderLine('X-Goog-Hash') === $hash; + }), Argument::any())->willReturn( + new \GuzzleHttp\Psr7\Response(200, [], '{"name":"' . $name . '","generation":"1"}') + ); + + $this->connection->projectId()->willReturn(self::PROJECT_ID); + $this->connection->requestWrapper()->willReturn($rw->reveal()); + + $uploader = new ResumableUploader( + $rw->reveal(), + $data, + 'http://example.com/upload', + [ + 'chunkSize' => 6, + 'contentType' => 'text/plain', + 'restOptions' => ['headers' => ['X-Goog-Hash' => $hash]] + ] + ); + + $this->connection->insertObject(Argument::any()) + ->willReturn($uploader); + + $bucket = $this->getBucket(); + $object = $bucket->upload($data, [ + 'name' => $name, + 'resumable' => true, + 'chunkSize' => 6, + ]); + + $this->assertInstanceOf(StorageObject::class, $object); + $this->assertEquals($name, $object->name()); + } + public function testGetObject() { $bucket = $this->getBucket(); diff --git a/Storage/tests/Unit/Connection/RestTest.php b/Storage/tests/Unit/Connection/RestTest.php index 7a2250edcb4a..55f69eea1754 100644 --- a/Storage/tests/Unit/Connection/RestTest.php +++ b/Storage/tests/Unit/Connection/RestTest.php @@ -532,6 +532,55 @@ public function insertObjectProvider() ]; } + public function testInsertObjectWithCalculatedXGoogHashHeader() + { + $rest = new RestCrc32cStub(); + $testData = 'some test data'; + $testStream = Utils::streamFor($testData); + $expectedCrc32c = $rest->getCrcFromStreamForTest($testStream); + $expectedMd5 = base64_encode(Utils::hash($testStream, 'md5', true)); + $expectedHashHeader = 'crc32c=' . $expectedCrc32c . ',md5=' . $expectedMd5; + + $actualRequest = null; + $response = new Response(200, ['Location' => 'http://www.mordor.com'], $this->successBody); + + $this->requestWrapper->send( + Argument::type(RequestInterface::class), + Argument::type('array') + )->will( + function ($args) use (&$actualRequest, $response) { + $actualRequest = $args[0]; + return $response; + } + ); + + $rest->extensionLoaded = false; + $rest->supportsBuiltin = true; + + $rest->setRequestWrapper($this->requestWrapper->reveal()); + + $uploadStream = Utils::streamFor($testData); + + $options = [ + 'bucket' => 'my-test-bucket', + 'name' => 'test-calculated-hash-file.txt', + 'data' => $uploadStream, + 'validate' => 'md5' + ]; + + $uploader = $rest->insertObject($options); + $this->assertInstanceOf(MultipartUploader::class, $uploader); + $uploader->upload(); + + $this->assertNotNull($actualRequest); + $this->assertTrue($actualRequest->hasHeader('X-Goog-Hash')); + $this->assertEquals([$expectedHashHeader], $actualRequest->getHeader('X-Goog-Hash')); + + list($contentType, $metadata) = $this->getContentTypeAndMetadata($actualRequest); + $this->assertEquals($expectedMd5, $metadata['md5Hash']); + $this->assertArrayNotHasKey('crc32c', $metadata); + } + /** * @dataProvider validationMethod */ @@ -592,6 +641,11 @@ public function validationMethod() true, true, false + ], [ + ['validate' => true, 'headers' => ['X-Goog-Hash' => 'crc32c=abc']], + true, + true, + false ] ]; } @@ -692,4 +746,21 @@ public function chooseValidationMethodProxy(array $args) $call = $chooseValidationMethod->bindTo($this, Rest::class); return $call($args); } + + /** + * Helper proxy to expose crcFromStream for testing. + * @param StreamInterface $data + * @return string + */ + public function getCrcFromStreamForTest(StreamInterface $data) + { + $data->rewind(); + + $crcFromStream = function (StreamInterface $data) { + return call_user_func_array([$this, 'crcFromStream'], func_get_args()); + }; + + $call = $crcFromStream->bindTo($this, Rest::class); + return $call($data); + } }