Skip to content

Commit 098feee

Browse files
salilg-engsalilg-eng
committed
feat(Storage): add support for idempotency token
1 parent 284f348 commit 098feee

5 files changed

Lines changed: 294 additions & 7 deletions

File tree

‎Storage/src/Connection/Rest.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?php
2+
23
/**
34
* Copyright 2015 Google Inc. All Rights Reserved.
45
*
@@ -19,7 +20,6 @@
1920

2021
use Google\Auth\GetUniverseDomainInterface;
2122
use Google\Cloud\Core\RequestBuilder;
22-
use Google\Cloud\Core\RequestWrapper;
2323
use Google\Cloud\Core\RestTrait;
2424
use Google\Cloud\Core\Retry;
2525
use Google\Cloud\Core\Upload\AbstractUploader;
@@ -141,7 +141,7 @@ public function __construct(array $config = [])
141141

142142
$this->apiEndpoint = $this->getApiEndpoint(null, $config, self::DEFAULT_API_ENDPOINT_TEMPLATE);
143143

144-
$this->setRequestWrapper(new RequestWrapper($config));
144+
$this->setRequestWrapper(new StorageRequestWrapper($config));
145145
$this->setRequestBuilder(new RequestBuilder(
146146
$config['serviceDefinitionPath'],
147147
$this->apiEndpoint

‎Storage/src/Connection/StorageRequestWrapper.php‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
<?php
2+
3+
/**
4+
* Copyright 2024 Google Inc. All Rights Reserved.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License");
7+
* you may not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
namespace Google\Cloud\Storage\Connection;
20+
21+
use Google\Cloud\Core\RequestWrapper;
22+
use Psr\Http\Message\RequestInterface;
23+
use Ramsey\Uuid\Uuid;
24+
25+
/**
26+
* A wrapper for requests which adds an Idempotency Token.
27+
*
28+
* @internal
29+
*/
30+
class StorageRequestWrapper extends RequestWrapper
31+
{
32+
/**
33+
* @param RequestInterface $request A PSR-7 request.
34+
* @param array $options [optional]
35+
* @return mixed
36+
*/
37+
public function send(RequestInterface $request, array $options = [])
38+
{
39+
$options = $this->addToken($request, $options);
40+
return parent::send($request, $options);
41+
}
42+
43+
/**
44+
* @param RequestInterface $request A PSR-7 request.
45+
* @param array $options [optional]
46+
* @return mixed
47+
*/
48+
public function sendAsync(RequestInterface $request, array $options = [])
49+
{
50+
$options = $this->addToken($request, $options);
51+
return parent::sendAsync($request, $options);
52+
}
53+
54+
/**
55+
* Helper to inject the token.
56+
*
57+
* @param RequestInterface $request
58+
* @param array $options
59+
* @return array
60+
*/
61+
private function addToken(RequestInterface $request, array $options)
62+
{
63+
$method = strtoupper($request->getMethod());
64+
if ($method === 'GET' || $method === 'HEAD' || $method === 'OPTIONS') {
65+
return $options;
66+
}
67+
68+
$hasTokenInOptions = false;
69+
if (isset($options['restOptions']['headers'])) {
70+
foreach ($options['restOptions']['headers'] as $key => $value) {
71+
if (strtolower($key) === 'x-goog-gcs-idempotency-token') {
72+
$hasTokenInOptions = true;
73+
break;
74+
}
75+
}
76+
}
77+
78+
if (!$hasTokenInOptions && !$request->hasHeader('x-goog-gcs-idempotency-token')) {
79+
$token = Uuid::uuid4()->toString();
80+
if (isset($options['retryHeaders'])) {
81+
foreach ($options['retryHeaders'] as $header) {
82+
if (strpos($header, 'gccl-invocation-id/') === 0) {
83+
$extractedToken = substr($header, 19);
84+
if ($extractedToken !== false && $extractedToken !== '') {
85+
$token = $extractedToken;
86+
}
87+
break;
88+
}
89+
}
90+
}
91+
$options['restOptions']['headers']['x-goog-gcs-idempotency-token'] = $token;
92+
}
93+
return $options;
94+
}
95+
}

‎Storage/tests/System/ManageObjectsTest.php‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,81 @@ public function testUploadAsync()
542542
$this->assertInstanceOf(StorageObject::class, $resp);
543543
}
544544

545+
public function testIdempotencyTokenRetries()
546+
{
547+
$name = uniqid(self::TESTING_PREFIX);
548+
$object = self::$bucket->upload('test data', [
549+
'name' => $name
550+
]);
551+
552+
$uuid = \Ramsey\Uuid\Uuid::uuid4()->toString();
553+
554+
// First delete will succeed
555+
$object->delete([
556+
'restOptions' => [
557+
'headers' => [
558+
'x-goog-gcs-idempotency-token' => $uuid
559+
]
560+
]
561+
]);
562+
563+
// Second delete uses the exact same UUID, simulating a network retry.
564+
// It should NOT throw a NotFoundException because the GCS backend
565+
// will recognize the token and return the cached success response.
566+
$object->delete([
567+
'restOptions' => [
568+
'headers' => [
569+
'x-goog-gcs-idempotency-token' => $uuid
570+
]
571+
]
572+
]);
573+
574+
$this->assertFalse($object->exists());
575+
}
576+
577+
public function testIdempotencyTokenUpdateRetriesWithPrecondition()
578+
{
579+
$name = uniqid(self::TESTING_PREFIX);
580+
$object = self::$bucket->upload('test data', [
581+
'name' => $name
582+
]);
583+
584+
$info = $object->info();
585+
$metageneration = $info['metageneration'];
586+
587+
$uuid = \Ramsey\Uuid\Uuid::uuid4()->toString();
588+
589+
$metadata = [
590+
'metadata' => [
591+
'location' => 'test'
592+
]
593+
];
594+
595+
// First update will succeed and increment the metageneration
596+
$object->update($metadata, [
597+
'ifMetagenerationMatch' => $metageneration,
598+
'restOptions' => [
599+
'headers' => [
600+
'x-goog-gcs-idempotency-token' => $uuid
601+
]
602+
]
603+
]);
604+
605+
// Second update uses the exact same UUID, simulating a network retry.
606+
// Even though the metageneration has changed, the backend recognizes
607+
// the idempotency token and returns 200 OK instead of 412 Precondition Failed.
608+
$object->update($metadata, [
609+
'ifMetagenerationMatch' => $metageneration,
610+
'restOptions' => [
611+
'headers' => [
612+
'x-goog-gcs-idempotency-token' => $uuid
613+
]
614+
]
615+
]);
616+
617+
$this->assertEquals('test', $object->info()['metadata']['location']);
618+
}
619+
545620
public function testUpdateObject()
546621
{
547622
$metadata = [

‎Storage/tests/Unit/Connection/RestTest.php‎

Lines changed: 120 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?php
2+
23
/**
34
* Copyright 2016 Google Inc.
45
*
@@ -21,12 +22,12 @@
2122
use Google\Cloud\Core\RequestBuilder;
2223
use Google\Cloud\Core\RequestWrapper;
2324
use Google\Cloud\Core\Retry;
24-
use Google\Cloud\Core\Testing\TestHelpers;
2525
use Google\Cloud\Core\Upload\MultipartUploader;
2626
use Google\Cloud\Core\Upload\ResumableUploader;
2727
use Google\Cloud\Core\Upload\StreamableUploader;
2828
use Google\Cloud\Storage\Connection\Rest;
2929
use Google\Cloud\Storage\Connection\RetryTrait;
30+
use Google\Cloud\Storage\Connection\StorageRequestWrapper;
3031
use GuzzleHttp\Client;
3132
use GuzzleHttp\Exception\RequestException;
3233
use GuzzleHttp\Promise\Create;
@@ -511,12 +512,12 @@ function ($args) use (
511512
$actualRequests[$requestIndex] = $args[0];
512513
$requestHeaders[$requestIndex] = $args[1]['headers'] ?? [];
513514
if ($requestIndex++ === 0) {
514-
throw new RequestException("Server error", $args[0], new Response($status1, [], $body1));
515+
throw new RequestException('Server error', $args[0], new Response($status1, [], $body1));
515516
}
516517
return new Response($status2, [], $body2);
517518
}
518519
);
519-
$requestWrapper = new RequestWrapper([
520+
$requestWrapper = new StorageRequestWrapper([
520521
'httpHandler' => new Guzzle7HttpHandler($mockClient->reveal()),
521522
'accessToken' => 'Fake token',
522523
'retries' => 3,
@@ -543,11 +544,13 @@ function ($args) use (
543544
$expectedUri,
544545
$actualUri1
545546
);
547+
unset($requestHeaders[0]['x-goog-gcs-idempotency-token']); // carefully remove just in case
546548
$this->assertEquals([], $requestHeaders[0]);
547549
$this->assertEquals(
548550
$expectedUri,
549551
$actualUri2
550552
);
553+
unset($requestHeaders[1]['x-goog-gcs-idempotency-token']);
551554
$this->assertEquals($expectedSecondRequestHeaders, $requestHeaders[1]);
552555
$this->assertEquals($expectedResult, $actualBody);
553556
}
@@ -964,7 +967,7 @@ public function validationMethod()
964967
true,
965968
true,
966969
false
967-
],[
970+
], [
968971
['validate' => null],
969972
true,
970973
true,
@@ -1030,6 +1033,119 @@ public function provideRetryHeaders()
10301033
];
10311034
}
10321035

1036+
public function testIdempotencyTokenHeaderAdded()
1037+
{
1038+
$mockClient = $this->prophesize(Client::class);
1039+
$mockClient->send(
1040+
Argument::type(RequestInterface::class),
1041+
Argument::that(function ($options) {
1042+
if (!isset($options['headers']['x-goog-gcs-idempotency-token'])) {
1043+
return false;
1044+
}
1045+
$token = $options['headers']['x-goog-gcs-idempotency-token'];
1046+
return preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/', $token) === 1;
1047+
})
1048+
)->willReturn(new Response(200, [], '{}'))->shouldBeCalled();
1049+
1050+
$rest = new Rest();
1051+
$rest->setRequestWrapper(new StorageRequestWrapper([
1052+
'httpHandler' => new Guzzle7HttpHandler($mockClient->reveal()),
1053+
'accessToken' => 'Fake token',
1054+
]));
1055+
1056+
$rest->insertBucket();
1057+
}
1058+
1059+
public function testIdempotencyTokenNotOverwrittenIfProvided()
1060+
{
1061+
$customToken = 'my-custom-uuid-1234';
1062+
1063+
$mockClient = $this->prophesize(Client::class);
1064+
$mockClient->send(
1065+
Argument::type(RequestInterface::class),
1066+
Argument::that(function ($options) use ($customToken) {
1067+
if (!isset($options['headers']['x-goog-gcs-idempotency-token'])) {
1068+
return false;
1069+
}
1070+
return $options['headers']['x-goog-gcs-idempotency-token'] === $customToken;
1071+
})
1072+
)->willReturn(new Response(200, [], '{}'))->shouldBeCalled();
1073+
1074+
$rest = new Rest();
1075+
$rest->setRequestWrapper(new StorageRequestWrapper([
1076+
'httpHandler' => new Guzzle7HttpHandler($mockClient->reveal()),
1077+
'accessToken' => 'Fake token',
1078+
]));
1079+
1080+
$rest->insertBucket([
1081+
'restOptions' => [
1082+
'headers' => [
1083+
'x-goog-gcs-idempotency-token' => $customToken
1084+
]
1085+
]
1086+
]);
1087+
}
1088+
1089+
public function testIdempotencyTokenNotOverwrittenIfProvidedWithMixedCase()
1090+
{
1091+
$customToken = 'my-custom-uuid-1234';
1092+
1093+
$mockClient = $this->prophesize(Client::class);
1094+
$mockClient->send(
1095+
Argument::type(RequestInterface::class),
1096+
Argument::that(function ($options) use ($customToken) {
1097+
if (isset($options['headers']['x-goog-gcs-idempotency-token'])) {
1098+
return false;
1099+
}
1100+
if (!isset($options['headers']['X-Goog-Gcs-Idempotency-Token'])) {
1101+
return false;
1102+
}
1103+
return $options['headers']['X-Goog-Gcs-Idempotency-Token'] === $customToken;
1104+
})
1105+
)->willReturn(new Response(200, [], '{}'))->shouldBeCalled();
1106+
1107+
$rest = new Rest();
1108+
$rest->setRequestWrapper(new StorageRequestWrapper([
1109+
'httpHandler' => new Guzzle7HttpHandler($mockClient->reveal()),
1110+
'accessToken' => 'Fake token',
1111+
]));
1112+
1113+
$rest->insertBucket([
1114+
'restOptions' => [
1115+
'headers' => [
1116+
'X-Goog-Gcs-Idempotency-Token' => $customToken
1117+
]
1118+
]
1119+
]);
1120+
}
1121+
1122+
public function testIdempotencyTokenGeneratedIfGcclInvocationIdMalformed()
1123+
{
1124+
$mockClient = $this->prophesize(Client::class);
1125+
$mockClient->send(
1126+
Argument::type(RequestInterface::class),
1127+
Argument::that(function ($options) {
1128+
if (!isset($options['headers']['x-goog-gcs-idempotency-token'])) {
1129+
return false;
1130+
}
1131+
$token = $options['headers']['x-goog-gcs-idempotency-token'];
1132+
return preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/', $token) === 1;
1133+
})
1134+
)->willReturn(new Response(200, [], '{}'))->shouldBeCalled();
1135+
1136+
$rest = new Rest();
1137+
$rest->setRequestWrapper(new StorageRequestWrapper([
1138+
'httpHandler' => new Guzzle7HttpHandler($mockClient->reveal()),
1139+
'accessToken' => 'Fake token',
1140+
]));
1141+
1142+
$rest->insertBucket([
1143+
'retryHeaders' => [
1144+
'gccl-invocation-id/'
1145+
]
1146+
]);
1147+
}
1148+
10331149
private function getContentTypeAndMetadata(RequestInterface $request)
10341150
{
10351151
// Resumable upload request

‎dev/tests/Snippet/ProductNeutralGuides/AuthenticationTest.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@ public function testAuthenticationCredentialsFetcherOption()
8282
$this->assertInstanceOf(StorageClient::class, $client);
8383
$connection = (new ReflectionClass($client))->getProperty('connection')->getValue($client);
8484
$requestWrapper = (new ReflectionClass($connection))->getProperty('requestWrapper')->getValue($connection);
85-
$creds = (new ReflectionClass($requestWrapper))->getProperty('credentialsFetcher')->getValue($requestWrapper);
85+
$requestWrapperReflection = new ReflectionClass(\Google\Cloud\Core\RequestWrapper::class);
86+
$creds = $requestWrapperReflection->getProperty('credentialsFetcher')->getValue($requestWrapper);
8687
$this->assertInstanceOf(ServiceAccountCredentials::class, $creds);
8788
$this->assertEquals($clientEmail, $creds->getClientName());
8889
}

0 commit comments

Comments
 (0)