Skip to content

Commit f4252ff

Browse files
brdeyobrdeyo
committed
Add VC metadata to blob uploads.
1 parent bcf652e commit f4252ff

6 files changed

Lines changed: 80 additions & 39 deletions

File tree

‎src/VirtualClient/VirtualClient.Core.UnitTests/BlobManagerTests.cs‎

Lines changed: 56 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public void SetupDefaultBehaviors()
5050
};
5151
};
5252

53-
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options) =>
53+
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options, metadata) =>
5454
{
5555
return new TestResponse<BlobContentInfo>()
5656
{
@@ -362,15 +362,15 @@ public void BlobManagerValidatesTheBlobNameBeforeUploadingABlob()
362362
this.mockDescriptor.Name = name;
363363

364364
Assert.ThrowsAsync<ArgumentException>(
365-
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, Policy.NoOpAsync()));
365+
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, retryPolicy: Policy.NoOpAsync()));
366366
});
367367

368368
validBlobNames.ForEach(name =>
369369
{
370370
this.mockDescriptor.Name = name;
371371

372372
Assert.DoesNotThrowAsync(
373-
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, Policy.NoOpAsync()));
373+
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, retryPolicy: Policy.NoOpAsync()));
374374
});
375375
}
376376
}
@@ -403,15 +403,15 @@ public void BlobManagerValidatesTheContainerNameBeforeUploadingABlob()
403403
this.mockDescriptor.ContainerName = name;
404404

405405
Assert.ThrowsAsync<ArgumentException>(
406-
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, Policy.NoOpAsync()));
406+
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, retryPolicy: Policy.NoOpAsync()));
407407
});
408408

409409
validContainerNames.ForEach(name =>
410410
{
411411
this.mockDescriptor.ContainerName = name;
412412

413413
Assert.DoesNotThrowAsync(
414-
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, Policy.NoOpAsync()));
414+
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, retryPolicy: Policy.NoOpAsync()));
415415
});
416416
}
417417
}
@@ -422,7 +422,7 @@ public async Task BlobManagerSupportsDependencyDescriptorBaseClassParameterConve
422422
using (MemoryStream uploadStream = new MemoryStream())
423423
{
424424
bool supported = false;
425-
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options) =>
425+
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options, metadata) =>
426426
{
427427
supported = true;
428428

@@ -436,7 +436,7 @@ public async Task BlobManagerSupportsDependencyDescriptorBaseClassParameterConve
436436
};
437437

438438
DependencyDescriptor baseDescriptor = new DependencyDescriptor(this.mockDescriptor);
439-
await this.blobManager.UploadBlobAsync(baseDescriptor, uploadStream, CancellationToken.None, Policy.NoOpAsync())
439+
await this.blobManager.UploadBlobAsync(baseDescriptor, uploadStream, CancellationToken.None, retryPolicy: Policy.NoOpAsync())
440440
.ConfigureAwait(false);
441441

442442
Assert.IsTrue(supported);
@@ -448,7 +448,7 @@ public async Task BlobManagerUploadsTheExpectedBlobFromAStream()
448448
{
449449
using (MemoryStream uploadStream = new MemoryStream())
450450
{
451-
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options) =>
451+
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options, metadata) =>
452452
{
453453
Assert.IsTrue(object.ReferenceEquals(this.mockDescriptor, descriptor));
454454
Assert.AreEqual(descriptor.ContentEncoding.WebName, options.HttpHeaders.ContentEncoding);
@@ -463,7 +463,7 @@ public async Task BlobManagerUploadsTheExpectedBlobFromAStream()
463463
};
464464
};
465465

466-
await this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, Policy.NoOpAsync())
466+
await this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, retryPolicy: Policy.NoOpAsync())
467467
.ConfigureAwait(false);
468468
}
469469
}
@@ -474,7 +474,7 @@ public async Task BlobManagerUsesETagsToEnableOptimisticConcurrencyOnBlobUploads
474474
using (MemoryStream uploadStream = new MemoryStream())
475475
{
476476
this.mockDescriptor.ETag = "\"0x123456789\"";
477-
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options) =>
477+
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options, metadata) =>
478478
{
479479
Assert.IsNotNull(options.Conditions);
480480
Assert.IsTrue(options.Conditions.IfMatch.HasValue);
@@ -489,7 +489,7 @@ public async Task BlobManagerUsesETagsToEnableOptimisticConcurrencyOnBlobUploads
489489
};
490490
};
491491

492-
await this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, Policy.NoOpAsync())
492+
await this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, retryPolicy: Policy.NoOpAsync())
493493
.ConfigureAwait(false);
494494
}
495495
}
@@ -499,7 +499,7 @@ public async Task BlobManagerReturnsTheExpectedBlobDescriptorOnUploads()
499499
{
500500
using (MemoryStream uploadStream = new MemoryStream())
501501
{
502-
BlobDescriptor actualDescriptor = await this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, Policy.NoOpAsync())
502+
BlobDescriptor actualDescriptor = await this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, retryPolicy: Policy.NoOpAsync())
503503
.ConfigureAwait(false) as BlobDescriptor;
504504

505505
Assert.IsNotNull(actualDescriptor);
@@ -511,12 +511,47 @@ public async Task BlobManagerReturnsTheExpectedBlobDescriptorOnUploads()
511511
}
512512
}
513513

514+
[Test]
515+
public async Task BlobManagerIncludesMetadataInBlobsUploadedWhenProvided()
516+
{
517+
using (MemoryStream uploadStream = new MemoryStream())
518+
{
519+
IDictionary<string, IConvertible> expectedMetadata = new Dictionary<string, IConvertible>
520+
{
521+
{ "Metadata1", "Value1" },
522+
{ "Metadata2", 1234 },
523+
{ "Metadata3", true },
524+
{ "Metadata4", Guid.NewGuid().ToString() }
525+
};
526+
527+
bool confirmed = false;
528+
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options, metadata) =>
529+
{
530+
CollectionAssert.AreEquivalent(expectedMetadata, metadata);
531+
confirmed = true;
532+
533+
return new TestResponse<BlobContentInfo>()
534+
{
535+
RawResponse = new TestResponse((int)HttpStatusCode.OK, "\"0x123456789\"")
536+
{
537+
ContentStream = new MemoryStream()
538+
}
539+
};
540+
};
541+
542+
BlobDescriptor actualDescriptor = await this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, expectedMetadata, Policy.NoOpAsync())
543+
.ConfigureAwait(false) as BlobDescriptor;
544+
545+
Assert.IsTrue(confirmed);
546+
}
547+
}
548+
514549
[Test]
515550
public void BlobManagerThrowsIfAFailedResponseIsReturnedOnAnAttemptToUploadABlob()
516551
{
517552
using (MemoryStream uploadStream = new MemoryStream())
518553
{
519-
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options) =>
554+
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options, metadata) =>
520555
{
521556
return new TestResponse<BlobContentInfo>()
522557
{
@@ -528,7 +563,7 @@ public void BlobManagerThrowsIfAFailedResponseIsReturnedOnAnAttemptToUploadABlob
528563
};
529564

530565
DependencyException error = Assert.ThrowsAsync<DependencyException>(
531-
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, Policy.NoOpAsync()));
566+
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, retryPolicy: Policy.NoOpAsync()));
532567

533568
Assert.IsTrue(error.Message.Contains("BadRequest"));
534569
}
@@ -540,7 +575,7 @@ public void BlobManagerAppliesTheRetryPolicyProvidedToHandleTransientErrorsWhenU
540575
using (MemoryStream uploadStream = new MemoryStream())
541576
{
542577
int uploadAttempts = 0;
543-
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options) =>
578+
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options, metadata) =>
544579
{
545580
uploadAttempts++;
546581
throw new RequestFailedException("Transient Error");
@@ -549,7 +584,7 @@ public void BlobManagerAppliesTheRetryPolicyProvidedToHandleTransientErrorsWhenU
549584
IAsyncPolicy expectedRetryPolicy = Policy.Handle<RequestFailedException>().RetryAsync(3);
550585

551586
Assert.ThrowsAsync<DependencyException>(
552-
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, expectedRetryPolicy));
587+
() => this.blobManager.UploadBlobAsync(this.mockDescriptor, uploadStream, CancellationToken.None, retryPolicy: expectedRetryPolicy));
553588

554589
Assert.IsTrue(uploadAttempts == 4);
555590
}
@@ -561,7 +596,7 @@ public void BlobManagerDefaultRetryPolicyDoesNotRetryNonTransientErrorsWhenUploa
561596
using (MemoryStream uploadStream = new MemoryStream())
562597
{
563598
int uploadAttempts = 0;
564-
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options) =>
599+
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options, metadata) =>
565600
{
566601
uploadAttempts++;
567602
throw new RequestFailedException(
@@ -582,7 +617,7 @@ public void BlobManagerDefaultRetryPolicyHandleSignatureMismatchErrors()
582617
using (MemoryStream uploadStream = new MemoryStream())
583618
{
584619
int uploadAttempts = 0;
585-
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options) =>
620+
this.blobManager.OnUploadFromStreamAsync = (descriptor, stream, options, metadata) =>
586621
{
587622
if (uploadAttempts <= 1)
588623
{
@@ -616,7 +651,7 @@ public TestBlobManager(DependencyBlobStore storeDescription)
616651

617652
public Func<BlobDescriptor, Stream, Response> OnDownloadToStreamAsync { get; set; }
618653

619-
public Func<BlobDescriptor, Stream, BlobUploadOptions, Response<BlobContentInfo>> OnUploadFromStreamAsync { get; set; }
654+
public Func<BlobDescriptor, Stream, BlobUploadOptions, IDictionary<string, IConvertible>, Response<BlobContentInfo>> OnUploadFromStreamAsync { get; set; }
620655

621656
public new BlobContainerClient CreateContainerClient(BlobDescriptor descriptor, DependencyBlobStore blobStore)
622657
{
@@ -632,10 +667,10 @@ protected override Task<Response> DownloadToStreamAsync(BlobDescriptor descripto
632667
return Task.FromResult(response);
633668
}
634669

635-
protected override Task<Response<BlobContentInfo>> UploadFromStreamAsync(BlobDescriptor descriptor, Stream stream, BlobUploadOptions uploadOptions, CancellationToken cancellationToken)
670+
protected override Task<Response<BlobContentInfo>> UploadFromStreamAsync(BlobDescriptor descriptor, Stream stream, BlobUploadOptions uploadOptions, CancellationToken cancellationToken, IDictionary<string, IConvertible> metadata = null)
636671
{
637672
Response<BlobContentInfo> response = this.OnUploadFromStreamAsync != null
638-
? this.OnUploadFromStreamAsync?.Invoke(descriptor, stream, uploadOptions)
673+
? this.OnUploadFromStreamAsync?.Invoke(descriptor, stream, uploadOptions, metadata)
639674
: null;
640675

641676
return Task.FromResult(response);

‎src/VirtualClient/VirtualClient.Core/BlobManager.cs‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ namespace VirtualClient
77
using System.Collections.Generic;
88
using System.Diagnostics.CodeAnalysis;
99
using System.IO;
10+
using System.Linq;
1011
using System.Net;
1112
using System.Text.RegularExpressions;
1213
using System.Threading;
1314
using System.Threading.Tasks;
1415
using Azure;
15-
using Azure.Core;
1616
using Azure.Storage.Blobs;
1717
using Azure.Storage.Blobs.Models;
1818
using Polly;
@@ -35,8 +35,6 @@ public class BlobManager : IBlobManager
3535
(int)HttpStatusCode.InternalServerError
3636
};
3737

38-
private static readonly char[] UriDelimiters = new char[] { '/', '\\' };
39-
4038
private static IAsyncPolicy defaultRetryPolicy = Policy.Handle<RequestFailedException>(error =>
4139
{
4240
return error.Status < 400 || BlobManager.RetryableCodes.Contains(error.Status)
@@ -88,8 +86,7 @@ public async Task<DependencyDescriptor> DownloadBlobAsync(DependencyDescriptor d
8886
return await (retryPolicy ?? BlobManager.defaultRetryPolicy).ExecuteAsync(async () =>
8987
{
9088
BlobDescriptor blobInfo = new BlobDescriptor(descriptor);
91-
Response response = await this.DownloadToStreamAsync(blobDescriptor, downloadStream, cancellationToken)
92-
.ConfigureAwait(false);
89+
Response response = await this.DownloadToStreamAsync(blobDescriptor, downloadStream, cancellationToken);
9390

9491
if (response.Status >= 300)
9592
{
@@ -107,7 +104,7 @@ public async Task<DependencyDescriptor> DownloadBlobAsync(DependencyDescriptor d
107104

108105
return blobInfo;
109106

110-
}).ConfigureAwait(false);
107+
});
111108
}
112109
catch (RequestFailedException exc) when (exc.Status == (int)HttpStatusCode.Forbidden)
113110
{
@@ -144,7 +141,7 @@ public async Task<DependencyDescriptor> DownloadBlobAsync(DependencyDescriptor d
144141
}
145142

146143
/// <inheritdoc />
147-
public async Task<DependencyDescriptor> UploadBlobAsync(DependencyDescriptor descriptor, Stream uploadStream, CancellationToken cancellationToken, IAsyncPolicy retryPolicy = null)
144+
public async Task<DependencyDescriptor> UploadBlobAsync(DependencyDescriptor descriptor, Stream uploadStream, CancellationToken cancellationToken, IDictionary<string, IConvertible> metadata = null, IAsyncPolicy retryPolicy = null)
148145
{
149146
descriptor.ThrowIfNull(nameof(descriptor));
150147
uploadStream.ThrowIfNull(nameof(uploadStream));
@@ -193,7 +190,8 @@ public async Task<DependencyDescriptor> UploadBlobAsync(DependencyDescriptor des
193190
ContentType = blobDescriptor.ContentType
194191
}
195192
},
196-
cancellationToken).ConfigureAwait(false);
193+
cancellationToken,
194+
metadata);
197195

198196
Response rawResponse = response.GetRawResponse();
199197
if (rawResponse.Status >= 300)
@@ -210,7 +208,7 @@ public async Task<DependencyDescriptor> UploadBlobAsync(DependencyDescriptor des
210208

211209
return blobInfo;
212210

213-
}).ConfigureAwait(false);
211+
});
214212
}
215213
catch (RequestFailedException exc) when (exc.Status == (int)HttpStatusCode.Forbidden)
216214
{
@@ -342,7 +340,7 @@ protected virtual Task<Response> DownloadToStreamAsync(BlobDescriptor descriptor
342340
/// <summary>
343341
/// Uploads the blob from the stream provided.
344342
/// </summary>
345-
protected virtual async Task<Response<BlobContentInfo>> UploadFromStreamAsync(BlobDescriptor descriptor, Stream stream, BlobUploadOptions uploadOptions, CancellationToken cancellationToken)
343+
protected virtual async Task<Response<BlobContentInfo>> UploadFromStreamAsync(BlobDescriptor descriptor, Stream stream, BlobUploadOptions uploadOptions, CancellationToken cancellationToken, IDictionary<string, IConvertible> metadata = null)
346344
{
347345
DependencyBlobStore blobStore = this.StoreDescription as DependencyBlobStore;
348346
BlobContainerClient containerClient = this.CreateContainerClient(descriptor, blobStore);
@@ -353,16 +351,21 @@ protected virtual async Task<Response<BlobContentInfo>> UploadFromStreamAsync(Bl
353351
// Container-specific SAS URIs do not allow the client to access container existence, properties or
354352
// to create the container. Furthermore, the container MUST already exist in order for this type of
355353
// SAS URI to be created from it.
356-
await containerClient.CreateIfNotExistsAsync(PublicAccessType.None)
357-
.ConfigureAwait(false);
354+
await containerClient.CreateIfNotExistsAsync(PublicAccessType.None);
358355
}
359356
catch
360357
{
361358
// Do nothing if identity doesn't have container access.
362359
}
363360

364-
return await blobClient.UploadAsync(stream, uploadOptions, cancellationToken)
365-
.ConfigureAwait(false);
361+
Response<BlobContentInfo> response = await blobClient.UploadAsync(stream, uploadOptions, cancellationToken);
362+
363+
if (metadata?.Any() == true)
364+
{
365+
blobClient.SetMetadata(metadata.ToDictionary(entry => entry.Key, entry => entry.Value?.ToString()));
366+
}
367+
368+
return response;
366369
}
367370

368371
private static string GetHttpStatusCodeName(int statusCode)

‎src/VirtualClient/VirtualClient.Core/IBlobManager.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33

44
namespace VirtualClient
55
{
6+
using System;
7+
using System.Collections.Generic;
68
using System.IO;
79
using System.Threading;
810
using System.Threading.Tasks;
@@ -36,7 +38,8 @@ public interface IBlobManager
3638
/// <param name="uploadStream">The stream that contains the blob binary content to upload.</param>
3739
/// <param name="cancellationToken">A token that can be used to cancel the operation.</param>
3840
/// <param name="retryPolicy">A policy to use for handling retries when transient errors/failures happen.</param>
41+
/// <param name="metadata">Metadata in which to tag the blob.</param>
3942
/// <returns>Full details for the blob as it exists in the store (e.g. name, content encoding, content type).</returns>
40-
Task<DependencyDescriptor> UploadBlobAsync(DependencyDescriptor descriptor, Stream uploadStream, CancellationToken cancellationToken, IAsyncPolicy retryPolicy = null);
43+
Task<DependencyDescriptor> UploadBlobAsync(DependencyDescriptor descriptor, Stream uploadStream, CancellationToken cancellationToken, IDictionary<string, IConvertible> metadata = null, IAsyncPolicy retryPolicy = null);
4144
}
4245
}

‎src/VirtualClient/VirtualClient.Core/Proxy/ProxyBlobManager.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ public async Task<DependencyDescriptor> DownloadBlobAsync(DependencyDescriptor d
123123
return descriptor;
124124
}
125125

126-
public async Task<DependencyDescriptor> UploadBlobAsync(DependencyDescriptor descriptor, Stream uploadStream, CancellationToken cancellationToken, IAsyncPolicy retryPolicy = null)
126+
public async Task<DependencyDescriptor> UploadBlobAsync(DependencyDescriptor descriptor, Stream uploadStream, CancellationToken cancellationToken, IDictionary<string, IConvertible> metadata = null, IAsyncPolicy retryPolicy = null)
127127
{
128128
descriptor.ThrowIfNull(nameof(descriptor));
129129
uploadStream.ThrowIfNull(nameof(uploadStream));

0 commit comments

Comments
 (0)