Rollback changes to the 'delete on upload' changes to support explicit definition in code, profile component-level and command line override.

Author: brdeyo

VERSION

@@ -1 +1 @@
-3.3.6
+3.3.7

src/VirtualClient/VirtualClient.Contracts/FileUploadDescriptor.cs

@@ -36,8 +36,10 @@ public class FileUploadDescriptor
         /// <param name="contentType">The web content type for the blob (e.g. text/plain, application/octet-stream).</param>
         /// <param name="filePath">The path to the file containing the content to upload.</param>
         /// <param name="manifest">Properties that define a manifest (e.g. metadata) for the file and its contents.</param>
+        /// <param name="deleteOnUpload">True/false whether the file should be deleted upon being successfully uploaded. Default = false.</param>
+        /// <param name="includeManifest">True/false whether a manifest should be included with the file uploaded.</param>
         [JsonConstructor]
-        public FileUploadDescriptor(string blobPath, string containerName, string contentEncoding, string contentType, string filePath, IDictionary<string, IConvertible> manifest = null)
+        public FileUploadDescriptor(string blobPath, string containerName, string contentEncoding, string contentType, string filePath, IDictionary<string, IConvertible> manifest = null, bool deleteOnUpload = false, bool includeManifest = false)
         {
             blobPath.ThrowIfNullOrWhiteSpace(nameof(blobPath));
             containerName.ThrowIfNullOrWhiteSpace(nameof(containerName));
@@ -51,7 +53,9 @@ public FileUploadDescriptor(string blobPath, string containerName, string conten
             this.ContainerName = containerName;
             this.ContentEncoding = contentEncoding;
             this.ContentType = contentType;
+            this.DeleteOnUpload = deleteOnUpload;
             this.FilePath = filePath;
+            this.IncludeManifest = includeManifest;
             this.Manifest = new Dictionary<string, IConvertible>(StringComparer.OrdinalIgnoreCase);
 
             if (manifest?.Any() == true)
@@ -90,12 +94,24 @@ public FileUploadDescriptor(string blobPath, string containerName, string conten
         [JsonProperty(PropertyName = "contentType", Required = Required.Always)]
         public string ContentType { get; }
 
+        /// <summary>
+        /// True/false whether the file should be deleted upon being successfully uploaded.
+        /// </summary>
+        [JsonProperty(PropertyName = "deleteOnUpload", Required = Required.Always)]
+        public bool DeleteOnUpload { get; set; }
+
         /// <summary>
         /// The full path to the file to upload.
         /// </summary>
         [JsonProperty(PropertyName = "filePath", Required = Required.Always)]
         public string FilePath { get; }
 
+        /// <summary>
+        /// True/false whether a manifest should be included alongside the file uploaded.
+        /// </summary>
+        [JsonProperty(PropertyName = "includeManifest", Required = Required.Always)]
+        public bool IncludeManifest { get; set; }
+
         /// <summary>
         /// Manifest/metadata information related to the file and. its contents
         /// </summary>

src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs

@@ -5,6 +5,7 @@ namespace VirtualClient.Contracts
 {
     using System;
     using System.Collections.Generic;
+    using System.Globalization;
     using System.Linq;
     using System.Reflection;
     using System.Runtime.InteropServices;
@@ -197,18 +198,36 @@ public string ContentPathTemplate
 
         /// <summary>
         /// True/false whether log files upload request processing should be deferred
-        /// for handling later when a content store is defined. Default = false.
+        /// for handling later when a content store is defined.
         /// </summary>
-        public bool DeferUploads
+        public bool? DeferUpload
         {
             get
             {
-                return this.Parameters.GetValue<bool>(nameof(this.DeferUploads), false);
+                this.Parameters.TryGetValue(nameof(this.DeferUpload), out IConvertible deferUpload);
+                return deferUpload?.ToBoolean(CultureInfo.InvariantCulture);
             }
 
             protected set
             {
-                this.Parameters[nameof(this.DeferUploads)] = false;
+                this.Parameters[nameof(this.DeferUpload)] = value;
+            }
+        }
+
+        /// <summary>
+        /// True/false whether log files should be deleted upon being successfully uploaded.
+        /// </summary>
+        public bool? DeleteOnUpload
+        {
+            get
+            {
+                this.Parameters.TryGetValue(nameof(this.DeleteOnUpload), out IConvertible delete);
+                return delete?.ToBoolean(CultureInfo.InvariantCulture);
+            }
+
+            protected set
+            {
+                this.Parameters[nameof(this.DeleteOnUpload)] = false;
             }
         }
 
@@ -320,6 +339,23 @@ protected set
             }
         }
 
+        /// <summary>
+        /// True/false whether a manifest should be included with file uploads.
+        /// </summary>
+        public bool? IncludeManifestOnUpload
+        {
+            get
+            {
+                this.Parameters.TryGetValue(nameof(this.IncludeManifestOnUpload), out IConvertible includeManifest);
+                return includeManifest?.ToBoolean(CultureInfo.InvariantCulture);
+            }
+
+            protected set
+            {
+                this.Parameters[nameof(this.IncludeManifestOnUpload)] = value;
+            }
+        }
+
         /// <summary>
         /// Metadata provided to the application on the command line.
         /// </summary>

src/VirtualClient/VirtualClient.Contracts/VirtualClientComponentExtensions.cs

@@ -146,6 +146,16 @@ public static FileUploadDescriptor CreateFileUploadDescriptor(this VirtualClient
                 component.ContentPathTemplate,
                 subPath);
 
+            if (component.DeleteOnUpload == true)
+            {
+                descriptor.DeleteOnUpload = true;
+            }
+
+            if (component.IncludeManifestOnUpload == true)
+            {
+                descriptor.IncludeManifest = true;
+            }
+
             return descriptor;
         }
 

src/VirtualClient/VirtualClient.Core.UnitTests/VirtualClientLoggingExtensionsTests.cs

@@ -14,7 +14,9 @@ namespace VirtualClient
     using Moq;
     using NUnit.Framework;
     using VirtualClient.Common;
+    using VirtualClient.Common.Contracts;
     using VirtualClient.Common.Telemetry;
+    using VirtualClient.Contracts;
 
     [TestFixture]
     [Category("Unit")]
@@ -551,5 +553,131 @@ public async Task LogProcessDetailsExtensionLogsToTheExpectedStandardErrorToFile
                 Assert.IsTrue(standardErrorConfirmed, "Standard error not confirmed.");
             }
         }
+
+        [Test]
+        public async Task LogProcessDetailsExtensionUploadsLogFilesByDefaultWhenAContentStoreIsProvided()
+        {
+            bool confirmed = false;
+            Mock<IProcessProxy> mockProcess = new Mock<IProcessProxy>().Setup();
+
+            using (TestExecutor component = new TestExecutor(this))
+            {
+                this.FileSystem
+                    .Setup(fs => fs.File.WriteAllTextAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
+                    .Callback<string, string, CancellationToken>((path, content, token) =>
+                    {
+                        if (path.StartsWith(this.PlatformSpecifics.ContentUploadsDirectory))
+                        {
+                            confirmed = true;
+                            FileUploadDescriptor descriptor = content.FromJson<FileUploadDescriptor>();
+                            Assert.IsNotNull(descriptor);
+                        }
+                    })
+                    .Returns(Task.CompletedTask);
+
+                await component.LogProcessDetailsAsync(mockProcess.Object, EventContext.None, logToTelemetry: false, logToFile: true);
+                Assert.IsTrue(confirmed);
+            }
+        }
+
+        [Test]
+        public async Task LogProcessDetailsExtensionSupportsDeferringLogFileUploads()
+        {
+            bool deferred = true;
+            Mock<IProcessProxy> mockProcess = new Mock<IProcessProxy>().Setup();
+
+            using (TestExecutor component = new TestExecutor(this))
+            {
+                component.Parameters[nameof(component.DeferUpload)] = true;
+
+                this.FileSystem
+                    .Setup(fs => fs.File.WriteAllTextAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
+                    .Callback<string, string, CancellationToken>((path, content, token) =>
+                    {
+                        if (path.StartsWith(this.PlatformSpecifics.ContentUploadsDirectory))
+                        {
+                            deferred = false;
+                        }
+                    })
+                    .Returns(Task.CompletedTask);
+
+                await component.LogProcessDetailsAsync(mockProcess.Object, EventContext.None, logToTelemetry: false, logToFile: true);
+                Assert.IsTrue(deferred);
+            }
+        }
+
+        [Test]
+        public async Task LogProcessDetailsExtensionDeletesLogFilesOnUploadsOnlyWhenExpected()
+        {
+            bool confirmed = false;
+            bool shouldBeDeleted = false;
+            Mock<IProcessProxy> mockProcess = new Mock<IProcessProxy>().Setup();
+
+            using (TestExecutor component = new TestExecutor(this))
+            {
+                // By default, log files should NOT be deleted on upload.
+                this.FileSystem
+                    .Setup(fs => fs.File.WriteAllTextAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
+                    .Callback<string, string, CancellationToken>((path, content, token) =>
+                    {
+                        if (path.StartsWith(this.PlatformSpecifics.ContentUploadsDirectory))
+                        {
+                            confirmed = true;
+                            FileUploadDescriptor descriptor = content.FromJson<FileUploadDescriptor>();
+                            Assert.IsNotNull(descriptor);
+                            Assert.AreEqual(shouldBeDeleted, descriptor.DeleteOnUpload, "Log file delete does not match expectation.");
+                        }
+                    })
+                    .Returns(Task.CompletedTask);
+
+                await component.LogProcessDetailsAsync(mockProcess.Object, EventContext.None, logToTelemetry: false, logToFile: true);
+                Assert.IsTrue(confirmed);
+
+                // If requested, however, log files should be deleted on upload.
+                confirmed = false;
+                shouldBeDeleted = true;
+                component.Parameters[nameof(component.DeleteOnUpload)] = true;
+
+                await component.LogProcessDetailsAsync(mockProcess.Object, EventContext.None, logToTelemetry: false, logToFile: true);
+                Assert.IsTrue(confirmed);
+            }
+        }
+
+        [Test]
+        public async Task LogProcessDetailsExtensionIncludesAManifestWithLogFilesOnUploadsOnlyWhenExpected()
+        {
+            bool confirmed = false;
+            bool shouldBeIncluded = false;
+            Mock<IProcessProxy> mockProcess = new Mock<IProcessProxy>().Setup();
+
+            using (TestExecutor component = new TestExecutor(this))
+            {
+                // By default, a manifest should NOT be included with log file uploads.
+                this.FileSystem
+                    .Setup(fs => fs.File.WriteAllTextAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
+                    .Callback<string, string, CancellationToken>((path, content, token) =>
+                    {
+                        if (path.StartsWith(this.PlatformSpecifics.ContentUploadsDirectory))
+                        {
+                            confirmed = true;
+                            FileUploadDescriptor descriptor = content.FromJson<FileUploadDescriptor>();
+                            Assert.IsNotNull(descriptor);
+                            Assert.AreEqual(shouldBeIncluded, descriptor.IncludeManifest, "Manifest inclusion does not match expectation.");
+                        }
+                    })
+                    .Returns(Task.CompletedTask);
+
+                await component.LogProcessDetailsAsync(mockProcess.Object, EventContext.None, logToTelemetry: false, logToFile: true);
+                Assert.IsTrue(confirmed);
+
+                // If requested, however, log files should be deleted on upload.
+                confirmed = false;
+                shouldBeIncluded = true;
+                component.Parameters[nameof(component.IncludeManifestOnUpload)] = true;
+
+                await component.LogProcessDetailsAsync(mockProcess.Object, EventContext.None, logToTelemetry: false, logToFile: true);
+                Assert.IsTrue(confirmed);
+            }
+        }
     }
 }

src/VirtualClient/VirtualClient.Core/VirtualClientLoggingExtensions.cs

@@ -408,7 +408,7 @@ await RetryPolicies.FileOperations.ExecuteAsync(async () =>
                         await fileSystem.File.WriteAllTextAsync(logFilePath, outputBuilder.ToString());
                     });
 
-                    if (upload && !component.DeferUploads && component.TryGetContentStoreManager(out IBlobManager blobManager))
+                    if (upload && component.DeferUpload != true && component.TryGetContentStoreManager(out IBlobManager blobManager))
                     {
                         string effectiveToolName = component.GetLogDirectoryName(processDetails.ToolName);
 

src/VirtualClient/VirtualClient.Main/profiles/EXECUTE-SCRIPT.json

@@ -27,7 +27,7 @@
                     "Parameters": {
                         "Scenario": "$.Parameters.Scenario",
                         "Command": "$.Parameters.Command",
-                        "DeferUploads": true,
+                        "DeferUpload": true,
                         "EnvironmentVariables": "EXPERIMENT_ID={ExperimentId},EXECUTION_SYSTEM={ExecutionSystem}",
                         "LogFileName": "$.Parameters.LogFileName",
                         "LogFolderName": "$.Parameters.LogFolderName",

src/VirtualClient/VirtualClient.Main/profiles/MONITORS-FILE-UPLOAD.json

@@ -5,15 +5,15 @@
         "SupportedOperatingSystems": "Azure Linux,CentOS,Debian,RedHat,Suse,Ubuntu,Windows"
     },
     "Parameters": {
-        "IncludeManifest": false
+        "DeleteOnUpload": null,
+        "IncludeManifestOnUpload": null
     },
     "Monitors": [
         {
             "Type": "FileUploadMonitor",
             "Parameters": {
-                "Scenario": "UploadContent",
-                "DeleteLogs": false,
-                "IncludeManifest": "$.Parameters.IncludeManifest"
+                "DeleteOnUpload": "$.Parameters.DeleteOnUpload",
+                "IncludeManifestOnUpload": "$.Parameters.IncludeManifestOnUpload"
             }
         }
     ]