Merge pull request #117 from cbovar/Dropout

Dropout fix in Flow

Author: cbovar

src/ConvNetSharp.Flow/Net.cs

@@ -35,6 +35,12 @@ public T Backward(Volume<T> y)
             throw new NotImplementedException();
         }
 
+        /// <summary>
+        /// Creates a dictionary containing input and evaluates the output Op
+        /// </summary>
+        /// <param name="input">input</param>
+        /// <param name="isTraining">isTraining has no use here.</param>
+        /// <returns></returns>
         public Volume<T> Forward(Volume<T> input, bool isTraining = false)
         {
             this._dico["input"] = input;

src/ConvNetSharp.Flow/Ops/DropoutGradient.cs

@@ -33,7 +33,7 @@ public override Volume<T> Evaluate(Session<T> session)
 
             var dropoutOutput = this._dropout.Evaluate(session);
             var dropoutInput = this._dropout.Parents[0].Evaluate(session);
-            var dropoutInputGradient = this._dropout.Derivate.Evaluate(session);
+            var dropoutOutputGradient = this._dropout.Derivate.Evaluate(session);
             var droupoutProb = this._dropout.DropoutProbability.Evaluate(session);
 
             if (this.Result == null || !Equals(this._lastInputShape, dropoutInput.Shape))
@@ -44,7 +44,7 @@ public override Volume<T> Evaluate(Session<T> session)
                 this.Result = BuilderInstance<T>.Volume.SameAs(dropoutInput.Shape);
             }
 
-            dropoutOutput.DoDropoutGradient(dropoutInput, this.Result, dropoutInputGradient, droupoutProb);
+            dropoutOutput.DoDropoutGradient(dropoutInput, dropoutOutputGradient, this.Result, droupoutProb);
 
             return base.Evaluate(session);
         }

src/ConvNetSharp.Flow/Session.cs

@@ -126,7 +126,12 @@ public void UpdatePlaceHolder(Op<T> fun, Dictionary<string, Volume<T>> dictionar
             {
                 if (op is PlaceHolder<T> placeHolder)
                 {
-                    placeHolder.SetValue(dictionary[placeHolder.Name]);
+                    if (!dictionary.TryGetValue(placeHolder.Name, out var volume))
+                    {
+                        throw new Exception($"Cannot find key '{placeHolder.Name}' in the provided dictionary");
+                    }
+                   
+                    placeHolder.SetValue(volume);
                 }
 
                 if (op is Variable<T> variable && variable.IsLearnable)

src/ConvNetSharp.Volume.Tests/VolumeTests.cs

@@ -1184,19 +1184,42 @@ public void ToArray()
             }
         }
 
+        /// <summary>
+        /// Dropout should let the volume go thru if drop probability is 0
+        /// </summary>
+        [TestMethod]
+        public void DropoutWith0Dropprob()
+        {
+            var volume = NewVolume(RandomUtilities.RandomDoubleArray(100), new Shape(100));
+            var result = NewVolume(new double[100], new Shape(100));
+            var dropprob = (T)Convert.ChangeType(0.0, typeof(T));
+
+            // Forward
+            volume.DoDropout(result, dropprob);
+            Assert.IsTrue(volume.ToArray().SequenceEqual(result.ToArray()));
+
+            // Backward
+            var inputGradient = BuilderInstance<T>.Volume.SameAs(volume.Storage, volume.Shape);
+            var outputActivationGradient = NewVolume(new double[100].Populate(1.0), new Shape(100));
+            volume.DoDropoutGradient(volume, outputActivationGradient, inputGradient, dropprob);
+
+            Assert.IsTrue(inputGradient.ToArray().SequenceEqual(outputActivationGradient.ToArray()));
+        }
+
         [TestMethod]
         public void Dropout()
         {
             var volume = NewVolume(new double[100].Populate(1.0), new Shape(100));
             var result = NewVolume(new double[100], new Shape(100));
 
-            var dropProb = 0.5;
+            var dropProb = 0.0;
             volume.DoDropout(result, (T)Convert.ChangeType(dropProb, typeof(T)));
 
             var array = result.Storage.ToArray();
             var c = array.Count(o => o.Equals(Ops<T>.Zero));
-            Assert.IsTrue(c > 0);
+            Assert.IsTrue(dropProb > 0 ? c > 0 : c >= 0);
 
+            // Check magnitude scale up
             var nonZeroEntry = array.First(o => !o.Equals(Ops<T>.Zero));
             AssertNumber.AreEqual(1.0 / (1 - dropProb), nonZeroEntry, 1e-6);
 

src/ConvNetSharp.Volume/Double/Volume.cs

@@ -266,13 +266,13 @@ public override void DoDivide(Volume<double> other, Volume<double> result)
 
         public override void DoDropout(Volume<double> result, double dropProbability)
         {
-            if (dropProbability > 0.0)
+            if (((NcwhVolumeStorage<double>)this.Storage).Dropped == null || ((NcwhVolumeStorage<double>)this.Storage).Dropped.Length != this.Shape.TotalLength)
             {
-                if (((NcwhVolumeStorage<double>)this.Storage).Dropped == null || ((NcwhVolumeStorage<double>)this.Storage).Dropped.Length != this.Shape.TotalLength)
-                {
-                    ((NcwhVolumeStorage<double>)this.Storage).Dropped = new bool[this.Shape.TotalLength];
-                }
+                ((NcwhVolumeStorage<double>)this.Storage).Dropped = new bool[this.Shape.TotalLength];
+            }
 
+            if (dropProbability > 0.0)
+            {
                 // do dropout
                 this.Storage.Map((x, i) =>
                 {

src/ConvNetSharp.Volume/Single/Volume.cs

@@ -267,13 +267,13 @@ public override void DoDivide(Volume<float> other, Volume<float> result)
 
         public override void DoDropout(Volume<float> result, float dropProbability)
         {
-            if (dropProbability > 0.0f)
+            if (((NcwhVolumeStorage<float>)this.Storage).Dropped == null || ((NcwhVolumeStorage<float>)this.Storage).Dropped.Length != this.Shape.TotalLength)
             {
-                if (((NcwhVolumeStorage<float>)this.Storage).Dropped == null || ((NcwhVolumeStorage<float>)this.Storage).Dropped.Length != this.Shape.TotalLength)
-                {
-                    ((NcwhVolumeStorage<float>)this.Storage).Dropped = new bool[this.Shape.TotalLength];
-                }
+                ((NcwhVolumeStorage<float>)this.Storage).Dropped = new bool[this.Shape.TotalLength];
+            }
 
+            if (dropProbability > 0.0f)
+            {
                 // do dropout
                 this.Storage.Map((x, i) =>
                 {