Merge pull request #79 from Sergio0694/feature_batch-normalization

Feature batch normalization

Author: Sergio0694

NeuralNetwork.NET/APIs/CuDnnNetworkLayers.cs

@@ -1,10 +1,8 @@
-using System;
-using System.Linq;
-using JetBrains.Annotations;
+using JetBrains.Annotations;
 using NeuralNetworkNET.APIs.Delegates;
 using NeuralNetworkNET.APIs.Enums;
 using NeuralNetworkNET.APIs.Structs;
-using NeuralNetworkNET.Extensions;
+using NeuralNetworkNET.cuDNN;
 using NeuralNetworkNET.Networks.Layers.Cuda;
 
 namespace NeuralNetworkNET.APIs
@@ -17,22 +15,7 @@ public static class CuDnnNetworkLayers
         /// <summary>
         /// Gets whether or not the Cuda acceleration is supported on the current system
         /// </summary>
-        public static bool IsCudaSupportAvailable
-        {
-            get
-            {
-                try
-                {
-                    // Calling this directly would could a crash in the <Module> loader due to the missing .dll files
-                    return CuDnnSupportHelper.IsGpuAccelerationSupported();
-                }
-                catch (TypeInitializationException)
-                {
-                    // Missing .dll file
-                    return false;
-                }
-            }
-        }
+        public static bool IsCudaSupportAvailable => CuDnnService.IsAvailable;
 
         /// <summary>
         /// Creates a new fully connected layer with the specified number of input and output neurons, and the given activation function
@@ -132,41 +115,14 @@ public static LayerFactory Convolutional(
         public static LayerFactory Inception(InceptionInfo info, BiasInitializationMode biasMode = BiasInitializationMode.Zero) 
             => input => new CuDnnInceptionLayer(input, info, biasMode);
 
-        #region Feature helper
-
         /// <summary>
-        /// A private class that is used to create a new standalone type that contains the actual test method (decoupling is needed to &lt;Module&gt; loading crashes)
+        /// Creates a new batch normalization layer
         /// </summary>
-        private static class CuDnnSupportHelper
-        {
-            /// <summary>
-            /// Checks whether or not the Cuda features are currently supported
-            /// </summary>
-            public static bool IsGpuAccelerationSupported()
-            {
-                try
-                {
-                    // CUDA test
-                    Alea.Gpu gpu = Alea.Gpu.Default;
-                    if (gpu == null) return false;
-                    if (!Alea.cuDNN.Dnn.IsAvailable) return false; // cuDNN
-                    using (Alea.DeviceMemory<float> sample_gpu = gpu.AllocateDevice<float>(1024))
-                    {
-                        Alea.deviceptr<float> ptr = sample_gpu.Ptr;
-                        void Kernel(int i) => ptr[i] = i;
-                        Alea.Parallel.GpuExtension.For(gpu, 0, 1024, Kernel); // JIT test
-                        float[] sample = Alea.Gpu.CopyToHost(sample_gpu);
-                        return Enumerable.Range(0, 1024).Select<int, float>(i => i).ToArray().ContentEquals(sample);
-                    }
-                }
-                catch
-                {
-                    // Missing .dll or other errors
-                    return false;
-                }
-            }
-        }
-
-        #endregion
+        /// <param name="mode">The normalization mode to use for the new layer</param>
+        /// <param name="activation">The desired activation function to use in the network layer</param>
+        [PublicAPI]
+        [Pure, NotNull]
+        public static LayerFactory BatchNormalization(NormalizationMode mode, ActivationType activation) 
+            => input => new CuDnnBatchNormalizationLayer(input, mode, activation);
     }
 }

NeuralNetwork.NET/APIs/Enums/LayerType.cs

@@ -30,6 +30,11 @@ public enum LayerType : byte
         /// </summary>
         Softmax,
 
+        /// <summary>
+        /// A batch normalization layer, used to scale the input batch into a 0-mean, 1-variance activations map
+        /// </summary>
+        BatchNormalization,
+
         /// <summary>
         /// An inception module, combining different kinds of convolution with a pooling operation
         /// </summary>

NeuralNetwork.NET/APIs/Enums/NormalizationMode.cs

@@ -0,0 +1,18 @@
+namespace NeuralNetworkNET.APIs.Enums
+{
+    /// <summary>
+    /// An <see langword="enum"/> indicating the normalization mode to apply to the input data of a layer
+    /// </summary>
+    public enum NormalizationMode : byte
+    {
+        /// <summary>
+        /// Activation-wise normalization, with a separate mean and variance value per activation
+        /// </summary>
+        PerActivation = 0,
+
+        /// <summary>
+        /// Spatial normalization, with a single mean and variance value per input channel (feature map)
+        /// </summary>
+        Spatial = 1
+    }
+}

NeuralNetwork.NET/APIs/NetworkLayers.cs

@@ -75,5 +75,15 @@ public static LayerFactory Convolutional(
         [PublicAPI]
         [Pure, NotNull]
         public static LayerFactory Pooling(ActivationType activation) => input => new PoolingLayer(input, PoolingInfo.Default, activation);
+
+        /// <summary>
+        /// Creates a new batch normalization layer
+        /// </summary>
+        /// <param name="mode">The normalization mode to use for the new layer</param>
+        /// <param name="activation">The desired activation function to use in the network layer</param>
+        [PublicAPI]
+        [Pure, NotNull]
+        public static LayerFactory BatchNormalization(NormalizationMode mode, ActivationType activation)
+            => input => new BatchNormalizationLayer(input, mode, activation);
     }
 }

NeuralNetwork.NET/APIs/NetworkLoader.cs

@@ -90,6 +90,7 @@ internal static INetworkLayer CpuLayerDeserialize([NotNull] Stream stream, Layer
                 case LayerType.Pooling: return PoolingLayer.Deserialize(stream);
                 case LayerType.Output: return OutputLayer.Deserialize(stream);
                 case LayerType.Softmax: return SoftmaxLayer.Deserialize(stream);
+                case LayerType.BatchNormalization: return BatchNormalizationLayer.Deserialize(stream);
                 default: throw new ArgumentOutOfRangeException(nameof(type), $"The {type} layer type is not supported by the default deserializer");
             }
         }

NeuralNetwork.NET/APIs/NetworkManager.cs

@@ -8,6 +8,7 @@
 using NeuralNetworkNET.APIs.Interfaces;
 using NeuralNetworkNET.APIs.Interfaces.Data;
 using NeuralNetworkNET.APIs.Results;
+using NeuralNetworkNET.APIs.Settings;
 using NeuralNetworkNET.APIs.Structs;
 using NeuralNetworkNET.Extensions;
 using NeuralNetworkNET.Networks.Graph;
@@ -141,13 +142,18 @@ private static TrainingSessionResult TrainNetworkCore(
             if (dropout < 0 || dropout >= 1) throw new ArgumentOutOfRangeException(nameof(dropout), "The dropout probability is invalid");
 
             // Start the training
-            return NetworkTrainer.TrainNetwork(
+            NetworkSettings.TrainingInProgress = NetworkSettings.TrainingInProgress
+                ? throw new InvalidOperationException("Can't train two networks at the same time") // This would cause problems with cuDNN
+                : true;
+            TrainingSessionResult result = NetworkTrainer.TrainNetwork(
                 network as NeuralNetworkBase ?? throw new ArgumentException("The input network instance isn't valid", nameof(network)), 
                 dataset as BatchesCollection ?? throw new ArgumentException("The input dataset instance isn't valid", nameof(dataset)),
                 epochs, dropout, algorithm, batchProgress, trainingProgress, 
                 validationDataset as ValidationDataset,
                 testDataset as TestDataset,
                 token);
+            NetworkSettings.TrainingInProgress = false;
+            return result;
         }
     }
 }

NeuralNetwork.NET/APIs/Settings/NetworkSettings.cs

@@ -35,5 +35,15 @@ public static AccuracyTester AccuracyTester
             get => _AccuracyTester;
             set => _AccuracyTester = value ?? throw new ArgumentNullException(nameof(AccuracyTester), "The input delegate can't be null");
         }
+
+        /// <summary>
+        /// Gets whether or not a neural network is currently being trained
+        /// </summary>
+        public static bool TrainingInProgress { get; internal set; }
+
+        /// <summary>
+        /// Gets whether or not a neural network is currently processing the training samples through backpropagation (as opposed to evaluating them)
+        /// </summary>
+        internal static bool BackpropagationInProgress { get; set; }
     }
 }

NeuralNetwork.NET/APIs/Structs/Tensor.cs

@@ -340,7 +340,7 @@ public static unsafe void TryFree([NotNull] params Tensor[] tensors)
         /// <summary>
         /// A proxy type to debug instances of the <see cref="Tensor"/> <see langword="struct"/>
         /// </summary>
-        private struct _TensorProxy
+        private readonly struct _TensorProxy
         {
             /// <summary>
             /// Gets a preview of the underlying memory area wrapped by this instance
@@ -352,7 +352,7 @@ private struct _TensorProxy
 
             private const int MaximumRowsCount = 10;
 
-            private const int MaximumItemsCount = 40000;
+            private const int MaximumItemsCount = 30000;
 
             [SuppressMessage("ReSharper", "UnusedMember.Local")]
             public _TensorProxy(Tensor obj)

NeuralNetwork.NET/Extensions/DebugExtensions.cs

@@ -22,7 +22,13 @@ public static unsafe bool ContentEquals(this Span<float> x1, Span<float> x2, flo
             fixed (float* p1 = &x1.DangerousGetPinnableReference(), p2 = &x2.DangerousGetPinnableReference())
             {
                 for (int i = 0; i < x1.Length; i++)
-                    if (!p1[i].EqualsWithDelta(p2[i], absolute, relative)) return false;
+                    if (!p1[i].EqualsWithDelta(p2[i], absolute, relative))
+                    {
+                        #if DEBUG
+                        System.Diagnostics.Debug.WriteLine($"[NO MATCH] {p1[i]} | {p2[i]} | diff: {(p1[i] - p2[i]).Abs()}");
+                        #endif
+                        return false;
+                    }
             }
             return true;
         }

NeuralNetwork.NET/Extensions/MiscExtensions.cs

@@ -79,6 +79,30 @@ public static ref T SwitchRef<T>(this bool flag, ref T left, ref T right)
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static float Abs(this float value) => value >= 0 ? value : -value;
 
+        /// <summary>
+        /// Returns the minimum possible upper <see cref="float"/> approximation of the given <see cref="double"/> value
+        /// </summary>
+        /// <param name="value">The value to approximate</param>
+        public static unsafe float ToApproximatedFloat(this double value)
+        {
+            // Get the bit representation of the double value
+            ulong bits = *((ulong*)&value);
+
+            // Extract and re-bias the exponent field
+            ulong exponent = ((bits >> 52) & 0x7FF) - 1023 + 127;
+
+            // Extract the significand bits and truncate the excess
+            ulong significand = (bits >> 29) & 0x7FFFFF;
+
+            // Assemble the result in 32-bit unsigned integer format, then add 1
+            ulong converted = (((bits >> 32) & 0x80000000u)
+                               | (exponent << 23)
+                               | significand) + 1;
+
+            // Reinterpret the bit pattern as a float
+            return *((float*)&converted);
+        }
+
         /// <summary>
         /// Calculates if two values are within a given distance from one another
         /// </summary>