Skip to content

Commit 5936f85

Browse files
Sergio0694Sergio0694
committed
Batch normalization layers adjusted
1 parent c59d64f commit 5936f85

File tree

5 files changed

+42
-18
lines changed

5 files changed

+42
-18
lines changed

‎NeuralNetwork.NET/Networks/Layers/Abstract/BatchNormalizationLayerBase.cs

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
using System;
22
using System.IO;
3+
using System.Runtime.CompilerServices;
34
using JetBrains.Annotations;
45
using NeuralNetworkNET.APIs.Enums;
6+
using NeuralNetworkNET.APIs.Interfaces;
57
using NeuralNetworkNET.APIs.Structs;
68
using NeuralNetworkNET.Extensions;
79
using NeuralNetworkNET.Helpers;
@@ -30,14 +32,20 @@ internal abstract class BatchNormalizationLayerBase : WeightedLayerBase
3032
[NotNull]
3133
public float[] Sigma2 { get; }
3234

33-
// The current iteration number (for the Cumulative Moving Average)
34-
private int _Iteration;
35+
/// <summary>
36+
/// Gets the current iteration number (for the Cumulative Moving Average)
37+
/// </summary>
38+
public int Iteration { get; private set; }
3539

3640
/// <summary>
3741
/// Gets the current CMA factor used to update the <see cref="Mu"/> and <see cref="Sigma2"/> tensors
3842
/// </summary>
3943
[JsonProperty(nameof(CumulativeMovingAverageFactor), Order = 6)]
40-
public float CumulativeMovingAverageFactor => 1f / (1 + _Iteration);
44+
public float CumulativeMovingAverageFactor
45+
{
46+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
47+
get => 1f / (1 + Iteration);
48+
}
4149

4250
/// <inheritdoc/>
4351
public override String Hash => Convert.ToBase64String(Sha256.Hash(Weights, Biases, Mu, Sigma2));
@@ -74,24 +82,26 @@ protected BatchNormalizationLayerBase(in TensorInfo shape, NormalizationMode mod
7482
NormalizationMode = mode;
7583
}
7684

77-
protected BatchNormalizationLayerBase(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation)
85+
protected BatchNormalizationLayerBase(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, int iteration, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation)
7886
: base(shape, shape, w, b, activation)
7987
{
8088
if (w.Length != b.Length) throw new ArgumentException("The size for both gamme and beta paarameters must be the same");
8189
if (mode == NormalizationMode.Spatial && w.Length != shape.Channels ||
8290
mode == NormalizationMode.PerActivation && w.Length != shape.Size)
8391
throw new ArgumentException("Invalid parameters size for the selected normalization mode");
92+
if (iteration < 0) throw new ArgumentOutOfRangeException(nameof(iteration), "The iteration value must be aat least equal to 0");
8493
if (mu.Length != w.Length || sigma2.Length != w.Length)
8594
throw new ArgumentException("The mu and sigma2 parameters must match the shape of the gamma and beta parameters");
8695
NormalizationMode = mode;
96+
Iteration = iteration;
8797
Mu = mu;
8898
Sigma2 = sigma2;
8999
}
90100

91101
/// <inheritdoc/>
92102
public override void Forward(in Tensor x, out Tensor z, out Tensor a)
93103
{
94-
if (NetworkTrainer.BackpropagationInProgress) ForwardTraining(1f / (1 + _Iteration++), x, out z, out a);
104+
if (NetworkTrainer.BackpropagationInProgress) ForwardTraining(1f / (1 + Iteration++), x, out z, out a);
95105
else ForwardInference(x, out z, out a);
96106
}
97107

@@ -112,11 +122,22 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)
112122
/// <param name="a">The output activation on the current layer</param>
113123
public abstract void ForwardTraining(float factor, in Tensor x, out Tensor z, out Tensor a);
114124

125+
/// <inheritdoc/>
126+
public override bool Equals(INetworkLayer other)
127+
{
128+
if (!base.Equals(other)) return false;
129+
return other is BatchNormalizationLayerBase layer &&
130+
Iteration == layer.Iteration &&
131+
Mu.ContentEquals(layer.Mu) &&
132+
Sigma2.ContentEquals(layer.Sigma2);
133+
}
134+
115135
/// <inheritdoc/>
116136
public override void Serialize(Stream stream)
117137
{
118138
base.Serialize(stream);
119139
stream.Write(NormalizationMode);
140+
stream.Write(Iteration);
120141
stream.Write(Mu.Length);
121142
stream.WriteShuffled(Mu);
122143
stream.Write(Sigma2.Length);

‎NeuralNetwork.NET/Networks/Layers/Cpu/BatchNormalizationLayer.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ internal sealed class BatchNormalizationLayer : BatchNormalizationLayerBase
1818
public BatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, ActivationType activation)
1919
: base(shape, mode, activation) { }
2020

21-
public BatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation)
22-
: base(shape, mode, w, b, mu, sigma2, activation) { }
21+
public BatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, int iteration, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation)
22+
: base(shape, mode, w, b, iteration, mu, sigma2, activation) { }
2323

2424
#region Implementation
2525

@@ -98,14 +98,15 @@ public static INetworkLayer Deserialize([NotNull] Stream stream)
9898
if (!stream.TryRead(out int bLength)) return null;
9999
float[] biases = stream.ReadUnshuffled(bLength);
100100
if (!stream.TryRead(out NormalizationMode mode)) return null;
101+
if (!stream.TryRead(out int iteration)) return null;
101102
if (!stream.TryRead(out int mLength)) return null;
102103
float[] mu = stream.ReadUnshuffled(mLength);
103104
if (!stream.TryRead(out int sLength)) return null;
104105
float[] sigma2 = stream.ReadUnshuffled(sLength);
105-
return new BatchNormalizationLayer(input, mode, weights, biases, mu, sigma2, activation);
106+
return new BatchNormalizationLayer(input, mode, weights, biases, iteration, mu, sigma2, activation);
106107
}
107108

108109
/// <inheritdoc/>
109-
public override INetworkLayer Clone() => new BatchNormalizationLayer(InputInfo, NormalizationMode, Weights.AsSpan().Copy(), Biases.AsSpan().Copy(), Mu.AsSpan().Copy(), Sigma2.AsSpan().Copy(), ActivationType);
110+
public override INetworkLayer Clone() => new BatchNormalizationLayer(InputInfo, NormalizationMode, Weights.AsSpan().Copy(), Biases.AsSpan().Copy(), Iteration, Mu.AsSpan().Copy(), Sigma2.AsSpan().Copy(), ActivationType);
110111
}
111112
}

‎NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnBatchNormalizationLayer.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ public CuDnnBatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode,
5252
SetupCuDnnInfo();
5353
}
5454

55-
public CuDnnBatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation)
56-
: base(shape, mode, w, b, mu, sigma2, activation)
55+
public CuDnnBatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, int iteration, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation)
56+
: base(shape, mode, w, b, iteration, mu, sigma2, activation)
5757
{
5858
Tensor.NewZeroed(1, Mu.Length, out SaveMean);
5959
Tensor.NewZeroed(1, Mu.Length, out SaveInvVariance);
@@ -158,15 +158,16 @@ public static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
158158
if (!stream.TryRead(out int bLength)) return null;
159159
float[] biases = stream.ReadUnshuffled(bLength);
160160
if (!stream.TryRead(out NormalizationMode mode)) return null;
161+
if (!stream.TryRead(out int iteration)) return null;
161162
if (!stream.TryRead(out int mLength)) return null;
162163
float[] mu = stream.ReadUnshuffled(mLength);
163164
if (!stream.TryRead(out int sLength)) return null;
164165
float[] sigma2 = stream.ReadUnshuffled(sLength);
165-
return new CuDnnBatchNormalizationLayer(input, mode, weights, biases, mu, sigma2, activation);
166+
return new CuDnnBatchNormalizationLayer(input, mode, weights, biases, iteration, mu, sigma2, activation);
166167
}
167168

168169
/// <inheritdoc/>
169-
public override INetworkLayer Clone() => new CuDnnBatchNormalizationLayer(InputInfo, NormalizationMode, Weights.AsSpan().Copy(), Biases.AsSpan().Copy(), Mu.AsSpan().Copy(), Sigma2.AsSpan().Copy(), ActivationType);
170+
public override INetworkLayer Clone() => new CuDnnBatchNormalizationLayer(InputInfo, NormalizationMode, Weights.AsSpan().Copy(), Biases.AsSpan().Copy(), Iteration, Mu.AsSpan().Copy(), Sigma2.AsSpan().Copy(), ActivationType);
170171

171172
#region IDisposable
172173

‎Unit/NeuralNetwork.NET.Cuda.Unit/CuDnnLayersTest.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ public void PerActivationBatchNormalizationForward()
176176
{
177177
BatchNormalizationLayerBase
178178
cpu = new BatchNormalizationLayer(TensorInfo.Linear(250), NormalizationMode.PerActivation, ActivationType.ReLU),
179-
gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.PerActivation, cpu.Weights, cpu.Biases, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
179+
gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.PerActivation, cpu.Weights, cpu.Biases, cpu.Iteration, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
180180
TestForward(cpu, gpu, 400);
181181
}
182182

@@ -185,7 +185,7 @@ public void PerActivationBatchNormalizationBackward()
185185
{
186186
BatchNormalizationLayerBase
187187
cpu = new BatchNormalizationLayer(TensorInfo.Linear(250), NormalizationMode.PerActivation, ActivationType.ReLU),
188-
gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.PerActivation, cpu.Weights, cpu.Biases, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
188+
gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.PerActivation, cpu.Weights, cpu.Biases, cpu.Iteration, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
189189
TestBackward(cpu, gpu, 400);
190190
}
191191

@@ -194,7 +194,7 @@ public void SpatialBatchNormalizationForward()
194194
{
195195
BatchNormalizationLayerBase
196196
cpu = new BatchNormalizationLayer(TensorInfo.Volume(12, 12, 13), NormalizationMode.Spatial, ActivationType.ReLU),
197-
gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.Spatial, cpu.Weights, cpu.Biases, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
197+
gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.Spatial, cpu.Weights, cpu.Biases, cpu.Iteration, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
198198
TestForward(cpu, gpu, 400);
199199
}
200200

@@ -203,7 +203,7 @@ public void SpatialBatchNormalizationBackward()
203203
{
204204
BatchNormalizationLayerBase
205205
cpu = new BatchNormalizationLayer(TensorInfo.Volume(12, 12, 13), NormalizationMode.Spatial, ActivationType.ReLU),
206-
gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.Spatial, cpu.Weights, cpu.Biases, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
206+
gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.Spatial, cpu.Weights, cpu.Biases, cpu.Iteration, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
207207
TestBackward(cpu, gpu, 400);
208208
}
209209

‎Unit/NeuralNetwork.NET.Unit/SerializationTest.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ public void NetworkSerialization()
6969
NetworkLayers.Convolutional((10, 10), 20, ActivationType.Identity),
7070
NetworkLayers.Pooling(ActivationType.ReLU),
7171
NetworkLayers.Convolutional((10, 10), 20, ActivationType.Identity),
72-
NetworkLayers.Pooling(ActivationType.ReLU),
72+
NetworkLayers.Pooling(ActivationType.Identity),
73+
NetworkLayers.BatchNormalization(NormalizationMode.Spatial, ActivationType.ReLU),
7374
NetworkLayers.FullyConnected(125, ActivationType.Tanh),
7475
NetworkLayers.Softmax(133));
7576
using (MemoryStream stream = new MemoryStream())

0 commit comments

Comments
 (0)