add ThresholdedReLU layer

fixed PReLU, ELU forward methods, mix name changes

Author: fohx13

include/af/autograd/Functions.hpp

@@ -40,6 +40,8 @@ namespace af {
         Variable operator >=(const Variable &lhs, const double &rhs);
         Variable operator <=(const Variable &lhs, const double &rhs);
 
+        Variable operator !(const Variable &input);
+
         Variable negate(const Variable &input);
         Variable reciprocal(const Variable &input);
 

include/af/nn/Modules/Activations.hpp

@@ -67,6 +67,17 @@ namespace af
 
             autograd::Variable forward(const autograd::Variable &input);
         };
+
+        class ThresholdReLU : public Module
+        {
+        private:
+            double m_threshold;
+        public:
+            ThresholdReLU(double threshold = 1.0);
+         
+            autograd::Variable forward(const autograd::Variable &input);
+        };
+
 
 
     }

src/autograd/Functions.cpp

@@ -119,26 +119,26 @@ namespace af {
 
         Variable max(const Variable &lhs, const Variable &rhs)
         {
-          auto mask = lhs > rhs;
-          auto result = max(lhs.array(), rhs.array());
-
-          auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
-            inputs[0].addGrad( inputs[2] * grad_output);
-            inputs[1].addGrad(!inputs[2] * grad_output);
-          };
-          return Variable(result, {lhs, rhs, mask}, grad_func);
+            auto mask = lhs > rhs;
+            auto result = max(lhs.array(), rhs.array());
+
+            auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
+                inputs[0].addGrad( inputs[2] * grad_output);
+                inputs[1].addGrad(!inputs[2] * grad_output);
+            };
+            return Variable(result, {lhs, rhs, mask}, grad_func);
         }
 
         Variable min(const Variable &lhs, const Variable &rhs)
         {
-          auto mask = lhs < rhs;
-          auto result = min(lhs.array(), rhs.array());
-
-          auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
-            inputs[0].addGrad( inputs[2] * grad_output);
-            inputs[1].addGrad(!inputs[2] * grad_output);
-          };
-          return Variable(result, {lhs, rhs, mask}, grad_func);
+            auto mask = lhs < rhs;
+            auto result = min(lhs.array(), rhs.array());
+
+            auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
+              inputs[0].addGrad( inputs[2] * grad_output);
+              inputs[1].addGrad(!inputs[2] * grad_output);
+            };
+            return Variable(result, {lhs, rhs, mask}, grad_func);
         }
 
 #define INSTANTIATE_FUNCTION(FN)                                        \

src/nn/Modules/Activations.cpp

@@ -60,22 +60,30 @@ namespace af
 
         Variable PReLU::forward(const Variable &input)
         {
-            auto tmp = max(input, 0.0);
-            auto res = expandAs(m_parameters[0],tmp) * tmp;
-            //TODO: Determine if doing the max after the mul is preferable
-            return res;
-
+            auto mask = input >= 0.0;
+            return (input * mask) + (input * !mask * expandAs(m_parameters[0],input));
         }
 
         ELU::ELU(double alpha) :
-          m_alpha(alpha)
+            m_alpha(alpha)
         {
         }
 
         Variable ELU::forward(const Variable &input)
         {
-            auto res = max(input, m_alpha * (exp(input) - 1));
-            return res;
+            auto mask = input >= 0.0;
+            return (mask * input) + (!mask * m_alpha * (exp(input)-1));
+        }
+
+        ThresholdReLU::ThresholdReLU(double threshold) :
+            m_threshold(threshold)
+        {
+        }
+
+        Variable ThresholdReLU::forward(const Variable &input)
+        {
+            auto mask = input >= m_threshold;
+            return input * mask;
         }
 
     }