From ebb52bb485224781b2bac3580b26731ece196e14 Mon Sep 17 00:00:00 2001 From: Patrick L Date: Fri, 7 Jul 2017 16:48:13 -0400 Subject: [PATCH 1/4] added min function --- include/af/autograd/Functions.hpp | 16 ++++++++++++++++ src/autograd/Functions.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/include/af/autograd/Functions.hpp b/include/af/autograd/Functions.hpp index 5ca6ff7..63439f3 100644 --- a/include/af/autograd/Functions.hpp +++ b/include/af/autograd/Functions.hpp @@ -17,16 +17,28 @@ namespace af { Variable operator *(const Variable &lhs, const Variable &rhs); Variable operator -(const Variable &lhs, const Variable &rhs); Variable operator /(const Variable &lhs, const Variable &rhs); + Variable operator >(const Variable &lhs, const Variable &rhs); + Variable operator <(const Variable &lhs, const Variable &rhs); + Variable operator >=(const Variable &lhs, const Variable &rhs); + Variable operator <=(const Variable &lhs, const Variable &rhs); Variable operator +(const double &lhs, const Variable &rhs); Variable operator *(const double &lhs, const Variable &rhs); Variable operator -(const double &lhs, const Variable &rhs); Variable operator /(const double &lhs, const Variable &rhs); + Variable operator >(const double &lhs, const Variable &rhs); + Variable operator <(const double &lhs, const Variable &rhs); + Variable operator >=(const double &lhs, const Variable &rhs); + Variable operator <=(const double &lhs, const Variable &rhs); Variable operator +(const Variable &lhs, const double &rhs); Variable operator *(const Variable &lhs, const double &rhs); Variable operator -(const Variable &lhs, const double &rhs); Variable operator /(const Variable &lhs, const double &rhs); + Variable operator >(const Variable &lhs, const double &rhs); + Variable operator <(const Variable &lhs, const double &rhs); + Variable operator >=(const Variable &lhs, const double &rhs); + Variable operator <=(const Variable &lhs, const double &rhs); Variable negate(const Variable &input); Variable reciprocal(const Variable &input); @@ -41,6 +53,10 @@ namespace af { Variable max(const Variable &lhs, const double &rhs); Variable max(const double &lhs, const Variable &rhs); + Variable min(const Variable &lhs, const Variable &rhs); + Variable min(const Variable &lhs, const double &rhs); + Variable min(const double &lhs, const Variable &rhs); + Variable transpose(const Variable &input); Variable expandAs(const Variable &input, const Variable &reference); Variable reduceAs(const Variable &input, const Variable &reference); diff --git a/src/autograd/Functions.cpp b/src/autograd/Functions.cpp index 87e0f7f..d85dabd 100644 --- a/src/autograd/Functions.cpp +++ b/src/autograd/Functions.cpp @@ -61,12 +61,26 @@ namespace af { return Variable(result, false); } + Variable operator <(const Variable &lhs, const Variable &rhs) + { + auto result = lhs.array() < rhs.array(); + return Variable(result, false); + } + + Variable operator >=(const Variable &lhs, const Variable &rhs) + { + auto result = lhs.array() >= rhs.array(); + return Variable(result, false); + } + Variable operator <=(const Variable &lhs, const Variable &rhs) { auto result = lhs.array() <= rhs.array(); return Variable(result, false); } + + #define INSTANTIATE_OPERATOR(OP) \ Variable operator OP(const double &lhs_val, const Variable &rhs) \ { \ @@ -91,6 +105,8 @@ namespace af { INSTANTIATE_OPERATOR(*) INSTANTIATE_OPERATOR(/) INSTANTIATE_OPERATOR(>) + INSTANTIATE_OPERATOR(<) + INSTANTIATE_OPERATOR(>=) INSTANTIATE_OPERATOR(<=) #undef INSTANTIATE_OPERATOR @@ -113,6 +129,18 @@ namespace af { return Variable(result, {lhs, rhs, mask}, grad_func); } + Variable min(const Variable &lhs, const Variable &rhs) + { + auto mask = lhs < rhs; + auto result = min(lhs.array(), rhs.array()); + + auto grad_func = [](std::vector &inputs, const Variable &grad_output) { + inputs[0].addGrad( inputs[2] * grad_output); + inputs[1].addGrad(!inputs[2] * grad_output); + }; + return Variable(result, {lhs, rhs, mask}, grad_func); + } + #define INSTANTIATE_FUNCTION(FN) \ Variable FN(const double &lhs_val, const Variable &rhs) \ { \ @@ -134,6 +162,7 @@ namespace af { INSTANTIATE_FUNCTION(max); + INSTANTIATE_FUNCTION(min); #undef INSTANTIATE_FUNCTION From d8a17c7cd25d80259635cc215211967bcf70878e Mon Sep 17 00:00:00 2001 From: Patrick L Date: Mon, 10 Jul 2017 15:40:50 -0400 Subject: [PATCH 2/4] Add PReLU layer --- include/af/nn/Modules/Activations.hpp | 10 ++++++++++ src/nn/Modules/Activations.cpp | 23 ++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/include/af/nn/Modules/Activations.hpp b/include/af/nn/Modules/Activations.hpp index 95beab9..29a68d1 100644 --- a/include/af/nn/Modules/Activations.hpp +++ b/include/af/nn/Modules/Activations.hpp @@ -48,5 +48,15 @@ namespace af autograd::Variable forward(const autograd::Variable &input); }; + + class PReLU : public Module + { + public: + PReLU(int size, double spread = 1.0); + PReLU(const autograd::Variable &w); + + autograd::Variable forward(const autograd::Variable &input); + }; + } } diff --git a/src/nn/Modules/Activations.cpp b/src/nn/Modules/Activations.cpp index 05b9510..9d76322 100644 --- a/src/nn/Modules/Activations.cpp +++ b/src/nn/Modules/Activations.cpp @@ -9,7 +9,7 @@ #include #include - +#include namespace af { namespace nn @@ -46,5 +46,26 @@ namespace af { return max(input, m_slope * input); } + + PReLU::PReLU(int size, double spread) + { + auto w = nn::weight(size, 1, spread); + setParams({w}); + } + + PReLU::PReLU(const Variable &w) : + Module({w}) + { + } + + Variable PReLU::forward(const Variable &input) + { + auto tmp = max(input, 0.0); + auto res = expandAs(m_parameters[0],tmp) * tmp; + //TODO: Determine if doing the max after the mul is preferable + return res; + + } + } } From 1c488443cb443660f1072c1ae89bb9ad914a1381 Mon Sep 17 00:00:00 2001 From: Patrick L Date: Mon, 10 Jul 2017 15:56:12 -0400 Subject: [PATCH 3/4] Add ELU layer --- include/af/nn/Modules/Activations.hpp | 11 +++++++++++ src/nn/Modules/Activations.cpp | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/include/af/nn/Modules/Activations.hpp b/include/af/nn/Modules/Activations.hpp index 29a68d1..5aa9aa1 100644 --- a/include/af/nn/Modules/Activations.hpp +++ b/include/af/nn/Modules/Activations.hpp @@ -58,5 +58,16 @@ namespace af autograd::Variable forward(const autograd::Variable &input); }; + class ELU : public Module + { + private: + double m_alpha; + public: + ELU(double alpha = 1.0); + + autograd::Variable forward(const autograd::Variable &input); + }; + + } } diff --git a/src/nn/Modules/Activations.cpp b/src/nn/Modules/Activations.cpp index 9d76322..27ad3a8 100644 --- a/src/nn/Modules/Activations.cpp +++ b/src/nn/Modules/Activations.cpp @@ -67,5 +67,16 @@ namespace af } + ELU::ELU(double alpha) : + m_alpha(alpha) + { + } + + Variable ELU::forward(const Variable &input) + { + auto res = max(input, m_alpha * (exp(input) - 1)); + return res; + } + } } From e1e64c1dbb5b358dd7bc2c53a8f685e4d62efa7b Mon Sep 17 00:00:00 2001 From: Patrick L Date: Mon, 10 Jul 2017 16:10:41 -0400 Subject: [PATCH 4/4] add ThresholdedReLU layer fixed PReLU, ELU forward methods, mix name changes --- include/af/autograd/Functions.hpp | 2 ++ include/af/nn/Modules/Activations.hpp | 11 +++++++++ src/autograd/Functions.cpp | 32 +++++++++++++-------------- src/nn/Modules/Activations.cpp | 24 +++++++++++++------- 4 files changed, 45 insertions(+), 24 deletions(-) diff --git a/include/af/autograd/Functions.hpp b/include/af/autograd/Functions.hpp index 63439f3..e272f60 100644 --- a/include/af/autograd/Functions.hpp +++ b/include/af/autograd/Functions.hpp @@ -40,6 +40,8 @@ namespace af { Variable operator >=(const Variable &lhs, const double &rhs); Variable operator <=(const Variable &lhs, const double &rhs); + Variable operator !(const Variable &input); + Variable negate(const Variable &input); Variable reciprocal(const Variable &input); diff --git a/include/af/nn/Modules/Activations.hpp b/include/af/nn/Modules/Activations.hpp index 5aa9aa1..2d00a90 100644 --- a/include/af/nn/Modules/Activations.hpp +++ b/include/af/nn/Modules/Activations.hpp @@ -67,6 +67,17 @@ namespace af autograd::Variable forward(const autograd::Variable &input); }; + + class ThresholdReLU : public Module + { + private: + double m_threshold; + public: + ThresholdReLU(double threshold = 1.0); + + autograd::Variable forward(const autograd::Variable &input); + }; + } diff --git a/src/autograd/Functions.cpp b/src/autograd/Functions.cpp index d85dabd..2be0d86 100644 --- a/src/autograd/Functions.cpp +++ b/src/autograd/Functions.cpp @@ -119,26 +119,26 @@ namespace af { Variable max(const Variable &lhs, const Variable &rhs) { - auto mask = lhs > rhs; - auto result = max(lhs.array(), rhs.array()); - - auto grad_func = [](std::vector &inputs, const Variable &grad_output) { - inputs[0].addGrad( inputs[2] * grad_output); - inputs[1].addGrad(!inputs[2] * grad_output); - }; - return Variable(result, {lhs, rhs, mask}, grad_func); + auto mask = lhs > rhs; + auto result = max(lhs.array(), rhs.array()); + + auto grad_func = [](std::vector &inputs, const Variable &grad_output) { + inputs[0].addGrad( inputs[2] * grad_output); + inputs[1].addGrad(!inputs[2] * grad_output); + }; + return Variable(result, {lhs, rhs, mask}, grad_func); } Variable min(const Variable &lhs, const Variable &rhs) { - auto mask = lhs < rhs; - auto result = min(lhs.array(), rhs.array()); - - auto grad_func = [](std::vector &inputs, const Variable &grad_output) { - inputs[0].addGrad( inputs[2] * grad_output); - inputs[1].addGrad(!inputs[2] * grad_output); - }; - return Variable(result, {lhs, rhs, mask}, grad_func); + auto mask = lhs < rhs; + auto result = min(lhs.array(), rhs.array()); + + auto grad_func = [](std::vector &inputs, const Variable &grad_output) { + inputs[0].addGrad( inputs[2] * grad_output); + inputs[1].addGrad(!inputs[2] * grad_output); + }; + return Variable(result, {lhs, rhs, mask}, grad_func); } #define INSTANTIATE_FUNCTION(FN) \ diff --git a/src/nn/Modules/Activations.cpp b/src/nn/Modules/Activations.cpp index 27ad3a8..adedf26 100644 --- a/src/nn/Modules/Activations.cpp +++ b/src/nn/Modules/Activations.cpp @@ -60,22 +60,30 @@ namespace af Variable PReLU::forward(const Variable &input) { - auto tmp = max(input, 0.0); - auto res = expandAs(m_parameters[0],tmp) * tmp; - //TODO: Determine if doing the max after the mul is preferable - return res; - + auto mask = input >= 0.0; + return (input * mask) + (input * !mask * expandAs(m_parameters[0],input)); } ELU::ELU(double alpha) : - m_alpha(alpha) + m_alpha(alpha) { } Variable ELU::forward(const Variable &input) { - auto res = max(input, m_alpha * (exp(input) - 1)); - return res; + auto mask = input >= 0.0; + return (mask * input) + (!mask * m_alpha * (exp(input)-1)); + } + + ThresholdReLU::ThresholdReLU(double threshold) : + m_threshold(threshold) + { + } + + Variable ThresholdReLU::forward(const Variable &input) + { + auto mask = input >= m_threshold; + return input * mask; } }