Skip to content

Add ELU, PReLU, ThresholdedReLU layers #32

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 10, 2017
Merged

Add ELU, PReLU, ThresholdedReLU layers #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ebb52bb
added min function
fohx13 Jul 7, 2017
d8a17c7
Add PReLU layer
fohx13 Jul 10, 2017
1c48844
Add ELU layer
fohx13 Jul 10, 2017
e1e64c1
add ThresholdedReLU layer
fohx13 Jul 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions include/af/autograd/Functions.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,30 @@ namespace af {
Variable operator *(const Variable &lhs, const Variable &rhs);
Variable operator -(const Variable &lhs, const Variable &rhs);
Variable operator /(const Variable &lhs, const Variable &rhs);
Variable operator >(const Variable &lhs, const Variable &rhs);
Variable operator <(const Variable &lhs, const Variable &rhs);
Variable operator >=(const Variable &lhs, const Variable &rhs);
Variable operator <=(const Variable &lhs, const Variable &rhs);

Variable operator +(const double &lhs, const Variable &rhs);
Variable operator *(const double &lhs, const Variable &rhs);
Variable operator -(const double &lhs, const Variable &rhs);
Variable operator /(const double &lhs, const Variable &rhs);
Variable operator >(const double &lhs, const Variable &rhs);
Variable operator <(const double &lhs, const Variable &rhs);
Variable operator >=(const double &lhs, const Variable &rhs);
Variable operator <=(const double &lhs, const Variable &rhs);

Variable operator +(const Variable &lhs, const double &rhs);
Variable operator *(const Variable &lhs, const double &rhs);
Variable operator -(const Variable &lhs, const double &rhs);
Variable operator /(const Variable &lhs, const double &rhs);
Variable operator >(const Variable &lhs, const double &rhs);
Variable operator <(const Variable &lhs, const double &rhs);
Variable operator >=(const Variable &lhs, const double &rhs);
Variable operator <=(const Variable &lhs, const double &rhs);

Variable operator !(const Variable &input);

Variable negate(const Variable &input);
Variable reciprocal(const Variable &input);
Expand All @@ -41,6 +55,10 @@ namespace af {
Variable max(const Variable &lhs, const double &rhs);
Variable max(const double &lhs, const Variable &rhs);

Variable min(const Variable &lhs, const Variable &rhs);
Variable min(const Variable &lhs, const double &rhs);
Variable min(const double &lhs, const Variable &rhs);

Variable transpose(const Variable &input);
Variable expandAs(const Variable &input, const Variable &reference);
Variable reduceAs(const Variable &input, const Variable &reference);
Expand Down
32 changes: 32 additions & 0 deletions include/af/nn/Modules/Activations.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,37 @@ namespace af

autograd::Variable forward(const autograd::Variable &input);
};

class PReLU : public Module
{
public:
PReLU(int size, double spread = 1.0);
PReLU(const autograd::Variable &w);

autograd::Variable forward(const autograd::Variable &input);
};

class ELU : public Module
{
private:
double m_alpha;
public:
ELU(double alpha = 1.0);

autograd::Variable forward(const autograd::Variable &input);
};

class ThresholdReLU : public Module
{
private:
double m_threshold;
public:
ThresholdReLU(double threshold = 1.0);

autograd::Variable forward(const autograd::Variable &input);
};



}
}
45 changes: 37 additions & 8 deletions src/autograd/Functions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,26 @@ namespace af {
return Variable(result, false);
}

Variable operator <(const Variable &lhs, const Variable &rhs)
{
auto result = lhs.array() < rhs.array();
return Variable(result, false);
}

Variable operator >=(const Variable &lhs, const Variable &rhs)
{
auto result = lhs.array() >= rhs.array();
return Variable(result, false);
}

Variable operator <=(const Variable &lhs, const Variable &rhs)
{
auto result = lhs.array() <= rhs.array();
return Variable(result, false);
}



#define INSTANTIATE_OPERATOR(OP) \
Variable operator OP(const double &lhs_val, const Variable &rhs) \
{ \
Expand All @@ -91,6 +105,8 @@ namespace af {
INSTANTIATE_OPERATOR(*)
INSTANTIATE_OPERATOR(/)
INSTANTIATE_OPERATOR(>)
INSTANTIATE_OPERATOR(<)
INSTANTIATE_OPERATOR(>=)
INSTANTIATE_OPERATOR(<=)

#undef INSTANTIATE_OPERATOR
Expand All @@ -103,14 +119,26 @@ namespace af {

Variable max(const Variable &lhs, const Variable &rhs)
{
auto mask = lhs > rhs;
auto result = max(lhs.array(), rhs.array());

auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
inputs[0].addGrad( inputs[2] * grad_output);
inputs[1].addGrad(!inputs[2] * grad_output);
};
return Variable(result, {lhs, rhs, mask}, grad_func);
auto mask = lhs > rhs;
auto result = max(lhs.array(), rhs.array());

auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
inputs[0].addGrad( inputs[2] * grad_output);
inputs[1].addGrad(!inputs[2] * grad_output);
};
return Variable(result, {lhs, rhs, mask}, grad_func);
}

Variable min(const Variable &lhs, const Variable &rhs)
{
auto mask = lhs < rhs;
auto result = min(lhs.array(), rhs.array());

auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
inputs[0].addGrad( inputs[2] * grad_output);
inputs[1].addGrad(!inputs[2] * grad_output);
};
return Variable(result, {lhs, rhs, mask}, grad_func);
}

#define INSTANTIATE_FUNCTION(FN) \
Expand All @@ -134,6 +162,7 @@ namespace af {


INSTANTIATE_FUNCTION(max);
INSTANTIATE_FUNCTION(min);

#undef INSTANTIATE_FUNCTION

Expand Down
42 changes: 41 additions & 1 deletion src/nn/Modules/Activations.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

#include <af/autograd/Functions.hpp>
#include <af/nn/Modules/Activations.hpp>

#include <af/nn/Types.hpp>
namespace af
{
namespace nn
Expand Down Expand Up @@ -46,5 +46,45 @@ namespace af
{
return max(input, m_slope * input);
}

PReLU::PReLU(int size, double spread)
{
auto w = nn::weight(size, 1, spread);
setParams({w});
}

PReLU::PReLU(const Variable &w) :
Module({w})
{
}

Variable PReLU::forward(const Variable &input)
{
auto mask = input >= 0.0;
return (input * mask) + (input * !mask * expandAs(m_parameters[0],input));
}

ELU::ELU(double alpha) :
m_alpha(alpha)
{
}

Variable ELU::forward(const Variable &input)
{
auto mask = input >= 0.0;
return (mask * input) + (!mask * m_alpha * (exp(input)-1));
}

ThresholdReLU::ThresholdReLU(double threshold) :
m_threshold(threshold)
{
}

Variable ThresholdReLU::forward(const Variable &input)
{
auto mask = input >= m_threshold;
return input * mask;
}

}
}