dnn_layer_act.h

Go to the documentation of this file.

// Copyright 2019 Claes Rolen (www.rolensystems.com)
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

#pragma once
namespace dnn
{

class layer_act: public layer
{
protected:
public:
    layer_act(void):layer() {};
    virtual void prop(void) =0;
    virtual void prop_mb(void) =0;
    virtual void backprop(void) =0;

    virtual void init(void)
    {
        // Get data from left
        layer::init();

        // Set right data to same as previous layer
        N_cols_right     = N_cols_left;
        N_rows_right     = N_rows_left;
        N_channels_right = N_channels_left;
        N_right          = N_left;
    }

    virtual void disp(void)
    {
        layer::disp();
        std::cout << "Nr of outputs:        " << get_nrof_outputs() << " ["<< N_rows_right << ","<<N_cols_right<<","<< N_channels_right<<"]"<< std::endl;
    }

}; // End class layer_act


class act_sigmoid: public layer_act
{
private:
public:

    act_sigmoid(void):layer_act()
    {
        type = "Sigmoid";
        id   = type;
    }

    void prop(void)
    {
        arma::Mat<DNN_Dtype> X1 = left->get_Y1();
        Y1 =  sigmoid(X1);
    }

    void prop_mb(void)
    {
        Y = sigmoid(*(left->get_Y_ptr()));
    }

    void backprop(void)
    {
        Dleft = Y%(1-Y)%(right->get_Dleft());
    }
}; // End class act_sigmoid


class act_softplus: public layer_act
{
private:
public:
    act_softplus(void):layer_act()
    {
        type = "Softplus";
        id   = type;
    }

    void prop(void)
    {
        const arma::Mat<DNN_Dtype> I(N_right,1,arma::fill::ones);
        *(this->get_Y1_ptr()) = arma::trunc_log(I+arma::trunc_exp(*(left->get_Y1_ptr())));
    }

    void prop_mb(void)
    {
        *(this->get_Y_ptr()) = arma::trunc_log(1+arma::trunc_exp(*(left->get_Y_ptr())));
    }

    void backprop(void)
    {
        *(this->get_Dleft_ptr()) = sigmoid(*(this->get_Y_ptr()))%(*(right->get_Dleft_ptr()));
    }
}; // End class act_softplus



class act_ReLU: public layer_act
{
private:
public:
    act_ReLU(void):layer_act()
    {
        type = "ReLU";
        id   = type;
    }

    void prop(void)
    {
        DNN_Dtype* x_ptr = left->get_Y1_memptr();
        DNN_Dtype* y_ptr = this->get_Y1_memptr();
        for(arma::uword n=0; n<N_left; n++)
        {
            *y_ptr++ = (*x_ptr <0) ? (DNN_Dtype)0.0 : *x_ptr;
            x_ptr++;
        }
    }

    void prop_mb(void)
    {
        DNN_Dtype* x_ptr = left->get_Y_memptr();
        DNN_Dtype* y_ptr = this->get_Y_memptr();
        const arma::uword N_ = N_left*N_batch;
        for(arma::uword n=0; n<N_; n++)
        {
            *y_ptr++ = (*x_ptr <0) ? (DNN_Dtype)0.0 : *x_ptr;
            x_ptr++;
        }
    }

    void backprop(void)
    {
        DNN_Dtype* x_ptr = left->get_Y_memptr();
        DNN_Dtype* y_ptr = this->get_Dleft_memptr();
        DNN_Dtype* d_ptr = right->get_Dleft_memptr();
        const arma::uword N_ = N_left*N_batch;
        for(arma::uword n=0; n<N_; n++)
        {
            *y_ptr++ = (*x_ptr<0 ) ? (DNN_Dtype)0.0: *d_ptr;
            d_ptr++;
            x_ptr++;
        }
    }
}; // End class act_ReLU


class act_LReLU: public layer_act
{
private:
    DNN_Dtype alpha=(DNN_Dtype)0.01;
public:
    act_LReLU(void):layer_act()
    {
        type = "Leaky ReLU";
        id   = type;
    }

    void prop(void)
    {
        arma::Mat<DNN_Dtype> X1 = left->get_Y1();
        for(arma::uword n=0; n<X1.n_elem; n++)
        {
            Y1(n) = X1(n)<0 ? X1(n)*alpha: X1(n);
        }
    }

    void prop_mb(void)
    {
        DNN_Dtype* x_ptr = left->get_Y_memptr();
        DNN_Dtype* y_ptr = this->get_Y_memptr();
        const arma::uword N_ = N_left*N_batch;
        for(arma::uword n=0; n<N_; n++)
        {
            *y_ptr++ = (*x_ptr <0) ? *x_ptr*alpha: *x_ptr;
            x_ptr++;
        }
    }

    void backprop(void)
    {
        DNN_Dtype* x_ptr = this->get_Y_memptr();
        DNN_Dtype* y_ptr = this->get_Dleft_memptr();
        DNN_Dtype* d_ptr = right->get_Dleft_memptr();
        const arma::uword N_ = N_left*N_batch;
        for(arma::uword n=0; n<N_; n++)
        {
            *y_ptr++ = (*x_ptr<0 ) ? *d_ptr*alpha: *d_ptr;
            d_ptr++;
            x_ptr++;
        }
    }

    void set_leak(DNN_Dtype a)
    {
        alpha = a;
    }

}; // End class act_LReLU


class act_tanh: public layer_act
{
private:
public:
    act_tanh(void):layer_act()
    {
        type = "tanh";
        id   = type;
    }

    void prop(void)
    {
        Y1 = arma::tanh(*(left->get_Y1_ptr()));
    }

    void prop_mb(void)
    {
        Y = arma::tanh(*(left->get_Y_ptr()));
    }

    void backprop(void)
    {
        Dleft  = (1-arma::square(Y))%(right->get_Dleft());
    }
}; // End class act_tanh


class act_softmax: public layer_act
{
private:
public:
    act_softmax(void):layer_act()
    {
        type = "Softmax";
        id   = type;
    }

    void prop(void)
    {
        Y1 = softmax(*(left->get_Y1_ptr()));
    }

    void prop_mb(void)
    {
        Y = softmax(*(left->get_Y_ptr()));
    }

    void backprop(void)
    {
        arma::Mat<DNN_Dtype> dO=*(right->get_Dleft_ptr());
        for(arma::uword k=0; k<N_batch; k++ )
        {
            arma::Mat<DNN_Dtype> jac = -Y.col(k)*(Y.col(k).t());  // Jacobian
            jac.diag() += Y.col(k);
            Dleft.col(k) = jac*dO.col(k);
        }
    }
}; // End class act_softmax
} // End namespace dnn