dnn_layer_base.h

Go to the documentation of this file.

// Copyright 2019 Claes Rolen (www.rolensystems.com)
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

#pragma once
#include <ios>
#include <iomanip>

namespace dnn
{

class layer
{
protected:
    // Network pointers
    layer* left;                    
    layer* right;                   
    // Misc parameters
    std::string id;                 
    std::string type;               
    arma::uword layer_ix;           
    bool train_par;                 
    bool add_bias;                  
    opt* opt_alg;                   
    PHASE phase;                    
    // Buffer dimensions
    arma::uword N_batch;            
    arma::uword N_rows_left;        
    arma::uword N_cols_left;        
    arma::uword N_channels_left;    
    arma::uword N_rows_right;       
    arma::uword N_cols_right;       
    arma::uword N_channels_right;   
    arma::uword N_left;             
    arma::uword N_right;            
    // Buffers - 'blob'
    arma::Mat<DNN_Dtype> Y;         
    arma::Mat<DNN_Dtype> Y1;        
    arma::Mat<DNN_Dtype> Dleft;     
    arma::Cube<DNN_Dtype> W;        
    arma::Mat<DNN_Dtype> B;         

public:
    layer(void)
    {
        left   = nullptr;
        right  = nullptr;
        id     = "ID";
        type   = "TYPE";
        layer_ix          = 0;
        N_batch           = 0;
        N_rows_left       = 0;
        N_cols_left       = 0;
        N_channels_left   = 0;
        N_rows_right      = 0;
        N_cols_right      = 0;
        N_channels_right  = 0;
        N_left            = 0;
        N_right           = 0;
        train_par         = false;
        add_bias          = true;
        opt_alg           = nullptr;
        phase             = PHASE::TRAIN;
    };

    ~layer() {};

    virtual void init(void)
    {
        N_left           = left->get_nrof_outputs();
        N_cols_left      = left->get_nrof_cols();
        N_rows_left      = left->get_nrof_rows();
        N_channels_left  = left->get_nrof_channels();
    }

    virtual void upd_buf_size(arma::uword nmb)
    {
        N_batch = nmb;
        Y.    zeros(N_right, N_batch);
        Y1.   zeros(N_right, 1);
        Dleft.zeros(N_left, N_batch);
    }

    friend std::ostream& operator<<(std::ostream& os, layer& lay)
    {
        // Dimension string
        std::string n_out_s="["+std::to_string(lay.N_rows_right)+","+
                                std::to_string(lay.N_cols_right)+","+
                                std::to_string(lay.N_channels_right)+"]";
        // Optimizer string
        std::string opt_s="";
        if(lay.opt_alg)
           opt_s = lay.opt_alg->get_algorithm();
        // Number of parameter string
        std::string param_s="";
        arma::uword p=(&lay)->get_nrof_params();
        if(p)
           param_s = std::to_string(p);

        // Put info to stream
        std::ios init(NULL);
        init.copyfmt(std::cout);  // Save stream format

        os << std::left << " "
           << std::setw(6)  << lay.layer_ix
           << std::setw(23) << lay.id
           << std::setw(30) << opt_s
           << std::setw(12) << n_out_s
           << std::setw(10) << param_s;

        std::cout.copyfmt(init); // Restore stream format

        return os;
    }

    virtual void prop(void)
    {
        Y1 = left->get_Y1();
    }

    virtual void prop_mb(void)
    {
        Y = *(left->get_Y_ptr());
    }

    virtual void backprop(void)
    {
        Dleft = *(right->get_Dleft_ptr());      // Back prop
    }

    virtual void update(void)
    {
    }

    virtual void disp(void)
    {
        std::cout << "====================== " << std::endl;
        std::cout << "Type:                 " << type << std::endl;
    }

    virtual void enable_training(void)
    {
        train_par = true;
    }

    virtual void disable_training(void)
    {
        train_par = false;
    }

    virtual void enable_bias(void)
    {
        add_bias = true;
    }

    virtual void disable_bias(void)
    {
        add_bias = false;
    }

    virtual arma::Mat<DNN_Dtype> get_B(void)
    {
        return B;
    }

    virtual arma::Cube<DNN_Dtype> get_W(void)
    {
        return W;
    }

    virtual void set_B(arma::Mat<DNN_Dtype>& b)
    {
        B = b;
    }

    virtual void set_W(arma::Cube<DNN_Dtype>& w)
    {
        W = w;
    }

    virtual arma::uword get_nrof_params(void)
    {
        return 0;
    }

    virtual void set_phase(PHASE p)
    {
        phase = p;
    }

    virtual void set_left(layer* lptr)
    {
        left  = lptr;
    }

    virtual void set_right(layer* rptr)
    {
        right  = rptr;
    }

    virtual void set_id(const std::string str)
    {
        id = str;
    }

    virtual std::string get_id(void)
    {
        return id;
    }

    virtual void set_type(const std::string str)
    {
        type = str;
    }

    virtual std::string get_type(void)
    {
        return type;
    }

    virtual void set_ix(const arma::uword n)
    {
        layer_ix = n;
    }

    virtual arma::uword get_ix(void)
    {
        return layer_ix;
    }

    virtual void set_batch_size(const arma::uword n)
    {
        N_batch = n;
        init();
    }

    virtual arma::uword get_batch_size(void)
    {
        return N_batch;
    }

    virtual void set_Y1(arma::Mat<DNN_Dtype>& y)
    {
        Y1 = y;
    }

    virtual void set_Y(arma::Mat<DNN_Dtype>& y)
    {
        Y = y;
    }

    virtual void set_Dleft(arma::Mat<DNN_Dtype>& d)
    {
        Dleft = d;
    }

    virtual void set_rows_in(arma::uword r)
    {
        N_rows_left = r;
    }

    virtual arma::uword  get_nrof_rows(void)
    {
        return N_rows_right;
    }

    virtual void set_cols_in(arma::uword c)
    {
        N_cols_left = c;
    }

    virtual arma::uword get_nrof_cols(void)
    {
        return N_cols_right;
    }

    virtual void set_channels_in(arma::uword l)
    {
        N_channels_left = l;
    }

    virtual arma::uword get_nrof_channels(void)
    {
        return N_channels_right;
    }

    virtual void set_opt_alg( opt& alg_class)
    {
        opt_alg  = &alg_class;
    }

    virtual opt* get_opt_alg( void)
    {
        return opt_alg;
    }

    virtual arma::uword get_nrof_inputs(void)
    {
        if(left!=NULL)
            return (left->get_nrof_rows())*(left->get_nrof_cols())*(left->get_nrof_channels());
        else
            return 0;
    }

    virtual arma::uword get_nrof_outputs(void)
    {
        return N_rows_right*N_cols_right*N_channels_right;
    }

    virtual arma::Mat<DNN_Dtype> get_Y1(void)
    {
        return Y1;
    }

    virtual arma::Mat<DNN_Dtype>* get_Y1_ptr(void)
    {
        return &Y1;
    }

    virtual DNN_Dtype* get_Y1_memptr(void)
    {
        return Y1.memptr();
    }

    virtual arma::Mat<DNN_Dtype> get_Y(void)
    {
        return Y;
    }

    virtual arma::Mat<DNN_Dtype>* get_Y_ptr(void)
    {
        return &Y;
    }

    virtual DNN_Dtype* get_Y_memptr(void)
    {
        return Y.memptr();
    }

    virtual DNN_Dtype* get_Y_colptr(const arma::uword n)
    {
        return Y.colptr(n);
    }

    virtual arma::Mat<DNN_Dtype> get_Dleft()
    {
        return Dleft;
    }

    virtual arma::Mat<DNN_Dtype>* get_Dleft_ptr(void)
    {
        return &Dleft;
    }

    virtual DNN_Dtype* get_Dleft_memptr(void)
    {
        return Dleft.memptr();
    }

    virtual DNN_Dtype* get_Dleft_colptr(const arma::uword n)
    {
        return Dleft.colptr(n);
    }

    virtual void save_layer_param( std::string name, PARAM_FORMAT f=PARAM_FORMAT::ASCII)
    {
        switch (f)
        {
        case PARAM_FORMAT::ASCII:
            W.save("dnn_w_" + name, arma::arma_ascii);
            B.save("dnn_b_" + name, arma::arma_ascii);
            break;
        case PARAM_FORMAT::BIN:
            W.save("dnn_w_" + name, arma::arma_binary);
            B.save("dnn_b_" + name, arma::arma_binary);
            break;
        default:
            std::cout << "Parameter format not implemented yet!" << std::endl;
        }
    }

    virtual void load_layer_param(std::string name, PARAM_FORMAT f = PARAM_FORMAT::AUTO)
    {
        bool ret = false;
        switch (f)
        {
        case PARAM_FORMAT::AUTO:
            ret = W.load("dnn_w_" + name, arma::auto_detect) &&
                  B.load("dnn_b_" + name, arma::auto_detect);
            break;
        case PARAM_FORMAT::ASCII:
            ret = W.load("dnn_w_" + name, arma::arma_ascii) &&
                  B.load("dnn_b_" + name, arma::arma_ascii);
            break;
        case PARAM_FORMAT::BIN:
            ret = W.load("dnn_w_" + name, arma::arma_binary) &&
                  B.load("dnn_b_" + name, arma::arma_binary);
            break;
        default:
            std::cout << "Parameter format not implemented yet!" << std::endl;
        }
        if (ret != true)
        {
            exit(EXIT_FAILURE);
        }
    }

    template <typename T>
    void init_weights(T& w, arma::uword fan_in, arma::uword fan_out, INIT_W_ALG alg, INIT_W_DIST dist)
    {
        if(dist == INIT_W_DIST::NORMAL)       w = (T) w.randn();
        if(dist == INIT_W_DIST::UNIFORM)      w = (T) w.randu()*((DNN_Dtype)std::sqrt(3.0));

        switch(alg)
        {
        case INIT_W_ALG::LECUN:
            w*=(DNN_Dtype)std::sqrt(1.0/fan_in);
            break;
        case INIT_W_ALG::XAVIER:
            w*= (DNN_Dtype)std::sqrt(2.0/(fan_in+fan_out));
            break;
        case INIT_W_ALG::HE:
            w*= (DNN_Dtype)std::sqrt(2.0/fan_in);
            break;
        case INIT_W_ALG::SIN:
            for(arma::uword k=0; k<w.n_elem; k++ )
            {
                w(k) = (DNN_Dtype)(sin(k*45191.0)/std::sqrt(fan_in+fan_out));
            }
            break;
        }
    }

    DNN_Dtype sigmoid(const DNN_Dtype x)
    {
        return 1/(1+std::exp(-x));
    }

    arma::Mat<DNN_Dtype> sigmoid(const arma::Mat<DNN_Dtype>& x)
    {
        return 1/(1+arma::trunc_exp(-x));
    }

    arma::Mat<DNN_Dtype> softmax(const arma::Mat<DNN_Dtype>& x)
    {
        // Numerical stability improvement by subtracting max before exp.
        arma::Mat<DNN_Dtype> y = arma::trunc_exp(x.each_row()-arma::max(x,0));
        return y.each_row()/arma::sum(y,0);
    }
}; // End class layer
} // End namespace dnn