dnn_layer_conv.h

Go to the documentation of this file.

// Copyright 2019 Claes Rolen (www.rolensystems.com)
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

#pragma once
namespace dnn
{

class layer_conv: public layer
{
private:
    arma::uword N_filter_size; 
    arma::uword N_pad;         
    arma::uword N_stride;      

    arma::Cube<DNN_Dtype> C;   
    arma::Mat<DNN_Dtype> Cbp;  

public:
    layer_conv(const arma::uword n_channels,
               const arma::uword n_kernel,
               const arma::uword n_pad = 0,
               const arma::uword n_stride = 1 ):layer()
    {
        N_channels_right = n_channels;
        N_filter_size    = n_kernel;
        N_pad            = n_pad;
        N_stride         = n_stride;
        type             = "Conv";
        id               = type;
    }

    void init(void)
    {
        layer::init();

        N_cols_right     = (N_cols_left-N_filter_size+2*N_pad)/N_stride + 1;
        N_rows_right     = (N_rows_left-N_filter_size+2*N_pad)/N_stride + 1;
        N_right          = N_cols_right*N_rows_right*N_channels_right;
        id = type+"("+std::to_string(N_filter_size)+"x"+std::to_string(N_filter_size)+","+
                      std::to_string(N_channels_left)+"*"+std::to_string(N_channels_right)+")";
        train_par        = true;

        W.     set_size(N_filter_size,N_filter_size,N_channels_left*N_channels_right);
        B.     set_size(N_channels_right,1);
        Cbp.   set_size(N_filter_size*N_filter_size*N_channels_right,N_rows_left*N_cols_left);

        init_weights(W,N_filter_size*N_filter_size*N_channels_left,
                     N_filter_size*N_filter_size*N_channels_right,
                     INIT_W_ALG::XAVIER,INIT_W_DIST::NORMAL);
        B.zeros();
    }

    void upd_buf_size(arma::uword nmb)
    {
        layer::upd_buf_size(nmb);
        C.zeros(N_filter_size*N_filter_size*N_channels_left,N_rows_right*N_cols_right,nmb);
    }

    arma::Mat<DNN_Dtype> rotate_180(const arma::Mat<DNN_Dtype>& M)
    {
        arma::Mat<DNN_Dtype> U(M);
        return arma::fliplr(arma::flipud(U));
    }

    arma::Cube<DNN_Dtype> rotate_180(const arma::Cube<DNN_Dtype>& C)
    {
        arma::Cube<DNN_Dtype> U(C);
        for (arma::uword n=0; n<C.n_slices; n++)
        {
            U.slice(n) = arma::fliplr(arma::flipud(C.slice(n)));
        }
        return U;
    }

    template <typename DNN_Dtype>
    void im2col_nzp( DNN_Dtype* Dptr,  const DNN_Dtype* Xptr,
                     const arma::uword XRows, const arma::uword XCols, const arma::uword XDepth,
                     const arma::uword KRows, const arma::uword KCols,
                     const arma::uword SRows, const arma::uword SCols)
    {
        const arma::uword XSize   = XRows*XCols;
        const arma::uword XC      = XCols-KCols;
        const arma::uword XR      = XRows-KRows;
        for (arma::uword x_col=0; x_col<=XC; x_col+=SCols)             // image cols
        {
            for(arma::uword x_row=0; x_row<=XR; x_row+=SRows )         // image rows
            {
                for (arma::uword layer=0; layer<XDepth; layer++)            // Depth (input layers/featuremaps)
                {
                    const arma::uword lay_offs = layer*XSize;
                    for(arma::uword k_col=0; k_col<KCols; k_col++ )         // kernel cols
                    {
                        const arma::uword col_offs = x_col+k_col;
                        const arma::uword l_c_offs = lay_offs+col_offs*XRows;
                        for(arma::uword k_row=0; k_row<KRows; k_row++ )     // kernel rows
                        {
                            *(Dptr++)=Xptr[l_c_offs+ x_row + k_row];
                        }
                    }
                }
            }
        }
    }

    template <typename DNN_Dtype>
    void im2col(DNN_Dtype* Dptr, const DNN_Dtype* Xptr,
                const arma::uword XRows, const arma::uword XCols, const arma::uword XDepth,
                const arma::uword KRows, const arma::uword KCols,
                const arma::uword PRows, const arma::uword PCols,
                const arma::uword SRows, const arma::uword SCols)
    {
        const arma::uword XSize = XRows * XCols;
        const arma::sword XC = XCols - KCols + PCols;
        const arma::sword XR = XRows - KRows + PRows;
        for (arma::sword x_col = -1*PCols; x_col <= XC; x_col += SCols)             // image cols
        {
            for (arma::sword x_row = -1*PRows; x_row <= XR; x_row += SRows)         // image rows
            {
                for (arma::uword layer = 0; layer < XDepth; layer++)     // Depth (input layers/featuremaps)
                {
                    const arma::uword lay_offs = layer * XSize;
                    for (arma::uword k_col = 0; k_col < KCols; k_col++)  // kernel cols
                    {
                        const arma::sword col_offs = x_col + k_col;
                        const bool c_ok = static_cast<unsigned int>(col_offs) < XCols;
                        if(c_ok)
                        {
                            const arma::uword l_c_offs = lay_offs + col_offs * XRows;
                            for (arma::uword k_row = 0; k_row < KRows; k_row++)     // kernel rows
                            {
                                const arma::sword row_offs = x_row + k_row;
                                const bool r_ok = static_cast<unsigned int>(row_offs) < XRows;
                                if(r_ok)
                                {
                                    *(Dptr++) = Xptr[l_c_offs + row_offs];
                                }
                                else
                                {
                                    *(Dptr++) = 0;
                                }
                            }
                        }
                        else
                        {
                            for (arma::uword k_row = 0; k_row < KRows; k_row++)     // kernel rows
                            {
                                *(Dptr++) = 0;
                            }
                        }
                    }
                }
            }
        }
    }

    template <typename DNN_Dtype>
    void im2row(DNN_Dtype* Dptr,  const DNN_Dtype* Xptr,
                const unsigned int XRows, const unsigned int XCols, const unsigned int XDepth,
                const unsigned int KRows, const unsigned int KCols,
                const unsigned int PRows, const unsigned int PCols,
                const unsigned int SRows, const unsigned int SCols)
    {
        const unsigned int XSize   = XRows*XCols;
        const int XC      = XCols-KCols+PCols;
        const int XR      = XRows-KRows+PRows;
        for (unsigned int layer=0; layer<XDepth; layer++)            // Depth (input layers/featuremaps)
        {
            const unsigned int lay_offs = layer*XSize;
            for(unsigned int k_col=0; k_col<KCols; k_col++ )         // kernel cols
            {
                for(unsigned int k_row=0; k_row<KRows; k_row++ )     // kernel rows
                {
                    for (int x_col=-PCols; x_col<=XC; x_col+=SCols)      // image cols
                    {
                        const int col_offs = x_col+k_col;
                        const int l_c_offs = lay_offs+col_offs*XRows;
                        for(int x_row=-PRows; x_row<=XR; x_row+=SRows )  // image rows
                        {
                            const int row_offs = x_row+k_row;
                            if((row_offs) >= 0  && static_cast<unsigned int>(row_offs) <  XRows &&
                               (col_offs) >= 0  && static_cast<unsigned int>(col_offs) <  XCols)
                            {
                                *(Dptr++)=Xptr[l_c_offs+row_offs];
                            }
                            else
                            {
                                *(Dptr++)=0;
                            }
                        }
                    }
                }
            }
        }
    }

    void disp(void)
    {
        layer::disp();
        std::cout << "Opt algorithm:        " << opt_alg->get_algorithm() << std::endl;
        std::cout << "Input size:           ["<< N_rows_left << "," << N_rows_left << "," << N_channels_left << "]" << std::endl;
        std::cout << "Filter size:          [" << N_filter_size << "," << N_filter_size << "," << N_channels_left*N_channels_right << "]" << std::endl;
        std::cout << "Padding:              " << N_pad << std::endl;
        std::cout << "Stride:               " << N_stride << std::endl;
        std::cout << "Output size:          ["<< N_rows_right << "," << N_rows_right << "," << N_channels_right << "]" << std::endl;
    }

    arma::uword get_nrof_params(void)
    {
        return N_filter_size*N_filter_size*N_channels_left*N_channels_right+N_channels_right;
    }

    void prop(void)
    {
        arma::Mat<DNN_Dtype> gg(W.memptr(), N_filter_size*N_filter_size*N_channels_left, N_channels_right, false, true);
        arma::Mat<DNN_Dtype> yy(N_rows_right*N_cols_right, N_channels_right);   // [RoCo,KKLi][KKLi,Lo]
        arma::Mat<DNN_Dtype> Bmat = arma::repmat(B.t(), N_rows_right*N_cols_right, 1);

        if (N_pad)
        {
            im2col(C.slice_memptr(0), left->get_Y1_memptr(), N_rows_left, N_cols_left, N_channels_left, N_filter_size, N_filter_size, N_pad, N_pad, N_stride, N_stride);
        }
        else
        {
            im2col_nzp(C.slice_memptr(0), left->get_Y1_memptr(), N_rows_left, N_cols_left, N_channels_left, N_filter_size, N_filter_size, N_stride, N_stride);
        }
        yy = ((C.slice(0)).t())*gg;   // [RoCo,KKLi][KKLi,Lo]
        yy += Bmat;
        Y1 = arma::vectorise(yy);
    }

    void prop_mb(void)
    {
        arma::Mat<DNN_Dtype> gg(W.memptr(),N_filter_size*N_filter_size*N_channels_left,N_channels_right,false,true);
        arma::Mat<DNN_Dtype> yy(N_rows_right*N_cols_right,N_channels_right);   // [RoCo,KKLi][KKLi,Lo]
        arma::Mat<DNN_Dtype> Bmat = arma::repmat(B.t(),N_rows_right*N_cols_right,1);
        for (arma::uword n=0; n<N_batch; n++)                                  // loop over every image in minibatch
        {
            if (N_pad)
            {
                im2col(C.slice_memptr(n), left->get_Y_colptr(n), N_rows_left, N_cols_left, N_channels_left, N_filter_size, N_filter_size, N_pad, N_pad, N_stride, N_stride);
            }
            else
            {
                im2col_nzp(C.slice_memptr(n), left->get_Y_colptr(n), N_rows_left, N_cols_left, N_channels_left, N_filter_size, N_filter_size, N_stride, N_stride);
            }
            yy=((C.slice(n)).t())*gg;                                          // [RoCo,KKLi][KKLi,Lo]
            yy += Bmat;
            Y.col(n) = arma::vectorise(yy);
        }
    }

    void backprop(void)
    {
        arma::Cube<DNN_Dtype> uu = rotate_180(W);
        arma::Mat<DNN_Dtype> gg(uu.memptr(),N_filter_size*N_filter_size*N_channels_right,N_channels_left,false,true); // [KKLo,Li]
        arma::Mat<DNN_Dtype> ii(N_rows_left*N_cols_left,N_channels_left); // [RiCi,Li]

        DNN_Dtype* C_ptr = Cbp.memptr();

        for (arma::uword n=0; n<N_batch; n++)                                            // loop over every image in minibatch
        {
            im2col(C_ptr, right->get_Dleft_colptr(n), N_rows_right, N_cols_right, N_channels_right, N_filter_size, N_filter_size, N_filter_size - 1, N_filter_size - 1, N_stride, N_stride);
            ii = Cbp.t()*gg; //  [KKLo,RiCi]'[KKLo,Li]
            Dleft.col(n) = arma::vectorise(ii);
        }
    }

    void update(void)
    {
        if(train_par==true)
        {
            arma::Mat<DNN_Dtype>  dG(N_filter_size*N_filter_size*N_channels_left,N_channels_right, arma::fill::zeros);
            const DNN_Dtype nrm = (DNN_Dtype)1.0/N_batch;
            arma::Mat<DNN_Dtype>  dB(1,N_channels_right, arma::fill::zeros);

            for (arma::uword n=0; n<N_batch; n++) // loop over every image in minibatch
            {
                arma::Mat<DNN_Dtype> d_o(right->get_Dleft_colptr(n),N_rows_right*N_cols_right,N_channels_right); // [RoCo,Lo]
                dG += C.slice(n) * d_o;      // [KKLi,RoCo][RoCo,Lo]
                if(add_bias) dB += arma::sum(d_o);
            }
            dG *=nrm;
            if(add_bias) dB *=nrm;

            arma::Cube<DNN_Dtype> g(W.memptr(),W.n_elem,1,1,false,true);
            arma::Cube<DNN_Dtype> dg(dG.memptr(),dG.n_elem,1,1,false,true);

            if(add_bias)
            {
                opt_alg->apply(g,B,dg,dB.t());
            }
            else
            {
                arma::Mat<DNN_Dtype> m={1.0};
                opt_alg->apply(g,m,dg,m);
            }
        }
    }

    arma::Mat<DNN_Dtype> weights2img(arma::uword IR)
    {
        arma::uword R = W.n_rows;
        arma::uword C = W.n_cols;
        arma::uword M = W.n_slices;
        arma::uword IC = static_cast<arma::uword>(ceil(M/(1.0*IR)));
        arma::Mat<DNN_Dtype> II(IR*R,IC*C,arma::fill::zeros);
        arma::uword r=0,c=0;
        for(arma::uword m=0; m<M; m++)
        {
            arma::Mat<DNN_Dtype> I = W.slice(m);
            I+=B(m);
            II(r,c,arma::size(R,C)) = I;
            c+=C;
            if(c >= IC*C)
            {
                r+=R;
                c=0;
            }
        }
        return II;
    }

    #ifdef SIGPACK_H
    void plotWeights(sp::gplot& gp, const arma::uword PR)
    {
        std::ostringstream tmp_s;
        arma::uword M = W.n_slices;
        arma::uword PC = ceil(M/(1.0*PR));

        gp.send2gp("unset key");
        gp.send2gp("set format xy \"\" ");
        DNN_Dtype p=arma::max(arma::abs(arma::vectorise(W)));
        tmp_s << "set cbrange [ " << -p << ":" << p << "]";
        gp.send2gp(tmp_s.str().c_str());
        tmp_s.str(""); // Clear buffer
        if(M>4)
        {
            gp.send2gp("unset colorbox");
            tmp_s << "set multiplot layout " << PR << "," << PC << " margins 0.01,0.99,0.01,0.99 spacing 0.01,0.01";
        }
        else
        {
            tmp_s << "set multiplot layout " << PR << "," << PC;
        }
        gp.send2gp(tmp_s.str().c_str());

        for(arma::uword m=0; m<M; m++)
        {
            arma::Mat<DNN_Dtype> Im = W.slice(m)+B(floor(m / N_channels_left));
            gp.image(arma::conv_to<arma::mat>::from(Im));
        }
        gp.send2gp("unset multiplot");
        gp.flush_buf();
    }
    #endif // SIGPACK_H

}; // End class layer_conv
} // End namespace dnn