dnn_opt.h

Go to the documentation of this file.

// Copyright 2019 Claes Rolen (www.rolensystems.com)
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

#pragma once
namespace dnn
{

class opt
{
protected:
    std::string alg;
    DNN_Dtype lr;         
    DNN_Dtype reg_lambda; 
    DNN_Dtype reg_alpha;  
    LR_ALG    lr_alg;     
    DNN_Dtype lr_0;       
    DNN_Dtype lr_a;       
    DNN_Dtype lr_b;       
    arma::uword it;       
public:
    opt()
    {
        lr_alg = LR_ALG::CONST;
        lr_0   = 1;
        it     = 0;
    };

    ~opt() {};

    virtual void apply(arma::Cube<DNN_Dtype>& W,
               arma::Mat<DNN_Dtype>& B,
               const arma::Cube<DNN_Dtype>& Wgrad,
               const arma::Mat<DNN_Dtype>& Bgrad ) = 0;

    virtual std::string get_algorithm(void)
    {
        return alg;
    }

    void set_learn_rate_alg(LR_ALG alg, DNN_Dtype a=0.0, DNN_Dtype b=10.0 )
    {
        lr_alg = alg;
        lr_a   = a;
        lr_b   = b;
        it     = 0;  // Reset counter
    }

    void update_learn_rate(void)
    {
        it++; // Increase counter
        switch (lr_alg)
        {
        case LR_ALG::TIME_DECAY:     // lr_a = time decay
            lr = lr_0/(1+lr_a*it);
            break;
        case LR_ALG::STEP_DECAY:     // lr_a = drop factor, lr_b = time interval
            lr = lr_0*std::pow(lr_a,std::floor(it/lr_b));
            break;
        case LR_ALG::EXP_DECAY:     // lr_p1 = dec rate
            lr = lr_0*std::exp(-lr_a*it);
            break;

        default:
            break;
        }
    }

    DNN_Dtype get_learn_rate(void)
    {
        return lr;
    }

}; // End class opt


class opt_SGD:public opt
{
public:
    opt_SGD(DNN_Dtype s, DNN_Dtype l=0.0, DNN_Dtype a=0.0):opt()
    {
        lr     = s;
        lr_0   = lr;
        reg_lambda  = l;
        reg_alpha  = a;
        alg = "SGD";
    };

    ~opt_SGD() {};

    void apply(arma::Cube<DNN_Dtype>& W,
               arma::Mat<DNN_Dtype>& B,
               const arma::Cube<DNN_Dtype>& Wgrad,
               const arma::Mat<DNN_Dtype>& Bgrad )
    {
        // Add regularization
        const arma::Cube<DNN_Dtype> dw=Wgrad+reg_lambda*(reg_alpha*arma::sign(W)+(1-reg_alpha)*W);

        // Update params
        W  = W - lr*dw;
        B  = B - lr*Bgrad;

        // Update learning rate
        update_learn_rate();
    }

    std::string get_algorithm(void)
    {
        char p[100];
        std::snprintf(p,100,"(%g)",lr);
        return alg+p;
    }
}; // End class opt_SGD


class opt_SGD_momentum:public opt
{
private:
    arma::Cube<DNN_Dtype> v;    
    arma::Mat<DNN_Dtype> vB;    
    DNN_Dtype mom;
public:
    opt_SGD_momentum(DNN_Dtype s, DNN_Dtype m, DNN_Dtype l=0.0, DNN_Dtype a=0.0):opt()
    {
        lr          = s;
        lr_0        = lr;
        reg_lambda  = l;
        reg_alpha   = a;
        mom         = m;
        alg         = "SGD_mom";
    };

    ~opt_SGD_momentum() {};

    void apply(arma::Cube<DNN_Dtype>& W,
               arma::Mat<DNN_Dtype>& B,
               const arma::Cube<DNN_Dtype>& Wgrad,
               const arma::Mat<DNN_Dtype>& Bgrad )
    {
        if(v.n_elem==0)
        {
            v.set_size(arma::size(W));
            v.zeros();
            vB.set_size(arma::size(B));
            vB.zeros();
        }

        // Add regularization
        const arma::Cube<DNN_Dtype> dw=Wgrad+reg_lambda*(reg_alpha*arma::sign(W)+(1-reg_alpha)*W);

        // Update params
        v  = mom*v - lr*dw;
        W  = W + v;
        vB = mom*vB - lr*Bgrad ;
        B  = B + vB;

        // Update learning rate
        update_learn_rate();
    }

    std::string get_algorithm(void)
    {
        char p[100];
        std::snprintf(p,100,"(%g,%g)",lr,mom);
        return alg+p;
    }
}; // End class opt_SGD_momentum


class opt_SGD_nesterov:public opt
{
private:
    arma::Cube<DNN_Dtype> v;
    arma::Cube<DNN_Dtype> vp;
    arma::Mat<DNN_Dtype> vB;
    arma::Mat<DNN_Dtype> vBp;
    DNN_Dtype mom;
public:
    opt_SGD_nesterov(DNN_Dtype s, DNN_Dtype m, DNN_Dtype l=0.0, DNN_Dtype a=0.0):opt()
    {
        lr         = s;
        lr_0       = lr;
        reg_lambda = l;
        reg_alpha  = a;
        mom        = m;
        alg        = "SGD_nest";
    };

    ~opt_SGD_nesterov() {};

    void apply(arma::Cube<DNN_Dtype>& W,
               arma::Mat<DNN_Dtype>& B,
               const arma::Cube<DNN_Dtype>& Wgrad,
               const arma::Mat<DNN_Dtype>& Bgrad )
    {
        if(v.n_elem==0)
        {
            v.set_size(arma::size(W));
            v.zeros();
            vp.set_size(arma::size(W));
            vp.zeros();
            vB.set_size(arma::size(B));
            vB.zeros();
            vBp.set_size(arma::size(B));
            vBp.zeros();
        }

        // Add regularization
        const arma::Cube<DNN_Dtype> dw=Wgrad+reg_lambda*(reg_alpha*arma::sign(W)+(1-reg_alpha)*W);

        // Update params
        vp = v;
        v  = mom*v - lr*dw;
        W  = W - mom*vp + (1+mom)*v;

        vBp = vB;
        vB  = mom*vB - lr*Bgrad ;
        B   = B -mom*vBp +(1+mom)*vB;

        // Update learning rate
        update_learn_rate();
    }

    std::string get_algorithm(void)
    {
        char p[100];
        std::snprintf(p,100,"(%g,%g)",lr,mom);
        return alg+p;
    }
}; // End class opt_SGD_nesterov


class opt_adam:public opt
{
private:
    arma::Cube<DNN_Dtype> v;
    arma::Cube<DNN_Dtype> m;
    arma::Mat<DNN_Dtype> vB;
    arma::Mat<DNN_Dtype> mB;
    DNN_Dtype beta1;
    DNN_Dtype beta2;
    DNN_Dtype eps;
public:

    opt_adam(DNN_Dtype s, DNN_Dtype l=0.0, DNN_Dtype a=0.0, DNN_Dtype b1=0.9,DNN_Dtype b2=0.999, DNN_Dtype e=1e-8):opt()
    {
        lr         = s;
        lr_0       = lr;
        reg_lambda = l;
        reg_alpha  = a;
        beta1      = b1;
        beta2      = b2;
        eps        = e;
        alg        = "ADAM";
    };

    ~opt_adam() {};

    void apply(arma::Cube<DNN_Dtype>& W,
               arma::Mat<DNN_Dtype>& B,
               const arma::Cube<DNN_Dtype>& Wgrad,
               const arma::Mat<DNN_Dtype>& Bgrad )
    {
        if(v.n_elem==0)
        {
            v.set_size(arma::size(W));
            v.zeros();
            m.set_size(arma::size(W));
            m.zeros();
            vB.set_size(arma::size(B));
            vB.zeros();
            mB.set_size(arma::size(B));
            mB.zeros();
        }

        // Add regularization
        const arma::Cube<DNN_Dtype> dw=Wgrad+reg_lambda*(reg_alpha*arma::sign(W)+(1-reg_alpha)*W);

        // Update params
        for (arma::uword k=0;k<v.n_elem;k++ )
        {
            m(k)  = beta1*m(k) +(1-beta1)*dw(k);
            v(k)  = beta2*v(k) +(1-beta2)*dw(k)*dw(k);
            W(k)  = W(k) - lr*(std::sqrt(1-beta2)/(1-beta1))*m(k)/(std::sqrt(v(k))+eps);
        }
        for (arma::uword k=0;k<vB.n_elem;k++ )
        {
            mB(k)  = beta1*mB(k) +(1-beta1)*Bgrad(k);
            vB(k)  = beta2*vB(k) +(1-beta2)*Bgrad(k)*Bgrad(k);
            B(k)   = B(k) - lr*(std::sqrt(1-beta2)/(1-beta1))*mB(k)/(std::sqrt(vB(k))+eps);
        }

        // Update learning rate
        update_learn_rate();
    }

    std::string get_algorithm(void)
    {
        char p[100];
        std::snprintf(p,100,"(%g)",lr);
        return alg+p;
    }
}; // End class opt_adam


class opt_adamax:public opt
{
private:
    arma::Cube<DNN_Dtype> v;
    arma::Cube<DNN_Dtype> m;
    arma::Mat<DNN_Dtype> vB;
    arma::Mat<DNN_Dtype> mB;
    DNN_Dtype beta1;
    DNN_Dtype beta2;
    DNN_Dtype eps;
public:

    opt_adamax(DNN_Dtype s, DNN_Dtype l=0.0, DNN_Dtype a=0.0, DNN_Dtype b1=0.9,DNN_Dtype b2=0.999, DNN_Dtype e=1e-8):opt()
    {
        lr         = s;
        lr_0       = lr;
        reg_lambda = l;
        reg_alpha  = a;
        beta1      = b1;
        beta2      = b2;
        eps        = e;
        alg        = "ADAmax";
    };

    ~opt_adamax() {};

    void apply(arma::Cube<DNN_Dtype>& W,
               arma::Mat<DNN_Dtype>& B,
               const arma::Cube<DNN_Dtype>& Wgrad,
               const arma::Mat<DNN_Dtype>& Bgrad )
    {
        if(v.n_elem==0)
        {
            v.set_size(arma::size(W));
            v.zeros();
            m.set_size(arma::size(W));
            m.zeros();
            vB.set_size(arma::size(B));
            vB.zeros();
            mB.set_size(arma::size(B));
            mB.zeros();
        }

        // Add regularization
        const arma::Cube<DNN_Dtype> dw=Wgrad+reg_lambda*(reg_alpha*arma::sign(W)+(1-reg_alpha)*W);

        // Update params
        for (arma::uword k=0;k<v.n_elem;k++ )
        {
            m(k) = beta1*m(k) +(1-beta1)*dw(k);
            v(k) = (beta2*v(k)> std::abs(dw(k))) ? beta2*v(k) : std::abs(dw(k));
            W(k) = W(k) - lr*(std::sqrt(1-beta2)/(1-beta1))*m(k)/(v(k)+eps);
        }
        for (arma::uword k=0;k<vB.n_elem;k++ )
        {
            mB(k) = beta1*mB(k) +(1-beta1)*Bgrad(k);
            vB(k) = (beta2*vB(k)> std::abs(Bgrad(k))) ? beta2*vB(k) : std::abs(Bgrad(k));
            B(k)  = B(k) - lr*(std::sqrt(1-beta2)/(1-beta1))*mB(k)/(vB(k)+eps);
        }

        // Update learning rate
        update_learn_rate();
    }

    std::string get_algorithm(void)
    {
        char p[100];
        std::snprintf(p,100,"(%g)",lr);
        return alg+p;
    }
}; // End class opt_adamax


class opt_adadelta:public opt
{
private:
    arma::Cube<DNN_Dtype> Ew;
    arma::Cube<DNN_Dtype> dW;
    arma::Mat<DNN_Dtype> Eb;
    arma::Mat<DNN_Dtype> dB;
    DNN_Dtype rho;
    DNN_Dtype eps;
public:
    opt_adadelta(DNN_Dtype r, DNN_Dtype s=1.0, DNN_Dtype l=0.0, DNN_Dtype a=0.0, DNN_Dtype e=1e-6):opt()
    {
        rho        = r;
        lr         = s;
        lr_0       = s;
        reg_lambda = l;
        reg_alpha  = a;
        eps        = e;
        alg        = "ADAdelta";
    };

    ~opt_adadelta(){};

    void apply(arma::Cube<DNN_Dtype>& W,
               arma::Mat<DNN_Dtype>& B,
               const arma::Cube<DNN_Dtype>& Wgrad,
               const arma::Mat<DNN_Dtype>& Bgrad )
    {
        if(Ew.n_elem==0)
        {
            Ew.set_size(arma::size(W));
            Ew.zeros();
            dW.set_size(arma::size(W));
            dW.zeros();
            Eb.set_size(arma::size(B));
            Eb.zeros();
            dB.set_size(arma::size(B));
            dB.zeros();
        }

        // Add regularization
        const arma::Cube<DNN_Dtype> w=Wgrad+reg_lambda*(reg_alpha*arma::sign(W)+(1-reg_alpha)*W);

        // Update params
        for (arma::uword k=0 ;k<W.n_elem ; k++ )
        {
            Ew(k) = rho*Ew(k)+(1-rho)*w(k)*w(k);

            DNN_Dtype upd = lr*w(k)*sqrt(dW(k)+eps)/sqrt(Ew(k)+eps);
            W(k)  = W(k)-upd;
            dW(k) = rho*dW(k)+(1-rho)*upd*upd;
        }
        for (arma::uword k=0 ;k<B.n_elem ; k++ )
        {
            Eb(k) = rho*Eb(k)+(1-rho)*Bgrad(k)*Bgrad(k);

            DNN_Dtype upd = lr*Bgrad(k)*sqrt(dB(k)+eps)/sqrt(Eb(k)+eps);;
            B(k)  = B(k)-upd;
            dB(k) = rho*dB(k)+(1-rho)*upd*upd;
        }

        // Update learning rate
        update_learn_rate();
    }

    std::string get_algorithm(void)
    {
        char p[100];
        std::snprintf(p,100,"(%g)",rho);
        return alg+p;
    }
}; // End class opt_adadelta


class opt_adagrad:public opt
{
private:
    arma::Cube<DNN_Dtype> v;
    arma::Mat<DNN_Dtype> vB;
    DNN_Dtype eps;
public:
    opt_adagrad(DNN_Dtype s, DNN_Dtype l=0.0, DNN_Dtype a=0.0, DNN_Dtype e=1e-8):opt()
    {
        lr         = s;
        lr_0       = lr;
        reg_lambda = l;
        reg_alpha  = a;
        eps        = e;
        alg        = "ADAgrad";
    };
    ~opt_adagrad() {};


    void apply(arma::Cube<DNN_Dtype>& W,
               arma::Mat<DNN_Dtype>& B,
               const arma::Cube<DNN_Dtype>& Wgrad,
               const arma::Mat<DNN_Dtype>& Bgrad )
    {
        if(v.n_elem==0)
        {
            v.set_size(arma::size(W));
            v.zeros();
            vB.set_size(arma::size(B));
            vB.zeros();
        }

        // Add regularization
        const arma::Cube<DNN_Dtype> dw=Wgrad+reg_lambda*(reg_alpha*arma::sign(W)+(1-reg_alpha)*W);

        // Update params
        for (arma::uword k=0 ;k<W.n_elem ; k++ )
        {
            v(k) = v(k)+dw(k)*dw(k);
            W(k) = W(k)-lr*dw(k)/std::sqrt(v(k)+eps);
        }
        for (arma::uword k=0 ;k<B.n_elem ; k++ )
        {
            vB(k) = vB(k)+Bgrad(k)*Bgrad(k);
            B(k) = B(k)-lr*Bgrad(k)/std::sqrt(vB(k)+eps);
        }

        // Update learning rate
        update_learn_rate();
    }

    std::string get_algorithm(void)
    {
        char p[100];
        std::snprintf(p,100,"(%g)",lr);
        return alg+p;
    }
}; // End class opt_adagrad


class opt_rmsprop:public opt
{
private:
    arma::Cube<DNN_Dtype> v;
    arma::Mat<DNN_Dtype> vB;
    DNN_Dtype beta;
    DNN_Dtype eps;
public:
    opt_rmsprop(DNN_Dtype s, DNN_Dtype l=0.0, DNN_Dtype a=0.0, const DNN_Dtype b=0.9, DNN_Dtype e=1e-8):opt()
    {
        lr         = s;
        lr_0       = lr;
        reg_lambda = l;
        reg_alpha  = a;
        beta       = b;
        eps        = e;
        alg        = "RMSprop";
    };

    ~opt_rmsprop() {};

    void apply(arma::Cube<DNN_Dtype>& W,
               arma::Mat<DNN_Dtype>& B,
               const arma::Cube<DNN_Dtype>& Wgrad,
               const arma::Mat<DNN_Dtype>& Bgrad )
    {
        if(v.n_elem==0)
        {
            v.set_size(arma::size(W));
            v.zeros();
            vB.set_size(arma::size(B));
            vB.zeros();
        }

        // Add regularization
        const arma::Cube<DNN_Dtype> dw=Wgrad+reg_lambda*(reg_alpha*arma::sign(W)+(1-reg_alpha)*W);

        // Update params
        for (arma::uword k=0 ;k<W.n_elem ; k++ )
        {
            v(k) = beta*v(k)+(1-beta)*dw(k)*dw(k);
            W(k) = W(k)-lr*dw(k)/std::sqrt(v(k)+eps);
        }
        for (arma::uword k=0 ;k<B.n_elem ; k++ )
        {
            vB(k) = beta*vB(k)+(1-beta)*Bgrad(k)*Bgrad(k);
            B(k) = B(k)-lr*Bgrad(k)/std::sqrt(vB(k)+eps);
        }

        // Update learning rate
        update_learn_rate();
    }

    std::string get_algorithm(void)
    {
        char p[100];
        std::snprintf(p,100,"(%g,%g)",lr,beta);
        return alg+p;
    }
}; // End class opt_rmsprop
} // End namespace dnn