---

gnn_momentum.c

Go to the documentation of this file.

/***************************************************************************
 *  @file gnn_momentum.c
 *  @brief Gradient Descent with Momentum Trainer Implementation.
 *
 *  @date   : 31-08-03 23:48
 *  @author : Pedro Ortega C. <peortega@dcc.uchile.cl>
 *  Copyright  2003  Pedro Ortega C.
 ****************************************************************************/
/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Library General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 */



/**
 * @defgroup gnn_momentum_doc gnn_momentum : Gradient Descent with Momentum Term.
 * @ingroup gnn_trainer_doc
 *
 * The present trainer provides an implementation of the gradient descent
 * with momentum term algorithm for parameter optimization. At each step,
 * the parameters are updated using the following rule:
 * \f[ \Delta w \leftarrow - \mu \frac{\partial E}{\partial w}
 *                         + \eta \Delta w
 * \f]
 * where \f$ \mu \f$ is the "learning rate", \f$ \eta \f$ is the
 * "momentum rate", and \f$\Delta w\f$ is the update.
 *
 * The learning rate \f$\mu\f$ is tipically one order of magnitude smaller than
 * the one used in the gradient descent algorithm. On the other hand, a tipical
 * choice of \f$\eta\f$ is \f$ \eta = 0.8 \f$.
 */



/******************************************/
/* Include Files                          */
/******************************************/

#include "gnn_momentum.h"



/******************************************/
/* Static Declaration                     */
/******************************************/

typedef struct _gnn_momentum gnn_momentum;

struct _gnn_momentum
{
    gnn_trainer trainer;
    gsl_vector *w;
    gsl_vector *mw;
    double mu;
    double eta;
};


static int
gnn_momentum_train (gnn_trainer   *trainer);

static void
gnn_momentum_destroy (gnn_trainer *trainer);



/******************************************/
/* Static Implementation                  */
/******************************************/

/**
 * @brief Reset function.
 * @ingroup gnn_momentum_doc
 *
 * @param  trainer A pointer to a \ref gnn_momentum.
 * @return Returns 0 if succeeded.
 */
static int
gnn_momentum_reset (gnn_trainer *trainer)
{
    gnn_momentum *mtrainer;

    assert (trainer != NULL);

    mtrainer = (gnn_momentum *) trainer;

    /* get parameters */
    gnn_node_param_get (trainer->node, mtrainer->w);

    /* reset momenta */
    gsl_vector_set_zero (mtrainer->mw);

    return 0;
}

/**
 * @brief Train function.
 * @ingroup gnn_momentum_doc
 *
 * @param  trainer A pointer to a \ref gnn_momentum.
 * @return Returns 0 if succeeded.
 */
static int
gnn_momentum_train (gnn_trainer   *trainer)
{
    gsl_vector *dw;
    gnn_momentum *mtrainer;

    /* get view */
    mtrainer = (gnn_momentum *) trainer;

    /* process minibatch */
    gnn_trainer_batch_process (trainer);
    
    /* get gradient */
    dw = gnn_trainer_batch_get_dw (trainer);
    
    /* move to next minibatch */
    gnn_trainer_batch_next (trainer);

    /* scale with learning factor */
    gsl_vector_scale (dw, - mtrainer->mu);

    /* modify momenta */
    gsl_vector_scale (mtrainer->mw, mtrainer->eta);
    gsl_vector_add (mtrainer->mw, dw);

    /* sum to parameters */
    gsl_vector_add (mtrainer->w, mtrainer->mw);

    /* update parameters */
    gnn_node_param_set (trainer->node, mtrainer->w);

    return 0;
}

/**
 * @brief Destructor.
 * @ingroup gnn_momentum_doc
 *
 * @param  trainer A pointer to a \ref gnn_momentum.
 */
static void
gnn_momentum_destroy (gnn_trainer *trainer)
{
    gnn_momentum *mtrainer;

    assert (trainer != NULL);

    mtrainer = (gnn_momentum *) trainer;

    if (mtrainer->w != NULL)
        gsl_vector_free (mtrainer->w);
    if (mtrainer->mw != NULL)
        gsl_vector_free (mtrainer->mw);

    return;
}



/******************************************/
/* Public Interface                       */
/******************************************/

/**
 * @brief Creates a new gradient descent with momentum trainer.
 * @ingroup gnn_momentum_doc
 *
 * This function creates a new gradient descent with momentum trainer
 * (\ref gnn_momentum). It uses the learning rate \f$\mu\f$ given by "mu"
 * and the momentum rate \f$\eta\f$ given by "eta", where \f$\mu > 0\f$ and
 * \f$\eta \geq 0\f$.
 *
 * @param  node A pointer to a \ref gnn_node.
 * @param  crit A pointer to a \ref gnn_criterion.
 * @param  data A pointer to a \ref gnn_dataset.
 * @param  mu   The learning rate \f$\mu\f$.
 * @param  eta  The momentum rate \f$\eta\f$.
 * @return Returns a pointer to a new \ref gnn_momentum trainer.
 */
gnn_trainer *
gnn_momentum_new (gnn_node *node,
                  gnn_criterion *crit,
                  gnn_dataset *data,
                  double mu,
                  double eta)
{
    int status;
    gnn_trainer *trainer;
    gnn_momentum *mtrainer;

    /* check that mu isn't negative */
    if (mu <= 0.0)
    {
        GSL_ERROR_VAL ("learning factor should be stricly positive",
                       GSL_EINVAL, NULL);
    }
    /* check that eta isn't negative */
    if (eta < 0.0)
    {
        GSL_ERROR_VAL ("momentum factor should be positive",
                       GSL_EINVAL, NULL);
    }

    /* allocate memory for the trainer */
    mtrainer = (gnn_momentum *) malloc (sizeof (gnn_momentum));
    if (mtrainer == NULL)
    {
        GSL_ERROR_VAL ("couldn't allocate memory for gnn_momentum",
                       GSL_ENOMEM, NULL);
    }

    /* get view as gnn_trainer */
    trainer = (gnn_trainer *) mtrainer;

    /* initialize */
    status = gnn_trainer_init (trainer,
                               "gnn_momentum",
                               node,
                               crit,
                               data,
                               gnn_momentum_reset,
                               gnn_momentum_train,
                               gnn_momentum_destroy);
    if (status)
    {
        GSL_ERROR_VAL ("couldn't initialize gnn_momentum",
                       GSL_EFAILED, NULL);
    }

    /* set fields */
    mtrainer->mu  = mu;
    mtrainer->eta = eta;
    mtrainer->w   = gsl_vector_alloc (gnn_node_param_get_size (node));
    mtrainer->mw  = gsl_vector_alloc (gnn_node_param_get_size (node));
    if (mtrainer->w == NULL || mtrainer->mw == NULL)
    {
        gnn_trainer_destroy (trainer);
        GSL_ERROR_VAL ("couldn't allocate memory for gnn_momentum",
                       GSL_ENOMEM, NULL);
    }

    return trainer;
}

/**
 * @brief Gets the learning rate.
 * @ingroup gnn_momentum_doc
 *
 * This function returns the learning rate \f$\mu\f$ used by the gradient
 * descent with momentum trainer.
 *
 * @param  trainer A pointer to a \ref gnn_momentum.
 * @return Returns the learning rate \f$\mu\f$.
 */
double
gnn_momentum_get_mu (gnn_trainer *trainer)
{
    gnn_momentum *mtrainer;

    assert (trainer != NULL);

    mtrainer = (gnn_momentum *) trainer;
    return mtrainer->mu;
}

/**
 * @brief Sets the learning rate.
 * @ingroup gnn_momentum_doc
 *
 * This function sets a new value for the learning rate \f$\mu\f$ used by
 * the gradient descent with momentum trainer. The learning rate should
 * be strictly positive.
 *
 * @param  trainer A pointer to a \ref gnn_momentum.
 * @param  mu      The learning rate \f$\mu\f$.
 * @return Returns 0 if suceeded.
 */
int
gnn_momentum_set_mu (gnn_trainer *trainer, double mu)
{
    gnn_momentum *mtrainer;

    assert (trainer != NULL);

    /* check learning factor */
    if (mu <= 0.0)
        GSL_ERROR ("learning factor should be stricly positive", GSL_EINVAL);

    /* get view */
    mtrainer = (gnn_momentum *) trainer;

    /* set new learning factor */
    mtrainer->mu = mu;

    return 0;
}

/**
 * @brief Gets the momentum rate.
 * @ingroup gnn_momentum_doc
 *
 * This function returns the momentum rate \f$\eta\f$ used by the gradient
 * descent with momentum trainer.
 *
 * @param  trainer A pointer to a \ref gnn_momentum.
 * @return Returns the learning rate \f$\eta\f$.
 */
double
gnn_momentum_get_eta (gnn_trainer *trainer)
{
    gnn_momentum *mtrainer;

    assert (trainer != NULL);

    mtrainer = (gnn_momentum *) trainer;
    return mtrainer->eta;
}

/**
 * @brief Sets the momentum rate.
 * @ingroup gnn_momentum_doc
 *
 * This function sets a new value for the momentum rate \f$\eta\f$ used by
 * the gradient descent with momentum trainer. The momentum rate should
 * be positive.
 *
 * @param  trainer A pointer to a \ref gnn_momentum.
 * @param  eta     The momentum rate \f$\eta\f$.
 * @return Returns 0 if suceeded.
 */
int
gnn_momentum_set_eta (gnn_trainer *trainer, double eta)
{
    gnn_momentum *mtrainer;

    assert (trainer != NULL);

    /* check learning factor */
    if (eta < 0.0)
        GSL_ERROR ("momentum factor should be positive", GSL_EINVAL);

    /* get view */
    mtrainer = (gnn_momentum *) trainer;

    /* set new learning factor */
    mtrainer->eta = eta;

    return 0;
}

---