---

gnn_gradient_descent.c

Go to the documentation of this file.

/***************************************************************************
 *  @file gnn_gradient_descent.c
 *  @brief Gradient Descent Trainer Implementation.
 *
 *  @date   : 31-08-03 21:32
 *  @author : Pedro Ortega C. <peortega@dcc.uchile.cl>
 *  Copyright  2003  Pedro Ortega C.
 ****************************************************************************/
/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Library General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 */



/**
 * @defgroup gnn_gradient_descent_doc gnn_gradient_descent : Gradient Descent Algorithm.
 * @ingroup gnn_trainer_doc
 *
 * The present trainer provides an implementation of the gradient descent
 * algorithm for parameter optimization. Gradient descent is the simplest
 * parameter optimization procedure based on the gradient. At each step,
 * the parameters are updated using the following rule:
 * \f[ \Delta w \leftarrow - \mu \frac{\partial E}{\partial w} \f]
 * where \f$ \mu \f$ is called the "learning rate", and determines the
 * step-size that are taken. The value of the learning rate depends on the
 * problem, but tipical values lie between \f$[0.0001, 0.1]\f$. Smaller values
 * tend to get trapped at local minima, but larger values often overshoot
 * optimum values.
 */



/******************************************/
/* Include Files                          */
/******************************************/

#include "gnn_gradient_descent.h"



/******************************************/
/* Static Declaration                     */
/******************************************/

typedef struct _gnn_gradient_descent gnn_gradient_descent;

struct _gnn_gradient_descent
{
    gnn_trainer trainer;
    gsl_vector *w;
    double mu;
};



static int
gnn_gradient_descent_train (gnn_trainer *trainer);

static void
gnn_gradient_descent_destroy (gnn_trainer *trainer);



/******************************************/
/* Static Implementation                  */
/******************************************/

/**
 * @brief The trainer's "reset" implementation.
 * @ingroup gnn_gradient_descent_doc
 *
 * @param  trainer A pointer to a \ref gnn_gradient_descent.
 * @return Returns 0 if suceeded.
 */
static int
gnn_gradient_descent_reset (gnn_trainer *trainer)
{
    gnn_gradient_descent *gdtrainer;
    
    assert (trainer != NULL);
    
    gdtrainer = (gnn_gradient_descent *) trainer;
    
    /* get parameters */
    gnn_node_param_get (trainer->node, gdtrainer->w);
    
    return 0;
}

/**
 * @brief The trainer's "train" implementation.
 * @ingroup gnn_gradient_descent_doc
 *
 * @param  trainer A pointer to a \ref gnn_gradient_descent.
 * @return Returns the mean cost.
 */
static int
gnn_gradient_descent_train (gnn_trainer   *trainer)
{
    gsl_vector *dw;
    gnn_gradient_descent *gdtrainer;
    
    /* get view */
    gdtrainer = (gnn_gradient_descent *) trainer;
    
    /* process minibatch */
    gnn_trainer_batch_process (trainer);
    
    /* move to next minibatch */
    gnn_trainer_batch_next (trainer);

    /* get gradient */
    dw = gnn_trainer_batch_get_dw (trainer);
    
    /* scale with learning factor */
    gsl_vector_scale (dw, - gdtrainer->mu);
    
    /* sum to parameters */
    gsl_vector_add (gdtrainer->w, dw);
    
    /* update parameters */
    gnn_node_param_set (trainer->node, gdtrainer->w);
    
    return 0;
}

/**
 * @brief The trainers "destroy" implementation.
 * @ingroup gnn_gradient_descent_doc
 *
 * @param trainer A pointer to a \ref gnn_gradient_descent.
 */
static void
gnn_gradient_descent_destroy (gnn_trainer *trainer)
{
    gnn_gradient_descent *gdtrainer;
    
    assert (trainer != NULL);

    gdtrainer = (gnn_gradient_descent *) trainer;
    
    gsl_vector_free (gdtrainer->w);
    
    return;
}



/******************************************/
/* Public Interface                       */
/******************************************/

/**
 * @brief Creates a new gradient descent trainer.
 * @ingroup gnn_gradient_descent_doc
 *
 * This function creates a new gradient descent trainer
 * (\ref gnn_gradient_descent). It uses the learning rate given by "mu".
 *
 * @param  node A pointer to a \ref gnn_node.
 * @param  crit A pointer to a \ref gnn_criterion.
 * @param  data A pointer to a \ref gnn_dataset.
 * @param  mu   The learning rate \f$\mu\f$.
 * @return Returns a pointer to a new \ref gnn_gradient_descent trainer.
 */
gnn_trainer *
gnn_gradient_descent_new (gnn_node *node,
                          gnn_criterion *crit,
                          gnn_dataset *data,
                          double mu)
{
    int status;
    gnn_trainer *trainer;
    gnn_gradient_descent *gdtrainer;
    
    /* check that mu isn't negative */
    if (mu <= 0.0)
    {
        GSL_ERROR_VAL ("learning factor should be stricly positive",
                       GSL_EINVAL, NULL);
    }
    
    /* allocate memory for the trainer */
    gdtrainer = (gnn_gradient_descent *) malloc (sizeof (gnn_gradient_descent));
    if (gdtrainer == NULL)
    {
        GSL_ERROR_VAL ("couldn't allocate memory for gnn_gradient_descent",
                       GSL_ENOMEM, NULL);
    }
    
    /* get view as gnn_trainer */
    trainer = (gnn_trainer *) gdtrainer;
    
    /* initialize */
    status = gnn_trainer_init (trainer,
                               "gnn_gradient_descent",
                               node,
                               crit,
                               data,
                               gnn_gradient_descent_reset,
                               gnn_gradient_descent_train,
                               gnn_gradient_descent_destroy);
    if (status)
    {
        GSL_ERROR_VAL ("couldn't initialize gnn_gradient_descent",
                       GSL_EFAILED, NULL);
    }
    
    /* set fields */
    gdtrainer->mu = mu;
    gdtrainer->w  = gsl_vector_alloc (gnn_node_param_get_size (node));
    if (gdtrainer->w == NULL)
    {
        GSL_ERROR_VAL ("couldn't allocate memory for gnn_gradient_descent",
                       GSL_ENOMEM, NULL);
    }

    return trainer;
}

/**
 * @brief Gets the learning rate.
 * @ingroup gnn_gradient_descent_doc
 *
 * This function returns the learning rate \f$\mu\f$ used by the gradient
 * descent trainer.
 *
 * @param  trainer A pointer to a \ref gnn_gradient_descent.
 * @return Returns the learning rate \f$\mu\f$.
 */
double
gnn_gradient_descent_get_mu (gnn_trainer *trainer)
{
    gnn_gradient_descent *gdtrainer;

    assert (trainer != NULL);

    gdtrainer = (gnn_gradient_descent *) trainer;
    return gdtrainer->mu;
}

/**
 * @brief Sets the learning rate.
 * @ingroup gnn_gradient_descent_doc
 *
 * This function sets a new value for the learning rate \f$\mu\f$ used by
 * the gradient descent trainer. The learning rate should be strictly positive.
 *
 * @param  trainer A pointer to a \ref gnn_gradient_descent.
 * @param  mu      The learning rate \f$\mu\f$.
 * @return Returns 0 if suceeded.
 */
int
gnn_gradient_descent_set_mu (gnn_trainer *trainer, double mu)
{
    gnn_gradient_descent *gdtrainer;

    assert (trainer != NULL);
    
    /* check learning factor */
    if (mu <= 0.0)
        GSL_ERROR ("learning factor should be stricly positive", GSL_EINVAL);

    /* get view */
    gdtrainer = (gnn_gradient_descent *) trainer;

    /* set new learning factor */
    gdtrainer->mu = mu;
    
    return 0;
}

---