---

gnn_rprop.c

Go to the documentation of this file.

/***************************************************************************
 *  @file gnn_rprop.c
 *  @brief Resilient Backpropagation Trainer Implementation.
 *
 *  @date   : 18-09-03 20:36
 *  @author : Pedro Ortega C. <peortega@dcc.uchile.cl>
 *  Copyright  2003  Pedro Ortega C.
 ****************************************************************************/
/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Library General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 */



/**
 * @brief Resilient Backpropagation.
 * @defgroup gnn_rprop_doc gnn_rprop : Resilient Error Backpropagation Trainer.
 * @ingroup gnn_trainer_doc
 *
 * The present trainer provides an implementation of the Resilient
 * Error-Backpropagation Algorithm.
 *
 * The weights are updated using the following rule:
 * \f[ \Delta w_k^{(t)} =
 *         \left \{
 *         \begin{array}{ccl}
 *         -\Delta_k^{(t)} &,&
 *             \textrm{if } \frac{\partial E}{\partial w_k}^{(t)} > 0\\
 *         +\Delta_k^{(t)} &,&
 *             \textrm{if } \frac{\partial E}{\partial w_k}^{(t)} < 0\\
 *         0 &,& \textrm{else}\\
 *         \end{array}
 *         \right .
 * \f]
 * where \f$\Delta_k^{(t)}\f$ is the 'update-value' for the parameter
 * \f$w_k\f$ at time \f$t\f$. These are computed by the following rule:
 * \f[ \Delta_k^{(t)} =
 *         \left \{
 *         \begin{array}{ccl}
 *         \nu^+ \cdot \Delta_k^{(t-1)} &,& \textrm{if }
 *             \frac{\partial E}{\partial w_k}
 *             \cdot \frac{\partial E}{\partial w_k} > 0 \\
 *         \nu^- \cdot \Delta_k^{(t-1)} &,& \textrm{if }
 *             \frac{\partial E}{\partial w_k}
 *             \cdot \frac{\partial E}{\partial w_k} < 0 \\
 *         \Delta_k^{(t-1)} &,& \textrm{else}\\
 *         \end{array}
 *         \right .
 * \f]
 * where \f$0 < \nu^- < 1 < \nu^+\f$.
 *
 * References:
 * [M. Riedmiller and H. Braun. "A direct adaptive method for faster
 *  backpropagation learning: The RPROP algorithm." In H. Ruspini, editor,
 *  Proceedings of the IEEE International Conference on Neural Networks (ICNN),
 *  pages 586 - 591, San Francisco, 1993.]
 */



/******************************************/
/* Include Files                          */
/******************************************/

#include "gnn_utilities.h"
#include "gnn_rprop.h"



/******************************************/
/* Static Declaration                     */
/******************************************/

typedef struct _gnn_rprop gnn_rprop;

struct _gnn_rprop
{
    gnn_trainer trainer;
    gsl_vector *w;
    gsl_vector *dwnew;
    gsl_vector *dwold;
    gsl_vector *deltaw;
    gsl_vector *delta;
    double nup;
    double num;
    double deltamax;
    double deltamin;
    double delta0;
};


static int
gnn_rprop_train (gnn_trainer   *trainer);

static void
gnn_rprop_destroy (gnn_trainer *trainer);



/******************************************/
/* Static Implementation                  */
/******************************************/

/**
 * @brief Reset function.
 * @ingroup gnn_rprop_doc
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @return Returns 0 if succeeded.
 */
static int
gnn_rprop_reset (gnn_trainer *trainer)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    rtrainer = (gnn_rprop *) trainer;

    /* get parameters */
    gnn_node_param_get (trainer->node, rtrainer->w);

    /* reset deltas */
    gsl_vector_set_all (rtrainer->delta, rtrainer->delta0);
    gsl_vector_set_zero (rtrainer->dwold);

    return 0;
}

/**
 * @brief Train function.
 * @ingroup gnn_rprop_doc
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @return Returns 0 if succeeded.
 */
static int
gnn_rprop_train (gnn_trainer *trainer)
{
    size_t k;
    size_t param_size;
    gnn_rprop *rtrainer;

    /* get view */
    rtrainer = (gnn_rprop *) trainer;

    /* process minibatch */
    gnn_trainer_batch_process (trainer);

    /* get a copy of the gradient */
    gsl_vector_memcpy (rtrainer->dwnew, gnn_trainer_batch_get_dw (trainer));

    /* move to next minibatch */
    gnn_trainer_batch_next (trainer);

    /* get parameter size */
    param_size = gnn_node_param_get_size (gnn_trainer_get_node (trainer));

    /* compute steps */
    for (k=0; k<param_size; ++k)
    {
        double dwnewk;
        double dwoldk;
        double deltak;
        double deltawk;
        double wk;

        dwnewk  = gsl_vector_get (rtrainer->dwnew, k);
        dwoldk  = gsl_vector_get (rtrainer->dwold, k);
        deltak  = gsl_vector_get (rtrainer->delta, k);
        wk      = gsl_vector_get (rtrainer->w, k);
        
        if (dwnewk * dwoldk > 0)
        {
            deltak  = GNN_MIN (rtrainer->nup * deltak, rtrainer->deltamax);
            deltawk = - GNN_SIGN (dwnewk) * deltak;
            wk = wk + deltawk;
            dwoldk = dwnewk;
        }
        else if (dwnewk * dwoldk < 0)
        {
            deltak = GNN_MAX (rtrainer->num * deltak, rtrainer->deltamin);
            dwoldk = 0;
        }
        else
        {
            deltawk = - GNN_SIGN (dwnewk) * deltak;
            wk = wk + deltawk;
            dwoldk = dwnewk;
        }
        
        gsl_vector_set (rtrainer->dwold, k, dwoldk);
        gsl_vector_set (rtrainer->deltaw, k, deltawk);
        gsl_vector_set (rtrainer->delta, k, deltak);
        gsl_vector_set (rtrainer->w, k, wk);
    }

    /* update parameters */
    gnn_node_param_set (trainer->node, rtrainer->w);

    return 0;
}

/**
 * @brief Destructor.
 * @ingroup gnn_rprop_doc
 *
 * @param  trainer A pointer to a \ref gnn_momentum.
 */
static void
gnn_rprop_destroy (gnn_trainer *trainer)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    rtrainer = (gnn_rprop *) trainer;

    if (rtrainer->w != NULL)
        gsl_vector_free (rtrainer->w);
    if (rtrainer->dwnew != NULL)
        gsl_vector_free (rtrainer->dwnew);
    if (rtrainer->dwold != NULL)
        gsl_vector_free (rtrainer->dwold);
    if (rtrainer->delta != NULL)
        gsl_vector_free (rtrainer->delta);

    return;
}



/******************************************/
/* Public Interface                       */
/******************************************/

/**
 * @brief Creates a new RPROP trainer.
 * @ingroup gnn_rprop_doc
 *
 * This function creates a RPROP trainer (\ref gnn_rprop). \f$\delta_0\f$
 * is the initial learning rate, and \f$\delta_{max}\f$ is the maximum
 * learning rate.
 *
 * @param  node A pointer to a \ref gnn_node.
 * @param  crit A pointer to a \ref gnn_criterion.
 * @param  data A pointer to a \ref gnn_dataset.
 * @param  delta0   Initial learning rate \f$\delta_0\f$.
 * @param  deltamax Maximum learnig rate \f$\delta_{max}\f$.
 * @return Returns a pointer to a new \ref gnn_rprop trainer.
 */
gnn_trainer *
gnn_rprop_new (gnn_node *node,
               gnn_criterion *crit,
               gnn_dataset *data,
               double delta0,
               double deltamax)
{
    int status;
    size_t param_size;
    gnn_trainer *trainer;
    gnn_rprop *rtrainer;

    /* check that delta0 isn't negative */
    if (delta0 <= 0.0)
    {
        GSL_ERROR_VAL ("delta0 should be stricly positive",
                       GSL_EINVAL, NULL);
    }
    /* check that deltamax isn't negative */
    if (deltamax <= 0.0)
    {
        GSL_ERROR_VAL ("deltamax should be strictly positive",
                       GSL_EINVAL, NULL);
    }

    /* allocate memory for the trainer */
    rtrainer = (gnn_rprop *) malloc (sizeof (gnn_rprop));
    if (rtrainer == NULL)
    {
        GSL_ERROR_VAL ("couldn't allocate memory for gnn_rprop",
                       GSL_ENOMEM, NULL);
    }

    /* get view as gnn_trainer */
    trainer = (gnn_trainer *) rtrainer;

    /* initialize */
    status = gnn_trainer_init (trainer,
                               "gnn_rprop",
                               node,
                               crit,
                               data,
                               gnn_rprop_reset,
                               gnn_rprop_train,
                               gnn_rprop_destroy);
    if (status)
    {
        GSL_ERROR_VAL ("couldn't initialize gnn_rprop",
                       GSL_EFAILED, NULL);
    }

    /* set fields */
    param_size = gnn_node_param_get_size (node);

    rtrainer->delta0   = delta0;
    rtrainer->deltamin = 0.000001;
    rtrainer->deltamax = deltamax;
    rtrainer->nup      = 1.2;
    rtrainer->num      = 0.5;
    rtrainer->w      = gsl_vector_alloc (param_size);
    rtrainer->dwold  = gsl_vector_alloc (param_size);
    rtrainer->dwnew  = gsl_vector_alloc (param_size);
    rtrainer->delta  = gsl_vector_alloc (param_size);
    rtrainer->deltaw = gsl_vector_alloc (param_size);
    if (   rtrainer->w      == NULL
        || rtrainer->dwold  == NULL
        || rtrainer->dwnew  == NULL
        || rtrainer->delta  == NULL
        || rtrainer->deltaw == NULL )
    {
        gnn_trainer_destroy (trainer);
        GSL_ERROR_VAL ("couldn't allocate memory for gnn_rprop",
                       GSL_ENOMEM, NULL);
    }

    return trainer;
}

/**
 * @brief Creates a standard RPROP trainer.
 * @ingroup gnn_rprop_doc
 *
 * This function creates a RPROP trainer (\ref gnn_rprop). The initial
 * learning rate \f$\delta_0\f$ is 0.1, and the maximum learning rate
 * \f$\delta_{max}\f$ is 50.
 *
 * @param  node A pointer to a \ref gnn_node.
 * @param  crit A pointer to a \ref gnn_criterion.
 * @param  data A pointer to a \ref gnn_dataset.
 * @return Returns a pointer to a new \ref gnn_rprop trainer.
 */
gnn_trainer *
gnn_rprop_standard_new (gnn_node *node,
                        gnn_criterion *crit,
                        gnn_dataset *data)
{
    return gnn_rprop_new (node, crit, data, 0.1, 50.0);
}


/**
 * @brief Sets \f$\delta_0\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function sets a new value for \f$\delta_0\f$, the
 * initial learning rate.
 *
 * Its value should be stricly positive.
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @param  delta0  The new value for \f$\delta_0\f$.
 * @return Returns 0 if suceeded.
 */
int
gnn_rprop_set_delta0 (gnn_trainer *trainer, double delta0)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    /* check value */
    if (delta0 <= 0.0)
        GSL_ERROR ("delta0 should be strictly positive", GSL_EINVAL);

    /* get view */
    rtrainer = (gnn_rprop *) trainer;

    /* set new delta0 */
    rtrainer->delta0 = delta0;

    return 0;
}

/**
 * @brief Gets \f$\delta_0\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function returns \f$\delta_0\f$, the initial learning rate.
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @return Returns \f$\delta_0\f$.
 */
double
gnn_rprop_get_delta0 (gnn_trainer *trainer)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    rtrainer = (gnn_rprop *) trainer;
    return rtrainer->delta0;
}

/**
 * @brief Sets \f$\delta_{min}\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function sets a new value for \f$\delta_{min}\f$, the
 * minimum learning rate.
 *
 * Its value should be stricly positive.
 *
 * @param  trainer   A pointer to a \ref gnn_rprop.
 * @param  deltamin  The new value for \f$\delta_{min}\f$.
 * @return Returns 0 if suceeded.
 */
int
gnn_rprop_set_deltamin (gnn_trainer *trainer, double deltamin)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    /* check value */
    if (deltamin <= 0.0)
        GSL_ERROR ("deltamin should be strictly positive", GSL_EINVAL);

    /* get view */
    rtrainer = (gnn_rprop *) trainer;

    /* set new deltamin */
    rtrainer->deltamin = deltamin;

    return 0;
}

/**
 * @brief Gets \f$\delta_{min}\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function returns \f$\delta_{min}\f$, the minimum learning rate.
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @return Returns \f$\delta_{min}\f$.
 */
double
gnn_rprop_get_deltamin (gnn_trainer *trainer)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    rtrainer = (gnn_rprop *) trainer;
    return rtrainer->deltamin;
}

/**
 * @brief Sets \f$\delta_{max}\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function sets a new value for \f$\delta_{max}\f$, the
 * maximum learning rate.
 *
 * Its value should be stricly positive.
 *
 * @param  trainer  A pointer to a \ref gnn_rprop.
 * @param  deltamax The new value for \f$\delta_{max}\f$.
 * @return Returns 0 if suceeded.
 */
int
gnn_rprop_set_deltamax (gnn_trainer *trainer, double deltamax)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    /* check value */
    if (deltamax <= 0.0)
        GSL_ERROR ("deltamax should be strictly positive", GSL_EINVAL);

    /* get view */
    rtrainer = (gnn_rprop *) trainer;

    /* set new learning factor */
    rtrainer->deltamax = deltamax;

    return 0;
}

/**
 * @brief Gets \f$\delta_{max}\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function returns \f$\delta_{max}\f$, the maximum learning rate.
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @return Returns \f$\delta_{max}\f$.
 */
double
gnn_rprop_get_deltamax (gnn_trainer *trainer)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    rtrainer = (gnn_rprop *) trainer;
    return rtrainer->deltamax;
}

/**
 * @brief Sets \f$\nu^+\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function sets a new value for \f$\nu^+\f$, the
 * increment rate.
 *
 * Its value should be greater than 1.
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @param  nup     The new value for \f$\nu^+\f$.
 * @return Returns 0 if suceeded.
 */
int
gnn_rprop_set_nup (gnn_trainer *trainer, double nup)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    /* check value */
    if (nup <= 1.0)
        GSL_ERROR ("nup should be greater than 1", GSL_EINVAL);

    /* get view */
    rtrainer = (gnn_rprop *) trainer;

    /* set new learning factor */
    rtrainer->nup = nup;

    return 0;
}

/**
 * @brief Gets \f$\nu^+\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function returns \f$\nu^+\f$, the increment rate.
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @return Returns \f$\nu^+f$.
 */
double
gnn_rprop_get_nup (gnn_trainer *trainer)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    rtrainer = (gnn_rprop *) trainer;
    return rtrainer->nup;
}

/**
 * @brief Sets \f$\nu^-\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function sets a new value for \f$\nu^-\f$, the
 * decrement rate.
 *
 * Its value should be within \f$(0;1)\f$.
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @param  num     The new value for \f$\nu^-\f$.
 * @return Returns 0 if suceeded.
 */
int
gnn_rprop_set_num (gnn_trainer *trainer, double num)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    /* check value */
    if (num <= 0.0 || num >= 1.0)
        GSL_ERROR ("num should be within (0,1)", GSL_EINVAL);

    /* get view */
    rtrainer = (gnn_rprop *) trainer;

    /* set new learning factor */
    rtrainer->num = num;

    return 0;
}

/**
 * @brief Gets \f$\nu^-\f$.
 * @ingroup gnn_rprop_doc
 *
 * This function returns \f$\nu^-\f$, the decrement rate.
 *
 * @param  trainer A pointer to a \ref gnn_rprop.
 * @return Returns \f$\nu^-f$.
 */
double
gnn_rprop_get_num (gnn_trainer *trainer)
{
    gnn_rprop *rtrainer;

    assert (trainer != NULL);

    rtrainer = (gnn_rprop *) trainer;
    return rtrainer->num;
}

---