---

gnn_weight_decay.c

Go to the documentation of this file.

/***************************************************************************
 *  @file gnn_weight_decay.c
 *  @brief gnn_weight_decay Implementation.
 *
 *  @date   : 03-10-03 21:21
 *  @author : Pedro Ortega C. <peortega@dcc.uchile.cl>
 *  Copyright  2003  Pedro Ortega C.
 ****************************************************************************/
/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Library General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 */

/**
 * @brief Weight Decay Regularization.
 * @defgroup gnn_weight_decay_doc gnn_weight_decay : Weight Decay Regularization.
 * @ingroup gnn_criterion_doc
 *
 * This datatype implements the \em Weight \em Decay Regularization, given
 * by
 * \f[ \Omega = \frac{1}{2} \sum_i w_i^2 \f]
 * Where the \f$w_i, i=0,\ldots,L\f$ are choosen parameters. They are usually
 * those of a \ref gnn_weight linear transform node.
 *
 * The gradient is:
 * \f[ \frac{\partial \widetilde{E}}{\partial w_l} =
 *         \frac{\partial E}{\partial w_l} + \nu w_l
 * \f]
 */



/******************************************/
/* Include Files                          */
/******************************************/

#include <math.h>
#include "gnn_weight_decay.h"



/******************************************/
/* Static Declaration                     */
/******************************************/

typedef struct _gnn_weight_decay gnn_weight_decay;

struct _gnn_weight_decay
{
    gnn_criterion  crit;
    gnn_criterion *subcrit;
    gnn_pbundle   *pb;
    gsl_vector    *w;
    gsl_vector    *dw;
    double         nu;
};


double
gnn_weight_decay_e (gnn_criterion *crit,
                    const gsl_vector *y,
                    const gsl_vector *t);

int
gnn_weight_decay_dy (gnn_criterion *crit,
                     const gsl_vector *y,
                     const gsl_vector *t,
                     gsl_vector * dy);

static void
gnn_weight_decay_destroy (gnn_criterion *crit);



/******************************************/
/* Static Implementation                  */
/******************************************/

/**
 * @brief The evaluation function.
 * @ingroup gnn_weight_decay_doc
 *
 * This function corresponds to the evaluation function.
 *
 * @param  crit A pointer to a \ref gnn_weight_decay criterion.
 * @param  y    A pointer to an estimation vector \f$y\f$.
 * @param  t    A pointer to the desired target vector \f$t\f$.
 * @return A real number corresponding to the value of the criterion.
 */
double
gnn_weight_decay_e (gnn_criterion *crit,
                    const gsl_vector *y,
                    const gsl_vector *t)
{
    int i;
    double E;
    double Ep;
    gnn_weight_decay *wc;

    assert (crit != NULL);
    assert (y != NULL);
    assert (t != NULL);

    /* get view as a gnn_weight_decay */
    wc = (gnn_weight_decay *) crit;

    /* check sizes */
    if (y->size != t->size)
        GSL_ERROR_VAL ("vector sizes should be the same", GSL_EINVAL, 0.0);

    /* compute subnode error */
    E = gnn_criterion_evaluate_e (wc->subcrit, y, t);

    /* get parameter vector */
    gnn_pbundle_get_w (wc->pb, wc->w);

    /* compute penalization term */
    Ep = 0.0;
    for (i=0; i<wc->w->size; ++i)
    {
        double wi;

        wi  = gsl_vector_get (wc->w, i);

        Ep += wi * wi;
    }

    return E + 0.5 * wc->nu * Ep;
}

/**
 * @brief The gradient evaluation function.
 * @ingroup gnn_weight_decay_doc
 *
 * This function implements criterion's gradient evaluation function.
 *
 * @param  crit A pointer to a \ref gnn_weight_decay criterion.
 * @param  y    A pointer to an estimation vector \f$y\f$.
 * @param  t    A pointer to the desired target vector \f$t\f$.
 * @param  dy   A pointer to a buffer vector where the result should be placed.
 * @return Returns 0 if succeeded.
 */
int
gnn_weight_decay_dy (gnn_criterion *crit,
                     const gsl_vector *y,
                     const gsl_vector *t,
                     gsl_vector * dy)
{
    int i;
    gnn_weight_decay *wc;

    assert (crit != NULL);
    assert (y    != NULL);
    assert (t    != NULL);
    assert (dy   != NULL);

    /* get view as a gnn_weight_decay */
    wc = (gnn_weight_decay *) crit;

    /* compute subnode dy */
    gnn_criterion_evaluate_dy (wc->subcrit, dy);

    /* get parameter gradient vector */
    gnn_pbundle_get_dw (wc->pb, wc->dw);

    /* compute gradient penalization terms */
    for (i=0; i<wc->w->size; ++i)
    {
        double wi;
        double dwi;

        wi  = gsl_vector_get (wc->w, i);
        dwi = gsl_vector_get (wc->dw, i);
        
        dwi += wc->nu * wi;

        gsl_vector_set (wc->dw, i, dwi);
    }

    /* set parameter gradient vector penalization */
    gnn_pbundle_set_dw (wc->pb, wc->dw);

    return 0;
}

/**
 * @brief The destroy function.
 * @ingroup gnn_weight_decay_doc
 *
 * This function implements destroy function.
 *
 * @param  crit A pointer to a \ref gnn_weight_decay criterion.
 */
static void
gnn_weight_decay_destroy (gnn_criterion *crit)
{
    gnn_weight_decay *wc;
    
    assert (crit != NULL);
    
    /* get view as a gnn_weight_decay */
    wc = (gnn_weight_decay *) crit;

    /* free allocate memory */
    if (wc->subcrit != NULL)
        gnn_criterion_destroy (wc->subcrit);
    if (wc->pb != NULL)
        gnn_pbundle_destroy (wc->pb);
    if (wc->w != NULL)
        gsl_vector_free (wc->w);
    if (wc->dw != NULL)
        gsl_vector_free (wc->dw);
}



/******************************************/
/* Public Interface                       */
/******************************************/

/**
 * @brief Creates a Weight Decay regularization for \ref gnn_weight.
 * @ingroup gnn_weight_decay_doc
 *
 * This function builds a new \ref gnn_weight_decay regularizator for
 * all \ref gnn_weight nodes contained in the given node \a node.
 *
 * @param  crit A pointer to a \ref gnn_weight_decay criterion.
 * @param  nu   The regularization coefficient \f$nu\f$.
 * @param  node A pointer to a \ref gnn_node.
 * @return Returns a pointer to a \ref gnn_weight_decay if succeeded or NULL.
 */
gnn_criterion *
gnn_weight_decay_new (gnn_criterion *crit, double nu, gnn_node *node)
{
    gnn_pbundle *pb;
    gnn_weight_decay *wc;

    assert (crit != NULL);
    assert (node != NULL);

    /* get parameter bundle consisting of gnn_weight nodes' params */
    pb = gnn_node_sub_search_params (node, "gnn_weight");

    /* create weight decay penalizer */
    wc = (gnn_weight_decay *) gnn_weight_decay_new_with_pbundle (crit, nu, pb);
    if (wc == NULL)
    {
        gnn_pbundle_destroy (pb);
        GSL_ERROR_VAL ("couldn't create gnn_weight_decay regularizer",
                       GSL_EFAILED, NULL);
    }

    return (gnn_criterion *) wc;
}

/**
 * @brief Creates a Weight Decay regularization for a given type.
 * @ingroup gnn_weight_decay_doc
 *
 * This function builds a new \ref gnn_weight_decay regularizator for
 * all nodes of type \a type contained in the given node \a node.
 *
 * @param  crit A pointer to a \ref gnn_weight_decay criterion.
 * @param  nu   The regularization coefficient \f$nu\f$.
 * @param  node A pointer to a \ref gnn_node.
 * @param  type A string containing the type.
 * @return Returns a pointer to a \ref gnn_weight_decay if succeeded or NULL.
 */
gnn_criterion *
gnn_weight_decay_new_with_type (gnn_criterion *crit,
                                double         nu,
                                gnn_node      *node,
                                const char    *type)
{
    gnn_pbundle *pb;
    gnn_weight_decay *wc;

    assert (crit != NULL);
    assert (node != NULL);
    assert (type != NULL);

    /* get parameter bundle consisting of  */
    /* the nodes' params of the given type */
    pb = gnn_node_sub_search_params (node, type);

    /* create weight decay penalizer */
    wc = (gnn_weight_decay *) gnn_weight_decay_new_with_pbundle (crit, nu, pb);
    if (wc == NULL)
    {
        gnn_pbundle_destroy (pb);
        GSL_ERROR_VAL ("couldn't create gnn_weight_decay regularizer",
                       GSL_EFAILED, NULL);
    }

    return (gnn_criterion *) wc;
}

/**
 * @brief Creates a Weight Decay regularization for a given parameter bundle.
 * @ingroup gnn_weight_decay_doc
 *
 * This function builds a new \ref gnn_weight_decay regularizator for
 * all parameters contained in the given parameter bundle \a pb.
 *
 * @param  crit A pointer to a \ref gnn_weight_decay criterion.
 * @param  nu   The regularization coefficient \f$nu\f$.
 * @param  pb   A \ref gnn_pbundle.
 * @return Returns a pointer to a \ref gnn_weight_decay if succeeded or NULL.
 */
gnn_criterion *
gnn_weight_decay_new_with_pbundle (gnn_criterion *crit,
                                   double         nu,
                                   gnn_pbundle   *pb)
{
    int status;
    size_t l;
    size_t size;
    gnn_criterion *c;
    gnn_weight_decay *wc;

    assert (crit != NULL);
    assert (pb != NULL);

    /* check for positive penalty term coefficient */
    if (nu < 0.0)
    {
        GSL_ERROR_VAL ("penalty term coefficient should be positive",
                       GSL_EINVAL, NULL);
    }

    /* alloc memory */
    wc = (gnn_weight_decay *) malloc (sizeof (gnn_weight_decay));
    if (wc == NULL)
    {
        GSL_ERROR_VAL ("couldn't alloc memory for gnn_weight_decay",
                       GSL_ENOMEM, NULL);
    }
    
    /* get view as gnn_criterion */
    c = (gnn_criterion *) wc;
    
    /* get size */
    size = gnn_criterion_get_size (crit);
    
    /* initialize */
    status = gnn_criterion_init  (c,
                                  "gnn_weight_decay",
                                  size,
                                  gnn_weight_decay_e,
                                  gnn_weight_decay_dy,
                                  gnn_weight_decay_destroy);
    if (status)
    {
        gnn_criterion_destroy (c);
        GSL_ERROR_VAL ("couldn't initialize gnn_weight_decay",
                       GSL_EFAILED, NULL);
    }

    /* get number of penalized parameters */
    l = gnn_pbundle_get_size (pb);

    /* set fields */
    wc->subcrit = crit;
    wc->pb      = pb;
    wc->w       = gsl_vector_alloc (l);
    wc->dw      = gsl_vector_alloc (l);
    wc->nu      = nu;
    
    if (wc->dw == NULL || wc->w == NULL)
    {
        gnn_criterion_destroy (c);
        GSL_ERROR_VAL ("couldn't allocate memory for internal "
                       "gnn_weight_decay buffer", GSL_EFAILED, NULL);
    }

    return c;
}

/**
 * @brief Sets the regularization coefficient.
 * @ingroup gnn_weight_decay_doc
 *
 * This function sets a new regularization coefficient for the
 * \ref gnn_weight_decay regularizer. It should be positive.
 *
 * @param  crit A pointer to a \ref gnn_weight_decay criterion.
 * @param  nu   The new regularization coefficient \f$nu\f$.
 * @return Returns 0 if succeeded.
 */
int
gnn_weight_decay_set_nu (gnn_criterion *crit, double nu)
{
    gnn_weight_decay *wc;

    assert (crit != NULL);

    if (nu < 0.0)
    {
        GSL_ERROR ("penalty term coefficient should be positive", GSL_EINVAL);
    }

    wc = (gnn_weight_decay *) crit;
    wc->nu = nu;

    return 0;
}

/**
 * @brief Gets the regularization coefficient.
 * @ingroup gnn_weight_decay_doc
 *
 * This function gets the currently used regularization coefficient of the
 * \ref gnn_weight_decay regularizer.
 *
 * @param  crit A pointer to a \ref gnn_weight_decay criterion.
 * @return Returns the regularization coefficient \f$nu\f$.
 */
double
gnn_weight_decay_get_nu (gnn_criterion *crit)
{
    gnn_weight_decay *wc;

    assert (crit != NULL);

    wc = (gnn_weight_decay *) crit;
    return wc->nu;
}

---