---

gnn_cross_entropy.c

Go to the documentation of this file.

/***************************************************************************
 *  @file gnn_cross_entropy.c
 *  @brief Cross Entropy Error Criterion.
 *
 *  @date   : 25-08-03 23:20
 *  @author : Pedro Ortega C. <peortega@dcc.uchile.cl>
 *  Copyright  2003  Pedro Ortega C.
 ****************************************************************************/
/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Library General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 */

/**
 * @brief Cross Entropy Error function.
 * @defgroup gnn_cross_entropy_doc gnn_cross_entropy : Cross-Entropy Error Criterion.
 * @ingroup gnn_criterion_doc
 *
 * This datatype implements the Cross-Entropy criterion, given by
 * \f[ E(y,t) =
       - \sum_i^m \left (
                  t_i \ln y_i + (1 - t_i) \ln (1 - y_i)
                  \right )
   \f]
 * As it can be easily seen, the cross entropy cost function depends on the
 * relative error (instead of the absolute error like in MSE), thus giving
 * to small and large errors te same weight.
 *
 * The cross entropy criterion fits well in classification problems, where
 * the output encodes the class using binary values; that is, where
 * \f$y_c = 1\f$ and \f$y_i = 0\f$ for all \f$i \neq c\f$ if the input
 * \f$x\f$ corresponds to the \f$c\f$-th class.
 *
 */



/******************************************/
/* Include Files                          */
/******************************************/

#include <math.h>
#include "gnn_cross_entropy.h"



/******************************************/
/* Static Declaration                     */
/******************************************/

typedef gnn_criterion gnn_cross_entropy;

double
gnn_cross_entropy_e (gnn_criterion *crit,
                     const gsl_vector *y,
                     const gsl_vector *t);

int
gnn_cross_entropy_dy (gnn_criterion *crit,
                      const gsl_vector *y,
                      const gsl_vector *t,
                      gsl_vector * dy);



/******************************************/
/* Static Implementation                  */
/******************************************/

/**
 * @brief The evaluation function.
 * @ingroup gnn_cross_entropy_doc
 *
 * This function corresponds to the evaluation of the cross entropy criterion.
 *
 * @param  crit A pointer to a \ref gnn_cross_entropy criterion.
 * @param  y    A pointer to an estimation vector \f$y\f$.
 * @param  t    A pointer to the desired target vector \f$t\f$.
 * @return A real number corresponding to the value of the criterion.
 */
double
gnn_cross_entropy_e (gnn_criterion *crit,
                     const gsl_vector *y,
                     const gsl_vector *t)
{
    int i;
    double E;

    assert (crit != NULL);
    assert (y != NULL);
    assert (t != NULL);

    /* check sizes */
    if (y->size != t->size)
        GSL_ERROR_VAL ("vector sizes should be the same", GSL_EINVAL, 0.0);

    /* compute error */
    E = 0.0;
    for (i=0; i<y->size; ++i)
    {
        double yi;
        double ti;
        double ei;

        yi = gsl_vector_get (y, i);
        ti = gsl_vector_get (t, i);

        ei  = (yi < GNN_CROSS_ENTROPY_EPS)?
                ti * log (GNN_CROSS_ENTROPY_EPS)
              : ti * log (yi);
              
        ei += (1.0 - yi < GNN_CROSS_ENTROPY_EPS)?
                (1.0 - ti) * log (1.0 - GNN_CROSS_ENTROPY_EPS)
              : (1.0 - ti) * log (1.0 - yi);

        E -= ei;
    }

    return E;
}

/**
 * @brief The gradient evaluation function.
 * @ingroup gnn_cross_entropy_doc
 *
 * This function implements the \ref gnn_cross_entropy criterion's gradient
 * evaluation function given by
 *
 * \f[ \frac{\partial E}{\partial y_i} =
       - \frac{y_i - t_i}{y_i (1 - y_i)} \f]
 *
 * @param  crit A pointer to a \ref gnn_cross_entropy criterion.
 * @param  y    A pointer to an estimation vector \f$y\f$.
 * @param  t    A pointer to the desired target vector \f$t\f$.
 * @param  dy   A pointer to a buffer vector where the result should be placed.
 * @return Returns 0 if succeeded.
 */
int
gnn_cross_entropy_dy (gnn_criterion *crit,
                      const gsl_vector *y,
                      const gsl_vector *t,
                      gsl_vector * dy)
{
    int i;

    assert (crit != NULL);
    assert (y != NULL);
    assert (t != NULL);
    assert (dy != NULL);

    /* check sizes */
    if (y->size != t->size || y->size != dy->size)
        GSL_ERROR ("vector sizes should be the same", GSL_EINVAL);

    /* compute dy */
    for (i=0; i<y->size; ++i)
    {
        double yi;
        double ti;
        double dyi;

        yi  = gsl_vector_get (y, i);
        ti  = gsl_vector_get (t, i);
        dyi = gsl_vector_get (dy, i);

        yi  = (yi > 1.0 - GNN_CROSS_ENTROPY_EPS)?
                 (1.0 - GNN_CROSS_ENTROPY_EPS)
               : yi;
        yi  = (yi < GNN_CROSS_ENTROPY_EPS)?
                 GNN_CROSS_ENTROPY_EPS
               : yi;

        dyi += (yi - ti) / (yi * (1.0 - yi));

        gsl_vector_set (dy, i, dyi);
    }

    return 0;
}


/******************************************/
/* Public Interface                       */
/******************************************/

/**
 * @brief Creates a new \ref gnn_cross_entropy criterion.
 * @ingroup gnn_cross_entropy_doc
 *
 * This function creates a new \ref gnn_cross_entropy criterion of the given
 * size.
 *
 * @param  size    The size of the estimation and the target vector \f$y\f$
 *                 and \f$t\f$.
 * @return Returns a pointer to a new \ref gnn_cross_entropy or NULL if failed.
 */
gnn_criterion *
gnn_cross_entropy_new (size_t size)
{
    int status;
    gnn_criterion *crit;

    /* check size */
    if (size < 1)
        GSL_ERROR_VAL ("size should be strictly positive", GSL_EINVAL, NULL);

    /* alloc memory */
    crit = (gnn_criterion *) malloc (sizeof (*crit));
    if (crit == NULL)
        GSL_ERROR_VAL ("couldn't alloc memory for gnn_cross_entropy",
                       GSL_ENOMEM, NULL);

    /* initialize */
    status = gnn_criterion_init  (crit,
                                  "gnn_cross_entropy",
                                  size,
                                  gnn_cross_entropy_e,
                                  gnn_cross_entropy_dy,
                                  NULL);
    if (status)
    {
        gnn_criterion_destroy (crit);
        GSL_ERROR_VAL ("couldn't initialize gnn_cross_entropy",
                       GSL_EFAILED, NULL);
    }

    return crit;
}

---