---

gnn_trainer.c

Go to the documentation of this file.

/***************************************************************************
 *  @file gnn_trainer.c
 *  @brief Trainer Implementation.
 *
 *  @date   : 29-08-03 21:45
 *  @author : Pedro Ortega C. <peortega@dcc.uchile.cl>
 *  Copyright  2003  Pedro Ortega C.
 ****************************************************************************/
/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Library General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 */



/**
 * @defgroup gnn_trainer_doc gnn_trainer : Trainers for Models.
 * @ingroup libgnn_trainer
 * @todo Provide an API for those who want to implement the training procedure
 *       from scratch.
 *
 * The \ref gnn_trainer type defines a common interface for handling training
 * algorithms. It provides the basic structure on which all training algorithms
 * are based upon. Like all basic structures in \ref libgnn, the
 * \ref gnn_trainer structure cannot be used by itself. Instead, implementations
 * of trainers fill in the necessary parts to become a fully-functional
 * \ref gnn_trainer.
 *
 * What is a \ref gnn_trainer? A \ref gnn_trainer is an object that, given
 * a Model (in form of a \ref gnn_node), a criterion (\ref gnn_criterion)
 * and a dataset (\ref gnn_dataset):
 * - presents the input patterns sequentially to the Model, that is
 *   \f$x^1, x^2, x^3, \ldots, x^N\f$, where \f$x^k\f$ denotes the \f$k\f$-th
 *   input pattern, and \f$N\f$ is the order of the dataset,
 * - obtains its outputs \f$y^k = f(x^k)\f$ for \f$k=0,\ldots,N-1\f$,
 * - compares them with the target patterns using the criterion
 * - computes the gradient with respect to the parameters and
 * - adjusts them.
 * These steps are performed in order to optimize the Model.
 *
 * The patterns are presented in order, one after the other, until the last
 * pattern has been presented. After then, the dataset is reset and the
 * procedure is repeated. This is what is called an epoch: a cycle where
 * each pattern is evaluated once.
 *
 * Although one pattern is presented at a time, the needed gradient
 * \f$\frac{\partial E}{\partial w}\f$ is estimated by perfoming a weighted
 * sum over the particular gradients \f$\frac{\partial E}{\partial w}|_{x^k}\f$
 * for each pattern \f$x^k\f$:
 * \f[ \frac{\partial \hat{E}}{\partial w} =
 *     \frac{ \sum_k p_k
 *            { \left . \frac{\partial E}{\partial w} \right | }_{x_k}
 *          }
 *          { \sum_k p_k }
 * \f]
 * where \f$p_k\f$ is the weight of the pattern \f$x^k\f$. How many patterns
 * are considered in this sum? Well, in the literature, when only one pattern
 * is considered, then the procedure is called on-line training, and in a
 * batch training, all patterns are taken. In \ref libgnn, this sum is taken
 * over so-called minibatches, which are simply groups of patterns of the same
 * size. E.g. if there a 10 patterns, and the minibatches are of size 4, then
 * the patterns are presented in the following groups:
 * \f[ \{x^1, x^2, x^3, x^4\}, \{x^5, x^6, x^7, x^8\}, \{x^9, x^{10}\} \f]
 * (Please note that the last minibatch has been cut down to fit.)
 * This operation is executed automatically by calling \ref gnn_trainer_train
 * on a \ref gnn_trainer. \ref gnn_trainer_train returns the mean cost
 * of all patterns presented to the model during the current epoch and adjusts
 * the patterns of the model.
 *
 * To reset a trainer and start over again (the parameters are kept),
 * call \ref gnn_trainer_reset.
 *
 * How to extend the \ref gnn_trainer structure to implement your own
 * trainer? In the same spirit as in all other basic structures, you can
 * use the \ref gnn_trainer structure itself or extend it like the following
 * hypothetical "my_trainer" structure:
 * \code
 *     typedef struct _my_trainer
 *     {
 *         gnn_trainer trainer;
 *
 *         // the specific data for "my_trainer"
 *         ...
 *     } my_trainer;
 * \endcode
 * This allows you to cast a "my_trainer" structure to a \ref gnn_trainer
 * structure whenever needed.
 *
 * Secondly, you have to provide at least four functions: a constructor,
 * which allocates and initializes a new "my_trainer" structure; a "reset"
 * function to reset the training procedure, which should be of type
 * \ref gnn_trainer_reset_type; a "train" function to train a minibatch
 * and update the parameters, which should be of type
 * \ref gnn_trainer_train_type; and a "destroy" function, which destroys
 * all additional data used by the "my_trainer" structure, of type
 * \ref gnn_trainer_destroy_type (alternatively, you can use the default
 * implementations for the "reset" and "destroy" functions).
 *
 * Please read the specific documentation for the function types that you
 * should provide: \ref gnn_trainer_reset_type and \ref gnn_trainer_train_type.
 */



/******************************************/
/* Include Files                          */
/******************************************/

#include <string.h>
#include "gnn_trainer.h"
#include "gnn_utilities.h"



/******************************************/
/* Static Declaration                     */
/******************************************/

static int
gnn_trainer_default_reset (gnn_trainer *trainer);

static void
gnn_trainer_default_destroy (gnn_trainer *trainer);



/******************************************/
/* Static Implementation                  */
/******************************************/

/**
 * @brief Default "reset" function for a trainer.
 * @ingroup gnn_trainer_doc
 *
 * This function is the default "reset" function for a trainer. It does nothing.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return 0 if succeeded.
 */
static int
gnn_trainer_default_reset (gnn_trainer *trainer)
{
    assert (trainer != NULL);

    return 0;
}

/**
 * @brief Default "destroy" function for a trainer.
 * @ingroup gnn_trainer_doc
 *
 * This function is the default "destroy" function for a trainer. It assumes
 * that there isn't any additional data for the specific trainer type, so
 * it actually just returns.
 *
 * @param trainer A pointer to a \ref gnn_trainer.
 */
static void
gnn_trainer_default_destroy (gnn_trainer *trainer)
{
    return;
}



/******************************************/
/* Public Interface                       */
/******************************************/

/**
 * @brief Get the trainer's type.
 * @ingroup gnn_trainer_doc
 *
 * This function returns a string which contains the name of the trainer's
 * type. The string should not be modified.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns a pointer to the trainer's own type string.
 */
const char *
gnn_trainer_get_type (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    return trainer->type;
}

/**
 * @brief Initializes a \ref gnn_trainer structure.
 * @ingroup gnn_trainer_doc
 *
 * This function initializes a \ref gnn_trainer structure by setting its
 * internal fields adecuately. Every extension to the \ref gnn_trainer
 * structure should have a constructor that calls this function to:
 *
 * - define the trainer's type,
 * - set the model to be trained (the node),
 * - set the criterion to be used,
 * - set the dataset with the patterns to be used during training,
 * - install the type-specific functions for "reset", "train" and "destroy"
 *
 * All arguments are mandatory, except for the "reset" and the "destroy"
 * functions: if the are specified as NULL, then the defaults are installed
 * instead (see \ref gnn_trainer_default_reset and
 * \ref gnn_trainer_default_destroy).
 *
 * The associated sizes should match. That is, the input pattern's sizes
 * contained in the dataset should corresponde to the node's input size.
 * Likewise, the output pattern's sizes should match with the node's output
 * size and the criterion's size.
 *
 * The following example should clarify its use. Suppose you want to create
 * a new trainer type, called "my_trainer". You have already implemented the
 * two of the needed functions, "my_trainer_reset", "my_trainer_train".
 * Actually, your special training function does not need any additional
 * data, and so the destructor isn't needed. Then, the following code could
 * be a possible implementation of the constructor:
 * \code
   gnn_trainer *
   my_trainer_new (gnn_node *model, gnn_criterion *crit, gnn_dataset *data)
   {
        int status;
        my_trainer  *t;     // this is just an extension of gnn_trainer
        gnn_trainer *tview; // points to the same structure, but uses another
                            // view
        
        // alloc space
        t = (my_trainer *) malloc (sizeof (*t));
        if (t == NULL)
            print_error_message_and_return_error_code ();

        // cast to a gnn_trainer
        tview = (gnn_trainer *) t;
        
        // initialize
        status = gnn_trainer_init (tview,
                                   "my_trainer",
                                   model,
                                   crit,
                                   data,
                                   my_trainer_reset,
                                   my_trainer_reset,
                                   NULL);
        if (status)
            print_error_message_and_return_error_code ();

        // finish initialization
        ...
        
        return tview;
   }
 * \endcode
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @param  type    A string to the typename. It will be duplicated.
 * @param  node    A pointer to a \ref gnn_node.
 * @param  crit    A pointer to a \ref gnn_criterion.
 * @param  data    A pointer to a \ref gnn_data.
 * @param  reset   A pointer to the "reset" function (or NULL).
 * @param  train   A pointer to the "train" function.
 * @param  destroy A pointer to the "destroy" function (or NULL).
 * @return Returns 0 if initialization succeeded.
 */
int
gnn_trainer_init (gnn_trainer             *trainer,
                  const char              *type,
                  gnn_node                *node,
                  gnn_criterion           *crit,
                  gnn_dataset             *data,
                  gnn_trainer_reset_type   reset,
                  gnn_trainer_train_type   train,
                  gnn_trainer_destroy_type destroy)
{
    size_t n;
    size_t m;
    size_t l;

    assert (trainer != NULL);
    
    /* check arguments */
    if (type == NULL)
        GSL_ERROR ("invalid trainer type", GSL_EINVAL);
    if (node == NULL)
        GSL_ERROR ("invalid node", GSL_EINVAL);
    if (crit == NULL)
        GSL_ERROR ("invalid criterion", GSL_EINVAL);
    if (data == NULL)
        GSL_ERROR ("invalid dataset", GSL_EINVAL);
    if (train == NULL)
        GSL_ERROR ("invalid train function", GSL_EINVAL);

    /* check sizes */
    if (gnn_dataset_input_get_size (data) != gnn_node_input_get_size (node))
        GSL_ERROR ("the input pattern's size does not match with the node's "
                   "input size", GSL_EINVAL);
    if (gnn_dataset_output_get_size (data) != gnn_node_output_get_size (node))
        GSL_ERROR ("the output pattern's size does not match with the node's "
                   "output size", GSL_EINVAL);
    if (gnn_dataset_output_get_size (data) != gnn_criterion_get_size (crit))
        GSL_ERROR ("the output pattern's sizes does not match with the "
                   "criterion's size", GSL_EINVAL);

    /* check if sizes are stricly positive */
    if (gnn_node_input_get_size (node) <= 0)
        GSL_ERROR ("the input size should be stricly positive",
                   GSL_EINVAL);
    if (gnn_node_output_get_size (node) <= 0)
        GSL_ERROR ("the output size should be stricly positive",
                   GSL_EINVAL);
    if (gnn_node_param_get_size (node) <= 0)
        GSL_ERROR ("the parameter size should be stricly positive",
                   GSL_EINVAL);

    /* get sizes */
    n = gnn_dataset_input_get_size (data);
    m = gnn_dataset_output_get_size (data);
    l = gnn_node_param_get_size (node);

    /* set fields */
    trainer->type    = strdup (type);
    trainer->node    = node;
    trainer->crit    = crit;
    trainer->data    = data;
    trainer->n       = 1;
    trainer->s       = 0;
    trainer->epoch   = 1;
    trainer->sume    = 0.0;
    trainer->sump    = 0.0;
    trainer->grad    = gnn_grad_new (node, crit, data);
    trainer->reset   = gnn_trainer_default_reset;
    trainer->train   = NULL;
    trainer->destroy = gnn_trainer_default_destroy;

    /* install functions */
    if (reset != NULL)
        trainer->reset = reset;
    if (train != NULL)
        trainer->train = train;
    if (destroy != NULL)
        trainer->destroy = destroy;

    return 0;
}

/**
 * @brief Destroys the trainer.
 * @ingroup gnn_trainer_doc
 *
 * This function destroys the \ref gnn_trainer by calling the installed
 * destructor function for the trainer type and by freeing the
 * \ref gnn_trainer's structure.
 *
 * It doesn't destroy the node, nor the criterion, nor the dataset. They
 * should be destroyed independently.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 */
void
gnn_trainer_destroy (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    /* destroy type-specific data */
    trainer->destroy (trainer);
    
    /* free structure */
    if (trainer->grad != NULL)
        gnn_grad_destroy (trainer->grad);
    if (trainer->type != NULL)
        free ((char *) trainer->type);
    free (trainer);
}

/**
 * @brief Resets the trainer.
 * @ingroup gnn_trainer_doc
 *
 * This function resets the trainer, i.e. the error is cleared, the epoch
 * reset, the pattern iterator set to the first pattern, etc.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns 0 if suceeded.
 */
int
gnn_trainer_reset (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    /* reset trainer */
    trainer->reset (trainer);
    
    /* reset common fields */
    trainer->s     = 0;
    trainer->epoch = 1;
    trainer->sume  = 0.0;
    trainer->sump  = 0.0;
    
    return 0;
}

/**
 * @brief Trains the node.
 * @ingroup gnn_trainer_doc
 *
 * This function executes a full iteration of the training algorithm on the
 * node. In this context, an iteration consists of presenting a batch of
 * patterns to the node and adjust its parameters accordingly.
 *
 * Every call to this function presents the next patterns until the end
 * is reached. The last batch will be of smaller if the number of total
 * patterns is not a multiple of the size of the batches. If the end is
 * reached, then the trainer goes to the next epoch and the dataset is
 * automatically reset.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns the mean value of the (by the criterion) computed cost for
 *         the current epoch.
 */
double
gnn_trainer_train (gnn_trainer *trainer)
{
    double cost;

    assert (trainer != NULL);

    /* train */
    cost = trainer->train (trainer);

    return cost;
}

/**
 * @brief Get the dataset.
 * @ingroup gnn_trainer_doc
 *
 * This function returns a pointer to the dataset that the trainer uses for
 * training.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns a pointer to a \ref gnn_dataset.
 */
gnn_dataset *
gnn_trainer_get_dataset (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    return trainer->data;
}

/**
 * @brief Get the model.
 * @ingroup gnn_trainer_doc
 *
 * This function returns a pointer to the model that the trainer trains.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns a pointer to a \ref gnn_node.
 */
gnn_node *
gnn_trainer_get_node (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    return trainer->node;
}

/**
 * @brief Get the criterion.
 * @ingroup gnn_trainer_doc
 *
 * This function returns a pointer to the criterion that the trainer uses for
 * training.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns a pointer to a \ref gnn_criterion.
 */
gnn_criterion *
gnn_trainer_get_criterion (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    return trainer->crit;
}

/**
 * @brief Set the size for minibatches.
 * @ingroup gnn_trainer_doc
 *
 * This function sets the size of the minibatches that should be processed
 * upon a call of \ref gnn_trainer_train. This size should be stricly positive.
 * Although it can be greather than the number of available patterns in the
 * dataset, the effective minibatch will be smaller.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @param  size    The size for minibatches.
 * @return Returns 0 if succeeded.
 */
int
gnn_trainer_batch_set_size (gnn_trainer *trainer, size_t size)
{
    assert (trainer != NULL);
    
    /* check batch size */
    if (size < 1)
    {
        GSL_ERROR ("the size of the batch should be stricly positive",
                   GSL_EINVAL);
    }
    
    /* set batchsize */
    trainer->n = size;
    
    return 0;
}

/**
 * @brief Get the size of the minibatches.
 * @ingroup gnn_trainer_doc
 *
 * This function returns a strictly positive number corresponding to the
 * size of the minibatches that are processed upon a call of
 * \ref gnn_trainer_train.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns the size of the minibatches.
 */
size_t
gnn_trainer_batch_get_size (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    return trainer->n;
}

/**
 * @brief Returns the index of the next minibatch.
 * @ingroup gnn_trainer_doc
 *
 * This function returns the index of the first pattern in the minibatch
 * to be processed.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns the index of the next pattern.
 */
size_t
gnn_trainer_get_pattern_index (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    return trainer->s;
}

/**
 * @brief Get the number of the current epoch.
 * @ingroup gnn_trainer_doc
 *
 * This function returns the number of the current epoch.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns the number of the current epoch.
 */
size_t
gnn_trainer_get_epoch (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    return trainer->epoch;
}

/**
 * @brief Get the mean cost.
 * @ingroup gnn_trainer_doc
 *
 * This function returns the mean cost of the patterns presented in the current
 * epoch. That is, if the patterns \f$ x^1, \ldots, x^{k'} \f$ have been
 * evaluated in the current epoch, then the value
 * \f[ <E>_{k'} = \frac{1}{\sum_{k=1}^{k'} p_k}
 *                     \sum_{k=1}^{k'} p_k E(y_k, t_k) \f]
 * where \f$p_k\f$ is the weight of the \f$k\f$-th pattern, will be returned.
 *
 * It is important to note that this value considers also the last evaluated
 * minibatch's cost.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns the weighted sum \f$<E>_{k'}\f$.
 */
double
gnn_trainer_get_epoch_cost (gnn_trainer *trainer)
{
    double Se;
    double Sp;
    
    assert (trainer != NULL);

    /* compute cost sum and weight sum */
    Se = trainer->sume + GNN_GRAD_SUME (trainer->grad);
    Sp = trainer->sump + GNN_GRAD_SUMP (trainer->grad);

    return Se / Sp;
}

/**
 * @brief Returns a pointer to the internal \ref gnn_grad buffer.
 * @ingroup gnn_trainer_doc
 *
 * This function returns a pointer to the internal error and gradients
 * evaluation buffer.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns a pointer to the \ref gnn_grad.
 */
gnn_grad *
gnn_trainer_batch_get_grad (gnn_trainer *trainer)
{
    return trainer->grad;
}

/**
 * @brief Returns the last evaluated batch's mean error.
 * @ingroup gnn_trainer_doc
 *
 * This function returns a the last evaluated batch's mean error \f$<E>\f$.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns \f$<E>\f$.
 */
double
gnn_trainer_batch_get_e (gnn_trainer *trainer)
{
    return GNN_GRAD_E (trainer->grad);
}

/**
 * @brief Returns the last batch's evaluated gradient dx.
 * @ingroup gnn_trainer_doc
 *
 * This function returns a the last evaluated batch's mean gradient with
 * respect to its inputs \f$<\frac{\partial E}{\partial x}>\f$.
 *
 * The returned vector can be freely accessed and its values modified, but
 * it should not be freed.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns \f$<\frac{\partial E}{\partial x}>\f$.
 */
gsl_vector *
gnn_trainer_batch_get_dx (gnn_trainer *trainer)
{
    return GNN_GRAD_DX (trainer->grad);
}

/**
 * @brief Returns the last batch's evaluated gradient dw.
 * @ingroup gnn_trainer_doc
 *
 * This function returns a the last evaluated batch's mean gradient with
 * respect to its parameters \f$<\frac{\partial E}{\partial w}>\f$.
 *
 * The returned vector can be freely accessed and its values modified, but
 * it should not be freed.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns \f$<\frac{\partial E}{\partial w}>\f$.
 */
gsl_vector *
gnn_trainer_batch_get_dw (gnn_trainer *trainer)
{
    return GNN_GRAD_DW (trainer->grad);
}

/**
 * @brief Processes a batch.
 * @ingroup gnn_trainer_doc
 *
 * This function processes the current minibatch of patterns, computing
 * the batch's mean cost and its gradients \f$\frac{\partial E}{\partial x}\f$
 * and \f$\frac{\partial E}{\partial w}\f$. They can be retrived afterthen
 * by invoking
 *
 * - \ref gnn_trainer_batch_get_e
 * - \ref gnn_trainer_batch_get_dx
 * - \ref gnn_trainer_batch_get_dw
 * - \ref gnn_trainer_get_epoch_cost
 *
 * Theese values are obtained by averaging over the batch's pattern.
 *
 * Several calls of this function will process the same minibatch. Only
 * a call of \ref gnn_trainer_batch_next moves onto the next one.
 *
 * This function is very handy when implementing your own trainers. It
 * provides an easy way to obtain the gradient and handles the necessary
 * field updates on the \ref gnn_trainer. Please refer to
 * \ref gnn_trainer_train_type for details.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns 0 if succeeded.
 */
int
gnn_trainer_batch_process (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    /* process batch */
    gnn_grad_pats (trainer->grad, gnnGradDw, trainer->s, trainer->n);

    return 0;
}

/**
 * @brief Moves onto the next batch.
 * @ingroup gnn_trainer_doc
 *
 * This function moves to the next batch. That is, if sets the trainers
 * internal state to prepare to process the next batch. If the end of the
 * dataset is reached, then the next epoch is initiated, the mean cost
 * info cleared, the dataset reset.
 *
 * This function is very handy when implementing your own trainers. It
 * should be used in conjunction with \ref gnn_trainer_batch_process.
 *
 * @param  trainer A pointer to a \ref gnn_trainer.
 * @return Returns 0 if suceeded.
 */
int
gnn_trainer_batch_next (gnn_trainer *trainer)
{
    assert (trainer != NULL);
    
    /* update current pattern index */
    trainer->s += trainer->n;

    /* update mean cost values */
    trainer->sume += GNN_GRAD_SUME (trainer->grad);
    trainer->sump += GNN_GRAD_SUMP (trainer->grad);

    /* if end reached, advance to next epoch */
    if (trainer->s >= gnn_dataset_get_size (trainer->data))
    {
        trainer->epoch++;
        trainer->s    = 0;
        trainer->sume = 0.0;
        trainer->sump = 0.0;
        gnn_dataset_reset (trainer->data);
    }
    
    return 0;
}

---