regularisation.h

Go to the documentation of this file.

/***************************************************************************
 *   Copyright (C) 2005 by Robot Group Leipzig                             *
 *    martius@informatik.uni-leipzig.de                                    *
 *    fhesse@informatik.uni-leipzig.de                                     *
 *    der@informatik.uni-leipzig.de                                        *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************
 *                                                                         *
 *  DESCRIPTION                                                            *
 *                                                                         *
 *  This file contains activation function and there derivatives in        *
 *  different regularised versions                                         *
 *                                                                         *
 *                                                                         *
 *   $Log: regularisation.h,v $
 *   Revision 1.3  2006/11/29 16:22:43  martius
 *   name is a variable of configurable and is used as such
 *
 *   Revision 1.2  2006/10/23 10:47:59  martius
 *   g and derivatives and inverses
 *
 *   Revision 1.1  2006/10/20 15:22:15  martius
 *   regularisation terms for g
 *
 *   Revision 1.2  2006/07/14 12:23:56  martius
 *   selforg becomes HEAD
 *
 *   Revision 1.1.2.1  2005/12/06 17:38:21  martius
 *   *** empty log message ***
 *
 *                                                                 *
 ***************************************************************************/
#ifndef __REGULARISATION_H
#define __REGULARISATION_H

double inline sqr(double x) { 
  return x*x; 
}

/// neuron transfer function
double inline g(double z)
{
  return tanh(z);
};

/// first dervative
double inline g_s(double z)
{
  double k=tanh(z); 
  return 1.025 - k*k; 
  // return 1/(1+z*z);    // softer
  //return 1/(1+log(1+z*z)); // even softer
};

/// inverse of the first derivative
double inline g_s_inv(double z)
{
  double k=tanh(z);
  return 1/(1.025 - k*k);
  // return 1+z*z; // softer
  //return 1+log(1+z*z); // even softer
};

/** \f[ g'(z+xsi) = 1-(tanh(z+xsi))^2 \f] with additional clipping */
double inline g_s(double z, double xsi) {
  double Z = clip(z, -3.0, 3.0) + clip(xsi, -1.0, 1.0);  
  double k=tanh(Z);  // approximation with Mittelwertsatz 
  return 1 - k*k;
};

/// an exact formula for g''/g', with clipped Z = z+xsi
double inline g_ss_div_s(double z, double xsi) { 
  // for consistency reasons we use the same clipped z as for g'.
  double Z = clip(z, -3.0, 3.0) + clip(xsi, -1.0, 1.0);  
  // approximation with Mittelwertsatz (z is clipped)
  return -2*g(Z);
};

/** with \f[ g'(z) = 1-(g(z+\xsi))^2 \f] we get 
    \f[\frac{\partial}{\partial z} \frac{1}{g'(Z)} = \frac{g''}{g'^2} \f] 
    again with clipped Z
 */
double inline derive_g_s_inv_exact_clip(double z, double xsi){
  double Z = clip(z, -3.0, 3.0) + clip(xsi, -1.0, 1.0);  
  double k=tanh(Z);  // approximation with Mittelwertsatz 
  return -2*k/(1-k*k);
}

/** \f[ g'(z) = 1-(z+\xsi)^2 \f] which is the series expansion to the second order
*/ 
double inline g_s_expand2(double z, double xsi){
  return 1-sqr(z+xsi);
}

/** \f[ \frac{1}{g'(z)} \approx 1+(z+\xsi)^2 \f] with geometric series approximation
*/ 
double inline g_s_inv_expand2(double z, double xsi){
  return 1+sqr(z+xsi);
}

/** with \f[ g'(z) = 1-(z+\xsi)^2 \f] which is the series expansion to the second order
    we get \f[ \frac{1}{g'(z)} \approx 1+(z+\xsi)^2 \f] and therewith
    \f[ \frac{\partial}{\partial z} \frac{1}{g'(Z)}  2(z+\xsi) \f]
 */
double inline derive_g_s_inv_expand2(double z, double xsi){
  return 2*(z+xsi);
}


#endif

---