regularisation.h

Go to the documentation of this file.

/***************************************************************************
 *   Copyright (C) 2005 by Robot Group Leipzig                             *
 *    martius@informatik.uni-leipzig.de                                    *
 *    fhesse@informatik.uni-leipzig.de                                     *
 *    der@informatik.uni-leipzig.de                                        *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************
 *                                                                         *
 *  DESCRIPTION                                                            *
 *                                                                         *
 *  This file contains activation function and there derivatives in        *
 *  different regularised versions                                         *
 *                                                                         *
 *                                                                         *
 *   $Log: regularisation.h,v $
 *   Revision 1.8  2008/05/02 17:20:04  martius
 *   *** empty log message ***
 *
 *   Revision 1.7  2008/02/05 10:08:47  der
 *   new function g_derivative
 *
 *   Revision 1.6  2008/01/29 09:38:02  der
 *   function g_s changed
 *
 *   Revision 1.5  2007/04/03 16:37:57  der
 *   *** empty log message ***
 *
 *   Revision 1.4  2007/02/23 09:25:52  der
 *   regularisation with taylor expansion
 *
 *   Revision 1.3  2006/11/29 16:22:43  martius
 *   name is a variable of configurable and is used as such
 *
 *   Revision 1.2  2006/10/23 10:47:59  martius
 *   g and derivatives and inverses
 *
 *   Revision 1.1  2006/10/20 15:22:15  martius
 *   regularisation terms for g
 *
 *   Revision 1.2  2006/07/14 12:23:56  martius
 *   selforg becomes HEAD
 *
 *   Revision 1.1.2.1  2005/12/06 17:38:21  martius
 *   *** empty log message ***
 *
 *                                                                 *
 ***************************************************************************/
#ifndef __REGULARISATION_H
#define __REGULARISATION_H

double inline sqr(double x) { 
  return x*x; 
}

/// neuron transfer function
double inline g(double z)
{
  return tanh(z);
};

/// first dervative
double inline g_s(double z)
{
  //double k=tanh(z); 
  // return 1.025 - k*k; 
   return 1/((1+0.5 * z*z)*(1+0.5 * z*z));    // softer
  //return 1/(1+log(1+z*z)); // even softer
};


/// first dervative with smoothing for large z
double inline g_derivative(double z)
{
  
   return 1/((1+0.5 * z*z)*(1+0.5 * z*z));  
};

/// inverse of the first derivative
double inline g_s_inv(double z)
{
  double k=tanh(z);
   return 1/(1.025 - k*k);
  // return 1+z*z; // softer
  //return 1+log(1+z*z); // even softer
};

/** \f[ g'(z+xsi) = 1-(tanh(z+xsi))^2 \f] with additional clipping */
double inline g_s(double z, double xsi) {
  double Z = clip(z, -3.0, 3.0) + clip(xsi, -1.0, 1.0);  
  // double k=tanh(Z);  // approximation with Mittelwertsatz 
  // return 1 - k*k;
  return 1/(1 + Z*Z);//TEST
};


/// an exact formula for g''/g', with clipped Z = z+xsi
double inline g_ss_div_s(double z, double xsi) { 
  // for consistency reasons we use the same clipped z as for g'.
  double Z = clip(z, -3.0, 3.0) + clip(xsi, -1.0, 1.0);  
  // approximation with Mittelwertsatz (z is clipped)
  //  return -2*g(Z);
  return -2*Z;//TEST
};

/** with \f[ g'(z) = 1-(g(z+\xi))^2 \f] we get 
    \f[\frac{\partial}{\partial z} \frac{1}{g'(Z)} = \frac{g''}{g'^2} \f] 
    again with clipped Z
 */
double inline derive_g_s_inv_exact_clip(double z, double xsi){
  double Z = clip(z, -3.0, 3.0) + clip(xsi, -1.0, 1.0);  
  double k=tanh(Z);  // approximation with Mittelwertsatz 
  return -2*k/(1-k*k);
}

/** \f[ g'(z) = 1-(z+\xi)^2 \f] which is the series expansion to the second order
*/ 
double inline g_s_expand2(double z, double xsi){
  double Z = z + clip(xsi, -fabs(z), fabs(z));  
  return 1/(1+sqr(Z));
}

/** \f[ \frac{1}{g'(z)} \approx 1+(z+\xi)^2 \f] with geometric series approximation
*/ 
double inline g_s_inv_expand2(double z, double xsi){
  double Z = z + clip(xsi, -fabs(z)/2.0, fabs(z)/2.0);  
  return 1+sqr(Z);
}

/** \f[ \frac{g''(z)}{g'(z)} \approx 2(z+\xi)(1+(z+\xi)^2) \f] with geometric series approximation
*/ 
double inline g_ss_div_s_expand2(double z, double xsi){
  double Z = z + clip(xsi, -fabs(z)/2.0, fabs(z)/2.0);  
  //  double Z = z + clip(xsi, -fabs(z), fabs(z));  
  return -2*tanh(Z); 
}


  /// squashing function (-0.1 to 0.1), to protect against to large weight updates
double inline squash(double z)
  {
    return clip(z, -0.1, 0.1);
    //return 0.1 * tanh(10.0 * z);
  };
 
  /// squashing function with adjustable clipping size, to protect against too large weight updates
double inline squash(void* d, double z) {
    double size = *((double*)d);
    return clip(z, -size, size);
  };



#endif

---