classicreinforce.h

Go to the documentation of this file.

/***************************************************************************
 *   Copyright (C) 2008 by Robot Group Leipzig                             *
 *    martius@informatik.uni-leipzig.de                                    *
 *    der@informatik.uni-leipzig.de                                        *
 *                                                                         *
 *   ANY COMMERCIAL USE FORBIDDEN!                                         *
 *   LICENSE:                                                              *
 *   This work is licensed under the Creative Commons                      *
 *   Attribution-NonCommercial-ShareAlike 2.5 License. To view a copy of   *
 *   this license, visit http://creativecommons.org/licenses/by-nc-sa/2.5/ *
 *   or send a letter to Creative Commons, 543 Howard Street, 5th Floor,   *
 *   San Francisco, California, 94105, USA.                                *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
 *                                                                         *
 *   This version of classicreinforce has the following features           *
 *    robot is controlled with decrete actions                             *
 *    external (e.g. keyboard) control possible                            *
 *    Q-learning used for action selection                                 *
 *     states: x, defined by subclass                                      *
 *     action: y, defined by subclass                                      *
 *                                                                         *
 *   $Log: classicreinforce.h,v $
 *   Revision 1.4  2008/07/01 12:56:21  martius
 *   *** empty log message ***
 *
 *   Revision 1.3  2008/05/30 11:58:27  martius
 *   use cmath instead of math.h
 *
 *   Revision 1.2  2008/04/17 14:54:44  martius
 *   randomGen added, which is a random generator with long period and an
 *    internal state. Each Agent has an instance and passed it to the controller
 *    and the wiring. This is good for
 *   a) repeatability on agent basis,
 *   b) parallel execution as done in ode_robots
 *
 *   Revision 1.1  2008/03/01 01:49:25  martius
 *   reinforcement learning with qlerning
 *
 *
 *
 ***************************************************************************/
#ifndef __CLASSICREINFORCE_H
#define __CLASSICREINFORCE_H

#include <selforg/abstractcontroller.h>

#include <assert.h>
#include <cmath>

#include <selforg/matrix.h>
#include <selforg/noisegenerator.h>
#include <selforg/qlearning.h>

typedef struct ClassicReinforceConf {
  unsigned short buffersize; ///< size of the ringbuffers for sensors, motors,...
  int    numContext;    ///< number of context sensors (ignored)
  int reinforce_interval; ///<  time between consecutive reinforcement selections
  
  QLearning* qlearning;      ///< QLearning instance
} ClassicReinforceConf;

/**
 * class for robot controller 
 * using several feedforward networks (satelite) and one selforg controller
 */
class ClassicReinforce : public AbstractController {

public:
  ClassicReinforce(const ClassicReinforceConf& conf = getDefaultConf());
  virtual void init(int sensornumber, int motornumber, RandGen* randGen = 0);

  virtual ~ClassicReinforce();

  /// returns the number of sensors the controller was initialised with or 0 if not initialised
  virtual int getSensorNumber() const { return number_sensors; }
  /// returns the mumber of motors the controller was initialised with or 0 if not initialised
  virtual int getMotorNumber() const  { return number_motors; }

  /// performs one step (includes learning). 
  /// Calulates motor commands from sensor inputs.
  virtual void step(const sensor* , int number_sensors, motor* , int number_motors);

  /// performs one step without learning. Calulates motor commands from sensor inputs.
  virtual void stepNoLearning(const sensor* , int number_sensors, 
                              motor* , int number_motors);

  // !!!!!!!!!!!!!!!!!!! MISC STUFF !!!!!!!!
  
  /** enables/disables manual control, action_ is the sat network number to be used
      if mControl is false, action is ignored
   */
  void setManualControl(bool mControl, int action_ = 0);


  /************** CONFIGURABLE ********************************/
  virtual paramval getParam(const paramkey& key) const;
  virtual bool setParam(const paramkey& key, paramval val);
  virtual paramlist getParamList() const;


  /**** STOREABLE ****/
  /** stores the controller values to a given file. */
  virtual bool store(FILE* f) const;
  /** loads the controller values from a given file. */
  virtual bool restore(FILE* f);  

  /**** INSPECTABLE ****/
  virtual std::list<iparamkey> getInternalParamNames() const;
  virtual std::list<iparamval> getInternalParams() const;  
  virtual std::list<ILayer> getStructuralLayers() const;
  virtual std::list<IConnection> getStructuralConnections() const;

  static ClassicReinforceConf getDefaultConf(){
    ClassicReinforceConf c;
    c.buffersize=10;
    c.numContext=0;
    c.reinforce_interval=10;
    c.qlearning=0;
    return c;
  }


protected:
  unsigned short number_sensors;
  unsigned short number_motors;
  
  // sensor, sensor-derivative and motor values storage
  unsigned short buffersize;
  matrix::Matrix* x_buffer;
  matrix::Matrix* y_buffer;
  matrix::Matrix* x_context_buffer;

  bool manualControl;          ///< True if actions (sats) are selected manually

  int action;                  ///< action
  int oldaction;               ///< old action
  int state;                   ///< current state
  double reward;               ///< current reward
  double oldreward;            ///< old reward (nicer for plotting)

  ClassicReinforceConf conf;
  bool initialised;
  int t;
  int managementInterval;       ///< interval between subsequent management calls

  /// returns number of state, to be overwritten
  virtual int getStateNumber() = 0;
  
  /// returns state, to be overwritten
  virtual int calcState() = 0;

  /// returns number of actions, to be overwritten
  virtual int getActionNumber() = 0;
  /// returns action Matrix from discrete actions, to be overwritten
  virtual matrix::Matrix calcMotor(int action) = 0;

  /// returns the reinforcement (reward), to be overwritten
  virtual double calcReinforcement() = 0;

  // put new value in ring buffer
  void putInBuffer(matrix::Matrix* buffer, const matrix::Matrix& vec, int delay = 0);

  /// puts the sensors in the ringbuffer
  virtual void fillSensorBuffer(const sensor* x_, int number_sensors);
  /// puts the motors in the ringbuffer
  virtual void fillMotorBuffer(const motor* y_, int number_motors);

  /// handles inhibition damping etc.
  virtual void management();
 
};

#endif

---