Robot Simulator of the Robotics Group for Self-Organization of Control  0.8.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
QLearning Class Reference

implements QLearning More...

#include <qlearning.h>

Inheritance diagram for QLearning:
Collaboration diagram for QLearning:

Public Member Functions

 QLearning (double eps, double discount, double exploration, int eligibility, bool random_initQ=false, bool useSARSA=false, int tau=1000)
 
virtual ~QLearning ()
 
virtual void init (unsigned int stateDim, unsigned int actionDim, RandGen *randGen=0)
 initialisation with the given number of action and states More...
 
virtual unsigned int select (unsigned int state)
 selection of action given current state. More...
 
virtual unsigned int select_sample (unsigned int state)
 selection of action given current state. More...
 
virtual unsigned int select_keepold (unsigned int state)
 select with preference to old (90% if good) and 30% second best More...
 
virtual double learn (unsigned int state, unsigned int action, double reward, double learnRateFactor=1)
 
matrix::Matrix getActionValues (unsigned int state)
 returns the vector of values for all actions given the current state More...
 
virtual void reset ()
 tells the q learning that the agent was reset, so that it forgets it memory. More...
 
virtual unsigned int getStateDim () const
 returns the number of states More...
 
virtual unsigned int getActionDim () const
 returns the number of actions More...
 
virtual double getCollectedReward () const
 returns the collectedReward reward More...
 
virtual const matrix::MatrixgetQ () const
 returns q table (mxn) == (states x actions) More...
 
virtual bool store (FILE *f) const
 stores the object to the given file stream (ASCII preferred). More...
 
virtual bool restore (FILE *f)
 loads the object from the given file stream (ASCII preferred). More...
 
- Public Member Functions inherited from Configurable
 Configurable ()
 
 Configurable (const std::string &name, const std::string &revision)
 intialise with name and revision (use "$ID$") More...
 
virtual ~Configurable ()
 
virtual void notifyOnChange (const paramkey &key)
 Is called when a parameter was changes via setParam(). More...
 
virtual void addParameter (const paramkey &key, paramval *val, paramval minBound, paramval maxBound, const paramdescr &descr=paramdescr())
 This is the new style for adding configurable parameters. More...
 
virtual void addParameter (const paramkey &key, paramval *val, const paramdescr &descr=paramdescr())
 See addParameter(const paramkey& key, paramval* val, paramval minBound, paramval maxBound, const paramdescr& descr) More...
 
virtual void addParameter (const paramkey &key, parambool *val, const paramdescr &descr=paramdescr())
 See addParameter(const paramkey& key, paramval* val) but for bool values. More...
 
virtual void addParameter (const paramkey &key, paramint *val, paramint minBound, paramint maxBound, const paramdescr &descr=paramdescr())
 See addParameter(const paramkey& key, paramval* val) but for int values. More...
 
virtual void addParameter (const paramkey &key, paramint *val, const paramdescr &descr=paramdescr())
 
virtual void addParameterDef (const paramkey &key, paramval *val, paramval def, paramval minBound, paramval maxBound, const paramdescr &descr=paramdescr())
 This function is only provided for convenience. More...
 
virtual void addParameterDef (const paramkey &key, paramval *val, paramval def, const paramdescr &descr=paramdescr())
 
virtual void addParameterDef (const paramkey &key, parambool *val, parambool def, const paramdescr &descr=paramdescr())
 See addParameterDef(const paramkey&, paramval*, paramval) More...
 
virtual void addParameterDef (const paramkey &key, paramint *val, paramint def, paramint minBound, paramint maxBound, const paramdescr &descr=paramdescr())
 See addParameterDef(const paramkey&, paramval*, paramval) More...
 
virtual void addParameterDef (const paramkey &key, paramint *val, paramint def, const paramdescr &descr=paramdescr())
 
virtual void setParamDescr (const paramkey &key, const paramdescr &descr, bool traverseChildren=true)
 sets a description for the given parameter More...
 
int getId () const
 return the id of the configurable objects, which is created by random on initialisation More...
 
virtual paramkey getName () const
 return the name of the object More...
 
virtual paramkey getRevision () const
 returns the revision of the object More...
 
virtual void setName (const paramkey &name, bool callSetNameOfInspectable=true)
 Sets the name of the configurable. More...
 
virtual void setRevision (const paramkey &revision)
 sets the revision Hint: { return "$ID$"; } More...
 
virtual paramval getParam (const paramkey &key, bool traverseChildren=true) const
 returns the value of the requested parameter or 0 (+ error message to stderr) if unknown. More...
 
virtual bool hasParam (const paramkey &key, bool traverseChildren=true) const
 Returns if the requested parameter is part of the configurable or their children. More...
 
virtual bool setParam (const paramkey &key, paramval val, bool traverseChildren=true)
 sets the value of the given parameter or does nothing if unknown. More...
 
virtual void setParamBounds (const paramkey &key, paramval minBound, paramval maxBound, bool traverseChildren=true)
 Sets the bounds (minBound and maxBound) of the given parameter. More...
 
virtual void setParamBounds (const paramkey &key, paramint minBound, paramint maxBound, bool traverseChildren=true)
 
virtual void setParamBounds (const paramkey &key, paramvalBounds bounds, bool traverseChildren=true)
 
virtual void setParamBounds (const paramkey &key, paramintBounds bounds, bool traverseChildren=true)
 
virtual paramlist getParamList () const
 The list of all parameters with there value as allocated lists. More...
 
virtual std::list< paramkeygetAllParamNames (bool traverseChildren=true)
 returns all names that are configureable More...
 
virtual parammap getParamValMap () const
 
virtual paramintmap getParamIntMap () const
 
virtual paramboolmap getParamBoolMap () const
 
virtual paramdescr getParamDescr (const paramkey &key, bool traverseChildren=true) const
 returns the description for the given parameter More...
 
virtual paramvalBounds getParamvalBounds (const paramkey &key, bool traverseChildren=true) const
 
virtual paramintBounds getParamintBounds (const paramkey &key, bool traverseChildren=true) const
 
virtual bool hasParamDescr (const paramkey &key, bool traverseChildren=true) const
 
virtual bool hasParamvalBounds (const paramkey &key, bool traverseChildren=true) const
 
virtual bool hasParamintBounds (const paramkey &key, bool traverseChildren=true) const
 
virtual bool storeCfg (const char *filenamestem, const std::list< std::string > &comments=std::list< std::string >())
 stores the key values paires into the file : filenamestem.cfg including the comments given in the list More...
 
virtual bool restoreCfg (const char *filenamestem)
 restores the key values paires from the file : filenamestem.cfg More...
 
void print (FILE *f, const char *prefix, int columns=90, bool traverseChildren=true) const
 prints the keys, values and descriptions to the file. Each line is prefixed More...
 
bool parse (FILE *f, const char *prefix=0, bool traverseChildren=true)
 parses the configuration from the given file More...
 
virtual void addConfigurable (Configurable *conf)
 Adds a configurable as a child object. More...
 
virtual void removeConfigurable (Configurable *conf)
 Removes a configurable as a child object. More...
 
virtual const configurableListgetConfigurables () const
 Returns the list containing all configurable children. More...
 
virtual void configurableChanged ()
 Indicates that the configurable itself or the configurable children attached to this configurable have changed. More...
 
- Public Member Functions inherited from BackCaller
 BackCaller ()
 
virtual ~BackCaller ()
 
virtual void addCallbackable (Callbackable *callbackableInstance, CallbackableType type=BackCaller::DEFAULT_CALLBACKABLE_TYPE)
 Adds a Callbackable instance to this caller instance. More...
 
virtual void removeCallbackable (Callbackable *callbackableInstance, CallbackableType type=BackCaller::DEFAULT_CALLBACKABLE_TYPE)
 Removes a Callbackable instance from this caller instance. More...
 
virtual void removeAllCallbackables (CallbackableType type)
 Removes all Callbackable instances from this caller instance. More...
 
virtual void callBack (CallbackableType type=BackCaller::DEFAULT_CALLBACKABLE_TYPE)
 Calls all registered callbackable classes of the determined type. More...
 
virtual void callBackQMP (CallbackableType type=BackCaller::DEFAULT_CALLBACKABLE_TYPE)
 Calls all registered callbackable classes of the determined type. More...
 
- Public Member Functions inherited from Storeable
virtual ~Storeable ()
 
bool storeToFile (const char *filename) const
 Provided for convenience. More...
 
bool restoreFromFile (const char *filename)
 Provided for convenience. More...
 

Static Public Member Functions

static int valInCrossProd (const std::list< std::pair< int, int > > &vals)
 expects a list of value,range and returns the associated state More...
 
static std::list< int > ConfInCrossProd (const std::list< int > &ranges, int val)
 expects a list of ranges and a state/action and return the configuration More...
 

Public Attributes

bool useSARSA
 if true, use SARSA strategy otherwise qlearning More...
 

Protected Attributes

double eps
 
double discount
 
double exploration
 
double eligibility
 
bool random_initQ
 
int tau
 time horizont for averaging the reward More...
 
matrix::Matrix Q
 
int * actions
 < Q table (mxn) == (states x actions) More...
 
int * states
 
double * rewards
 
int ringbuffersize
 
double * longrewards
 
int t
 
bool initialised
 
double collectedReward
 
RandGenrandGen
 

Additional Inherited Members

- Public Types inherited from Configurable
typedef std::string paramkey
 
typedef std::string paramdescr
 
typedef double paramval
 
typedef std::list< std::pair
< paramkey, paramval > > 
paramlist
 
typedef std::map< paramkey,
paramval * > 
parammap
 
typedef bool parambool
 
typedef std::list< std::pair
< paramkey, parambool > > 
paramboollist
 
typedef std::map< paramkey,
parambool * > 
paramboolmap
 
typedef int paramint
 
typedef std::list< std::pair
< paramkey, paramint > > 
paramintlist
 
typedef std::map< paramkey,
paramint * > 
paramintmap
 
typedef std::map< paramkey,
paramdescr
paramdescrmap
 
typedef std::pair< paramval,
paramval
paramvalBounds
 
typedef std::map< paramkey,
paramvalBounds
paramvalBoundsMap
 
typedef std::pair< paramint,
paramint
paramintBounds
 
typedef std::map< paramkey,
paramintBounds
paramintBoundsMap
 
typedef std::pair< paramkey,
paramval * > 
paramvalpair
 
typedef std::pair< paramkey,
parambool * > 
paramboolpair
 
typedef std::pair< paramkey,
paramint * > 
paramintpair
 
typedef std::vector
< Configurable * > 
configurableList
 
- Public Types inherited from BackCaller
typedef unsigned long CallbackableType
 
- Static Public Attributes inherited from Configurable
static const CallbackableType CALLBACK_CONFIGURABLE_CHANGED = 11
 
- Static Public Attributes inherited from BackCaller
static const CallbackableType DEFAULT_CALLBACKABLE_TYPE = 0
 This is the default Callbackable type. More...
 
- Protected Member Functions inherited from Configurable
void copyParameters (const Configurable &, bool traverseChildren=true)
 copies the internal params of the given configurable More...
 
void printdescr (FILE *f, const char *prefix, const paramkey &key, int columns, int indent) const
 

Detailed Description

implements QLearning

Constructor & Destructor Documentation

QLearning ( double  eps,
double  discount,
double  exploration,
int  eligibility,
bool  random_initQ = false,
bool  useSARSA = false,
int  tau = 1000 
)
Parameters
epslearning rate (typically 0.1)
discountdiscount factor for Q-values (typically 0.9)
explorationexploration rate (typically 0.02)
eligibilitynumber of steps to update backwards in time
random_initQif true Q table is filled with small random numbers at the start (default: false)
useSARSAif true, use SARSA strategy otherwise qlearning (default: false)
taunumber of time steps to average over reward for col_rew
~QLearning ( )
virtual

Member Function Documentation

std::list< int > ConfInCrossProd ( const std::list< int > &  ranges,
int  val 
)
static

expects a list of ranges and a state/action and return the configuration

unsigned int getActionDim ( ) const
virtual

returns the number of actions

matrix::Matrix getActionValues ( unsigned int  state)

returns the vector of values for all actions given the current state

double getCollectedReward ( ) const
virtual

returns the collectedReward reward

virtual const matrix::Matrix& getQ ( ) const
inlinevirtual

returns q table (mxn) == (states x actions)

unsigned int getStateDim ( ) const
virtual

returns the number of states

void init ( unsigned int  stateDim,
unsigned int  actionDim,
RandGen randGen = 0 
)
virtual

initialisation with the given number of action and states

Parameters
actionDimnumber of actions
stateDimnumber of states
unit_mapif 0 the parametes are choosen randomly. Otherwise the model is initialised to represent a unit_map with the given response strength.
double learn ( unsigned int  state,
unsigned int  action,
double  reward,
double  learnRateFactor = 1 
)
virtual
void reset ( )
virtual

tells the q learning that the agent was reset, so that it forgets it memory.

please note, that updating the Q-table is one step later, so in case of a reward you should call learn one more time before reset.

bool restore ( FILE *  f)
virtual

loads the object from the given file stream (ASCII preferred).

Implements Storeable.

unsigned int select ( unsigned int  state)
virtual

selection of action given current state.

The policy is to take the actions with the highest value, or a random action at the rate of exploration

unsigned int select_keepold ( unsigned int  state)
virtual

select with preference to old (90% if good) and 30% second best

unsigned int select_sample ( unsigned int  state)
virtual

selection of action given current state.

The policy is to sample from the above average actions, with bias to the old action (also exploration included).

bool store ( FILE *  f) const
virtual

stores the object to the given file stream (ASCII preferred).

Implements Storeable.

int valInCrossProd ( const std::list< std::pair< int, int > > &  vals)
static

expects a list of value,range and returns the associated state

Member Data Documentation

int* actions
protected

< Q table (mxn) == (states x actions)

double collectedReward
protected
double discount
protected
double eligibility
protected
double eps
protected
double exploration
protected
bool initialised
protected
double* longrewards
protected
matrix::Matrix Q
protected
RandGen* randGen
protected
bool random_initQ
protected
double* rewards
protected
int ringbuffersize
protected
int* states
protected
int t
protected
int tau
protected

time horizont for averaging the reward

bool useSARSA

if true, use SARSA strategy otherwise qlearning


The documentation for this class was generated from the following files: