19 #ifndef __MULTIREINFORCE_H
20 #define __MULTIREINFORCE_H
22 #include <selforg/abstractcontroller.h>
23 #include <selforg/multilayerffnn.h>
28 #include <selforg/matrix.h>
29 #include <selforg/noisegenerator.h>
30 #include <selforg/multilayerffnn.h>
31 #include <selforg/qlearning.h>
65 virtual void init(
int sensornumber,
int motornumber,
RandGen* randGen = 0);
85 static std::list<std::string>
createFileList(
const char* filestem,
int n);
87 void restoreSats(
const std::list<std::string>& files);
89 void storeSats(
const std::list<std::string>& files);
103 virtual bool store(FILE* f)
const;
matrix::Matrix satInput
input to satelite networks (x_{t-1}, xp_{t-1}, y_{t-1})^T
Definition: multireinforce.h:145
int managementInterval
interval between subsequent management calls
Definition: multireinforce.h:162
virtual void fillSensorBuffer(const sensor *x_, int number_sensors)
puts the sensors in the ringbuffer
Definition: multireinforce.cpp:284
Matrix type.
Definition: matrix.h:65
virtual std::list< IConnection > getStructuralConnections() const
Specifies which parameter matrix forms a connection between layers (in terms of a neural network) The...
Definition: multireinforce.cpp:480
int newaction
index of new controlling network
Definition: multireinforce.h:147
unsigned short buffersize
Definition: multireinforce.h:136
bool useDerive
input to sat network includes derivatives
Definition: multireinforce.h:38
void storeSats(const std::list< std::string > &files)
stores the sats into the given files
Definition: multireinforce.cpp:414
MultiReinforce(const MultiReinforceConf &conf=getDefaultConf())
Definition: multireinforce.cpp:33
int phase
current phase of the controller: 0: action just selected 1:state changed first time 2:state changed s...
Definition: multireinforce.h:152
unsigned short number_sensors
Definition: multireinforce.h:132
Abstract class for robot controller (with some basic functionality).
Definition: abstractcontroller.h:46
matrix::Matrix satAvgErrors
averaged errors of the sats
Definition: multireinforce.h:156
matrix::Matrix * y_buffer
Definition: multireinforce.h:139
unsigned short buffersize
size of the ringbuffers for sensors, motors,...
Definition: multireinforce.h:34
static MultiReinforceConf getDefaultConf()
Definition: multireinforce.h:113
implements QLearning
Definition: qlearning.h:33
charArray paramkey
Definition: avrtypes.h:36
class for robot controller using several feedforward networks (satelite) and one selforg controller ...
Definition: multireinforce.h:61
virtual int getMotorNumber() const
returns the mumber of motors the controller was initialised with or 0 if not initialised ...
Definition: multireinforce.h:72
virtual double calcReinforcement()=0
returns the reinforcement (reward), to be overwritten
matrix::Matrix * xp_buffer
Definition: multireinforce.h:138
bool initialised
Definition: multireinforce.h:160
virtual std::list< iparamval > getInternalParams() const
Definition: multireinforce.cpp:458
double sensor
Definition: types.h:29
random generator with 48bit integer arithmentic
Definition: randomgenerator.h:34
int t
Definition: multireinforce.h:161
bool manualControl
True if actions (sats) are selected manually.
Definition: multireinforce.h:143
Satelite network struct.
Definition: multireinforce.h:50
virtual void step(const sensor *, int number_sensors, motor *, int number_motors)
performs one step (includes learning).
Definition: multireinforce.cpp:115
matrix::Matrix satErrors
actual errors of the sats
Definition: multireinforce.h:155
virtual bool store(FILE *f) const
stores the controller values to a given file.
Definition: multireinforce.cpp:340
virtual void fillMotorBuffer(const motor *y_, int number_motors)
puts the motors in the ringbuffer
Definition: multireinforce.cpp:298
struct MultiReinforceConf MultiReinforceConf
QLearning * qlearning
QLearning instance.
Definition: multireinforce.h:45
virtual void stepNoLearning(const sensor *, int number_sensors, motor *, int number_motors)
performs one step without learning. Calulates motor commands from sensor inputs.
Definition: multireinforce.cpp:274
virtual int getSensorNumber() const
returns the number of sensors the controller was initialised with or 0 if not initialised ...
Definition: multireinforce.h:70
double sensor
Definition: abstractcontroller.h:48
static std::list< std::string > createFileList(const char *filestem, int n)
restores the sat networks from seperate files
Definition: multireinforce.cpp:433
virtual int getStateNumber()=0
returns number of state, to be overwritten
virtual void notifyOnChange(const paramkey &key)
Is called when a parameter was changes via setParam().
Definition: multireinforce.cpp:327
virtual void management()
handles inhibition damping etc.
Definition: multireinforce.cpp:323
int oldaction
index of old controlling network
Definition: multireinforce.h:148
double eps
Definition: multireinforce.h:53
void putInBuffer(matrix::Matrix *buffer, const matrix::Matrix &vec, int delay=0)
Definition: multireinforce.cpp:109
int reinforce_interval
time between consecutive reinforcement selections
Definition: multireinforce.h:43
virtual void init(int sensornumber, int motornumber, RandGen *randGen=0)
initialisation of the controller with the given sensor/ motornumber Must be called before use...
Definition: multireinforce.cpp:65
matrix::Matrix statesbins
bins with counts for each state
Definition: multireinforce.h:157
matrix::Matrix nomSatOutput
norminal output of satelite networks (x_t,y_t)^T
Definition: multireinforce.h:144
virtual bool restore(FILE *f)
loads the controller values from a given file.
Definition: multireinforce.cpp:359
void setManualControl(bool mControl, int action_=0)
enables/disables manual control, action_ is the sat network number to be used if mControl is false...
Definition: multireinforce.cpp:306
struct Sat Sat
Satelite network struct.
double lifetime
Definition: multireinforce.h:54
virtual std::list< iparamkey > getInternalParamNames() const
The list of the names of all internal parameters given by getInternalParams().
Definition: multireinforce.cpp:443
double reward
current reward
Definition: multireinforce.h:150
virtual int calcState()=0
returns state, to be overwritten
std::vector< Sat > sats
satelite networks
Definition: multireinforce.h:142
double motor
Definition: types.h:30
double oldreward
old reward (nicer for plotting)
Definition: multireinforce.h:151
matrix::Matrix * x_buffer
Definition: multireinforce.h:137
Sat(MultiLayerFFNN *_net, double _eps)
Definition: multireinforce.cpp:26
int numSats
filenames for sat networks
Definition: multireinforce.h:37
int numContext
number of context sensors (ignored)
Definition: multireinforce.h:35
virtual ~MultiReinforce()
Definition: multireinforce.cpp:50
multi layer neural network with configurable activation functions
Definition: multilayerffnn.h:35
int state
current state
Definition: multireinforce.h:149
double motor
Definition: abstractcontroller.h:49
int action
index of controlling network
Definition: multireinforce.h:146
matrix::Matrix * x_context_buffer
Definition: multireinforce.h:140
matrix::Matrix * actioncorrel
Definition: multireinforce.h:46
std::list< std::string > satFiles
Definition: multireinforce.h:36
matrix::Matrix calcDerivatives(const matrix::Matrix *buffer, int delay)
Calculates first and second derivative and returns both in on matrix (above).
Definition: multireinforce.cpp:315
MultiLayerFFNN * net
Definition: multireinforce.h:52
int phasecnt
counts number of steps in one phase.
Definition: multireinforce.h:153
void restoreSats(const std::list< std::string > &files)
restores the sat networks from seperate files
Definition: multireinforce.cpp:394
int c
Definition: hexapod.cpp:56
virtual std::list< ILayer > getStructuralLayers() const
Specifies which parameter vector forms a structural layer (in terms of a neural network) The ordering...
Definition: multireinforce.cpp:472
unsigned short number_motors
Definition: multireinforce.h:133
MultiReinforceConf conf
Definition: multireinforce.h:159
Definition: multireinforce.h:33
bool useY
input to sat network includes y (motor values)
Definition: multireinforce.h:39