GPLIB++
NeuralNetwork.cpp
Go to the documentation of this file.
1 #include "NeuralNetwork.h"
2 #include "UniformRNG.h"
3 #include "Adaptors.h"
4 #include "FatalException.h"
5 #include <algorithm>
6 #include <iostream>
7 #include <fstream>
8 
9 using namespace std;
10 
11 namespace gplib
12  {
13  NeuralNetwork::NeuralNetwork(const int inputsize, const int outputsize) :
14  AdaptiveFilter(inputsize, outputsize), alpha(0.0), mu(0.0), LocInput(
15  inputsize, 0.0), LocOutput(outputsize, 0.0), LocDesired(outputsize,
16  0.0)
17  {
18  }
19 
20  NeuralNetwork::NeuralNetwork(const int inputsize, const int outputsize,
21  const double mu_, const ttypeArray &Layersetup, const double maxinit,
22  bool cachedoutput) :
23  AdaptiveFilter(inputsize, outputsize), alpha(0.0), mu(mu_), LocInput(
24  inputsize, 0.0), LocOutput(outputsize, 0.0), LocDesired(outputsize,
25  0.0)
26  {
27  SetLayers(Layersetup, cachedoutput);
28  InitWeights(maxinit, maxinit);
29  }
30 
32  {
33  }
34 
35  void NeuralNetwork::AdaptFilter(const gplib::rvec &Input,
36  const gplib::rvec &Desired)
37  {
38  if (Desired.size() != LocDesired.size())
39  throw FatalException("Input does not match network geometry !");
40  copy(Desired.begin(), Desired.end(), LocDesired.begin());
41  AdaptWeights();
42  SetEpsilon(Desired - GetFilterOutput());
43  }
44 
45  void NeuralNetwork::CalcOutput(const gplib::rvec &Input,
46  gplib::rvec &Output)
47  {
48  if (Input.size() != LocInput.size())
49  throw FatalException("Input does not match network geometry !");
50  copy(Input.begin(), Input.end(), LocInput.begin());
51  CalcOutput();
52  if (Output.size() != LocOutput.size())
53  throw FatalException("Output does not match network geometry !");
54  copy(LocOutput.begin(), LocOutput.end(), Output.begin());
55  SetOutput(Output);
56  }
57 
58  const gplib::rvec &NeuralNetwork::GetWeightsAsVector()
59  {
60  size_t size = 0;
61  for (size_t i = 1; i < Layers.size(); ++i)
62  {
63  for (size_t j = 0; j < Layers.at(i).size(); ++j)
64  size += Layers.at(i).at(j)->GetWeights().size();
65  }
66  gplib::rvec temp(size);
67  size_t currstart = 0;
68  for (size_t i = 1; i < Layers.size(); ++i)
69  {
70  for (size_t j = 0; j < Layers.at(i).size(); ++j)
71  {
72  for (size_t k = 0; k < Layers.at(i).at(j)->GetWeights().size(); ++k)
73  temp(currstart + k) = Layers.at(i).at(j)->GetWeights().at(k);
74  currstart += Layers.at(i).at(j)->GetWeights().size();
75  }
76  }
77  WeightsAsVector = temp;
78  return WeightsAsVector;
79  }
80 
81  void NeuralNetwork::SetLayers(ttypeArray typeArray, bool cachedoutput)
82  {
83  tNeuralLayer CurrentLayer;
85 
86  const size_t nlayers = typeArray.size();
87  LocOutput.assign(typeArray.back().size(), 0);
88  LocDesired.assign(typeArray.back().size(), 0);
89  for (size_t i = 0; i < LocInput.size(); ++i)
90  CurrentLayer.push_back(boost::shared_ptr<GeneralNeuron>(
91  new InputNeuron(LocInput.at(i))));
92  Layers.push_back(CurrentLayer);
93  CurrentLayer.clear();
94  cout << "Nlayers: " << nlayers << endl;
95  for (size_t i = 0; i < nlayers; ++i)
96  {
97  for (size_t j = 0; j < typeArray.at(i).size(); ++j)
98  {
99  CurrentLayer.push_back(boost::shared_ptr<GeneralNeuron>(
100  new SigmoidalNeuron(typeArray.at(i).at(j), cachedoutput)));
101  }
102  Layers.push_back(CurrentLayer);
103  CurrentLayer.clear();
104  }
105  for (size_t i = 1; i < nlayers + 1; ++i)
106  {
107  inputvector.clear();
108  const size_t prevsize = Layers.at(i - 1).size();
109  cout << "Previous layer size: " << prevsize << endl;
110  for (size_t j = 0; j < prevsize; ++j)
111  {
112  inputvector.push_back(Layers.at(i - 1).at(j));
113  }
114  for (size_t j = 0; j < Layers.at(i).size(); ++j)
115  {
116  Layers.at(i).at(j)->SetInput(inputvector);
117  Layers.at(i).at(j)->SetOldDelta().assign(prevsize, 0.0);
118  }
119  }
120  }
121 
122  void NeuralNetwork::InitWeights(const double MaxWeight,
123  const double MaxBias)
124  {
125  UniformRNG Random;
126  for (size_t i = 1; i < Layers.size(); ++i)
127  for (size_t j = 0; j < Layers.at(i).size(); ++j)
128  {
129  for (size_t k = 0; k < Layers.at(i).at(j)->GetWeights().size(); ++k)
130  {
131  Layers.at(i).at(j)->SetWeights().at(k) = Random.GetNumber(
132  -MaxWeight, MaxWeight);
133  Layers.at(i).at(j)->SetOldDelta().at(k) = 0.0;
134  }
135  Layers.at(i).at(j)->SetBias(Random.GetNumber(-MaxBias, MaxBias));
136  }
137  }
138 
139  std::vector<double> &NeuralNetwork::CalcOutput()
140  {
141  gplib::rvec temp(LocOutput.size());
142  for (size_t i = 0; i < Layers.back().size(); ++i)
143  {
144  LocOutput.at(i) = Layers.back().at(i)->GetOutput();
145  }
146  return LocOutput;
147  }
148 
149  void NeuralNetwork::AdaptWeights()
150  {
151  tNeuralArray::iterator previouslayer = (Layers.end() - 2);
152  tNeuralArray::iterator currlayer = (Layers.end() - 1);
153  double correction = 0.0;
154  for (size_t i = 0; i < previouslayer->size(); ++i)
155  previouslayer->at(i)->SetDelta(0.0);
156  for (size_t j = 0; j < currlayer->size(); ++j)
157  {
158  // Delta for the last layer depends on the difference between output and reference
159  currlayer->at(j)->SetDelta(Layers.back().at(j)->CalcDeriv(
160  Layers.back().at(j)->GetNet()) * (LocDesired.at(j)
161  - LocOutput.at(j)));
162  //follow all the weights to update delta for the previous layer
163  for (size_t i = 0; i < currlayer->at(j)->GetWeights().size(); ++i)
164  {
165  currlayer->at(j)->GetInput().at(i)->SetDelta(
166  currlayer->at(j)->GetInput().at(i)->GetDelta()
167  + currlayer->at(j)->GetWeights().at(i) * currlayer->at(
168  j)->GetDelta());
169  //calculate the correction for the last layer
170  correction = alpha * currlayer->at(j)->GetOldDelta().at(i) + mu
171  * (previouslayer->at(i)->GetLastOutput())
172  * Layers.back().at(j)->GetDelta();
173  currlayer->at(j)->SetWeights().at(i) += correction;
174  currlayer->at(j)->SetOldDelta().at(i) = correction;
175  }
176  currlayer->at(j)->SetBias(Layers.back().at(j)->GetBias() + mu
177  * Layers.back().at(j)->GetDelta());
178  }
179  //for all layers but the last and the first one
180  for (size_t i = Layers.size() - 2; i >= 1; --i)
181  {
182  //initialize delta for previous layer
183  for (size_t j = 0; j < Layers.at(i - 1).size(); ++j)
184  Layers.at(i - 1).at(j)->SetDelta(0.0);
185  //for all neurons in the current layer
186  for (tNeuralLayer::iterator currneuron = Layers.at(i).begin(); currneuron
187  < Layers.at(i).end(); ++currneuron)
188  {
189  //multiply delta of current neuron by derivative of activation function
190  currneuron->get()->SetDelta(currneuron->get()->GetDelta()
191  * currneuron->get()->CalcDeriv(currneuron->get()->GetNet()));
192  //for all weights of the current neuron
193  for (size_t k = 0; k < currneuron->get()->GetWeights().size(); ++k)
194  {
195  //propagate delta along the weight to the previous layer
196  currneuron->get()->GetInput().at(k)->SetDelta(
197  currneuron->get()->GetInput().at(k)->GetDelta()
198  + currneuron->get()->GetWeights().at(k)
199  * currneuron->get()->GetDelta());
200  //calculate the correction for the current layer
201  correction
202  = alpha * currneuron->get()->GetOldDelta().at(k)
203  + mu
204  * currneuron->get()->GetInput().at(k)->GetLastOutput()
205  * currneuron->get()->GetDelta();
206  //apply weight correction
207  currneuron->get()->SetWeights().at(k) += correction;
208  //store weight correction for feedback loop
209  currneuron->get()->SetOldDelta().at(k) = correction;
210  }
211 
212  currneuron->get()->SetBias(currneuron->get()->GetBias() + mu
213  * currneuron->get()->GetDelta());
214  }
215  }
216  //we do not have to consider the first layer, it only gets the input
217  }
218 
219  void NeuralNetwork::PrintWeights(ostream &output)
220  {
221  for (size_t i = 1; i < Layers.size(); ++i)
222  {
223  for (size_t j = 0; j < Layers.at(i).size(); ++j)
224  {
225  copy(Layers.at(i).at(j)->GetWeights().begin(), Layers.at(i).at(
226  j)->GetWeights().end(), ostream_iterator<double> (output,
227  " "));
228  }
229  }
230  output << endl;
231  }
232 
233  void NeuralNetwork::PrintTopology(std::string filename)
234  {
235  gplib::rvec WeightVector = GetWeightsAsVector();
236  const double maxpower = std::abs(*max_element(WeightVector.begin(),
237  WeightVector.end(), gplib::absLess<double, double>()));
238  std::ofstream output(filename.c_str());
239  output << "digraph network {" << endl;
240  for (size_t i = 0; i < Layers.front().size(); ++i)
241  {
242  output << "node [shape=point];" << std::endl;
243  output << "i" << i << " -> input" << i << ";" << std::endl;
244  output << "input" << i << " [shape=circle];" << std::endl;
245  for (size_t j = 0; j < Layers.at(1).size(); ++j)
246  {
247  output << "input" << i << " -> n1" << j;
248  //output << "[label =\" " << Layers.at(1).at(j)->GetWeights().at(i) << " \"]";
249  output << "[color =\" 0.7 1.0 " << 0.1 + std::abs(
250  Layers.at(1).at(j)->GetWeights().at(i)) / maxpower
251  << " \"]";
252  output << ";" << std::endl;
253  }
254  }
255  for (size_t i = 1; i < Layers.size() - 1; ++i)
256  {
257  output << "{ rank=same;" << std::endl;
258  for (size_t j = 0; j < Layers.at(i).size(); ++j)
259  {
260  output << "n" << i << j << " [shape=circle];" << std::endl;
261  //output << "b" << i << j << " [shape=point];"<<std::endl;
262  //output << "b" << i << j << " -> n" << i << j;
263  //output << "[label =\" " << Layers.at(i).at(j)->GetBias() << " \"] ";
264  //output << "[color =\" " << Layers.at(1).at(j)->GetBias()/maxpower << " 1.0 1.0\"]";
265  //output << ";" << std::endl;
266  }
267  output << "};" << std::endl;
268  for (size_t j = 0; j < Layers.at(i).size(); ++j)
269  for (size_t k = 0; k < Layers.at(i + 1).size(); ++k)
270  {
271  output << "n" << i << j << " -> n" << i + 1 << k;
272  //output << "[label =\" " << Layers.at(i+1).at(k)->GetWeights().at(j) << " \"]";
273  output << "[color =\" 0.7 1.0 " << 0.1 + std::abs(
274  Layers.at(i + 1).at(k)->GetWeights().at(j)) / maxpower
275  << " \"]";
276  output << ";" << std::endl;
277  }
278  }
279  output << "{ rank=same;" << std::endl;
280  for (size_t i = 0; i < Layers.back().size(); ++i)
281  {
282  output << "n" << Layers.size() - 1 << i << " [shape=circle];"
283  << std::endl;
284  //output << "b" << Layers.size()-1 << i << " -> n" << Layers.size()-1 << i;
285  //output << "[color =\" " << Layers.at(Layers.size()-1).at(i)->GetBias()/maxpower << " 1.0 1.0\"]";
286  //output << "[label =\" " << Layers.at(Layers.size()-1).at(i)->GetBias() << " \"]";
287  //output << ";" << std::endl;
288  }
289  output << "};" << std::endl;
290  for (size_t i = 0; i < Layers.back().size(); ++i)
291  {
292  output << "output" << i << " [shape=point];" << std::endl;
293  for (size_t j = 0; j < Layers.at(Layers.size() - 1).size(); ++j)
294  {
295  output << "n" << Layers.size() - 1 << j << " -> output" << i
296  << ";" << std::endl;
297  }
298  }
299  output << "}";
300  }
301  }
void InitWeights(const double MaxWeight, const double MaxBias)
Initialize the weights with random values with the specified maxima.
std::vector< boost::shared_ptr< GeneralNeuron > > tNeuralLayer
Definition: NeuralNetwork.h:20
void SetLayers(ttypeArray typeArray, bool cachedoutput=false)
Configure the layers of the network according to the types in typeArray.
void SetEpsilon(const gplib::rvec &MyEps)
Possibility for derived classes to set estimation error.
Generate uniformly distributed random numbers, this is basically a wrapper for the boost random numbe...
Definition: UniformRNG.h:19
virtual void PrintWeights(std::ostream &output)
Print the weights of the network to the specified output stream.
std::vector< ttypeVector > ttypeArray
Definition: NeuralNetwork.h:23
virtual void AdaptFilter(const gplib::rvec &Input, const gplib::rvec &Desired)
Adapt the Filter with the current input and desired.
A generic base class for all types of adaptive filters.
const gplib::rvec & GetFilterOutput() const
Access to the last calculated output (not sure if needed)
NeuralNetwork(const int inputsize, const int outputsize)
The minium values for the network are the length of the input and output.
float GetNumber(const float low, const float high)
Return a random float between low and high.
Definition: UniformRNG.cpp:21
virtual const gplib::rvec & GetWeightsAsVector()
Return the network weights as a single vector.
void SetOutput(const gplib::rvec &Out)
Possibility for derived classes to set output.
SigmoidalNeuron implements the main functionality of neurons in a neural network. ...
void PrintTopology(std::string filename)
Print the topology and weights of the network for plotting with the dot program.
const int size
Definition: perftest.cpp:14
std::vector< boost::shared_ptr< GeneralNeuron > > tinvector
The basic exception class for all errors that arise in gplib.