NeuralNetwork.cpp

Go to the documentation of this file.

#include "NeuralNetwork.h"
#include "UniformRNG.h"
#include "Adaptors.h"
#include "FatalException.h"
#include <algorithm>
#include <iostream>
#include <fstream>

using namespace std;

namespace gplib
  {
    NeuralNetwork::NeuralNetwork(const int inputsize, const int outputsize) :
      AdaptiveFilter(inputsize, outputsize), alpha(0.0), mu(0.0), LocInput(
          inputsize, 0.0), LocOutput(outputsize, 0.0), LocDesired(outputsize,
          0.0)
      {
      }

    NeuralNetwork::NeuralNetwork(const int inputsize, const int outputsize,
        const double mu_, const ttypeArray &Layersetup, const double maxinit,
        bool cachedoutput) :
      AdaptiveFilter(inputsize, outputsize), alpha(0.0), mu(mu_), LocInput(
          inputsize, 0.0), LocOutput(outputsize, 0.0), LocDesired(outputsize,
          0.0)
      {
        SetLayers(Layersetup, cachedoutput);
        InitWeights(maxinit, maxinit);
      }

    NeuralNetwork::~NeuralNetwork()
      {
      }

    void NeuralNetwork::AdaptFilter(const gplib::rvec &Input,
        const gplib::rvec &Desired)
      {
        if (Desired.size() != LocDesired.size())
          throw FatalException("Input does not match network geometry !");
        copy(Desired.begin(), Desired.end(), LocDesired.begin());
        AdaptWeights();
        SetEpsilon(Desired - GetFilterOutput());
      }

    void NeuralNetwork::CalcOutput(const gplib::rvec &Input,
        gplib::rvec &Output)
      {
        if (Input.size() != LocInput.size())
          throw FatalException("Input does not match network geometry !");
        copy(Input.begin(), Input.end(), LocInput.begin());
        CalcOutput();
        if (Output.size() != LocOutput.size())
          throw FatalException("Output does not match network geometry !");
        copy(LocOutput.begin(), LocOutput.end(), Output.begin());
        SetOutput(Output);
      }

    const gplib::rvec &NeuralNetwork::GetWeightsAsVector()
      {
        size_t size = 0;
        for (size_t i = 1; i < Layers.size(); ++i)
          {
            for (size_t j = 0; j < Layers.at(i).size(); ++j)
              size += Layers.at(i).at(j)->GetWeights().size();
          }
        gplib::rvec temp(size);
        size_t currstart = 0;
        for (size_t i = 1; i < Layers.size(); ++i)
          {
            for (size_t j = 0; j < Layers.at(i).size(); ++j)
              {
                for (size_t k = 0; k < Layers.at(i).at(j)->GetWeights().size(); ++k)
                  temp(currstart + k) = Layers.at(i).at(j)->GetWeights().at(k);
                currstart += Layers.at(i).at(j)->GetWeights().size();
              }
          }
        WeightsAsVector = temp;
        return WeightsAsVector;
      }

    void NeuralNetwork::SetLayers(ttypeArray typeArray, bool cachedoutput)
      {
        tNeuralLayer CurrentLayer;
        GeneralLinearCombiner::tinvector inputvector;

        const size_t nlayers = typeArray.size();
        LocOutput.assign(typeArray.back().size(), 0);
        LocDesired.assign(typeArray.back().size(), 0);
        for (size_t i = 0; i < LocInput.size(); ++i)
          CurrentLayer.push_back(boost::shared_ptr<GeneralNeuron>(
              new InputNeuron(LocInput.at(i))));
        Layers.push_back(CurrentLayer);
        CurrentLayer.clear();
        cout << "Nlayers: " << nlayers << endl;
        for (size_t i = 0; i < nlayers; ++i)
          {
            for (size_t j = 0; j < typeArray.at(i).size(); ++j)
              {
                CurrentLayer.push_back(boost::shared_ptr<GeneralNeuron>(
                    new SigmoidalNeuron(typeArray.at(i).at(j), cachedoutput)));
              }
            Layers.push_back(CurrentLayer);
            CurrentLayer.clear();
          }
        for (size_t i = 1; i < nlayers + 1; ++i)
          {
            inputvector.clear();
            const size_t prevsize = Layers.at(i - 1).size();
            cout << "Previous layer size: " << prevsize << endl;
            for (size_t j = 0; j < prevsize; ++j)
              {
                inputvector.push_back(Layers.at(i - 1).at(j));
              }
            for (size_t j = 0; j < Layers.at(i).size(); ++j)
              {
                Layers.at(i).at(j)->SetInput(inputvector);
                Layers.at(i).at(j)->SetOldDelta().assign(prevsize, 0.0);
              }
          }
      }

    void NeuralNetwork::InitWeights(const double MaxWeight,
        const double MaxBias)
      {
        UniformRNG Random;
        for (size_t i = 1; i < Layers.size(); ++i)
          for (size_t j = 0; j < Layers.at(i).size(); ++j)
            {
              for (size_t k = 0; k < Layers.at(i).at(j)->GetWeights().size(); ++k)
                {
                  Layers.at(i).at(j)->SetWeights().at(k) = Random.GetNumber(
                      -MaxWeight, MaxWeight);
                  Layers.at(i).at(j)->SetOldDelta().at(k) = 0.0;
                }
              Layers.at(i).at(j)->SetBias(Random.GetNumber(-MaxBias, MaxBias));
            }
      }

    std::vector<double> &NeuralNetwork::CalcOutput()
      {
        gplib::rvec temp(LocOutput.size());
        for (size_t i = 0; i < Layers.back().size(); ++i)
          {
            LocOutput.at(i) = Layers.back().at(i)->GetOutput();
          }
        return LocOutput;
      }

    void NeuralNetwork::AdaptWeights()
      {
        tNeuralArray::iterator previouslayer = (Layers.end() - 2);
        tNeuralArray::iterator currlayer = (Layers.end() - 1);
        double correction = 0.0;
        for (size_t i = 0; i < previouslayer->size(); ++i)
          previouslayer->at(i)->SetDelta(0.0);
        for (size_t j = 0; j < currlayer->size(); ++j)
          {
            // Delta for the last layer depends on the difference between output and reference
            currlayer->at(j)->SetDelta(Layers.back().at(j)->CalcDeriv(
                Layers.back().at(j)->GetNet()) * (LocDesired.at(j)
                - LocOutput.at(j)));
            //follow all the weights to update delta for the previous layer
            for (size_t i = 0; i < currlayer->at(j)->GetWeights().size(); ++i)
              {
                currlayer->at(j)->GetInput().at(i)->SetDelta(
                    currlayer->at(j)->GetInput().at(i)->GetDelta()
                        + currlayer->at(j)->GetWeights().at(i) * currlayer->at(
                            j)->GetDelta());
                //calculate the correction for the last layer
                correction = alpha * currlayer->at(j)->GetOldDelta().at(i) + mu
                    * (previouslayer->at(i)->GetLastOutput())
                    * Layers.back().at(j)->GetDelta();
                currlayer->at(j)->SetWeights().at(i) += correction;
                currlayer->at(j)->SetOldDelta().at(i) = correction;
              }
            currlayer->at(j)->SetBias(Layers.back().at(j)->GetBias() + mu
                * Layers.back().at(j)->GetDelta());
          }
        //for all layers but the last and the first one
        for (size_t i = Layers.size() - 2; i >= 1; --i)
          {
            //initialize delta for previous layer
            for (size_t j = 0; j < Layers.at(i - 1).size(); ++j)
              Layers.at(i - 1).at(j)->SetDelta(0.0);
            //for all neurons in the current layer
            for (tNeuralLayer::iterator currneuron = Layers.at(i).begin(); currneuron
                < Layers.at(i).end(); ++currneuron)
              {
                //multiply delta of current neuron by derivative of activation function
                currneuron->get()->SetDelta(currneuron->get()->GetDelta()
                    * currneuron->get()->CalcDeriv(currneuron->get()->GetNet()));
                //for all weights of the current neuron
                for (size_t k = 0; k < currneuron->get()->GetWeights().size(); ++k)
                  {
                    //propagate delta along the weight to the previous layer
                    currneuron->get()->GetInput().at(k)->SetDelta(
                        currneuron->get()->GetInput().at(k)->GetDelta()
                            + currneuron->get()->GetWeights().at(k)
                                * currneuron->get()->GetDelta());
                    //calculate the correction for the current layer
                    correction
                        = alpha * currneuron->get()->GetOldDelta().at(k)
                            + mu
                                * currneuron->get()->GetInput().at(k)->GetLastOutput()
                                * currneuron->get()->GetDelta();
                    //apply weight correction
                    currneuron->get()->SetWeights().at(k) += correction;
                    //store weight correction for feedback loop
                    currneuron->get()->SetOldDelta().at(k) = correction;
                  }

                currneuron->get()->SetBias(currneuron->get()->GetBias() + mu
                    * currneuron->get()->GetDelta());
              }
          }
        //we do not have to consider the first layer, it only gets the input
      }

    void NeuralNetwork::PrintWeights(ostream &output)
      {
        for (size_t i = 1; i < Layers.size(); ++i)
          {
            for (size_t j = 0; j < Layers.at(i).size(); ++j)
              {
                copy(Layers.at(i).at(j)->GetWeights().begin(), Layers.at(i).at(
                    j)->GetWeights().end(), ostream_iterator<double> (output,
                    " "));
              }
          }
        output << endl;
      }

    void NeuralNetwork::PrintTopology(std::string filename)
      {
        gplib::rvec WeightVector = GetWeightsAsVector();
        const double maxpower = std::abs(*max_element(WeightVector.begin(),
            WeightVector.end(), gplib::absLess<double, double>()));
        std::ofstream output(filename.c_str());
        output << "digraph network {" << endl;
        for (size_t i = 0; i < Layers.front().size(); ++i)
          {
            output << "node [shape=point];" << std::endl;
            output << "i" << i << " -> input" << i << ";" << std::endl;
            output << "input" << i << " [shape=circle];" << std::endl;
            for (size_t j = 0; j < Layers.at(1).size(); ++j)
              {
                output << "input" << i << " -> n1" << j;
                //output <<  "[label =\" " << Layers.at(1).at(j)->GetWeights().at(i) << " \"]";
                output << "[color =\" 0.7 1.0 " << 0.1 + std::abs(
                    Layers.at(1).at(j)->GetWeights().at(i)) / maxpower
                    << " \"]";
                output << ";" << std::endl;
              }
          }
        for (size_t i = 1; i < Layers.size() - 1; ++i)
          {
            output << "{ rank=same;" << std::endl;
            for (size_t j = 0; j < Layers.at(i).size(); ++j)
              {
                output << "n" << i << j << " [shape=circle];" << std::endl;
                //output << "b" << i << j << " [shape=point];"<<std::endl;
                //output << "b" << i << j << " -> n" << i << j;
                //output << "[label =\" " << Layers.at(i).at(j)->GetBias() << " \"] ";
                //output <<  "[color =\" " << Layers.at(1).at(j)->GetBias()/maxpower << " 1.0 1.0\"]";
                //output << ";" << std::endl;
              }
            output << "};" << std::endl;
            for (size_t j = 0; j < Layers.at(i).size(); ++j)
              for (size_t k = 0; k < Layers.at(i + 1).size(); ++k)
                {
                  output << "n" << i << j << " -> n" << i + 1 << k;
                  //output <<  "[label =\" " << Layers.at(i+1).at(k)->GetWeights().at(j) << " \"]";
                  output << "[color =\" 0.7 1.0 " << 0.1 + std::abs(
                      Layers.at(i + 1).at(k)->GetWeights().at(j)) / maxpower
                      << " \"]";
                  output << ";" << std::endl;
                }
          }
        output << "{ rank=same;" << std::endl;
        for (size_t i = 0; i < Layers.back().size(); ++i)
          {
            output << "n" << Layers.size() - 1 << i << " [shape=circle];"
                << std::endl;
            //output << "b" << Layers.size()-1 << i << " -> n" << Layers.size()-1 << i;
            //output <<  "[color =\" " << Layers.at(Layers.size()-1).at(i)->GetBias()/maxpower << " 1.0 1.0\"]";
            //output <<  "[label =\" " << Layers.at(Layers.size()-1).at(i)->GetBias() << " \"]";
            //output << ";" << std::endl;
          }
        output << "};" << std::endl;
        for (size_t i = 0; i < Layers.back().size(); ++i)
          {
            output << "output" << i << " [shape=point];" << std::endl;
            for (size_t j = 0; j < Layers.at(Layers.size() - 1).size(); ++j)
              {
                output << "n" << Layers.size() - 1 << j << " -> output" << i
                    << ";" << std::endl;
              }
          }
        output << "}";
      }
  }