Merge pull request #62 from MatthieuHernandez/Improve_Data

Improve Data
MatthieuHernandez · Nov 3, 2024 · 37a5c8d · 37a5c8d
2 parents 2c1b7e4 + 8b6beee
commit 37a5c8d
Show file tree

Hide file tree

Showing 23 changed files with 157 additions and 73 deletions.
diff --git a/.github/workflows/dataset_tests.yml b/.github/workflows/dataset_tests.yml
@@ -14,6 +14,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    - name: Set up GCC
+      uses: egor-tensin/setup-gcc@v1.3
+      with:
+        version: 13
+        platform: x64
     - name: Checkout projet
       uses: actions/checkout@v2
     - name: Checkout datasets
@@ -24,7 +29,7 @@ jobs:
     - name: CMake
       run: |
         cd ./build
-        CC=gcc-10 CXX=/usr/bin/g++-10 cmake -G"Unix Makefiles" ./.. -DCMAKE_BUILD_TYPE=Release
+        CXX=/usr/bin/g++-13 cmake -G"Unix Makefiles" ./.. -DCMAKE_BUILD_TYPE=Release
     - name: make
       run: |
         cd ./build

diff --git a/.github/workflows/unit_tests_linux.yml b/.github/workflows/unit_tests_linux.yml
@@ -8,12 +8,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    - name: Set up GCC
+      uses: egor-tensin/setup-gcc@v1.3
+      with:
+        version: 13
+        platform: x64
     - name: Checkout the project
       uses: actions/checkout@v2
     - name: CMake
       run: |
         cd ./build
-        CC=gcc-10 CXX=/usr/bin/g++-10 cmake -G"Unix Makefiles" ./.. -DCMAKE_BUILD_TYPE=Release
+        CXX=/usr/bin/g++-13 cmake -G"Unix Makefiles" ./.. -DCMAKE_BUILD_TYPE=Release
     - name: Make
       run: |
         cd ./build

diff --git a/build/BestNeuralNetworkForAudioCatsAndDogs.snn b/build/BestNeuralNetworkForAudioCatsAndDogs.snn
diff --git a/src/data/Data.cpp b/src/data/Data.cpp
@@ -145,15 +145,14 @@ void Data::initialize(vector<vector<float>>& trainingInputs,
         throw NotImplementedException();
     }
 
-    this->normalization(0, 1);
-    tools::log<minimal>("Data loaded");
-
     int err = this->isValid();
     if (err != 0)
     {
         string message = string("Error ") + to_string(err) + ": Wrong parameter in the creation of data";
         throw runtime_error(message);
     }
+
+    tools::log<minimal>("Data loaded");
 }
 
 void Data::flatten(set set, vector<vector<vector<float>>>& input3D)
@@ -211,7 +210,7 @@ void Data::flatten(vector<vector<vector<float>>>& input3D)
     this->sets[testing].size = (int)this->sets[testing].inputs.size();
 }
 
-void Data::normalization(const float min, const float max)
+void Data::normalize(const float min, const float max)
 {
     try
     {
@@ -273,9 +272,7 @@ int Data::isValid()
     {
         for (auto& value : input)
         {
-            if (value < -1
-                || value > 1
-                || isnan(value))
+            if (isnan(value))
             {
                 return 401;
             }

diff --git a/src/data/Data.hpp b/src/data/Data.hpp
@@ -79,7 +79,7 @@ namespace snn
 
         virtual ~Data() = default;
 
-        void normalization(float min, float max);
+        void normalize(float min, float max);
 
         void shuffle();
         void unshuffle();

diff --git a/src/neural_network/StraightforwardNeuralNetwork.cpp b/src/neural_network/StraightforwardNeuralNetwork.cpp
@@ -95,16 +95,8 @@ void StraightforwardNeuralNetwork::trainSync(Data& data, Wait wait, const int ba
     this->numberOfTrainingsBetweenTwoEvaluations = data.sets[training].size;
     this->wantToStopTraining = false;
     this->isIdle = false;
-
     if (evaluationFrequency > 0)
-    {
-        this->evaluate(data);
-        log<minimal>("Epoch: ", toConstSizeString(this->epoch, 2),
-                     " - Accuracy: ", toConstSizeString(this->getGlobalClusteringRate(), 6),
-                     " - MAE: ", toConstSizeString(this->getMeanAbsoluteError(), 7),
-                     " - Time: ", toConstSizeString(wait.getDurationSinceLastTime(), 2), "s");
-    }
-
+        this->evaluate(data, &wait);
     for (this->epoch = 1; this->continueTraining(wait); this->epoch++)
     {
         data.shuffle();
@@ -123,27 +115,16 @@ void StraightforwardNeuralNetwork::trainSync(Data& data, Wait wait, const int ba
                                 data.getTrainingOutputs(this->index, batchSize), data.isFirstTrainingDataOfTemporalSequence(this->index));
             else
                 this->outputForTraining(data.getTrainingData(this->index, batchSize), data.isFirstTrainingDataOfTemporalSequence(this->index));
+            this->logInProgress<minimal>(wait, data, training);
         }
         if (evaluationFrequency > 0 && this->epoch % evaluationFrequency == 0)
-        {
-            this->evaluate(data);
-            log<minimal, false>("Epoch: ", toConstSizeString(this->epoch, 2),
-                                " - Accuracy: ", toConstSizeString(this->getGlobalClusteringRate(), 6),
-                                " - MAE: ", toConstSizeString(this->getMeanAbsoluteError(), 7),
-                                " - Time: ", toConstSizeString(wait.getDurationSinceLastTime(), 2), "s");
-            if (this->autoSaveWhenBetter && this->globalClusteringRateIsBetterThanMax)
-            {
-                this->saveSync(autoSaveFilePath);
-                log<minimal, false>(" - Saved");
-            }
-            log<minimal>();
-        }
+            this->evaluate(data, &wait);
     }
     this->resetTrainingValues();
     log<minimal>("Stop training");
 }
 
-void StraightforwardNeuralNetwork::evaluate(const Data& data)
+void StraightforwardNeuralNetwork::evaluate(const Data& data, Wait* wait)
 {
     this->startTesting();
     for (this->index = 0; this->index < data.sets[testing].size; this->index++)
@@ -159,8 +140,19 @@ void StraightforwardNeuralNetwork::evaluate(const Data& data)
             this->evaluateOnce(data);
         else
             this->output(data.getTestingData(this->index), data.isFirstTestingDataOfTemporalSequence(this->index));
+        this->logInProgress<minimal>(*wait, data, testing);
     }
     this->stopTesting();
+    if (this->autoSaveWhenBetter && this->globalClusteringRateIsBetterThanMax)
+    {
+        this->saveSync(autoSaveFilePath);
+        if(wait != nullptr)
+            this->logAccuracy<minimal>(*wait, true);
+    }
+    else
+    {
+        this->logAccuracy<minimal>(*wait, false);
+    }
 }
 
 inline

diff --git a/src/neural_network/StraightforwardNeuralNetwork.hpp b/src/neural_network/StraightforwardNeuralNetwork.hpp
@@ -27,11 +27,17 @@ namespace snn
 
         void trainSync(Data& data, Wait wait, int batchSize, int evaluationFrequency);
         void saveSync(std::string filePath);
+        void evaluate(const Data& data, Wait* wait);
         void evaluateOnce(const Data& data);
 
         bool continueTraining(Wait wait) const;
         void validData(const Data& data, int batchSize) const;
 
+        template <logLevel T>
+        void logAccuracy(Wait& wait, const bool hasSaved) const;
+        template <logLevel T>
+        void logInProgress(Wait& wait, const Data& data, set set) const;
+
         friend class boost::serialization::access;
         template <class Archive>
         void serialize(Archive& ar, unsigned version);
@@ -56,7 +62,7 @@ namespace snn
         void waitFor(Wait wait) const;
         void train(Data& data, Wait wait, int batchSize = 1, int evaluationFrequency = 1);
 
-        void evaluate(const Data& data);
+        void evaluate(const Data& data) { return this->evaluate(data, nullptr); }
 
         std::vector<float> computeOutput(const std::vector<float>& inputs, bool temporalReset = false);
         int computeCluster(const std::vector<float>& inputs, bool temporalReset = false);
@@ -82,6 +88,38 @@ namespace snn
         bool operator!=(const StraightforwardNeuralNetwork& neuralNetwork) const;
     };
 
+
+    template <logLevel T>
+    void StraightforwardNeuralNetwork::logAccuracy(Wait& wait, const bool hasSaved) const
+    {
+        if constexpr (T > none && T <= verbose)
+        {
+            tools::log<T, false>("\rEpoch: ", tools::toConstSizeString(this->epoch, 2),
+                                " - Accuracy: ", tools::toConstSizeString<2>(this->getGlobalClusteringRate(), 4),
+                                " - MAE: ", tools::toConstSizeString<4>(this->getMeanAbsoluteError(), 7),
+                                " - Time: ", tools::toConstSizeString<0>(wait.getDurationAndReset(), 2), "s");
+            if (hasSaved)
+                tools::log<T, false>(" - Saved");
+            tools::log<T>();
+        }
+    }
+
+    template <logLevel T>
+    void StraightforwardNeuralNetwork::logInProgress(Wait& wait, const Data& data, set set) const
+    {
+        if constexpr (T > none && T <= verbose)
+        {
+            if (wait.tick() >= 100)
+            {
+                const std::string name = set == training ? "Training  " : "Evaluation";
+                const int progress = static_cast<int>(this->index / static_cast<float>(data.sets[set].size) * 100);
+                tools::log<T, false>("\rEpoch: ", tools::toConstSizeString(this->epoch, 2),
+                    " - ", name, " in progress... ", tools::toConstSizeString(progress, 2), "%",
+                    " - Time: ", tools::toConstSizeString<0>(wait.getDuration(), 2), "s");
+            }
+        }
+    }
+
     template <class Archive>
     void StraightforwardNeuralNetwork::serialize(Archive& ar, [[maybe_unused]] const unsigned version)
     {

diff --git a/src/neural_network/Wait.cpp b/src/neural_network/Wait.cpp
@@ -58,7 +58,8 @@ Wait& Wait::operator&&(const Wait& wait)
 void Wait::startClock()
 {
     this->start = system_clock::now();
-    this->last = this->start;
+    this->lastTick = this->start;
+    this->lastReset = this->start;
 }
 
 bool Wait::isOver(int currentEpochs, float CurrentAccuracy, float currentMae) const
@@ -86,12 +87,28 @@ bool Wait::isOver(int currentEpochs, float CurrentAccuracy, float currentMae) co
     return false;
 }
 
-int Wait::getDurationSinceLastTime()
+int Wait::tick()
 {
     const auto now = system_clock::now();
-    const auto currentDuration = static_cast<int>(duration_cast<seconds>(now - this->last).count());
-    this->last = now;
-    return currentDuration;
+    const auto tickDuration = static_cast<int>(duration_cast<milliseconds>(now - this->lastTick).count());
+    return tickDuration;
+}
+
+float Wait::getDuration()
+{
+    const auto now = system_clock::now();
+    const auto currentDuration = static_cast<int>(duration_cast<milliseconds>(now - this->lastReset).count());
+    this->lastTick = now;
+    return currentDuration / 1000.0f;
+}
+
+float Wait::getDurationAndReset()
+{
+    const auto now = system_clock::now();
+    const auto currentDuration = static_cast<int>(duration_cast<milliseconds>(now - this->lastReset).count());
+    this->lastTick = now;
+    this->lastReset = now;
+    return currentDuration / 1000.0f;
 }
 
 Wait snn::operator""_ep(unsigned long long value)

diff --git a/src/neural_network/Wait.hpp b/src/neural_network/Wait.hpp
@@ -17,13 +17,16 @@ namespace snn
         float mae = -1;
         int duration = -1;
         std::chrono::time_point<std::chrono::system_clock> start;
-        std::chrono::time_point<std::chrono::system_clock> last;
+        std::chrono::time_point<std::chrono::system_clock> lastReset;
+        std::chrono::time_point<std::chrono::system_clock> lastTick;
         waitOperator op = waitOperator::noneOp;
         Wait& operator||(const Wait& wait);
         Wait& operator&&(const Wait& wait);
         void startClock();
-        bool isOver(int currentEpochs, float CurrentAccuracy, float currentMae) const;
-        int getDurationSinceLastTime();
+        [[nodiscard]] bool isOver(int currentEpochs, float CurrentAccuracy, float currentMae) const;
+        [[nodiscard]] int tick(); // Time since last tick in milliseconds
+        [[nodiscard]] float getDuration();
+        [[nodiscard]] float getDurationAndReset();
     };
 
     extern Wait operator""_ep(unsigned long long value);

diff --git a/src/neural_network/layer/SimpleLayer.hpp b/src/neural_network/layer/SimpleLayer.hpp
@@ -20,7 +20,7 @@ namespace snn::internal
     protected:
         [[nodiscard]] std::vector<float> computeBackOutput(std::vector<float>& inputErrors) final;
         [[nodiscard]] std::vector<float> computeOutput(const std::vector<float>& inputs, bool temporalReset) final;
-        virtual void computeTrain(std::vector<float>& inputErrors) final;
+        void computeTrain(std::vector<float>& inputErrors) final;
 
     public:
         SimpleLayer() = default;  // use restricted to Boost library only

diff --git a/src/tools/Tools.hpp b/src/tools/Tools.hpp
@@ -1,4 +1,5 @@
 #pragma once
+#include <format>
 #include <chrono>
 #include <iostream>
 #include <random>
@@ -117,6 +118,8 @@ namespace snn::tools
             (std::cout << ... << messages);
             if constexpr (endLine)
                 std::cout << std::endl;
+            else
+                std::cout << std::flush;
         }
     }
 
@@ -129,12 +132,21 @@ namespace snn::tools
         return str;
     }
 
-    inline
+    template<int T>
     std::string toConstSizeString(float value, size_t length)
     {
-        auto str = std::to_string(value);
+        std::string str;
+        if constexpr (T == 0)
+            str = std::format("{:.0f}", value);
+        else if constexpr (T == 2)
+            str = std::format("{:.2f}", value);
+        else if constexpr (T == 4)
+            str = std::format("{:.4f}", value);
+        else
+            throw std::exception();
+
         while (str.length() < length)
-            str += "0";
+            str = " " + str;
         return str;
     }
 

diff --git a/tests/audiofile/AudioFile.hpp b/tests/audiofile/AudioFile.hpp
@@ -23,6 +23,7 @@
 #ifndef _AS_AudioFile_h
 #define _AS_AudioFile_h
 
+#include <cstdint>
 #include <iostream>
 #include <vector>
 #include <assert.h>

diff --git a/tests/dataset_tests/CIFAR-10/Cifar10.cpp b/tests/dataset_tests/CIFAR-10/Cifar10.cpp
@@ -32,6 +32,7 @@ void Cifar10::loadData(string folderPath)
     vector2D<float> testingInputs = readImages(testFilePaths, 1, testingLabels);
 
     this->data = make_unique<Data>(problem::classification, trainingInputs, trainingLabels, testingInputs, testingLabels);
+    this->data->normalize(0, 1);
 }
 
 vector2D<float> Cifar10::readImages(string filePaths[], size_t size, vector2D<float>& labels) const

diff --git a/tests/dataset_tests/Fashion-MNIST/FashionMnist.cpp b/tests/dataset_tests/Fashion-MNIST/FashionMnist.cpp
@@ -20,6 +20,7 @@ void FashionMnist::loadData(string folderPath)
     vector2D<float> testingLabels = readLabels(folderPath + "/t10k-labels-idx1-ubyte", 10000);
 
     this->data = make_unique<Data>(problem::classification, trainingInputs, trainingLabels, testingInputs, testingLabels);
+    this->data->normalize(0, 1);
 }
 
 vector2D<float> FashionMnist::readImages(string filePath, int size)

diff --git a/tests/dataset_tests/Fashion-MNIST/FashionMnistTest.cpp b/tests/dataset_tests/Fashion-MNIST/FashionMnistTest.cpp
@@ -55,13 +55,14 @@ TEST_F(FashionMnistTest, convolutionNeuralNetwork)
 {
     StraightforwardNeuralNetwork neuralNetwork({
         Input(1, 28, 28),
-        Convolution(4, 5, activation::ReLU),
+        Convolution(8, 3, activation::ReLU),
+        Convolution(24, 5, activation::ReLU),
         FullyConnected(10)
     },
-        StochasticGradientDescent(0.002f, 0.90f));
-    neuralNetwork.train(*data, 1_ep || 20_s);
+        StochasticGradientDescent(0.0002f, 0.80f));
+    neuralNetwork.train(*data, 1_ep || 30_s);
     auto accuracy = neuralNetwork.getGlobalClusteringRate();
-    ASSERT_ACCURACY(accuracy, 0.75f);
+    ASSERT_ACCURACY(accuracy, 0.70f);
 }
 
 TEST_F(FashionMnistTest, DISABLED_trainBestNeuralNetwork)

diff --git a/tests/dataset_tests/MNIST/Mnist.cpp b/tests/dataset_tests/MNIST/Mnist.cpp
@@ -20,6 +20,7 @@ void Mnist::loadData(string folderPath)
     vector2D<float> testingLabels = readLabels(folderPath + "/t10k-labels.idx1-ubyte", 10000);
 
     this->data = make_unique<Data>(problem::classification, trainingInputs, trainingLabels, testingInputs, testingLabels);
+    this->data->normalize(0, 1);
 }
 
 vector2D<float> Mnist::readImages(string filePath, int size)

diff --git a/tests/dataset_tests/Wine/Wine.cpp b/tests/dataset_tests/Wine/Wine.cpp
@@ -52,4 +52,5 @@ void Wine::loadData(string folderPath)
     }
     file.close();
     this->data = make_unique<Data>(problem::classification, inputs, labels);
+    this->data->normalize(0, 1);
 }