feat: added network_main for network usage

src/utils/AI/network_main.c

@@ -0,0 +1,106 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <err.h>
+#include <string.h>
+#include "../Application/ApplicationUtils.h"
+#include "neural_utils.h"
+
+void network_train(neural_network* network, char* training_data_dir, char* save_path, size_t iteration, size_t warmup, size_t warmup_iteration, double learning_rate, size_t AdaFactor)
+{
+    network->nb_input = 169;
+    network->hidden_height = 30;
+    network->hidden_depth = 2;
+    network->nb_output = 26 * AdaFactor;
+
+    init_neural_network(network);
+
+    size_t data_len;
+
+    training_data* training_datas = load_dataset(training_data_dir, AdaFactor, &data_len);
+
+    train_network(network, training_datas, data_len, learning_rate, warmup, warmup_iteration, iteration);
+
+    save_neural_network(network, save_path);
+}
+
+void network_use(neural_network* network, char** inputs)
+{
+    for (size_t i = 0; i < network->nb_input; i++)
+    {
+        network->inputs[i].activation = atof(inputs[i]);
+    }
+
+    process_network(network);
+
+    printf("Predicted character: %c", get_network_char_prediction(network, network->nb_output / 26));
+}
+
+void network_main(int argc, char* argv[])
+{
+    (void)argc;
+    char* network_application_directory = path_get_directory(argv[0]);
+
+    if (argc < 2)
+        errx(EXIT_FAILURE, "missing arguments, usage: ./network <train or use>");
+
+    char* action = argv[1];
+
+    neural_network network;
+    
+    if (strcmp(action, "train") == 0)       //train network: ./network train <network.csv> <data directory> <iterations> <warmup> <warmup iterations> [learning_rate] [AdaFactor]
+    {
+        if (argc < 7)
+            errx(EXIT_FAILURE, "missing arguments, usage: ./network train <network.csv> <data directory> <iterations> <warmup> <warmup iterations> [learning_rate] [AdaFactor]");
+        
+        char* network_path = combine_path(network_application_directory, argv[2]);
+
+        char* data_dir = combine_path(network_application_directory, argv[3]);
+        size_t iterations = (size_t)atoi(argv[4]);
+        size_t warmup = (size_t)atoi(argv[5]);
+        size_t warmup_iterations = (size_t)atoi(argv[6]);
+
+        double learning_rate = 0.1;
+        if (argc > 7)
+            learning_rate = atof(argv[7]);
+
+        size_t AdaFactor = 1;
+        if (argc > 8)
+            AdaFactor = (size_t)atoi(argv[8]);
+
+        network_train(&network, data_dir, network_path, iterations, warmup, warmup_iterations, learning_rate, AdaFactor);
+    }
+    else if (strcmp(action, "use") == 0)    //use network: ./network use <network.csv> input1,input2,...,inputx
+    {
+        if (argc < 3)
+            errx(EXIT_FAILURE, "missing arguments, usage: ./network use <network.csv> input1,input2,...,inputx");
+
+        char* network_path = combine_path(network_application_directory, argv[2]);
+
+        char* input_str = argv[3];
+
+        size_t nb_input = 0;
+
+        char** input_array = string_split(input_str, ',', &nb_input);
+
+        neural_network network;
+
+        load_neural_network(&network, read_file(network_path));
+
+        if (nb_input != network.nb_input)
+            errx(EXIT_FAILURE, "inputs are not valid");
+
+        network_use(&network, input_array);
+    }
+
+    /*printf("Succes rate: %i\n", (int)(get_network_success_rate(&network, training_datas, data_len, 1) * 100));
+
+    training_data* benchmark_datas = load_dataset(combine_path(network_application_directory, "../examples/dataset/validation"), 1, &data_len);
+
+    network_set_input_data(&network, benchmark_datas[0]);
+
+    process_network(&network);
+
+    printf("Character: a, predicted character: %c\n", get_network_char_prediction(&network));
+
+    save_neural_network(&network, combine_path(network_application_directory, "../network.csv"));*/
+}

src/utils/AI/neural_utils.c

@@ -8,6 +8,7 @@
 #include <math.h>
 #include <err.h>
 #include <dirent.h>
+#include <time.h>
 
 double sigmoid(double x)
 {
@@ -137,10 +138,10 @@ void free_neural_network(neural_network* network)
     
     for (size_t i = 0; i < network->hidden_depth; i++)
     {
-        free(network->hidden + i);
+        free(network->hidden[i]);
     }
 
-    //free(network->hidden);
+    free(network->hidden);
     free(network->inputs);
     free(network->outputs);
 }
@@ -392,28 +393,17 @@ void process_network(neural_network* network)
     process_output_layer(network);
 }
 
-double get_network_cost(neural_network* network, training_data* expected_data, size_t data_len)
+double get_network_cost(neural_network* network, training_data expected_data)
 {
     double cost = 0.0;
-    for (size_t d = 0; d < data_len; d++)
+
+    for (size_t i = 0; i < network->nb_output; i++)
     {
-        /*for (size_t i = 0; i < network->nb_input; i++)
-        {
-            network->inputs[i].activation = expected_data[d].inputs[i];
-        }*/
+        neuron* curr = &network->outputs[i];
 
-        network_set_input_data(network, expected_data[d]);
+        double err = expected_data.outputs[i] - curr->activation;
 
-        process_network(network);
-
-        for (size_t i = 0; i < network->nb_output; i++)
-        {
-            neuron* curr = &network->outputs[i];
-
-            double err = expected_data[d].outputs[i] - curr->activation;
-
-            cost += err * err;
-        }
+        cost += err * err;
     }
 
     return 1.0/2.0 * cost;
@@ -520,20 +510,31 @@ void network_set_input_data(neural_network* network, training_data data)
     }
 }
 
-void network_train_data(neural_network* network, training_data data, double learning_rate)
+void network_train_data(neural_network* network, training_data data, double learning_rate, double* cost)
 {
     network_set_input_data(network, data);
     
     process_network(network);
 
     network_back_propagation(network, data.outputs, learning_rate);
+
+    if (cost != NULL)
+    {
+        process_network(network);
+
+        *cost = get_network_cost(network, data);
+    }
 }
 
-void network_process_epoche(neural_network* network, training_data* data, size_t data_len, double learning_rate)
+void network_process_epoche(neural_network* network, training_data* data, size_t data_len, double learning_rate, double* total_cost)
 {
     for (size_t i = 0; i < data_len; i++)
     {
-        network_train_data(network, data[i], learning_rate);
+        double i_cost;
+        network_train_data(network, data[i], learning_rate, &i_cost);
+
+        if (total_cost != NULL)
+            *total_cost += i_cost;
     }
 }
 
@@ -546,20 +547,64 @@ double get_network_total_cost(neural_network* network, training_data* datas, siz
 
         process_network(network);
         
-        cost += get_network_cost(network, datas, data_len);
+        cost += get_network_cost(network, datas[i]);
     }
 
     return cost;
 }
 
+char get_data_char_prediction(training_data data, size_t nb_output)
+{
+    char res = '\0';
+    double max_prediction = 0.0;
+
+    for (size_t i = 0; i < nb_output; i++)
+    {
+        if (data.outputs[i] > max_prediction)
+        {
+            max_prediction = data.outputs[i];
+            res = 'a' + (i % 26);
+        }
+    }
+
+    return res;
+}
+
+char get_network_char_prediction(neural_network* network, size_t AdaFactor)
+{
+    (void) AdaFactor;
+    char res = '\0';
+    double max_prediction = 0.0;
+
+    for (size_t i = 0; i < network->nb_output; i++)
+    {
+        if (network->outputs[i].activation > max_prediction)
+        {
+            max_prediction = network->outputs[i].activation;
+            res = 'a' + (i % 26);
+        }
+    }
+
+    return res;
+}
+
 void train_network(neural_network* network, training_data* datas, size_t data_len, float learning_rate, size_t warmup, size_t warmup_iterations, size_t iterations)
 {
+    time_t start;
+
     if (warmup > 0)
     {
+        printf("Warming uloading...\n");
         neural_network networks[warmup];
         double min_net_cost = -1;
         size_t min_net = 0;
 
+        (void)min_net_cost;
+
+        printf("Network warming up...\n");
+
+        start = time(NULL);
+
         for (size_t n = 0; n < warmup; n++)
         {
             networks[n].nb_input = network->nb_input;
@@ -569,13 +614,22 @@ void train_network(neural_network* network, training_data* datas, size_t data_le
 
             init_neural_network(networks + n);
 
+            double cost = 0;
+
             for (size_t i = 0; i < warmup_iterations; i++)
             {
-                network_process_epoche(&(networks[n]), datas, data_len, learning_rate);
+                network_process_epoche(&(networks[n]), datas, data_len, learning_rate, &cost);
+                if (warmup_iterations * warmup < 100 || (n * warmup_iterations + i) % ((warmup_iterations * warmup) / 100) == 0)
+                {
+                    time_t time_pos = time(NULL);
+                    double delta = difftime(time_pos, start);
+                    int poucent = (int)(((float)(n * warmup_iterations + i) / (float)(warmup_iterations * warmup)) * 100);
+                    double total_time = (round(delta) / (double)poucent) * 100.0;
+                    int eta = (int)(round(total_time * (1-((double)poucent / (double)100))));
+                    printf("Warming %i%% (ETA: %is)\n", poucent, eta <= 0 ? 0 : eta);
+                }
             }
 
-            double cost = get_network_total_cost(&networks[n], datas, data_len);
-
             if (min_net_cost < 0 || cost < min_net_cost)
             {
                 min_net_cost = cost;
@@ -594,13 +648,21 @@ void train_network(neural_network* network, training_data* datas, size_t data_le
         }
     }
 
+    start = time(NULL);
+
     for (size_t i = 0; i < iterations; i++)
     {
-        network_process_epoche(network, datas, data_len, learning_rate);
+        network_process_epoche(network, datas, data_len, learning_rate, NULL);
         
         if (i % (iterations / 100) == 0) //Debug
         {
-            printf("Training %i%% (ETA: %s)\n", (int)(((float)i / (float)iterations) * 100), "1 m");
+            time_t time_pos = time(NULL);
+            double delta = difftime(time_pos, start);
+            int poucent = (int)(((float)i / (float)iterations) * 100);
+            double total_time = (round(delta) / (double)poucent) * 100.0;
+            int eta = (int)(round(total_time * (1-((double)poucent / (double)100))));
+
+            printf("Training %i%% (ETA: %is)\n", poucent, eta <= 0 ? 0: eta);
             /*printf("----------------------------------------\n");
             printf("\t\tEpoche %li:\n", i);
             print_training_debug(network, datas, 4);*/
@@ -610,7 +672,7 @@ void train_network(neural_network* network, training_data* datas, size_t data_le
     }
 }
 
-double get_network_success_rate(neural_network* network, training_data* datas, size_t data_len)
+double get_network_success_rate(neural_network* network, training_data* datas, size_t data_len, size_t AdaFactor)
 {
     int success = 0;
 
@@ -620,8 +682,13 @@ double get_network_success_rate(neural_network* network, training_data* datas, s
 
         process_network(network);
 
-        if ((int)datas[i].outputs[0] == (int)round(network->outputs[0].activation))
-            success++;
+        for (size_t o = 0; o < data_len; o++)
+        {
+            if (get_network_char_prediction(network, AdaFactor) != get_data_char_prediction(datas[i], network->nb_output))
+                continue;
+        }
+
+        success++;
     }
 
     return (double)success / (double)data_len;
@@ -720,23 +787,6 @@ training_data* load_dataset(const char* directory, size_t AdaFactor, size_t* nb_
     return datas;
 }
 
-char get_network_char_prediction(neural_network* network)
-{
-    char res = '\0';
-    double max_prediction = 0.0;
-
-    for (size_t i = 0; i < network->nb_output; i++)
-    {
-        if (network->outputs[i].activation > max_prediction)
-        {
-            max_prediction = network->outputs[i].activation;
-            res = 'a' + (i % 26);
-        }
-    }
-
-    return res;
-}
-
 void print_network_activations(neural_network* network)
 {
     for (size_t y = 0; y < network->hidden_height; y++)

src/utils/AI/neural_utils.h

@@ -42,32 +42,36 @@ void init_neural_network(neural_network* network);
 
 void save_neural_network(neural_network* network, const char* file_path);
 
+char** string_split(const char* string, char separator, size_t* res_len);
+
 void load_neural_network(neural_network* network, const char* content);
 
 char* read_file(const char* file);
 
 void process_network(neural_network* network);
 
-double get_network_cost(neural_network* network, training_data* expected_data, size_t data_len);
+double get_network_cost(neural_network* network, training_data expected_data);
 
 void network_back_propagation(neural_network* network, double* expected_data, double learning_rate);
 
-void network_process_epoche(neural_network* network, training_data* data, size_t data_len, double learning_rate);
+void network_process_epoche(neural_network* network, training_data* data, size_t data_len, double learning_rate, double* total_cost);
 
 double get_network_total_cost(neural_network* network, training_data* datas, size_t data_len);
 
 void train_network(neural_network* network, training_data* datas, size_t data_len, float learning_rate, size_t warmup, size_t warmup_iterations, size_t iterations);
 
-double get_network_success_rate(neural_network* network, training_data* datas, size_t data_len);
+double get_network_success_rate(neural_network* network, training_data* datas, size_t data_len, size_t AdaFactor);
 
 training_data* load_dataset(const char* directory, size_t AdaFactor, size_t* nb_data);
 
-char get_network_char_prediction(neural_network* network);
+char get_network_char_prediction(neural_network* network, size_t AdaFactor);
 
 void print_network_activations(neural_network* network);
 
 void network_set_input_data(neural_network* network, training_data data);
 
+void network_train_data(neural_network* network, training_data data, double learning_rate, double* cost);
+
 void print_network_state(neural_network *network);
 
 void print_training_debug(neural_network* network, training_data* data, size_t data_len);