You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

genann.c 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405
  1. /*
  2. * GENANN - Minimal C Artificial Neural Network
  3. *
  4. * Copyright (c) 2015-2018 Lewis Van Winkle
  5. *
  6. * http://CodePlea.com
  7. *
  8. * This software is provided 'as-is', without any express or implied
  9. * warranty. In no event will the authors be held liable for any damages
  10. * arising from the use of this software.
  11. *
  12. * Permission is granted to anyone to use this software for any purpose,
  13. * including commercial applications, and to alter it and redistribute it
  14. * freely, subject to the following restrictions:
  15. *
  16. * 1. The origin of this software must not be misrepresented; you must not
  17. * claim that you wrote the original software. If you use this software
  18. * in a product, an acknowledgement in the product documentation would be
  19. * appreciated but is not required.
  20. * 2. Altered source versions must be plainly marked as such, and must not be
  21. * misrepresented as being the original software.
  22. * 3. This notice may not be removed or altered from any source distribution.
  23. *
  24. */
  25. #include "genann.h"
  26. #include <assert.h>
  27. #include <errno.h>
  28. #include <math.h>
  29. #include <stdio.h>
  30. #include <stdlib.h>
  31. #include <string.h>
  32. #ifndef genann_act
  33. #define genann_act_hidden genann_act_hidden_indirect
  34. #define genann_act_output genann_act_output_indirect
  35. #else
  36. #define genann_act_hidden genann_act
  37. #define genann_act_output genann_act
  38. #endif
  39. #define LOOKUP_SIZE 4096
  40. double genann_act_hidden_indirect(const struct genann *ann, double a) {
  41. return ann->activation_hidden(ann, a);
  42. }
  43. double genann_act_output_indirect(const struct genann *ann, double a) {
  44. return ann->activation_output(ann, a);
  45. }
  46. const double sigmoid_dom_min = -15.0;
  47. const double sigmoid_dom_max = 15.0;
  48. double interval;
  49. double lookup[LOOKUP_SIZE];
  50. #ifdef __GNUC__
  51. #define likely(x) __builtin_expect(!!(x), 1)
  52. #define unlikely(x) __builtin_expect(!!(x), 0)
  53. #define unused __attribute__((unused))
  54. #else
  55. #define likely(x) x
  56. #define unlikely(x) x
  57. #define unused
  58. #pragma warning(disable : 4996) /* For fscanf */
  59. #endif
  60. double genann_act_sigmoid(const genann *ann unused, double a) {
  61. if (a < -45.0) return 0;
  62. if (a > 45.0) return 1;
  63. return 1.0 / (1 + exp(-a));
  64. }
  65. void genann_init_sigmoid_lookup(const genann *ann) {
  66. const double f = (sigmoid_dom_max - sigmoid_dom_min) / LOOKUP_SIZE;
  67. int i;
  68. interval = LOOKUP_SIZE / (sigmoid_dom_max - sigmoid_dom_min);
  69. for (i = 0; i < LOOKUP_SIZE; ++i) {
  70. lookup[i] = genann_act_sigmoid(ann, sigmoid_dom_min + f * i);
  71. }
  72. }
  73. double genann_act_sigmoid_cached(const genann *ann unused, double a) {
  74. assert(!isnan(a));
  75. if (a < sigmoid_dom_min) return lookup[0];
  76. if (a >= sigmoid_dom_max) return lookup[LOOKUP_SIZE - 1];
  77. size_t j = (size_t)((a-sigmoid_dom_min)*interval+0.5);
  78. /* Because floating point... */
  79. if (unlikely(j >= LOOKUP_SIZE)) return lookup[LOOKUP_SIZE - 1];
  80. return lookup[j];
  81. }
  82. double genann_act_linear(const struct genann *ann unused, double a) {
  83. return a;
  84. }
  85. double genann_act_threshold(const struct genann *ann unused, double a) {
  86. return a > 0;
  87. }
  88. genann *genann_init(int inputs, int hidden_layers, int hidden, int outputs) {
  89. if (hidden_layers < 0) return 0;
  90. if (inputs < 1) return 0;
  91. if (outputs < 1) return 0;
  92. if (hidden_layers > 0 && hidden < 1) return 0;
  93. const int hidden_weights = hidden_layers ? (inputs+1) * hidden + (hidden_layers-1) * (hidden+1) * hidden : 0;
  94. const int output_weights = (hidden_layers ? (hidden+1) : (inputs+1)) * outputs;
  95. const int total_weights = (hidden_weights + output_weights);
  96. const int total_neurons = (inputs + hidden * hidden_layers + outputs);
  97. /* Allocate extra size for weights, outputs, and deltas. */
  98. const int size = sizeof(genann) + sizeof(double) * (total_weights + total_neurons + (total_neurons - inputs));
  99. genann *ret = malloc(size);
  100. if (!ret) return 0;
  101. ret->inputs = inputs;
  102. ret->hidden_layers = hidden_layers;
  103. ret->hidden = hidden;
  104. ret->outputs = outputs;
  105. ret->total_weights = total_weights;
  106. ret->total_neurons = total_neurons;
  107. /* Set pointers. */
  108. ret->weight = (double*)((char*)ret + sizeof(genann));
  109. ret->output = ret->weight + ret->total_weights;
  110. ret->delta = ret->output + ret->total_neurons;
  111. genann_randomize(ret);
  112. ret->activation_hidden = genann_act_sigmoid_cached;
  113. ret->activation_output = genann_act_sigmoid_cached;
  114. genann_init_sigmoid_lookup(ret);
  115. return ret;
  116. }
  117. genann *genann_read(FILE *in) {
  118. int inputs, hidden_layers, hidden, outputs;
  119. int rc;
  120. errno = 0;
  121. rc = fscanf(in, "%d %d %d %d", &inputs, &hidden_layers, &hidden, &outputs);
  122. if (rc < 4 || errno != 0) {
  123. perror("fscanf");
  124. return NULL;
  125. }
  126. genann *ann = genann_init(inputs, hidden_layers, hidden, outputs);
  127. int i;
  128. for (i = 0; i < ann->total_weights; ++i) {
  129. errno = 0;
  130. rc = fscanf(in, " %le", ann->weight + i);
  131. if (rc < 1 || errno != 0) {
  132. perror("fscanf");
  133. genann_free(ann);
  134. return NULL;
  135. }
  136. }
  137. return ann;
  138. }
  139. genann *genann_copy(genann const *ann) {
  140. const int size = sizeof(genann) + sizeof(double) * (ann->total_weights + ann->total_neurons + (ann->total_neurons - ann->inputs));
  141. genann *ret = malloc(size);
  142. if (!ret) return 0;
  143. memcpy(ret, ann, size);
  144. /* Set pointers. */
  145. ret->weight = (double*)((char*)ret + sizeof(genann));
  146. ret->output = ret->weight + ret->total_weights;
  147. ret->delta = ret->output + ret->total_neurons;
  148. return ret;
  149. }
  150. void genann_randomize(genann *ann) {
  151. int i;
  152. for (i = 0; i < ann->total_weights; ++i) {
  153. double r = GENANN_RANDOM();
  154. /* Sets weights from -0.5 to 0.5. */
  155. ann->weight[i] = r - 0.5;
  156. }
  157. }
  158. void genann_free(genann *ann) {
  159. /* The weight, output, and delta pointers go to the same buffer. */
  160. free(ann);
  161. }
  162. double const *genann_run(genann const *ann, double const *inputs) {
  163. double const *w = ann->weight;
  164. double *o = ann->output + ann->inputs;
  165. double const *i = ann->output;
  166. /* Copy the inputs to the scratch area, where we also store each neuron's
  167. * output, for consistency. This way the first layer isn't a special case. */
  168. memcpy(ann->output, inputs, sizeof(double) * ann->inputs);
  169. int h, j, k;
  170. if (!ann->hidden_layers) {
  171. double *ret = o;
  172. for (j = 0; j < ann->outputs; ++j) {
  173. double sum = *w++ * -1.0;
  174. for (k = 0; k < ann->inputs; ++k) {
  175. sum += *w++ * i[k];
  176. }
  177. *o++ = genann_act_output(ann, sum);
  178. }
  179. return ret;
  180. }
  181. /* Figure input layer */
  182. for (j = 0; j < ann->hidden; ++j) {
  183. double sum = *w++ * -1.0;
  184. for (k = 0; k < ann->inputs; ++k) {
  185. sum += *w++ * i[k];
  186. }
  187. *o++ = genann_act_hidden(ann, sum);
  188. }
  189. i += ann->inputs;
  190. /* Figure hidden layers, if any. */
  191. for (h = 1; h < ann->hidden_layers; ++h) {
  192. for (j = 0; j < ann->hidden; ++j) {
  193. double sum = *w++ * -1.0;
  194. for (k = 0; k < ann->hidden; ++k) {
  195. sum += *w++ * i[k];
  196. }
  197. *o++ = genann_act_hidden(ann, sum);
  198. }
  199. i += ann->hidden;
  200. }
  201. double const *ret = o;
  202. /* Figure output layer. */
  203. for (j = 0; j < ann->outputs; ++j) {
  204. double sum = *w++ * -1.0;
  205. for (k = 0; k < ann->hidden; ++k) {
  206. sum += *w++ * i[k];
  207. }
  208. *o++ = genann_act_output(ann, sum);
  209. }
  210. /* Sanity check that we used all weights and wrote all outputs. */
  211. assert(w - ann->weight == ann->total_weights);
  212. assert(o - ann->output == ann->total_neurons);
  213. return ret;
  214. }
  215. void genann_train(genann const *ann, double const *inputs, double const *desired_outputs, double learning_rate) {
  216. /* To begin with, we must run the network forward. */
  217. genann_run(ann, inputs);
  218. int h, j, k;
  219. /* First set the output layer deltas. */
  220. {
  221. double const *o = ann->output + ann->inputs + ann->hidden * ann->hidden_layers; /* First output. */
  222. double *d = ann->delta + ann->hidden * ann->hidden_layers; /* First delta. */
  223. double const *t = desired_outputs; /* First desired output. */
  224. /* Set output layer deltas. */
  225. if (genann_act_output == genann_act_linear ||
  226. ann->activation_output == genann_act_linear) {
  227. for (j = 0; j < ann->outputs; ++j) {
  228. *d++ = *t++ - *o++;
  229. }
  230. } else {
  231. for (j = 0; j < ann->outputs; ++j) {
  232. *d++ = (*t - *o) * *o * (1.0 - *o);
  233. ++o; ++t;
  234. }
  235. }
  236. }
  237. /* Set hidden layer deltas, start on last layer and work backwards. */
  238. /* Note that loop is skipped in the case of hidden_layers == 0. */
  239. for (h = ann->hidden_layers - 1; h >= 0; --h) {
  240. /* Find first output and delta in this layer. */
  241. double const *o = ann->output + ann->inputs + (h * ann->hidden);
  242. double *d = ann->delta + (h * ann->hidden);
  243. /* Find first delta in following layer (which may be hidden or output). */
  244. double const * const dd = ann->delta + ((h+1) * ann->hidden);
  245. /* Find first weight in following layer (which may be hidden or output). */
  246. double const * const ww = ann->weight + ((ann->inputs+1) * ann->hidden) + ((ann->hidden+1) * ann->hidden * (h));
  247. for (j = 0; j < ann->hidden; ++j) {
  248. double delta = 0;
  249. for (k = 0; k < (h == ann->hidden_layers-1 ? ann->outputs : ann->hidden); ++k) {
  250. const double forward_delta = dd[k];
  251. const int windex = k * (ann->hidden + 1) + (j + 1);
  252. const double forward_weight = ww[windex];
  253. delta += forward_delta * forward_weight;
  254. }
  255. *d = *o * (1.0-*o) * delta;
  256. ++d; ++o;
  257. }
  258. }
  259. /* Train the outputs. */
  260. {
  261. /* Find first output delta. */
  262. double const *d = ann->delta + ann->hidden * ann->hidden_layers; /* First output delta. */
  263. /* Find first weight to first output delta. */
  264. double *w = ann->weight + (ann->hidden_layers
  265. ? ((ann->inputs+1) * ann->hidden + (ann->hidden+1) * ann->hidden * (ann->hidden_layers-1))
  266. : (0));
  267. /* Find first output in previous layer. */
  268. double const * const i = ann->output + (ann->hidden_layers
  269. ? (ann->inputs + (ann->hidden) * (ann->hidden_layers-1))
  270. : 0);
  271. /* Set output layer weights. */
  272. for (j = 0; j < ann->outputs; ++j) {
  273. *w++ += *d * learning_rate * -1.0;
  274. for (k = 1; k < (ann->hidden_layers ? ann->hidden : ann->inputs) + 1; ++k) {
  275. *w++ += *d * learning_rate * i[k-1];
  276. }
  277. ++d;
  278. }
  279. assert(w - ann->weight == ann->total_weights);
  280. }
  281. /* Train the hidden layers. */
  282. for (h = ann->hidden_layers - 1; h >= 0; --h) {
  283. /* Find first delta in this layer. */
  284. double const *d = ann->delta + (h * ann->hidden);
  285. /* Find first input to this layer. */
  286. double const *i = ann->output + (h
  287. ? (ann->inputs + ann->hidden * (h-1))
  288. : 0);
  289. /* Find first weight to this layer. */
  290. double *w = ann->weight + (h
  291. ? ((ann->inputs+1) * ann->hidden + (ann->hidden+1) * (ann->hidden) * (h-1))
  292. : 0);
  293. for (j = 0; j < ann->hidden; ++j) {
  294. *w++ += *d * learning_rate * -1.0;
  295. for (k = 1; k < (h == 0 ? ann->inputs : ann->hidden) + 1; ++k) {
  296. *w++ += *d * learning_rate * i[k-1];
  297. }
  298. ++d;
  299. }
  300. }
  301. }
  302. void genann_write(genann const *ann, FILE *out) {
  303. fprintf(out, "%d %d %d %d", ann->inputs, ann->hidden_layers, ann->hidden, ann->outputs);
  304. int i;
  305. for (i = 0; i < ann->total_weights; ++i) {
  306. fprintf(out, " %.20e", ann->weight[i]);
  307. }
  308. }