Leaky ReLUs

CMakeLists.txt

@@ -66,7 +66,7 @@ INCLUDE(DefineInstallationPaths)
 
 
 configure_file (cmake/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/src/include/config.h)
-include_directories (${CMAKE_CURRENT_BINARY_DIR}/src/include/)
+include_directories (${CMAKE_CURRENT_BINARY_DIR}/src/include/ ${CMAKE_CURRENT_BINARY_DIR}/src/include/optimized/opencl/)
 
 configure_file (cmake/fann.pc.cmake ${CMAKE_CURRENT_BINARY_DIR}/fann.pc @ONLY)
 

examples/mkfile

@@ -0,0 +1,11 @@
+</$objtype/mkfile
+
+CC=pcc
+CFLAGS=-D_PLAN9_SOURCE -c
+NOMK=(parallel_train.c)
+TARG=`{ ls *.c | grep -v '^('$NOMK')$' | sed 's,.c$,,' }
+PROGS=${TARG:%=$O.%}
+
+all:V: $PROGS
+
+</sys/src/cmd/mkmany

src/CMakeLists.txt

@@ -10,8 +10,15 @@ ELSE()
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
 ENDIF(NOT OPENMP_FOUND OR DISABLE_PARALLEL_FANN)
 
+FIND_PACKAGE(OpenCL)
+IF(OPENCL_FOUND)
+  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I${OpenCL_INCLUDE_DIR}")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${OpenCL_INCLUDE_DIR}")
+ENDIF(OPENCL_FOUND)
+
 ADD_SUBDIRECTORY( include )
 
+#INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/include/optimized/opencl)
 INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/include)
 ADD_DEFINITIONS(-D_REENTRANT)
 if (WIN32)
@@ -81,6 +88,12 @@ INSTALL(TARGETS fixedfann fixedfann_static LIBRARY DESTINATION ${LIB_INSTALL_DIR
 
 SET(fann_LIB_SRCS
         floatfann.c
+#	optimized/opencl/fann.c
+#	optimized/opencl/fann_cl.cpp
+#	optimized/opencl/fann_cl_kernel.c
+#	optimized/opencl/fann_cl_train.c
+#	optimized/opencl/fann_cl_ann.c
+#	optimized/opencl/fann_cl_run.c	
         )
 
 ADD_LIBRARY(fann SHARED ${fann_LIB_SRCS})

src/doublefann.c

@@ -19,6 +19,10 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 /* Easy way to allow for build of multiple binaries */
 
+#ifdef PLAN9
+#include <stdio.h>
+#endif
+
 #include "config.h"
 #include "doublefann.h"
 
@@ -28,4 +32,6 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 #include "fann_train_data.c"
 #include "fann_error.c"
 #include "fann_cascade.c"
+#ifndef PLAN9
 #include "parallel_fann.c"
+#endif

src/fann.c

@@ -17,12 +17,21 @@
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
 
+#ifdef PLAN9
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <time.h>
+#include <math.h>
+#else
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
 #include <time.h>
 #include <math.h>
+#endif
 
 #include "config.h"
 #include "fann.h"
@@ -654,7 +663,8 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input)
 						break;
 				}
 
-				for(; i != num_connections; i += 4)
+				#pragma omp parallel for reduction(+:neuron_sum)
+				for(i = num_connections & 3; i < num_connections; i += 4)
 				{
 					neuron_sum +=
 						fann_mult(weights[i], neurons[i].value) +
@@ -688,7 +698,8 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input)
 						break;
 				}
 
-				for(; i != num_connections; i += 4)
+				#pragma omp parallel for reduction(+:neuron_sum)
+				for(i = num_connections & 3; i < num_connections; i += 4)
 				{
 					neuron_sum +=
 						fann_mult(weights[i], neuron_pointers[i]->value) +
@@ -769,6 +780,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input)
 				case FANN_LINEAR_PIECE_SYMMETRIC:
 					neuron_it->value = (fann_type)((neuron_sum < -multiplier) ? -multiplier : (neuron_sum > multiplier) ? multiplier : neuron_sum);
 					break;
+				case FANN_LINEAR_PIECE_LEAKY:
+					neuron_it->value = (fann_type)((neuron_sum < 0) ? 0.01 * neuron_sum: neuron_sum);
+					break;
+				case FANN_LINEAR_PIECE_RECT:
+					neuron_it->value = (fann_type)((neuron_sum < 0) ? 0: neuron_sum);
+					break;
 				case FANN_ELLIOT:
 				case FANN_ELLIOT_SYMMETRIC:
 				case FANN_GAUSSIAN:
@@ -1828,7 +1845,7 @@ FANN_EXTERNAL void FANN_API fann_enable_seed_rand()
 /* INTERNAL FUNCTION
    Seed the random function.
  */
-void fann_seed_rand()
+void fann_seed_rand(void)
 {
 #ifndef _WIN32
 	FILE *fp = fopen("/dev/urandom", "r");

src/fann_cascade.c

@@ -17,6 +17,12 @@
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
 
+#ifdef PLAN9
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#endif
+
 #include "config.h"
 #include "fann.h"
 #include "string.h"
@@ -741,6 +747,8 @@ fann_type fann_train_candidates_epoch(struct fann *ann, struct fann_train_data *
 				case FANN_GAUSSIAN_STEPWISE:
 				case FANN_ELLIOT:
 				case FANN_LINEAR_PIECE:
+				case FANN_LINEAR_PIECE_LEAKY:
+				case FANN_LINEAR_PIECE_RECT:
 				case FANN_SIN:
 				case FANN_COS:
 					break;

src/fann_io.c

@@ -19,9 +19,14 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#ifndef PLAN9
 #include <stdarg.h>
+#endif
 #include <string.h>
 #include <limits.h>
+#ifdef PLAN9
+#include <math.h>
+#endif
 
 #include "config.h"
 #include "fann.h"

src/fann_train.c

@@ -17,6 +17,10 @@
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
 
+#ifdef PLAN9
+#define sqrtf sqrt
+#endif
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
@@ -42,6 +46,10 @@ fann_type fann_activation_derived(unsigned int activation_function,
 		case FANN_LINEAR_PIECE:
 		case FANN_LINEAR_PIECE_SYMMETRIC:
 			return (fann_type) fann_linear_derive(steepness, value);
+		case FANN_LINEAR_PIECE_LEAKY:
+			return (fann_type) ((value<0)? steepness * 0.01: steepness);
+		case FANN_LINEAR_PIECE_RECT:
+			return (fann_type) ((value<0)? 0: steepness);
 		case FANN_SIGMOID:
 		case FANN_SIGMOID_STEPWISE:
 			value = fann_clip(value, 0.01f, 0.99f);
@@ -133,6 +141,8 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron* neuron, fann_typ
 		case FANN_LINEAR_PIECE:
 		case FANN_SIN:
 		case FANN_COS:
+		case FANN_LINEAR_PIECE_LEAKY:
+		case FANN_LINEAR_PIECE_RECT:
 			break;
 	}
 

src/fann_train_data.c

@@ -21,6 +21,10 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
+#ifdef PLAN9
+#include <math.h>
+#define sqrtf sqrt
+#endif
 
 #include "config.h"
 #include "fann.h"

src/fixedfann.c

@@ -19,6 +19,10 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 /* Easy way to allow for build of multiple binaries */
 
+#ifdef PLAN9
+#include <stdio.h>
+#endif
+
 #include "config.h"
 #include "fixedfann.h"
 

src/floatfann.c

@@ -19,6 +19,10 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 /* Easy way to allow for build of multiple binaries */
 
+#ifdef PLAN9
+#include <stdio.h>
+#endif
+
 #include "config.h"
 #include "floatfann.h"
 
@@ -28,4 +32,6 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 #include "fann_train_data.c"
 #include "fann_error.c"
 #include "fann_cascade.c"
+#ifndef PLAN9
 #include "parallel_fann.c"
+#endif

src/include/fann.h

@@ -37,6 +37,10 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 /* Group: Creation, Destruction & Execution */
 
 #ifndef FANN_INCLUDE
+#ifdef _PLAN9_SOURCE
+#pragma lib "/$M/lib/ape/libfann.a"
+#endif
+
 /* just to allow for inclusion of fann.h in normal stuations where only floats are needed */ 
 #ifdef FIXEDFANN
 #include "fixedfann.h"
@@ -45,7 +49,9 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 #endif	/* FIXEDFANN  */
 
 #else
-
+
+#ifndef PLAN9
+
 /* COMPAT_TIME REPLACEMENT */ 
 #ifndef _WIN32
 #include <sys/time.h>
@@ -58,6 +64,8 @@ extern unsigned long __stdcall GetTickCount(void);
 #include <windows.h>
 #endif	/* _MSC_EXTENSIONS */
 #endif	/* _WIN32 */
+
+#endif
 
 #ifndef __fann_h__
 #define __fann_h__
@@ -113,11 +121,14 @@ extern "C"
  to use dll's. To use dll's FANN_USE_DLL has to be defined before
  including the fann headers.
 */ 
+#ifndef PLAN9
 #if defined(_MSC_VER) && (_MSC_VER > 1300)
 #ifndef FANN_NO_DLL
 #define FANN_USE_DLL
 #endif	/* FANN_USE_LIB */
 #endif	/* _MSC_VER */
+#endif
+#ifndef PLAN9
 #if defined(_MSC_VER) && (defined(FANN_USE_DLL) || defined(FANN_DLL_EXPORTS))
 #ifdef FANN_DLL_EXPORTS
 #define FANN_EXTERNAL __declspec(dllexport)
@@ -126,9 +137,12 @@ extern "C"
 #endif	/* FANN_DLL_EXPORTS*/
 #define FANN_API __stdcall
 #else							/*  */
+#ifndef PLAN9
 #define FANN_EXTERNAL
 #define FANN_API
+#endif
 #endif	/* _MSC_VER */
+#endif
 /* ----- End of macros used to define DLL external entrypoints ----- */ 
 
 #include "fann_error.h"

src/include/fann_activation.h

@@ -33,13 +33,19 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 __doublefann_h__ is not defined
 */
 #ifndef __doublefann_h__
+#ifndef PLAN9
 #define FANN_EXP(x) expf(x)
-#define FANN_SIN(x) sinf(x)
-#define FANN_COS(x) cosf(x)
+#define _FANN_SIN(x) sinf(x)
+#define _FANN_COS(x) cosf(x)
 #else
 #define FANN_EXP(x) exp(x)
-#define FANN_SIN(x) sin(x)
-#define FANN_COS(x) cos(x)
+#define _FANN_SIN(x) sin(x)
+#define _FANN_COS(x) cos(x)
+#endif
+#else
+#define FANN_EXP(x) exp(x)
+#define _FANN_SIN(x) sin(x)
+#define _FANN_COS(x) cos(x)
 #endif
 
 #define fann_linear_func(v1, r1, v2, r2, sum) (((((r2)-(r1)) * ((sum)-(v1)))/((v2)-(v1))) + (r1))
@@ -80,19 +86,19 @@ __doublefann_h__ is not defined
 #define fann_elliot_symmetric_derive(steepness, value, sum) (steepness * 1.0f / ((1.0f + fann_abs(sum)) * (1.0f + fann_abs(sum))))
 
 /* FANN_SIN_SYMMETRIC */
-#define fann_sin_symmetric_real(sum) (FANN_SIN(sum))
+#define fann_sin_symmetric_real(sum) (_FANN_SIN(sum))
 #define fann_sin_symmetric_derive(steepness, sum) (steepness*cos(steepness*sum))
 
 /* FANN_COS_SYMMETRIC */
-#define fann_cos_symmetric_real(sum) (FANN_COS(sum))
+#define fann_cos_symmetric_real(sum) (_FANN_COS(sum))
 #define fann_cos_symmetric_derive(steepness, sum) (steepness*-sin(steepness*sum))
 
 /* FANN_SIN */
-#define fann_sin_real(sum) (FANN_SIN(sum)/2.0f+0.5f)
+#define fann_sin_real(sum) (_FANN_SIN(sum)/2.0f+0.5f)
 #define fann_sin_derive(steepness, sum) (steepness*cos(steepness*sum)/2.0f)
 
 /* FANN_COS */
-#define fann_cos_real(sum) (FANN_COS(sum)/2.0f+0.5f)
+#define fann_cos_real(sum) (_FANN_COS(sum)/2.0f+0.5f)
 #define fann_cos_derive(steepness, sum) (steepness*-sin(steepness*sum)/2.0f)
 
 #define fann_activation_switch(activation_function, value, result) \
@@ -152,6 +158,12 @@ switch(activation_function) \
 	case FANN_GAUSSIAN_STEPWISE: \
         result = 0; \
         break; \
+	case FANN_LINEAR_PIECE_LEAKY: \
+		result = (fann_type)((value < 0) ? value*0.01 : value); \
+        break; \
+	case FANN_LINEAR_PIECE_RECT: \
+		result = (fann_type)((value < 0) ? 0 : value); \
+        break; \
 }
 
 #endif

src/include/fann_data.h

@@ -197,6 +197,16 @@ static char const *const FANN_TRAIN_NAMES[] = {
 	 * span: 0 <= y <= 1
 	 * y = cos(x*s)/2+0.5
 	 * d = s*-sin(x*s)/2
+
+	FANN_LINEAR_PIECE_LEAKY - leaky ReLU
+	 * span: -inf < y < inf
+	 * y = x<0? 0.01*x: x
+	 * d = x<0? 0.01: 1
+
+	FANN_LINEAR_PIECE_RECT - ReLU
+	 * span: -inf < y < inf
+	 * y = x<0? 0: x
+	 * d = x<0? 0: 1
 
 	See also:
    	<fann_set_activation_function_layer>, <fann_set_activation_function_hidden>,
@@ -226,7 +236,9 @@ enum fann_activationfunc_enum
 	FANN_SIN_SYMMETRIC,
 	FANN_COS_SYMMETRIC,
 	FANN_SIN,
-	FANN_COS
+	FANN_COS,
+	FANN_LINEAR_PIECE_LEAKY,
+	FANN_LINEAR_PIECE_RECT,
 };
 
 /* Constant: FANN_ACTIVATIONFUNC_NAMES
@@ -258,7 +270,9 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = {
 	"FANN_SIN_SYMMETRIC",
 	"FANN_COS_SYMMETRIC",
 	"FANN_SIN",
-	"FANN_COS"
+	"FANN_COS",
+	"FANN_LINEAR_PIECE_LEAKY",
+	"FANN_LINEAR_PIECE_RECT"
 };
 
 /* Enum: fann_errorfunc_enum