changeset 43:2f0caec46e26

[activation functions] add ReLU
author Jeff Hammel <k0scist@gmail.com>
date Mon, 04 Sep 2017 14:53:32 -0700
parents 38aa9098bf2d
children 857a606783e1
files tvii/activation.py
diffstat 1 files changed, 12 insertions(+), 1 deletions(-) [+]
line wrap: on
line diff
--- a/tvii/activation.py	Mon Sep 04 14:34:00 2017 -0700
+++ b/tvii/activation.py	Mon Sep 04 14:53:32 2017 -0700
@@ -2,4 +2,15 @@
 activation functions
 """
 
-# PLACEHOLDER
+# tanh:
+# g(z) = tanh(z) = (exp(z) - exp(-z))/(exp(z) + exp(-z))
+# g'(z) = 1 - (tanh(z))**2
+
+# ReLU
+def ReLU(z):
+    return max((0, z))
+
+def ReLUprime(z):
+    return 1. if z > 0 else 0.
+
+