# HG changeset patch
# User Jeff Hammel <k0scist@gmail.com>
# Date 1504562012 25200
# Node ID 2f0caec46e26f1e97f30821b2bfe8a472fdc51c5
# Parent  38aa9098bf2d793a8624c44bcd52d01ea8495549
[activation functions] add ReLU

diff -r 38aa9098bf2d -r 2f0caec46e26 tvii/activation.py
--- a/tvii/activation.py	Mon Sep 04 14:34:00 2017 -0700
+++ b/tvii/activation.py	Mon Sep 04 14:53:32 2017 -0700
@@ -2,4 +2,15 @@
 activation functions
 """
 
-# PLACEHOLDER
+# tanh:
+# g(z) = tanh(z) = (exp(z) - exp(-z))/(exp(z) + exp(-z))
+# g'(z) = 1 - (tanh(z))**2
+
+# ReLU
+def ReLU(z):
+    return max((0, z))
+
+def ReLUprime(z):
+    return 1. if z > 0 else 0.
+
+