[ACTF1] - Activation functions

Some activation functions, with their derivatives.

Objectives :

• View the main activation functions

Les fonctions d'activation dans Keras :
https://www.tensorflow.org/api_docs/python/tf/keras/activations

What we're going to do :

• Juste visualiser les principales fonctions d'activation

import numpy as np
import matplotlib
import matplotlib.pyplot as plt

import math
from math import erfc, sqrt, exp
from math import pi as PI
from math import e as E
import sys

import fidle

# Init Fidle environment
run_id, run_dir, datasets_dir = fidle.init('ACTF1')

FIDLE - Environment initialization

Version              : 2.3.0
Run id               : ACTF1
Run dir              : ./run/ACTF1
Datasets dir         : /gpfswork/rech/mlh/uja62cb/fidle-project/datasets-fidle
Start time           : 03/03/24 21:26:52
Hostname             : r3i7n8 (Linux)
Tensorflow log level : Info + Warning + Error  (=0)
Update keras cache   : False
Update torch cache   : False
Save figs            : ./run/ACTF1/figs (True)
numpy                : 1.24.4
sklearn              : 1.3.2
yaml                 : 6.0.1
matplotlib           : 3.8.2
pandas               : 2.1.3

SELU_A = -sqrt(2/PI)/(erfc(1/sqrt(2))*exp(1/2)-1)
SELU_L = (1-erfc(1/sqrt(2))*sqrt(E))*sqrt(2*PI) / (2*erfc(sqrt(2))*E*E+PI*erfc(1/sqrt(2))**2*E-2*(2+PI)*erfc(1/sqrt(2))*sqrt(E)+PI+2)**0.5


def heaviside(z):
    return np.where(z<0,0,1)

def sign(z):
    return np.where(z<0,-1,1)
#    return np.sign(z)

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def tanh(z):
    return np.tanh(z)

def relu(z):
    return np.maximum(0, z)

def leaky_relu(z,a=0.05):
    return np.maximum(a*z, z)

def elu(z,a=1):
    #y=z.copy()
    y=a*(np.exp(z)-1)
    y[z>0]=z[z>0]
    return y

def selu(z):
    return SELU_L*elu(z,a=SELU_A)

def derivative(f, z, eps=0.000001):
    return (f(z + eps) - f(z - eps))/(2 * eps)

pw=5
ph=5

z = np.linspace(-5, 5, 200)


# ------ Heaviside
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(0, 0, "rx", markersize=10)
ax.plot(z, heaviside(z),              linestyle='-',  label="Heaviside")
ax.plot(z, derivative(heaviside, z),  linewidth=3, alpha=0.6, label="dHeaviside/dx")
# ax.plot(z, sign(z),                  label="Heaviside")
ax.set_title("Heaviside")
fidle.scrawler.save_fig('Heaviside')
plt.show()


# ----- Logit/Sigmoid
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, sigmoid(z),             label="Sigmoid")
ax.plot(z, derivative(sigmoid, z), linewidth=3, alpha=0.6, label="dSigmoid/dx")
ax.set_title("Logit")
fidle.scrawler.save_fig('Logit')
plt.show()

# ----- Tanh
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, tanh(z),             label="Tanh")
ax.plot(z, derivative(tanh, z), linewidth=3, alpha=0.6, label="dTanh/dx")
ax.set_title("Tanh")
fidle.scrawler.save_fig('Tanh')
plt.show()

# ----- Relu
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, relu(z),             label="ReLU")
ax.plot(z, derivative(relu, z), linewidth=3, alpha=0.6, label="dReLU/dx")
ax.set_title("ReLU")
fidle.scrawler.save_fig('ReLU')
plt.show()

# ----- Leaky Relu
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, leaky_relu(z),              label="Leaky ReLU")
ax.plot(z, derivative( leaky_relu, z), linewidth=3, alpha=0.6, label="dLeakyReLU/dx")
ax.set_title("Leaky ReLU (α=0.05)")
fidle.scrawler.save_fig('LeakyReLU')
plt.show()

# ----- Elu
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, elu(z),              label="ReLU")
ax.plot(z, derivative( elu, z), linewidth=3, alpha=0.6, label="dExpReLU/dx")
ax.set_title("ELU (α=1)")
fidle.scrawler.save_fig('ELU')
plt.show()

# ----- Selu
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, selu(z),              label="SeLU")
ax.plot(z, derivative( selu, z), linewidth=3, alpha=0.6, label="dSeLU/dx")
ax.set_title("ELU (SELU)")
fidle.scrawler.save_fig('SeLU')
plt.show()

Saved: ./run/ACTF1/figs/Heaviside

Saved: ./run/ACTF1/figs/Logit

Saved: ./run/ACTF1/figs/Tanh

Saved: ./run/ACTF1/figs/ReLU

Saved: ./run/ACTF1/figs/LeakyReLU

Saved: ./run/ACTF1/figs/ELU

Saved: ./run/ACTF1/figs/SeLU

---