Activation Functions

Most common functions:

1. Sigmoid

2. Tanh

3. ReLu

4. Leaky ReLu

import numpy as np
import seaborn as sns
sns.set(rc={'figure.figsize':(11.7,8.27)})
x = np.linspace(-10, 10, 1000)

Sigmoid

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

sns.lineplot(x, sigmoid(x))

/home/chansoo/projects/statsbook/.venv/lib/python3.8/site-packages/seaborn/_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(

<Axes: >

Tanh

def tanh(x):
    return (np.exp(x) - np.exp(-x)) / (np.exp(x) + np.exp(-x))

sns.lineplot(x, tanh(x))

/home/chansoo/projects/statsbook/.venv/lib/python3.8/site-packages/seaborn/_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(

<Axes: >

ReLu

def relu(x):
    return [max(0, _) for _ in x]

sns.lineplot(x, relu(x))

/home/chansoo/projects/statsbook/.venv/lib/python3.8/site-packages/seaborn/_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(

<Axes: >

Leaky ReLu

def leakyRelu(x, e=0.05):
    return [max(e*z, z) for z in x]

sns.lineplot(x, leakyRelu(x))

/home/chansoo/projects/statsbook/.venv/lib/python3.8/site-packages/seaborn/_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(

<Axes: >