code-paul-creator · Nov 11, 2026
diff --git a/neural_network/activation_functions/exponential_linear_unit.py b/neural_network/activation_functions/exponential_linear_unit.py
@@ -29,7 +29,54 @@ def exponential_linear_unit(vector: np.ndarray, alpha: float) -> np.ndarray:
     >>> exponential_linear_unit(vector=np.array([-9.2,-0.3,0.45,-4.56]), alpha=0.067)
     array([-0.06699323, -0.01736518,  0.45      , -0.06629904])
 
+    Edge values: zero maps to zero for any alpha, and large negative inputs
+    saturate at -alpha (the exponential underflows to 0):
 
+    >>> exponential_linear_unit(vector=np.array([0.0]), alpha=0.3)
+    array([0.])
+    >>> exponential_linear_unit(vector=np.array([-1000.0]), alpha=0.3)
+    array([-0.3])
+    >>> exponential_linear_unit(vector=np.array([-50.0, 50.0]), alpha=1.0)
+    array([-1., 50.])
+
+    With alpha = 0 the negative side is flat zero (a plain ReLU):
+
+    >>> exponential_linear_unit(vector=np.array([-2.0, 0.0, 2.0]), alpha=0.0)
+    array([-0.,  0.,  2.])
+
+    SELU is ELU multiplied by a scale, so with scale (lambda_) 1 and the same alpha
+    the two functions are identical:
+
+    >>> from neural_network.activation_functions.scaled_exponential_linear_unit import (
+    ...     scaled_exponential_linear_unit,
+    ... )
+    >>> x = np.array([-5.0, -1.0, -0.1, 0.0, 0.1, 1.0, 5.0])
+    >>> bool(np.array_equal(
+    ...     exponential_linear_unit(x, alpha=0.3),
+    ...     scaled_exponential_linear_unit(x, alpha=0.3, lambda_=1.0),
+    ... ))
+    True
+
+    At SELU's documented constants (alpha=1.6732, lambda_=1.0507), SELU equals
+    1.0507 * ELU(x, alpha=1.6732):
+
+    >>> bool(np.allclose(
+    ...     scaled_exponential_linear_unit(x),
+    ...     1.0507 * exponential_linear_unit(x, alpha=1.6732),
+    ... ))
+    True
+
+    For positive inputs ELU and Leaky ReLU are both the identity, for any alpha:
+
+    >>> from neural_network.activation_functions.leaky_rectified_linear_unit import (
+    ...     leaky_rectified_linear_unit,
+    ... )
+    >>> positive = np.array([0.1, 1.0, 2.5, 100.0])
+    >>> bool(np.array_equal(
+    ...     exponential_linear_unit(positive, alpha=0.3),
+    ...     leaky_rectified_linear_unit(positive, alpha=0.3),
+    ... ))
+    True
     """
     return np.where(vector > 0, vector, (alpha * (np.exp(vector) - 1)))
 

diff --git a/work/R23-T054/COMPARE.md b/work/R23-T054/COMPARE.md
@@ -0,0 +1,63 @@
+# R23-T054: ELU vs SELU vs Leaky ReLU
+
+Files compared (all in `neural_network/activation_functions/`):
+`exponential_linear_unit.py`, `scaled_exponential_linear_unit.py`,
+`leaky_rectified_linear_unit.py`.
+Evidence: `compare_test.py` (19 checks, all pass) and `bench_compare.py`.
+
+## API differences
+| | ELU | SELU | Leaky ReLU |
+|---|---|---|---|
+| Function | `exponential_linear_unit(vector, alpha)` | `scaled_exponential_linear_unit(vector, alpha=1.6732, lambda_=1.0507)` | `leaky_rectified_linear_unit(vector, alpha)` |
+| Input arg | `vector` | `vector` | `vector` |
+| `alpha` | required, no default | optional (default 1.6732) | required, no default |
+| What alpha means | negative side saturates at `-alpha` | same as ELU | slope for negative inputs |
+| Extra arg | none | `lambda_` (scale; trailing `_` because `lambda` is a Python keyword) | none |
+| Positive side | `x` | `lambda_ * x` | `x` |
+| Negative side | `alpha * (e^x - 1)` | `lambda_ * alpha * (e^x - 1)` | `alpha * x` |
+
+All three are typed `np.ndarray -> np.ndarray` and use `np.where`. None validates
+its input. A plain Python float is accepted by accident and returns a 0-d array
+(`exponential_linear_unit(2.0, 0.3)` -> `array(2.)`); an int array returns floats.
+The ELU docstring says `alpha >= 0` but this is not enforced (and NaN in gives NaN out).
+
+## Results: where they agree
+(`compare_test.py`, 14 shared inputs from -1000 to 700, alphas 0, 0.067, 0.3, 1.0, 1.6732)
+- **ELU == SELU** exactly (`np.array_equal`) when SELU's `lambda_` is 1 and alpha is the same.
+- **SELU defaults == 1.0507 * ELU(x, alpha=1.6732)** (`np.allclose`).
+- **ELU == Leaky ReLU == x for every positive input**, for any alpha.
+- **All three give 0 at x = 0.**
+- They cannot agree on negative inputs: ELU curves toward `-alpha`, Leaky ReLU keeps a straight line of slope alpha.
+
+## Edge values
+| input | ELU(alpha=0.3) | Leaky ReLU(alpha=0.3) |
+|---|---|---|
+| 0 | 0 | 0 |
+| -1 | -0.1896 | -0.3 |
+| -1000 (large negative) | -0.3 (saturates at -alpha) | -300 (unbounded) |
+| 700 | 700 | 700 |
+| 1000 (large positive) | 1000, **but** `RuntimeWarning: overflow encountered in exp` | 1000, no warning |
+
+The warning happens because `np.where` evaluates both branches: `exp(1000)` overflows
+to `inf` in the unused negative branch. The returned value is still correct. Both ELU
+and SELU have this; Leaky ReLU has no `exp`, so it does not. (Not changed in this PR,
+which only adds doctests.)
+
+## Speed
+Min of 5 repeats, ms per call, random normal inputs (one run on a cloud machine, so
+treat as rough):
+
+| size | ELU | SELU | Leaky ReLU |
+|---|---|---|---|
+| 1,000 | 0.004 | 0.004 | 0.002 |
+| 100,000 | 0.184 | 0.216 | 0.101 |
+| 1,000,000 | 2.440 | 2.818 | 1.754 |
+
+Leaky ReLU is fastest (no `exp`). SELU is a little slower than ELU because it does one
+extra multiplication of the whole array.
+
+## Recommendation for a beginner
+**Leaky ReLU.** It is one line (`x` if positive else `alpha * x`), has no exponential,
+no overflow warning, and is the fastest. Move to ELU when you want smooth negative
+outputs that saturate; use SELU only when you want its self-normalising behaviour
+with its fixed constants.
diff --git a/work/R23-T054/bench_compare.py b/work/R23-T054/bench_compare.py
@@ -0,0 +1,36 @@
+"""Benchmark ELU, SELU and Leaky ReLU on the same inputs (R23-T054).
+
+Run from the repo root:  python work/R23-T054/bench_compare.py
+"""
+
+import sys
+import timeit
+
+import numpy as np
+
+sys.path.insert(0, ".")
+from neural_network.activation_functions.exponential_linear_unit import (
+    exponential_linear_unit,
+)
+from neural_network.activation_functions.leaky_rectified_linear_unit import (
+    leaky_rectified_linear_unit,
+)
+from neural_network.activation_functions.scaled_exponential_linear_unit import (
+    scaled_exponential_linear_unit,
+)
+
+rng = np.random.default_rng(0)
+print("| size | elu (ms) | selu (ms) | leaky relu (ms) |")
+print("|---|---|---|---|")
+for n in (1_000, 100_000, 1_000_000):
+    x = rng.normal(0, 3, n)
+    reps = 200 if n <= 100_000 else 20
+    t = [
+        min(timeit.repeat(lambda f=f: f(x), number=reps, repeat=5)) / reps * 1000
+        for f in (
+            lambda v: exponential_linear_unit(v, 1.6732),
+            scaled_exponential_linear_unit,
+            lambda v: leaky_rectified_linear_unit(v, 0.01),
+        )
+    ]
+    print(f"| {n:,} | {t[0]:.3f} | {t[1]:.3f} | {t[2]:.3f} |")
diff --git a/work/R23-T054/compare_test.py b/work/R23-T054/compare_test.py
@@ -0,0 +1,80 @@
+"""Compare ELU with SELU and Leaky ReLU (R23-T054).
+
+Run from the repo root:  python -m pytest work/R23-T054/compare_test.py -v
+"""
+
+import numpy as np
+import pytest
+
+from neural_network.activation_functions.exponential_linear_unit import (
+    exponential_linear_unit,
+)
+from neural_network.activation_functions.leaky_rectified_linear_unit import (
+    leaky_rectified_linear_unit,
+)
+from neural_network.activation_functions.scaled_exponential_linear_unit import (
+    scaled_exponential_linear_unit,
+)
+
+# 14 shared inputs: negatives, zero, positives, tiny and large values.
+SHARED = np.array(
+    [-1000.0, -50.0, -5.0, -2.0, -1.0, -0.1, 0.0, 0.1, 0.5, 1.0, 2.0, 5.0, 50.0, 700.0]
+)
+POSITIVE = SHARED[SHARED > 0]
+ALPHAS = [0.0, 0.067, 0.3, 1.0, 1.6732]
+
+
+@pytest.mark.parametrize("alpha", ALPHAS)
+def test_elu_equals_selu_with_scale_one(alpha):
+    # SELU(x, alpha, lambda_=1) is exactly ELU(x, alpha).
+    assert np.array_equal(
+        exponential_linear_unit(SHARED, alpha),
+        scaled_exponential_linear_unit(SHARED, alpha, 1.0),
+    )
+
+
+def test_selu_defaults_are_scaled_elu():
+    # SELU's documented constants: SELU(x) = 1.0507 * ELU(x, alpha=1.6732).
+    assert np.allclose(
+        scaled_exponential_linear_unit(SHARED),
+        1.0507 * exponential_linear_unit(SHARED, 1.6732),
+    )
+
+
+@pytest.mark.parametrize("alpha", ALPHAS)
+def test_elu_equals_leaky_relu_for_positive_inputs(alpha):
+    assert np.array_equal(
+        exponential_linear_unit(POSITIVE, alpha),
+        leaky_rectified_linear_unit(POSITIVE, alpha),
+    )
+    assert np.array_equal(exponential_linear_unit(POSITIVE, alpha), POSITIVE)
+
+
+@pytest.mark.parametrize("alpha", ALPHAS)
+def test_zero_maps_to_zero_in_all_three(alpha):
+    zero = np.array([0.0])
+    assert exponential_linear_unit(zero, alpha)[0] == 0
+    assert leaky_rectified_linear_unit(zero, alpha)[0] == 0
+    assert scaled_exponential_linear_unit(zero, alpha, 1.0)[0] == 0
+
+
+def test_large_negative_saturates_for_elu_but_not_leaky_relu():
+    x = np.array([-1000.0])
+    assert exponential_linear_unit(x, 0.3)[0] == pytest.approx(-0.3)
+    assert leaky_rectified_linear_unit(x, 0.3)[0] == pytest.approx(-300.0)
+
+
+def test_they_disagree_on_negative_inputs():
+    # Why the three cannot agree everywhere: different negative branches.
+    x = np.array([-1.0])
+    elu = exponential_linear_unit(x, 0.3)[0]
+    leaky = leaky_rectified_linear_unit(x, 0.3)[0]
+    assert elu != pytest.approx(leaky)
+
+
+def test_large_positive_gives_right_answer_but_warns():
+    # np.where evaluates both branches, so exp(1000) overflows (inf) and warns,
+    # even though the positive branch is the one returned.
+    with pytest.warns(RuntimeWarning, match="overflow"):
+        result = exponential_linear_unit(np.array([1000.0]), 0.3)
+    assert result[0] == 1000.0