teeny_core/nn/activation/
misc.rs

1/*
2 * Copyright (c) 2026 Teenygrad.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *   http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use core::marker::PhantomData;
18
19use crate::{
20    dtype::{EagerTensor, Float, Tensor},
21    nn::Layer,
22};
23
24/// Leaky ReLU activation layer: `x` for `x >= 0`, `negative_slope * x` otherwise.
25pub struct LeakyRelu<D: Float, T, const RANK: usize> {
26    /// The slope applied to negative inputs.
27    pub negative_slope: f64,
28    _pd: PhantomData<(D, T)>,
29}
30
31impl<D: Float, T, const RANK: usize> LeakyRelu<D, T, RANK> {
32    /// Creates a new `LeakyRelu` layer with the given negative slope.
33    pub fn new(negative_slope: f64) -> Self {
34        Self {
35            negative_slope,
36            _pd: PhantomData,
37        }
38    }
39}
40
41impl<D: Float, T: Tensor<D, RANK> + EagerTensor, const RANK: usize> Layer<T>
42    for LeakyRelu<D, T, RANK>
43{
44    type Output = T;
45    fn call(&self, _input: T) -> Self::Output {
46        todo!()
47    }
48}
49
50/// Threshold activation layer: passes `x` through if `x > threshold`, else outputs `value`.
51pub struct Threshold<D: Float, T, const RANK: usize> {
52    /// The threshold.
53    pub threshold: f64,
54    /// The replacement value for inputs at or below the threshold.
55    pub value: f64,
56    _pd: PhantomData<(D, T)>,
57}
58
59impl<D: Float, T, const RANK: usize> Threshold<D, T, RANK> {
60    /// Creates a new `Threshold` layer.
61    pub fn new(threshold: f64, value: f64) -> Self {
62        Self {
63            threshold,
64            value,
65            _pd: PhantomData,
66        }
67    }
68}
69
70impl<D: Float, T: Tensor<D, RANK> + EagerTensor, const RANK: usize> Layer<T>
71    for Threshold<D, T, RANK>
72{
73    type Output = T;
74    fn call(&self, _input: T) -> Self::Output {
75        todo!()
76    }
77}
78
79/// Softsign activation layer: `x / (1 + |x|)`.
80pub struct Softsign<D: Float, T, const RANK: usize> {
81    _pd: PhantomData<(D, T)>,
82}
83
84impl<D: Float, T, const RANK: usize> Softsign<D, T, RANK> {
85    /// Creates a new `Softsign` layer.
86    pub fn new() -> Self {
87        Self { _pd: PhantomData }
88    }
89}
90
91impl<D: Float, T, const RANK: usize> Default for Softsign<D, T, RANK> {
92    fn default() -> Self {
93        Self::new()
94    }
95}
96
97impl<D: Float, T: Tensor<D, RANK> + EagerTensor, const RANK: usize> Layer<T>
98    for Softsign<D, T, RANK>
99{
100    type Output = T;
101    fn call(&self, _input: T) -> Self::Output {
102        todo!()
103    }
104}
105
106/// Softshrink activation layer: shrinks values toward zero by `lambda`.
107pub struct Softshrink<D: Float, T, const RANK: usize> {
108    /// The shrinkage threshold.
109    pub lambda: f64,
110    _pd: PhantomData<(D, T)>,
111}
112
113impl<D: Float, T, const RANK: usize> Softshrink<D, T, RANK> {
114    /// Creates a new `Softshrink` layer.
115    pub fn new(lambda: f64) -> Self {
116        Self {
117            lambda,
118            _pd: PhantomData,
119        }
120    }
121}
122
123impl<D: Float, T: Tensor<D, RANK> + EagerTensor, const RANK: usize> Layer<T>
124    for Softshrink<D, T, RANK>
125{
126    type Output = T;
127    fn call(&self, _input: T) -> Self::Output {
128        todo!()
129    }
130}
131
132/// Softplus activation layer: a smooth ReLU approximation, `(1/beta) * ln(1 + e^(beta*x))`,
133/// reverting to linear above `threshold` for numerical stability.
134pub struct Softplus<D: Float, T, const RANK: usize> {
135    /// The `beta` sharpness parameter.
136    pub beta: f64,
137    /// Above this value, the layer reverts to linear (`x`) to avoid overflow in `exp`.
138    pub threshold: f64,
139    _pd: PhantomData<(D, T)>,
140}
141
142impl<D: Float, T, const RANK: usize> Softplus<D, T, RANK> {
143    /// Creates a new `Softplus` layer with the given `beta`/`threshold`.
144    pub fn new(beta: f64, threshold: f64) -> Self {
145        Self {
146            beta,
147            threshold,
148            _pd: PhantomData,
149        }
150    }
151}
152
153impl<D: Float, T: Tensor<D, RANK> + EagerTensor, const RANK: usize> Layer<T>
154    for Softplus<D, T, RANK>
155{
156    type Output = T;
157    fn call(&self, _input: T) -> Self::Output {
158        todo!()
159    }
160}
teeny_core/nn/activation/misc.rs

teeny_core/nn/activation/
misc.rs
