teeny_core/nn/
layernorm.rs

1/*
2 * Copyright (c) 2026 Teenygrad.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *   http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use core::marker::PhantomData;
18
19use crate::{
20    dtype::{Dtype, EagerTensor, Tensor},
21    nn::Layer,
22};
23
24/// Layer normalisation over the last `normalized_shape.len()` dimensions.
25///
26/// Input shape:  `[*, normalized_shape[0], ..., normalized_shape[n-1]]`
27/// Output shape: same as input.
28///
29/// Parameters:
30/// - `normalized_shape` — the shape of the axes to normalise over (e.g. `[C]`
31///   for 2-D input `[N, C]`, or `[H, W]` for spatial normalisation)
32/// - `eps`     — numerical stability constant (default 1e-5)
33/// - `affine`  — if true, learns per-element γ and β (default true)
34pub struct LayerNorm<D: Dtype, IT, OT, const RANK: usize> {
35    /// The shape of the trailing axes to normalize over.
36    pub normalized_shape: alloc::vec::Vec<usize>,
37    /// Numerical stability constant added to the variance.
38    pub eps: f64,
39    /// Whether to learn per-element scale (gamma) and shift (beta) parameters.
40    pub affine: bool,
41    _pd: PhantomData<(D, IT, OT)>,
42}
43
44impl<D: Dtype, IT, OT, const RANK: usize> LayerNorm<D, IT, OT, RANK> {
45    /// Creates a new layer with default `eps`, affine on.
46    pub fn new(normalized_shape: impl Into<alloc::vec::Vec<usize>>) -> Self {
47        Self {
48            normalized_shape: normalized_shape.into(),
49            eps: 1e-5,
50            affine: true,
51            _pd: PhantomData,
52        }
53    }
54
55    /// Sets the numerical stability constant.
56    pub fn with_eps(mut self, eps: f64) -> Self {
57        self.eps = eps;
58        self
59    }
60
61    /// Sets whether to learn per-element scale/shift parameters.
62    pub fn with_affine(mut self, affine: bool) -> Self {
63        self.affine = affine;
64        self
65    }
66}
67
68impl<D: Dtype, IT: Tensor<D, RANK> + EagerTensor, OT: Tensor<D, RANK>, const RANK: usize> Layer<IT>
69    for LayerNorm<D, IT, OT, RANK>
70{
71    type Output = OT;
72
73    fn call(&self, _input: IT) -> Self::Output {
74        todo!()
75    }
76}
teeny_core/nn/layernorm.rs

teeny_core/nn/
layernorm.rs
