teeny_core/nn/
rmsnorm.rs

1/*
2 * Copyright (c) 2026 Teenygrad.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *   http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17use core::marker::PhantomData;
18
19use crate::{
20    dtype::{Dtype, EagerTensor, Tensor},
21    nn::Layer,
22};
23
24/// Root Mean Square normalisation.
25///
26/// Normalises over the last `normalized_shape.len()` dimensions by dividing by
27/// the RMS (no mean subtraction):
28///   `y = x / rms(x) * γ`
29///
30/// Output shape equals input shape.
31///
32/// Parameters:
33/// - `normalized_shape` — axes to normalise over
34/// - `eps`    — numerical stability constant (default 1e-8)
35/// - `affine` — if true, learns per-element γ (default true)
36pub struct RmsNorm<D: Dtype, IT, OT, const RANK: usize> {
37    /// The shape of the trailing axes to normalize over.
38    pub normalized_shape: alloc::vec::Vec<usize>,
39    /// Numerical stability constant.
40    pub eps: f64,
41    /// Whether to learn a per-element scale (gamma) parameter.
42    pub affine: bool,
43    _pd: PhantomData<(D, IT, OT)>,
44}
45
46impl<D: Dtype, IT, OT, const RANK: usize> RmsNorm<D, IT, OT, RANK> {
47    /// Creates a new layer with default `eps`, affine on.
48    pub fn new(normalized_shape: impl Into<alloc::vec::Vec<usize>>) -> Self {
49        Self {
50            normalized_shape: normalized_shape.into(),
51            eps: 1e-8,
52            affine: true,
53            _pd: PhantomData,
54        }
55    }
56
57    /// Sets the numerical stability constant.
58    pub fn with_eps(mut self, eps: f64) -> Self {
59        self.eps = eps;
60        self
61    }
62
63    /// Sets whether to learn a per-element scale parameter.
64    pub fn with_affine(mut self, affine: bool) -> Self {
65        self.affine = affine;
66        self
67    }
68}
69
70impl<D: Dtype, IT: Tensor<D, RANK> + EagerTensor, OT: Tensor<D, RANK>, const RANK: usize> Layer<IT>
71    for RmsNorm<D, IT, OT, RANK>
72{
73    type Output = OT;
74
75    fn call(&self, _input: IT) -> Self::Output {
76        todo!()
77    }
78}
teeny_core/nn/rmsnorm.rs

teeny_core/nn/
rmsnorm.rs
