teeny_cache/
lib.rs

1/*
2 * Copyright (c) 2026 Teenygrad.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *   http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//! Key-value cache utilities for LLM inference in [teenygrad](https://teenygrad.org) — currently
18//! [`DynamicCache`], tracking sequence length and per-layer sliding-window/compileability state
19//! across generation steps.
20
21#![warn(missing_docs)]
22
23/// Tracks per-layer KV cache state (sequence length, sliding-window/compileability) across
24/// generation steps.
25#[derive(Debug, Clone, Default)]
26pub struct DynamicCache {}
27
28impl DynamicCache {
29    /// Creates an empty cache.
30    pub fn new() -> Self {
31        Self {}
32    }
33
34    /// Returns the number of tokens currently cached.
35    ///
36    /// Not yet implemented.
37    pub fn get_sequence_length(&self) -> usize {
38        todo!()
39    }
40
41    /// Per-layer sliding-window state: `(window_size, is_sliding)` for each layer, or `None` if
42    /// no layer uses a sliding window.
43    ///
44    /// Not yet implemented.
45    pub fn is_sliding(&self) -> Option<&[(usize, bool)]> {
46        todo!()
47    }
48
49    /// Whether this cache's current state is safe to pass into a compiled (AOT/JIT) kernel path.
50    ///
51    /// Not yet implemented.
52    pub fn is_compileable(&self) -> bool {
53        todo!()
54    }
55}
teeny_cache/lib.rs

teeny_cache/
lib.rs
