teeny_nlp/tokenizer/
mod.rs

1/*
2 * Copyright (c) 2026 Teenygrad.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *   http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//! Tokenizer trait and chat-message types.
18
19use serde::{Deserialize, Serialize};
20
21/// A single chat-conversation turn: a `role` (e.g. `"user"`, `"assistant"`, `"system"`) paired
22/// with its `content`.
23#[derive(Debug, Clone, Serialize, Deserialize)]
24pub struct Message {
25    /// Who sent this turn (e.g. `"user"`, `"assistant"`, `"system"`).
26    pub role: String,
27    /// The turn's text content.
28    pub content: String,
29}
30
31impl Message {
32    /// Creates a message from `role`/`content` string slices.
33    pub fn new(role: &str, content: &str) -> Self {
34        Self {
35            role: role.to_string(),
36            content: content.to_string(),
37        }
38    }
39}
40
41/// Text tokenization and chat-templating, implemented per model/tokenizer family.
42pub trait Tokenizer {
43    /// Renders `messages` through `chat_template` (a Jinja-style template string, per the Hugging
44    /// Face chat-template convention), optionally tokenizing the result (`tokenize`), appending a
45    /// generation prompt (`add_generation_prompt`), and enabling "thinking"/reasoning mode
46    /// (`enable_thinking`) for models that support it.
47    fn apply_chat_template(
48        &self,
49        messages: &[Message],
50        chat_template: &str,
51        tokenize: bool,
52        add_generation_prompt: bool,
53        enable_thinking: bool,
54    ) -> String;
55
56    /// Encodes `texts` into a flat sequence of token IDs.
57    fn encode(&self, texts: &[String]) -> Vec<usize>;
58
59    /// Decodes a sequence of token IDs back into text.
60    fn decode(&self, ids: &[usize]) -> String;
61}
teeny_nlp/tokenizer/mod.rs

teeny_nlp/tokenizer/
mod.rs
