Skip to main content

Api: tokenizer

Api members in the tokenizer family.

Member
tokenizer_apply_chat_template_messages_add_generation_prompt / f_tokenizer_apply_chat_template_messages_add_generation_prompt
tokenizer_bos_id
tokenizer_decode / f_tokenizer_decode
tokenizer_decode_batch_encoded_skip_special_tokens / f_tokenizer_decode_batch_encoded_skip_special_tokens
tokenizer_decode_batch_ids_cu_seqlens / f_tokenizer_decode_batch_ids_cu_seqlens
tokenizer_decode_batch_ids_skip_special_tokens / f_tokenizer_decode_batch_ids_skip_special_tokens
tokenizer_encode / f_tokenizer_encode
tokenizer_encode_batch / f_tokenizer_encode_batch
tokenizer_encode_chat_messages / f_tokenizer_encode_chat_messages
tokenizer_encode_text / f_tokenizer_encode_text
tokenizer_eos_id
tokenizer_free
tokenizer_from_bert_vocab / f_tokenizer_from_bert_vocab
tokenizer_from_file / f_tokenizer_from_file
tokenizer_from_gpt2 / f_tokenizer_from_gpt2
tokenizer_from_huggingface / f_tokenizer_from_huggingface
tokenizer_from_sentencepiece / f_tokenizer_from_sentencepiece
tokenizer_from_tekken / f_tokenizer_from_tekken
tokenizer_from_tiktoken / f_tokenizer_from_tiktoken
tokenizer_has_chat_template
tokenizer_id_to_token / f_tokenizer_id_to_token
tokenizer_pad_id
tokenizer_streaming_decoder
tokenizer_token_to_id / f_tokenizer_token_to_id
tokenizer_tokenize / f_tokenizer_tokenize
tokenizer_vocab_size

tokenizer_apply_chat_template_messages_add_generation_prompt / f_tokenizer_apply_chat_template_messages_add_generation_prompt​

fn f_tokenizer_apply_chat_template_messages_add_generation_prompt(self, self_: &Tokenizer, messages_json: &str, add_generation_prompt: bool) -> Result<String>
fn tokenizer_apply_chat_template_messages_add_generation_prompt(self, self_: &Tokenizer, messages_json: &str, add_generation_prompt: bool) -> String

Wraps api-table member tokenizer_apply_chat_template_messages_add_generation_prompt. ClikaRT::tokenizer::Tokenizer::apply_chat_template <- ClikaRT/tokenizer/tokenizer.h

tokenizer_bos_id​

fn tokenizer_bos_id(self, self_: &Tokenizer) -> i64

Wraps api-table member tokenizer_bos_id. ClikaRT::tokenizer::Tokenizer::bos_id <- ClikaRT/tokenizer/tokenizer.h

tokenizer_decode / f_tokenizer_decode​

fn f_tokenizer_decode(self, self_: &Tokenizer, ids: &[i32], skip_special_tokens: bool) -> Result<String>
fn tokenizer_decode(self, self_: &Tokenizer, ids: &[i32], skip_special_tokens: bool) -> String

Wraps api-table member tokenizer_decode. ClikaRT::tokenizer::Tokenizer::decode <- ClikaRT/tokenizer/tokenizer.h

tokenizer_decode_batch_encoded_skip_special_tokens / f_tokenizer_decode_batch_encoded_skip_special_tokens​

fn f_tokenizer_decode_batch_encoded_skip_special_tokens(self, self_: &Tokenizer, encoded: &Encoded, skip_special_tokens: bool) -> Result<StringList>
fn tokenizer_decode_batch_encoded_skip_special_tokens(self, self_: &Tokenizer, encoded: &Encoded, skip_special_tokens: bool) -> StringList

Wraps api-table member tokenizer_decode_batch_encoded_skip_special_tokens. ClikaRT::tokenizer::Tokenizer::decode_batch <- ClikaRT/tokenizer/tokenizer.h

tokenizer_decode_batch_ids_cu_seqlens / f_tokenizer_decode_batch_ids_cu_seqlens​

fn f_tokenizer_decode_batch_ids_cu_seqlens(self, self_: &Tokenizer, ids: &Tensor, cu_seqlens: &Tensor, skip_special_tokens: bool) -> Result<StringList>
fn tokenizer_decode_batch_ids_cu_seqlens(self, self_: &Tokenizer, ids: &Tensor, cu_seqlens: &Tensor, skip_special_tokens: bool) -> StringList

Wraps api-table member tokenizer_decode_batch_ids_cu_seqlens. ClikaRT::tokenizer::Tokenizer::decode_batch <- ClikaRT/tokenizer/tokenizer.h

tokenizer_decode_batch_ids_skip_special_tokens / f_tokenizer_decode_batch_ids_skip_special_tokens​

fn f_tokenizer_decode_batch_ids_skip_special_tokens(self, self_: &Tokenizer, ids: &Tensor, skip_special_tokens: bool) -> Result<StringList>
fn tokenizer_decode_batch_ids_skip_special_tokens(self, self_: &Tokenizer, ids: &Tensor, skip_special_tokens: bool) -> StringList

Wraps api-table member tokenizer_decode_batch_ids_skip_special_tokens. ClikaRT::tokenizer::Tokenizer::decode_batch <- ClikaRT/tokenizer/tokenizer.h

tokenizer_encode / f_tokenizer_encode​

fn f_tokenizer_encode(self, self_: &Tokenizer, text: &str, add_special_tokens: bool) -> Result<Vec<i32>>
fn tokenizer_encode(self, self_: &Tokenizer, text: &str, add_special_tokens: bool) -> Vec<i32>

Wraps api-table member tokenizer_encode. ClikaRT::tokenizer::Tokenizer::encode <- ClikaRT/tokenizer/tokenizer.h

tokenizer_encode_batch / f_tokenizer_encode_batch​

fn f_tokenizer_encode_batch(self, self_: &Tokenizer, texts: &[&str], options: Option<&clika_rt_encode_options>) -> Result<Encoded>
fn tokenizer_encode_batch(self, self_: &Tokenizer, texts: &[&str], options: Option<&clika_rt_encode_options>) -> Encoded

Wraps api-table member tokenizer_encode_batch. ClikaRT::tokenizer::Tokenizer::encode_batch <- ClikaRT/tokenizer/tokenizer.h

tokenizer_encode_chat_messages / f_tokenizer_encode_chat_messages​

fn f_tokenizer_encode_chat_messages(self, self_: &Tokenizer, messages_json: &str, add_generation_prompt: bool, add_special_tokens: bool) -> Result<Vec<i32>>
fn tokenizer_encode_chat_messages(self, self_: &Tokenizer, messages_json: &str, add_generation_prompt: bool, add_special_tokens: bool) -> Vec<i32>

Wraps api-table member tokenizer_encode_chat_messages. ClikaRT::tokenizer::Tokenizer::encode_chat <- ClikaRT/tokenizer/tokenizer.h

tokenizer_encode_text / f_tokenizer_encode_text​

fn f_tokenizer_encode_text(self, self_: &Tokenizer, text: &str, options: Option<&clika_rt_encode_options>) -> Result<Encoded>
fn tokenizer_encode_text(self, self_: &Tokenizer, text: &str, options: Option<&clika_rt_encode_options>) -> Encoded

Wraps api-table member tokenizer_encode_text. ClikaRT::tokenizer::Tokenizer::encode_text <- ClikaRT/tokenizer/tokenizer.h

tokenizer_eos_id​

fn tokenizer_eos_id(self, self_: &Tokenizer) -> i64

Wraps api-table member tokenizer_eos_id. ClikaRT::tokenizer::Tokenizer::eos_id <- ClikaRT/tokenizer/tokenizer.h

tokenizer_free​

fn tokenizer_free(self, h: &Tokenizer)

Wraps api-table member tokenizer_free.

tokenizer_from_bert_vocab / f_tokenizer_from_bert_vocab​

fn f_tokenizer_from_bert_vocab(self, dir: &str) -> Result<Tokenizer>
fn tokenizer_from_bert_vocab(self, dir: &str) -> Tokenizer

Wraps api-table member tokenizer_from_bert_vocab. ClikaRT::tokenizer::Tokenizer::from_bert_vocab <- ClikaRT/tokenizer/tokenizer.h

tokenizer_from_file / f_tokenizer_from_file​

fn f_tokenizer_from_file(self, path: &str) -> Result<Tokenizer>
fn tokenizer_from_file(self, path: &str) -> Tokenizer

Wraps api-table member tokenizer_from_file. ClikaRT::tokenizer::Tokenizer::from_file <- ClikaRT/tokenizer/tokenizer.h

tokenizer_from_gpt2 / f_tokenizer_from_gpt2​

fn f_tokenizer_from_gpt2(self, dir: &str) -> Result<Tokenizer>
fn tokenizer_from_gpt2(self, dir: &str) -> Tokenizer

Wraps api-table member tokenizer_from_gpt2. ClikaRT::tokenizer::Tokenizer::from_gpt2 <- ClikaRT/tokenizer/tokenizer.h

tokenizer_from_huggingface / f_tokenizer_from_huggingface​

fn f_tokenizer_from_huggingface(self, path: &str) -> Result<Tokenizer>
fn tokenizer_from_huggingface(self, path: &str) -> Tokenizer

Wraps api-table member tokenizer_from_huggingface. ClikaRT::tokenizer::Tokenizer::from_huggingface <- ClikaRT/tokenizer/tokenizer.h

tokenizer_from_sentencepiece / f_tokenizer_from_sentencepiece​

fn f_tokenizer_from_sentencepiece(self, path: &str) -> Result<Tokenizer>
fn tokenizer_from_sentencepiece(self, path: &str) -> Tokenizer

Wraps api-table member tokenizer_from_sentencepiece. ClikaRT::tokenizer::Tokenizer::from_sentencepiece <- ClikaRT/tokenizer/tokenizer.h

tokenizer_from_tekken / f_tokenizer_from_tekken​

fn f_tokenizer_from_tekken(self, path: &str) -> Result<Tokenizer>
fn tokenizer_from_tekken(self, path: &str) -> Tokenizer

Wraps api-table member tokenizer_from_tekken. ClikaRT::tokenizer::Tokenizer::from_tekken <- ClikaRT/tokenizer/tokenizer.h

tokenizer_from_tiktoken / f_tokenizer_from_tiktoken​

fn f_tokenizer_from_tiktoken(self, path: &str) -> Result<Tokenizer>
fn tokenizer_from_tiktoken(self, path: &str) -> Tokenizer

Wraps api-table member tokenizer_from_tiktoken. ClikaRT::tokenizer::Tokenizer::from_tiktoken <- ClikaRT/tokenizer/tokenizer.h

tokenizer_has_chat_template​

fn tokenizer_has_chat_template(self, self_: &Tokenizer) -> bool

Wraps api-table member tokenizer_has_chat_template. ClikaRT::tokenizer::Tokenizer::has_chat_template <- ClikaRT/tokenizer/tokenizer.h

tokenizer_id_to_token / f_tokenizer_id_to_token​

fn f_tokenizer_id_to_token(self, self_: &Tokenizer, id: i32) -> Result<String>
fn tokenizer_id_to_token(self, self_: &Tokenizer, id: i32) -> String

Wraps api-table member tokenizer_id_to_token. ClikaRT::tokenizer::Tokenizer::id_to_token <- ClikaRT/tokenizer/tokenizer.h

tokenizer_pad_id​

fn tokenizer_pad_id(self, self_: &Tokenizer) -> i64

Wraps api-table member tokenizer_pad_id. ClikaRT::tokenizer::Tokenizer::pad_id <- ClikaRT/tokenizer/tokenizer.h

tokenizer_streaming_decoder​

fn tokenizer_streaming_decoder(self, self_: &Tokenizer, skip_special_tokens: bool) -> Option<StreamingDecoder>

Wraps api-table member tokenizer_streaming_decoder. ClikaRT::tokenizer::Tokenizer::streaming_decoder <- ClikaRT/tokenizer/tokenizer.h A NULL return maps to None.

tokenizer_token_to_id / f_tokenizer_token_to_id​

fn f_tokenizer_token_to_id(self, self_: &Tokenizer, token: &str) -> Result<i32>
fn tokenizer_token_to_id(self, self_: &Tokenizer, token: &str) -> i32

Wraps api-table member tokenizer_token_to_id. ClikaRT::tokenizer::Tokenizer::token_to_id <- ClikaRT/tokenizer/tokenizer.h

tokenizer_tokenize / f_tokenizer_tokenize​

fn f_tokenizer_tokenize(self, self_: &Tokenizer, text: &str, add_special_tokens: bool) -> Result<TokenList>
fn tokenizer_tokenize(self, self_: &Tokenizer, text: &str, add_special_tokens: bool) -> TokenList

Wraps api-table member tokenizer_tokenize. ClikaRT::tokenizer::Tokenizer::tokenize <- ClikaRT/tokenizer/tokenizer.h

tokenizer_vocab_size​

fn tokenizer_vocab_size(self, self_: &Tokenizer) -> i64

Wraps api-table member tokenizer_vocab_size. ClikaRT::tokenizer::Tokenizer::vocab_size <- ClikaRT/tokenizer/tokenizer.h