bytepiece-rs

Usage

```rust use bytepice_rs::Tokenizer;

let tokenizer = Tokenizer::new(); // or load a custom model let tokenizer = Tokenizer::buildfrom(modelpath); let text = "今天天气不错"; let ids = tokenizer.encode(text, false, false, 0.0); asserteq!(ids, vec![40496, 45268, 39432]); let text2 = tokenizer.decode(ids); asserteq!(text2, text); ```