strizer is a minimal and fast library for text tokenization.
Add this to your Cargo.toml:
toml
[dependencies]
strizer = "0.1.0"
```rust use std::fs::File; use std::io::BufReader; use strizer::{StreamTokenizer, Token, TokenKind};
fn main() -> std::io::Result<()> { // read contest to a reader buffer let file = File::open("log.txt")?; let mut reader = BufReader::new(file);
// tokenize BufRead, and count number of "ERROR" words let errorcount = StreamTokenizer::new(&mut reader, &[]) .filter(|(, _, slice)| slice == "ERROR") .count();
println!("number of error logs: {}", error_count); Ok(()) } ```
```rust use strizer::StringTokenizer;
fn main() -> std::io::Result<()> { // tokenize input string and count the amount of words let token_count = StringTokenizer::new("hello world", &[]).count();
println!("number of words: {}", token_count); Ok(()) } ```