Tokenizer-lib

Tokenization utilities for building parsers in Rust

Examples

Static token channel:

rs let mut stc = StaticTokenChannel::new(); stc.push(Token(12, Span(0, 2))); stc.push(Token(32, Span(2, 4))); stc.push(Token(52, Span(4, 8))); assert_eq!(stc.next().unwrap(), Token(12, Span(0, 2))); assert_eq!(stc.next().unwrap(), Token(32, Span(2, 4))); assert_eq!(stc.next().unwrap(), Token(52, Span(4, 8))); assert_eq!(stc.next(), None);

Streamed token channel:

```rs let (mut sender, mut reader) = getstreamedtoken_channel(); std::thread::spawn(move || { sender.push(Token(12, Span(0, 2))); sender.push(Token(32, Span(2, 4))); sender.push(Token(52, Span(4, 8))); });

asserteq!(reader.next().unwrap(), Token(12, Span(0, 2))); asserteq!(reader.next().unwrap(), Token(32, Span(2, 4))); asserteq!(reader.next().unwrap(), Token(52, Span(4, 8))); asserteq!(reader.next(), None); ```