Tokenization utilities for building parsers in Rust
Buffered token channel:
```rust use tokenizer_lib::{BufferedTokenQueue, Token, TokenReader, TokenSender, TokenTrait};
struct Span(pub u32, pub u32);
struct N(pub u32);
impl TokenTrait for N {}
let mut btq = BufferedTokenQueue::new(); btq.push(Token(N(12), Span(0, 2))); btq.push(Token(N(32), Span(2, 4))); btq.push(Token(N(52), Span(4, 8))); asserteq!(btq.next().unwrap().0, N(12)); asserteq!(btq.next().unwrap().0, N(32)); asserteq!(btq.next().unwrap().0, N(52)); assert!(btq.next().isnone()); ```
(Multi-thread safe) Parallel token queue:
```rust use tokenizer_lib::{ParallelTokenQueue, Token, TokenReader, TokenSender, TokenTrait};
struct Span(pub u32, pub u32);
struct N(pub u32);
impl TokenTrait for N {}
let (mut sender, mut reader) = ParallelTokenQueue::new(); std::thread::spawn(move || { sender.push(Token(N(12), Span(0, 2))); sender.push(Token(N(32), Span(2, 4))); sender.push(Token(N(52), Span(4, 8))); });
asserteq!(reader.next().unwrap().0, N(12)); asserteq!(reader.next().unwrap().0, N(32)); asserteq!(reader.next().unwrap().0, N(52)); assert!(reader.next().isnone()); ```
Generator token queue:
```rust use tokenizer_lib::{GeneratorTokenQueue, GeneratorTokenQueueBuffer, Token, TokenReader, TokenSender, TokenTrait};
struct N(pub u32);
impl TokenTrait for N {}
fn lexer(state: &mut u32, sender: &mut GeneratorTokenQueueBuffer
let mut reader = GeneratorTokenQueue::new(lexer, 0);
asserteq!(reader.next().unwrap().0, N(2)); asserteq!(reader.next().unwrap().0, N(4)); asserteq!(reader.next().unwrap().0, N(6)); assert!(reader.next().isnone()); ```
Provides utilities such as peek
, peek_n
and scan
for lookahead. Also expect_next
for expecting a token value and conditional_next
for advancing on a predicate.