Charred

This crate provides a generic asynchronous lexer that operates on files with tokio. Tokens are parsed with provided async closures.

Usage

```rust use crate::error::TapeResult; use crate::input_reader::InputReader; use crate::lexer::Lexer; use crate::token::{Token, TokenCheckerFn}; use std::io::Cursor; use std::sync::Arc;

struct NumberToken(i32); struct StringToken(String); struct WhiteSpaceToken;

async fn parsenumbertoken(reader: &mut InputReader) -> TapeResult> { let mut num = String::new(); while !reader.checkeof().await && reader.peek().await?.isnumeric() { num.push(reader.consume().await?); } if num.is_empty() { Ok(None) } else { Ok(Some(Token::new(NumberToken(num.parse::().unwrap())))) } }

async fn parsewhitespacetoken(reader: &mut InputReader) -> TapeResult> { let mut count = 0; while !reader.checkeof().await && reader.peek().await?.iswhitespace() { reader.consume().await?; count += 1; } if count > 0 { Ok(Some(Token::new(WhiteSpaceToken))) } else { Ok(None) } }

async fn parsestringtoken(reader: &mut InputReader) -> TapeResult> { let mut value = String::new(); while !reader.checkeof().await && !reader.peek().await?.isnumeric() && !reader.peek().await?.iswhitespace() { value.push(reader.consume().await?); } if value.isempty() { Ok(None) } else { Ok(Some(Token::new(StringToken(value)))) } }

[tokio::main]

async fn main() { // functions that try to parse the token into an object let checkers: Vec = vec![ Arc::new(|reader| Box::pin(parsenumbertoken(reader))), Arc::new(|reader| Box::pin(parsewhitespacetoken(reader))), Arc::new(|reader| Box::pin(parsestringtoken(reader))), ]; // input reader encapsulates (almost) any type that implements AsyncBufRead let inputreader = InputReader::new(Cursor::new("Word 12")); let mut lexer = Lexer::new(inputreader, checkers);

// scan starts scanning the provided input
let tokens = lexer.scan().await.unwrap();
assert!(!tokens.is_empty());

let mut tokens = tokens.into_iter();
// use the is, try_as and try_into methods on the token type to get the underlying value
assert!(tokens.next().unwrap().is::<StringToken>());
assert!(tokens.next().unwrap().is::<WhiteSpaceToken>());
assert!(tokens.next().unwrap().is::<NumberToken>());

} ```

License

Apache-2.0