generic-lexer

A generic lexer in Rust using a simple match function or closure

```rust use generic_lexer::{Lexer, LexerInput};

[derive(Debug)]

enum TokenKind { Int, Float, Name, Plus, Minus, Star, Slash, Semicolon, Equals, }

fn lexint(input: &mut LexerInput) -> TokenKind { input.nextwhile(u8::isasciidigit); if input.accept(b'.') { return lex_float(input); } TokenKind::Int }

fn lexfloat(input: &mut LexerInput) -> TokenKind { input.nextwhile(u8::isasciidigit); TokenKind::Float }

fn lexname(input: &mut LexerInput) -> TokenKind { input.nextwhile(|&b| b == b'' || b.isascii_alphabetic()); TokenKind::Name }

fn lex(byte: u8, input: &mut LexerInput) -> Result { let kind = match byte { b'+' => TokenKind::Plus, b'-' => TokenKind::Minus, b'*' => TokenKind::Star, b'/' => TokenKind::Slash, b';' => TokenKind::Semicolon, b'=' => TokenKind::Equals,

    b if b.is_ascii_digit() => lex_int(input),
    b if b.is_ascii_alphabetic() => lex_name(input),

    _ => return Err(format!("Unexpected '{}'", char::from(byte)))
};

Ok(kind)

}

fn main() -> Result<(), String> { let input = "a = 420 + 69 * 3.14;"; let lexer = Lexer::new(&input, &lex, true); let tokens = lexer.collect::, _>>()?; println!("{:#?}", tokens); Ok(()) } ```

Token { kind: Name, text: "a" } Token { kind: Equals, text: "=" } Token { kind: Int, text: "420" } Token { kind: Plus, text: "+" } Token { kind: Int, text: "69" } Token { kind: Star, text: "*" } Token { kind: Float, text: "3.14" } Token { kind: Semicolon, text: ";" }