reflex

A simple flex-like lexing/tokenizing library written in Rust

Workflow:

• Define a Token enum with all possible tokens that implements Clone
• Create a Ruleset with Ruleset::<Token>::new()
• Add token rules with add_rule(), add_simple(), and add_noop()
• Call lex() with the string to be tokenized
• Use the resultant lazy iterator however you like

Example code for tokenizing a simple calculator language:

```rust extern crate reflex;

use std::io; use std::fmt; use reflex::{Ruleset, lex};

[derive(Clone)]

enum Token { Number(f64), OpAdd, OpSub, OpMul, OpDiv, OpPow, }

impl fmt::Display for Token { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", match self { Token::Number(n) => n.to_string(), Token::OpAdd => "+".to_string(), Token::OpSub => "-".to_string(), Token::OpMul => "".tostring(), Token::OpDiv => "/".tostring(), Token::OpPow => "^".tostring(), }.tostring()) } }

fn main() { let mut program = String::new(); io::stdin().read_line(&mut program).unwrap();

let mut ruleset: Ruleset<Token> = Ruleset::new();
ruleset.add_rule(r"-?[0-9]*\.?[0-9]+", |token| Token::Number(token.parse().unwrap_or(0.0)));
ruleset.add_simple(r"\+", Token::OpAdd);
ruleset.add_simple(r"-", Token::OpSub);
ruleset.add_simple(r"\*", Token::OpMul);
ruleset.add_simple(r"/", Token::OpDiv);
ruleset.add_simple(r"\^", Token::OpPow);
ruleset.add_noop(r"(?s)\s");

for token in lex(&ruleset, program) {
    println!("{}", token.unwrap());
}

} ```