>>
{
Tokens::new_filter_map(input, |x| Some(x))
}
pub fn new_filter(input: &'i str, mut predicate: P)
-> Tokens<'i, impl FnMut(Token<'i>) -> Option>>
where
P: FnMut(&Token<'i>) -> bool,
{
Tokens::new_filter_map(input,
move |x|
{
if predicate(&x)
{
Some(x)
}
else
{
None
}
})
}
}
impl<'i, F> Tokens<'i, F>
{
pub fn new_filter_map(input: &'i str, functor: F) -> Self
{
Self
{
original_input: input,
input,
previous_index: 0,
reached_end_of_stream: false,
functor,
}
}
fn next_token(&mut self) -> Option), crate::parse::Error>>
{
self.input = trim_start(self.input);
let index_left = substring_offset(self.input, self.original_input);
let first_character = match self.input.chars().next()
{
None => return None,
Some(first_character) => first_character,
};
if self.input.starts_with(")")
{
return Some(Err(crate::parse::Error::new_unmatched_parenthesis(
crate::parse::error::Location::new(0, Some(1)))));
}
match parenthesized_expression(self.input)
{
Ok(Some((parenthesized_expression, remaining_input))) =>
{
self.input = remaining_input;
let index_right = substring_offset(self.input, self.original_input);
return Some(Ok((index_left, index_right,
Token::ParenthesizedExpression(parenthesized_expression))));
},
Ok(None) => (),
Err(error) => return Some(Err(error)),
}
match number(self.input)
{
Ok(Some((number, remaining_input))) =>
{
self.input = remaining_input;
let index_right = substring_offset(self.input, self.original_input);
return Some(Ok((index_left, index_right, Token::Number(number))));
},
Ok(None) => (),
Err(error) => return Some(Err(error)),
}
if let Some((identifier, remaining_input)) = identifier(self.input)
{
self.input = remaining_input;
let index_right = substring_offset(self.input, self.original_input);
return Some(Ok((index_left, index_right, Token::Identifier(identifier))));
}
if let Some((symbol, remaining_input)) = symbol(self.input)
{
self.input = remaining_input;
let index_right = substring_offset(self.input, self.original_input);
return Some(Ok((index_left, index_right, Token::Symbol(symbol))));
}
return Some(Err(crate::parse::Error::new_character_not_allowed(first_character,
crate::parse::error::Location::new(0, Some(0)))));
}
pub fn remaining_input(&mut self) -> Option<&'i str>
{
if self.reached_end_of_stream
{
return None;
}
let remaining_input = self.original_input[self.previous_index..].trim();
self.reached_end_of_stream = true;
Some(remaining_input)
}
pub fn split(self) -> TokenSplit
{
TokenSplit::new(self)
}
}
impl<'i, F, G> std::iter::Iterator for Tokens<'i, F>
where
F: FnMut(Token<'i>) -> Option,
{
type Item = Result<(&'i str, G), crate::parse::Error>;
fn next(&mut self) -> Option
{
if self.previous_index == self.original_input.len()
{
return None;
}
loop
{
match self.next_token()
{
Some(Ok((index_left, index_right, token))) =>
{
let token = match (self.functor)(token)
{
None => continue,
Some(token) => token,
};
let input_left = self.original_input[self.previous_index..index_left].trim();
self.previous_index = index_right;
return Some(Ok((input_left, token)));
},
Some(Err(error)) => return Some(Err(error)),
None => return None,
}
}
}
}
pub(crate) struct TokenSplit
{
tokens: T,
}
impl TokenSplit<()>
{
pub fn new(tokens: T) -> TokenSplit
{
TokenSplit
{
tokens,
}
}
}
impl<'i, F, G> std::iter::Iterator for TokenSplit>
where
F: FnMut(Token<'i>) -> Option,
{
type Item = Result<&'i str, crate::parse::Error>;
fn next(&mut self) -> Option
{
match self.tokens.next()
{
Some(Ok((input_before, _))) => Some(Ok(input_before)),
Some(Err(error)) => Some(Err(error)),
None => match self.tokens.remaining_input()
{
Some(remaining_input) => Some(Ok(remaining_input)),
None => None,
},
}
}
}
#[cfg(test)]
mod tests
{
use super::*;
#[test]
fn tokenize_identifier()
{
assert_eq!(identifier("test").unwrap(), ("test", ""));
assert_eq!(identifier("test2").unwrap(), ("test2", ""));
assert_eq!(identifier("Test").unwrap(), ("Test", ""));
assert_eq!(identifier("Test2").unwrap(), ("Test2", ""));
assert_eq!(identifier("_test").unwrap(), ("_test", ""));
assert_eq!(identifier("_test2").unwrap(), ("_test2", ""));
assert_eq!(identifier("__test").unwrap(), ("__test", ""));
assert_eq!(identifier("__test2").unwrap(), ("__test2", ""));
assert_eq!(identifier("test, test").unwrap(), ("test", ", test"));
assert_eq!(identifier("test2, test").unwrap(), ("test2", ", test"));
assert_eq!(identifier("Test, Test").unwrap(), ("Test", ", Test"));
assert_eq!(identifier("Test2, Test").unwrap(), ("Test2", ", Test"));
assert_eq!(identifier("_test, _test").unwrap(), ("_test", ", _test"));
assert_eq!(identifier("_test2, _test").unwrap(), ("_test2", ", _test"));
assert_eq!(identifier("__test, __test").unwrap(), ("__test", ", __test"));
assert_eq!(identifier("__test2, __test").unwrap(), ("__test2", ", __test"));
assert!(identifier("2test, test").is_none());
assert!(identifier("#test, test").is_none());
assert!(identifier("$test, test").is_none());
assert!(identifier(",test, test").is_none());
}
#[test]
fn tokenize_primitives()
{
assert_eq!(parenthesized_expression("(foo bar baz) test").unwrap(),
Some(("foo bar baz", " test")));
assert!(parenthesized_expression("( | asd#0231(asd|asd) test").is_err());
assert_eq!(parenthesized_expression("( | asd#0231(asd|asd) ) test").unwrap(),
Some((" | asd#0231(asd|asd) ", " test")));
assert_eq!(parenthesized_expression("( | a)sd#0231(asd|asd) test").unwrap(),
Some((" | a", "sd#0231(asd|asd) test")));
assert_eq!(number("1234, ").unwrap(), Some((1234, ", ")));
assert_eq!(number("1234.5, ").unwrap(), Some((1234, ".5, ")));
assert_eq!(number("-1234, ").unwrap(), None);
assert_eq!(number("a1234, ").unwrap(), None);
assert_eq!(symbol("<-"), Some((Symbol::ArrowLeft, "")));
assert_eq!(symbol("<->"), Some((Symbol::ArrowLeftAndRight, "")));
assert_eq!(symbol("->"), Some((Symbol::ArrowRight, "")));
assert_eq!(symbol(","), Some((Symbol::Comma, "")));
assert_eq!(symbol("/"), Some((Symbol::Division, "")));
assert_eq!(symbol("="), Some((Symbol::Equal, "")));
assert_eq!(symbol("**"), Some((Symbol::Exponentiation, "")));
assert_eq!(symbol(">"), Some((Symbol::Greater, "")));
assert_eq!(symbol(">="), Some((Symbol::GreaterOrEqual, "")));
assert_eq!(symbol("<"), Some((Symbol::Less, "")));
assert_eq!(symbol("<="), Some((Symbol::LessOrEqual, "")));
assert_eq!(symbol("-"), Some((Symbol::Minus, "")));
assert_eq!(symbol("*"), Some((Symbol::Multiplication, "")));
assert_eq!(symbol("!="), Some((Symbol::NotEqual, "")));
assert_eq!(symbol("+"), Some((Symbol::Plus, "")));
assert_eq!(symbol("|"), Some((Symbol::VerticalBar, "")));
assert_eq!(symbol("<-a"), Some((Symbol::ArrowLeft, "a")));
assert_eq!(symbol("<->a"), Some((Symbol::ArrowLeftAndRight, "a")));
assert_eq!(symbol("->a"), Some((Symbol::ArrowRight, "a")));
assert_eq!(symbol(",a"), Some((Symbol::Comma, "a")));
assert_eq!(symbol("/a"), Some((Symbol::Division, "a")));
assert_eq!(symbol("=a"), Some((Symbol::Equal, "a")));
assert_eq!(symbol("**a"), Some((Symbol::Exponentiation, "a")));
assert_eq!(symbol(">a"), Some((Symbol::Greater, "a")));
assert_eq!(symbol(">=a"), Some((Symbol::GreaterOrEqual, "a")));
assert_eq!(symbol("