From e0c91f04e961e854abcb071c26c5855815184b58 Mon Sep 17 00:00:00 2001 From: Codex Yore Date: Mon, 5 Jun 2023 10:20:40 +0545 Subject: [PATCH 1/2] Changed Parsing Method of Identifiers --- src/error/display.rs | 1 + src/error/mod.rs | 3 +++ src/token/mod.rs | 44 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/error/display.rs b/src/error/display.rs index d7a4e758..4df398ef 100644 --- a/src/error/display.rs +++ b/src/error/display.rs @@ -120,6 +120,7 @@ impl fmt::Display for EvalexprError { write!(f, "This context does not allow disabling builtin functions") }, IllegalEscapeSequence(string) => write!(f, "Illegal escape sequence: {}", string), + IllegalIdentifierSequence => write!(f, "Illegal Identifier Sequence"), CustomMessage(message) => write!(f, "Error: {}", message), } } diff --git a/src/error/mod.rs b/src/error/mod.rs index e896973f..ad893a07 100644 --- a/src/error/mod.rs +++ b/src/error/mod.rs @@ -209,6 +209,9 @@ pub enum EvalexprError { /// This context does not allow disabling builtin functions. BuiltinFunctionsCannotBeDisabled, + /// Error parsing identifier + IllegalIdentifierSequence, + /// A custom error explained by its message. CustomMessage(String), } diff --git a/src/token/mod.rs b/src/token/mod.rs index f1194fb3..9421dd8e 100644 --- a/src/token/mod.rs +++ b/src/token/mod.rs @@ -348,7 +348,19 @@ fn partial_tokens_to_tokens(mut tokens: &[PartialToken]) -> EvalexprResult { cutoff = 1; - if let Ok(number) = parse_dec_or_hex(&literal) { + let starts_with_alphabet_or_underscore = + literal.starts_with(|x: char| x.is_alphabetic() || x == '_'); + let contains_alphanumeric_only = + literal.chars().all(|x| x.is_alphanumeric() || x == '_'); + let is_not_underscore = literal != "_"; + let contains_alphabet = literal.contains(|x: char| x.is_alphabetic()); + if starts_with_alphabet_or_underscore + && contains_alphanumeric_only + && is_not_underscore + && contains_alphabet + { + Some(Token::Identifier(literal)) + } else if let Ok(number) = parse_dec_or_hex(&literal) { Some(Token::Int(number)) } else if let Ok(number) = literal.parse::() { Some(Token::Float(number)) @@ -370,10 +382,10 @@ fn partial_tokens_to_tokens(mut tokens: &[PartialToken]) -> EvalexprResult Some(Token::Identifier(literal.to_string())), + _ => return Err(EvalexprError::IllegalIdentifierSequence), } } }, @@ -495,4 +507,30 @@ mod tests { ] ); } + + #[test] + fn wrong_identifier_sequence() { + assert_eq!( + tokenize("1b + 1"), + Err(crate::EvalexprError::IllegalIdentifierSequence) + ); + assert_eq!( + tokenize("_b + 1"), + Ok(vec![ + Token::Identifier(String::from("_b")), + Token::Plus, + Token::Int(1) + ]) + ); + assert_eq!( + tokenize("1.0 1e+5 _1a2 1"), + Ok(vec![ + Token::Float(1.0), + Token::Float(100000.0), + Token::Identifier(String::from("_1a2")), + Token::Int(1) + ]) + ); + // assert_eq!(tokenize("string")) + } } From 8431650ef544586ae6c2d2b454d52904a7b75c9f Mon Sep 17 00:00:00 2001 From: Codex Yore Date: Mon, 5 Jun 2023 11:02:19 +0545 Subject: [PATCH 2/2] Changed,removed tests with illegal identifiers. --- src/error/display.rs | 4 ++- src/error/mod.rs | 2 +- src/token/mod.rs | 28 ++++++++++++------ tests/integration.rs | 69 +++++++++++++++----------------------------- 4 files changed, 46 insertions(+), 57 deletions(-) diff --git a/src/error/display.rs b/src/error/display.rs index 4df398ef..9a07129b 100644 --- a/src/error/display.rs +++ b/src/error/display.rs @@ -120,7 +120,9 @@ impl fmt::Display for EvalexprError { write!(f, "This context does not allow disabling builtin functions") }, IllegalEscapeSequence(string) => write!(f, "Illegal escape sequence: {}", string), - IllegalIdentifierSequence => write!(f, "Illegal Identifier Sequence"), + IllegalIdentifierSequence(string) => { + write!(f, "Illegal Identifier Sequence: {}", string) + }, CustomMessage(message) => write!(f, "Error: {}", message), } } diff --git a/src/error/mod.rs b/src/error/mod.rs index ad893a07..1bd1053c 100644 --- a/src/error/mod.rs +++ b/src/error/mod.rs @@ -210,7 +210,7 @@ pub enum EvalexprError { BuiltinFunctionsCannotBeDisabled, /// Error parsing identifier - IllegalIdentifierSequence, + IllegalIdentifierSequence(String), /// A custom error explained by its message. CustomMessage(String), diff --git a/src/token/mod.rs b/src/token/mod.rs index 9421dd8e..4ebaaaff 100644 --- a/src/token/mod.rs +++ b/src/token/mod.rs @@ -350,12 +350,16 @@ fn partial_tokens_to_tokens(mut tokens: &[PartialToken]) -> EvalexprResult() { + Some(Token::Boolean(boolean)) + } else if starts_with_alphabet_or_underscore + && contains_only_valid_chars && is_not_underscore && contains_alphabet { @@ -364,8 +368,6 @@ fn partial_tokens_to_tokens(mut tokens: &[PartialToken]) -> EvalexprResult() { Some(Token::Float(number)) - } else if let Ok(boolean) = literal.parse::() { - Some(Token::Boolean(boolean)) } else { // If there are two tokens following this one, check if the next one is // a plus or a minus. If so, then attempt to parse all three tokens as a @@ -382,10 +384,16 @@ fn partial_tokens_to_tokens(mut tokens: &[PartialToken]) -> EvalexprResult return Err(EvalexprError::IllegalIdentifierSequence), + _ => { + return Err(EvalexprError::IllegalIdentifierSequence( + literal.to_string(), + )) + }, } } }, @@ -512,7 +520,9 @@ mod tests { fn wrong_identifier_sequence() { assert_eq!( tokenize("1b + 1"), - Err(crate::EvalexprError::IllegalIdentifierSequence) + Err(crate::EvalexprError::IllegalIdentifierSequence( + String::from("1b") + )) ); assert_eq!( tokenize("_b + 1"), diff --git a/tests/integration.rs b/tests/integration.rs index 94e26a18..ff267f93 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -615,7 +615,7 @@ fn test_shortcut_functions() { ); assert_eq!( eval_string("3..3"), - Err(EvalexprError::VariableIdentifierNotFound("3..3".to_owned())) + Err(EvalexprError::IllegalIdentifierSequence("3..3".to_owned())) ); assert_eq!( eval_string_with_context("string", &context), @@ -629,7 +629,7 @@ fn test_shortcut_functions() { ); assert_eq!( eval_string_with_context("3..3", &context), - Err(EvalexprError::VariableIdentifierNotFound("3..3".to_owned())) + Err(EvalexprError::IllegalIdentifierSequence("3..3".to_owned())) ); assert_eq!( eval_string_with_context_mut("string", &mut context), @@ -643,7 +643,7 @@ fn test_shortcut_functions() { ); assert_eq!( eval_string_with_context_mut("3..3", &mut context), - Err(EvalexprError::VariableIdentifierNotFound("3..3".to_owned())) + Err(EvalexprError::IllegalIdentifierSequence("3..3".to_owned())) ); assert_eq!(eval_float("3.3"), Ok(3.3)); @@ -689,7 +689,7 @@ fn test_shortcut_functions() { ); assert_eq!( eval_int("(,);."), - Err(EvalexprError::VariableIdentifierNotFound(".".to_owned())) + Err(EvalexprError::IllegalIdentifierSequence(".".to_owned())) ); assert_eq!(eval_int_with_context("3", &context), Ok(3)); assert_eq!( @@ -700,7 +700,7 @@ fn test_shortcut_functions() { ); assert_eq!( eval_int_with_context("(,);.", &context), - Err(EvalexprError::VariableIdentifierNotFound(".".to_owned())) + Err(EvalexprError::IllegalIdentifierSequence(".".to_owned())) ); assert_eq!(eval_int_with_context_mut("3", &mut context), Ok(3)); assert_eq!( @@ -711,7 +711,7 @@ fn test_shortcut_functions() { ); assert_eq!( eval_int_with_context_mut("(,);.", &mut context), - Err(EvalexprError::VariableIdentifierNotFound(".".to_owned())) + Err(EvalexprError::IllegalIdentifierSequence(".".to_owned())) ); assert_eq!(eval_number("3"), Ok(3.0)); @@ -802,7 +802,7 @@ fn test_shortcut_functions() { ); assert_eq!( eval_tuple("3a3"), - Err(EvalexprError::VariableIdentifierNotFound("3a3".to_owned())) + Err(EvalexprError::IllegalIdentifierSequence("3a3".to_owned())) ); assert_eq!( eval_tuple_with_context("3,3", &context), @@ -816,7 +816,7 @@ fn test_shortcut_functions() { ); assert_eq!( eval_tuple_with_context("3a3", &context), - Err(EvalexprError::VariableIdentifierNotFound("3a3".to_owned())) + Err(EvalexprError::IllegalIdentifierSequence("3a3".to_owned())) ); assert_eq!( eval_tuple_with_context_mut("3,3", &mut context), @@ -830,7 +830,7 @@ fn test_shortcut_functions() { ); assert_eq!( eval_tuple_with_context_mut("3a3", &mut context), - Err(EvalexprError::VariableIdentifierNotFound("3a3".to_owned())) + Err(EvalexprError::IllegalIdentifierSequence("3a3".to_owned())) ); assert_eq!(eval_empty(""), Ok(EMPTY_VALUE)); @@ -889,8 +889,8 @@ fn test_shortcut_functions() { }) ); assert_eq!( - build_operator_tree("3..3").unwrap().eval_string(), - Err(EvalexprError::VariableIdentifierNotFound("3..3".to_owned())) + build_operator_tree("3..3"), + Err(EvalexprError::IllegalIdentifierSequence("3..3".to_owned())) ); assert_eq!( build_operator_tree("string") @@ -907,10 +907,8 @@ fn test_shortcut_functions() { }) ); assert_eq!( - build_operator_tree("3..3") - .unwrap() - .eval_string_with_context(&context), - Err(EvalexprError::VariableIdentifierNotFound("3..3".to_owned())) + build_operator_tree("3..3"), + Err(EvalexprError::IllegalIdentifierSequence("3..3".to_owned())) ); assert_eq!( build_operator_tree("string") @@ -926,12 +924,6 @@ fn test_shortcut_functions() { actual: Value::Float(3.3) }) ); - assert_eq!( - build_operator_tree("3..3") - .unwrap() - .eval_string_with_context_mut(&mut context), - Err(EvalexprError::VariableIdentifierNotFound("3..3".to_owned())) - ); assert_eq!(build_operator_tree("3.3").unwrap().eval_float(), Ok(3.3)); assert_eq!( @@ -993,8 +985,8 @@ fn test_shortcut_functions() { }) ); assert_eq!( - build_operator_tree("(,);.").unwrap().eval_int(), - Err(EvalexprError::VariableIdentifierNotFound(".".to_owned())) + build_operator_tree("(,);."), + Err(EvalexprError::IllegalIdentifierSequence(".".to_owned())) ); assert_eq!( build_operator_tree("3") @@ -1011,10 +1003,8 @@ fn test_shortcut_functions() { }) ); assert_eq!( - build_operator_tree("(,);.") - .unwrap() - .eval_int_with_context(&context), - Err(EvalexprError::VariableIdentifierNotFound(".".to_owned())) + build_operator_tree("(,);."), + Err(EvalexprError::IllegalIdentifierSequence(".".to_owned())) ); assert_eq!( build_operator_tree("3") @@ -1030,12 +1020,6 @@ fn test_shortcut_functions() { actual: Value::Float(3.3) }) ); - assert_eq!( - build_operator_tree("(,);.") - .unwrap() - .eval_int_with_context_mut(&mut context), - Err(EvalexprError::VariableIdentifierNotFound(".".to_owned())) - ); assert_eq!(build_operator_tree("3").unwrap().eval_number(), Ok(3.0)); assert_eq!( @@ -1161,8 +1145,8 @@ fn test_shortcut_functions() { }) ); assert_eq!( - build_operator_tree("3a3").unwrap().eval_tuple(), - Err(EvalexprError::VariableIdentifierNotFound("3a3".to_owned())) + build_operator_tree("3a3"), + Err(EvalexprError::IllegalIdentifierSequence("3a3".to_owned())) ); assert_eq!( build_operator_tree("3,3") @@ -1178,12 +1162,7 @@ fn test_shortcut_functions() { actual: Value::Int(33) }) ); - assert_eq!( - build_operator_tree("3a3") - .unwrap() - .eval_tuple_with_context(&context), - Err(EvalexprError::VariableIdentifierNotFound("3a3".to_owned())) - ); + assert_eq!( build_operator_tree("3,3") .unwrap() @@ -1199,10 +1178,8 @@ fn test_shortcut_functions() { }) ); assert_eq!( - build_operator_tree("3a3") - .unwrap() - .eval_tuple_with_context_mut(&mut context), - Err(EvalexprError::VariableIdentifierNotFound("3a3".to_owned())) + build_operator_tree("3a3"), + Err(EvalexprError::IllegalIdentifierSequence("3a3".to_owned())) ); assert_eq!( @@ -2301,6 +2278,6 @@ fn test_hex() { eval("0x"), // The "VariableIdentifierNotFound" error is what evalexpr currently returns, // but ideally it would return more specific errors for "illegal" literals. - Err(EvalexprError::VariableIdentifierNotFound("0x".into())) + Err(EvalexprError::IllegalIdentifierSequence("0x".into())) ); }