From b809543496c7dc35f203462b203560399761883d Mon Sep 17 00:00:00 2001 From: miroiu Date: Tue, 1 Jul 2025 17:44:01 +0300 Subject: [PATCH] Fix binary + unary operator without space not parsing correctly --- StringMath.Benchmarks/Benchmarks.cs | 9 +++++--- StringMath.Tests/Extensions.cs | 4 ++-- StringMath.Tests/MathExprTests.cs | 3 ++- StringMath.Tests/ParserTests.cs | 32 ++++++++++++++++------------- StringMath.Tests/TokenizerTests.cs | 26 ++++++++++++++++------- StringMath/Extensions.cs | 2 +- StringMath/Parser/SourceText.cs | 2 +- StringMath/Parser/Tokenizer.cs | 29 ++++++++++++++++++++++++-- 8 files changed, 76 insertions(+), 31 deletions(-) diff --git a/StringMath.Benchmarks/Benchmarks.cs b/StringMath.Benchmarks/Benchmarks.cs index 32e3d5c..a629286 100644 --- a/StringMath.Benchmarks/Benchmarks.cs +++ b/StringMath.Benchmarks/Benchmarks.cs @@ -10,7 +10,9 @@ public class Benchmarks [Benchmark] public void Tokenize() { - var tokenizer = new Tokenizer("1.23235456576878798 - ((3 + {b}) max .1) ^ sqrt(-999 / 2 * 3 max 5) + !5 - 0.00000000002 / {ahghghh}"); + var context = MathContext.Default; + + var tokenizer = new Tokenizer("1.23235456576878798 - ((3 + {b}) max .1) ^ sqrt(-999 / 2 * 3 max 5) + !5 - 0.00000000002 / {ahghghh}", context); Token token; @@ -24,8 +26,9 @@ public void Tokenize() [Benchmark] public void Parse() { - var tokenizer = new Tokenizer("1.23235456576878798 - ((3 + {b}) max .1) ^ sqrt(-999 / 2 * 3 max 5) + !5 - 0.00000000002 / {ahghghh}"); - var parser = new Parser(tokenizer, MathContext.Default); + var context = MathContext.Default; + var tokenizer = new Tokenizer("1.23235456576878798 - ((3 + {b}) max .1) ^ sqrt(-999 / 2 * 3 max 5) + !5 - 0.00000000002 / {ahghghh}", context); + var parser = new Parser(tokenizer, context); _ = parser.Parse(); } diff --git a/StringMath.Tests/Extensions.cs b/StringMath.Tests/Extensions.cs index aa3c6d3..c90f5e3 100644 --- a/StringMath.Tests/Extensions.cs +++ b/StringMath.Tests/Extensions.cs @@ -4,9 +4,9 @@ namespace StringMath.Tests { static class Extensions { - public static List ReadAllTokens(this string input) + public static List ReadAllTokens(this string input, IMathContext context) { - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, context); List tokens = new List(); Token t; diff --git a/StringMath.Tests/MathExprTests.cs b/StringMath.Tests/MathExprTests.cs index 670f5e1..def060e 100644 --- a/StringMath.Tests/MathExprTests.cs +++ b/StringMath.Tests/MathExprTests.cs @@ -154,10 +154,11 @@ public void SetOperator_Unary_Should_Not_Overwrite_Global_Operator() [TestCase("1 + sqrt 4", 3)] [TestCase("sind(90) + sind 30", 1.5)] [TestCase("((1 + 1) + ((1 + 1) + (((1) + 1)) + 1))", 7)] + [TestCase("719.04+sin(60)", 718.735d)] public void Evaluate(string input, double expected) { double result = input.Eval(); - Assert.AreEqual(expected, result); + Assert.AreEqual(expected, result, 0.001); } [TestCase("{b}+3*{a}", 3, 2, 11)] diff --git a/StringMath.Tests/ParserTests.cs b/StringMath.Tests/ParserTests.cs index b1821a4..eabe901 100644 --- a/StringMath.Tests/ParserTests.cs +++ b/StringMath.Tests/ParserTests.cs @@ -23,9 +23,12 @@ public void Setup() [TestCase("1.15215345346", "1.15215345346")] [TestCase("0", "0")] [TestCase("!2", "2!")] + [TestCase("--1", "-(-1)")] + [TestCase("1+sin(3)", "1 + sin(3)")] + [TestCase("1+sin 3", "1 + sin(3)")] public void ParseMathExpression(string input, string expected) { - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); Parser parser = new Parser(tokenizer, _context); IExpression result = parser.Parse(); @@ -56,8 +59,9 @@ public void ParseMathExpression(string input, string expected) [TestCase("1+")] [TestCase("1.")] [TestCase("1..1")] - [TestCase("--1")] + [TestCase("-*1")] [TestCase("-+1")] + [TestCase("+-1")] [TestCase("{")] [TestCase("}")] [TestCase("asd")] @@ -67,7 +71,7 @@ public void ParseMathExpression(string input, string expected) [TestCase("1 + 2 1")] public void ParseBadExpression_Exception(string input) { - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); Parser parser = new Parser(tokenizer, _context); MathException exception = Assert.Throws(() => parser.Parse()); @@ -84,7 +88,7 @@ public void ParseExpression_CustomOperators(string input, string expected) context.RegisterBinary("pow", (a, b) => a); context.RegisterUnary("rand", (a) => a); - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); Parser parser = new Parser(tokenizer, context); IExpression result = parser.Parse(); @@ -100,7 +104,7 @@ public void ParseExpression_CustomOperators_Exception(string expected) { MathContext context = new MathContext(); - Tokenizer tokenizer = new Tokenizer(expected); + Tokenizer tokenizer = new Tokenizer(expected, _context); Parser parser = new Parser(tokenizer, context); MathException exception = Assert.Throws(() => parser.Parse()); @@ -119,7 +123,7 @@ public void ParseExpression_CustomOperators_Exception(string expected) [TestCase("{a13}", "a13")] public void ParseVariableExpression(string expected, string name) { - Tokenizer tokenizer = new Tokenizer(expected); + Tokenizer tokenizer = new Tokenizer(expected, _context); Parser parser = new Parser(tokenizer, _context); IExpression result = parser.Parse(); @@ -139,7 +143,7 @@ public void ParseVariableExpression(string expected, string name) [TestCase("{-a}")] public void ParseVariableExpression_Exception(string expected) { - Tokenizer tokenizer = new Tokenizer(expected); + Tokenizer tokenizer = new Tokenizer(expected, _context); Parser parser = new Parser(tokenizer, _context); MathException exception = Assert.Throws(() => parser.Parse()); @@ -159,7 +163,7 @@ public void ParseVariableExpression_Exception(string expected) [TestCase("1 / 2 / 3", "1 / 2 / 3")] public void ParseBinaryExpression(string input, string expected) { - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); Parser parser = new Parser(tokenizer, _context); IExpression result = parser.Parse(); @@ -183,7 +187,7 @@ public void ParseBinaryExpression(string input, string expected) [TestCase("sqrt{a}", "sqrt({a})")] public void ParseUnaryExpression(string input, string expected) { - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); Parser parser = new Parser(tokenizer, _context); IExpression result = parser.Parse(); @@ -199,7 +203,7 @@ public void ParseUnaryExpression(string input, string expected) [TestCase("+5")] public void ParseUnaryExpression_Exception(string input) { - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); Parser parser = new Parser(tokenizer, _context); MathException exception = Assert.Throws(() => parser.Parse()); @@ -213,7 +217,7 @@ public void ParseUnaryExpression_Exception(string input) [TestCase("9999999", "9999999")] public void ParseConstantExpression(string input, string expected) { - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); Parser parser = new Parser(tokenizer, _context); IExpression result = parser.Parse(); @@ -231,7 +235,7 @@ public void ParseConstantExpression(string input, string expected) [TestCase("9.01+")] public void ParseConstantExpression_Exception(string expected) { - Tokenizer tokenizer = new Tokenizer(expected); + Tokenizer tokenizer = new Tokenizer(expected, _context); Parser parser = new Parser(tokenizer, _context); MathException exception = Assert.Throws(() => parser.Parse()); @@ -250,7 +254,7 @@ public void ParseConstantExpression_Exception(string expected) [TestCase("((5 - 2) + ((-1 + 2) * 3))", "5 - 2 + (-1 + 2) * 3")] public void ParseGroupingExpression(string input, string expected) { - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); Parser parser = new Parser(tokenizer, _context); IExpression result = parser.Parse(); @@ -271,7 +275,7 @@ public void ParseGroupingExpression(string input, string expected) [TestCase("({a} + (1 + 2)")] public void ParseGroupingExpression_Fail(string expected) { - Tokenizer tokenizer = new Tokenizer(expected); + Tokenizer tokenizer = new Tokenizer(expected, _context); Parser parser = new Parser(tokenizer, _context); MathException exception = Assert.Throws(() => parser.Parse()); diff --git a/StringMath.Tests/TokenizerTests.cs b/StringMath.Tests/TokenizerTests.cs index a9759ea..d295e44 100644 --- a/StringMath.Tests/TokenizerTests.cs +++ b/StringMath.Tests/TokenizerTests.cs @@ -7,15 +7,25 @@ namespace StringMath.Tests [TestFixture] internal class TokenizerTests { + private IMathContext _context; + + [OneTimeSetUp] + public void Setup() + { + _context = MathContext.Default; + } + [Test] [TestCase("-1 * 3.5", new[] { TokenType.Operator, TokenType.Number, TokenType.Operator, TokenType.Number })] [TestCase("2 pow 3", new[] { TokenType.Number, TokenType.Operator, TokenType.Number })] [TestCase("{a} + 2", new[] { TokenType.Identifier, TokenType.Operator, TokenType.Number })] [TestCase("(-1) + 2", new[] { TokenType.OpenParen, TokenType.Operator, TokenType.Number, TokenType.CloseParen, TokenType.Operator, TokenType.Number })] [TestCase("5!", new[] { TokenType.Number, TokenType.Exclamation })] + [TestCase("1+sin(3)", new[] { TokenType.Number, TokenType.Operator, TokenType.Operator, TokenType.OpenParen, TokenType.Number, TokenType.CloseParen })] + [TestCase("1+sin 3", new[] { TokenType.Number, TokenType.Operator, TokenType.Operator, TokenType.Number })] public void ReadToken(string input, TokenType[] expected) { - IEnumerable actualTokens = input.ReadAllTokens() + IEnumerable actualTokens = input.ReadAllTokens(_context) .Where(token => token.Type != TokenType.EndOfCode) .Select(t => t.Type); Assert.That(actualTokens, Is.EquivalentTo(expected)); @@ -29,7 +39,7 @@ public void ReadToken(string input, TokenType[] expected) public void ReadToken_IgnoresWhitespace(string input) { // Arrange - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); // Act Token token1 = tokenizer.ReadToken(); @@ -53,7 +63,7 @@ public void ReadToken_IgnoresWhitespace(string input) public void ReadIdentifier(string input) { // Arrange - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); // Act Token token = tokenizer.ReadToken(); @@ -75,7 +85,7 @@ public void ReadIdentifier(string input) public void ReadIdentifier_Exception(string input) { // Arrange - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); // Act & Assert MathException exception = Assert.Throws(() => tokenizer.ReadToken()); @@ -91,8 +101,10 @@ public void ReadIdentifier_Exception(string input) [TestCase("a@a")] public void ReadOperator(string input) { + _context.RegisterBinary("**", (a, b) => a * b, Precedence.Multiplication); + // Arrange - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); // Act Token token = tokenizer.ReadToken(); @@ -113,7 +125,7 @@ public void ReadOperator(string input) public void ReadNumber(string input) { // Arrange - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); // Act Token token = tokenizer.ReadToken(); @@ -133,7 +145,7 @@ public void ReadNumber(string input) public void ReadNumber_Exception(string input) { // Arrange - Tokenizer tokenizer = new Tokenizer(input); + Tokenizer tokenizer = new Tokenizer(input, _context); // Act & Assert MathException exception = Assert.Throws(() => tokenizer.ReadToken()); diff --git a/StringMath/Extensions.cs b/StringMath/Extensions.cs index 3469763..aa5e39e 100644 --- a/StringMath/Extensions.cs +++ b/StringMath/Extensions.cs @@ -40,7 +40,7 @@ public static IExpression Parse(this string text, IMathContext context) { text.EnsureNotNull(nameof(text)); - Tokenizer tokenizer = new Tokenizer(text); + Tokenizer tokenizer = new Tokenizer(text, context); Parser parser = new Parser(tokenizer, context); return parser.Parse(); } diff --git a/StringMath/Parser/SourceText.cs b/StringMath/Parser/SourceText.cs index 3fda914..8d7b2a7 100644 --- a/StringMath/Parser/SourceText.cs +++ b/StringMath/Parser/SourceText.cs @@ -6,7 +6,7 @@ namespace StringMath internal sealed class SourceText : IEnumerator { public string Text { get; } - public int Position { get; private set; } + public int Position { get; set; } public char Current => Text[Position]; object IEnumerator.Current => Current; diff --git a/StringMath/Parser/Tokenizer.cs b/StringMath/Parser/Tokenizer.cs index c24eb49..48e11e0 100644 --- a/StringMath/Parser/Tokenizer.cs +++ b/StringMath/Parser/Tokenizer.cs @@ -7,6 +7,7 @@ namespace StringMath internal sealed partial class Tokenizer { private readonly SourceText _text; + private readonly IMathContext _context; // Excluded characters for custom operators private static readonly HashSet _invalidOperatorCharacters = new HashSet @@ -16,14 +17,17 @@ internal sealed partial class Tokenizer /// Creates a new instance of the tokenizer. /// The text to tokenize. - public Tokenizer(SourceText text) + /// The math context. + public Tokenizer(SourceText text, IMathContext context) { _text = text; + _context = context; } /// Creates a new instance of the tokenizer. /// The text to tokenize. - public Tokenizer(string text) : this(new SourceText(text)) + /// The math context. + public Tokenizer(string text, IMathContext context) : this(new SourceText(text), context) { } @@ -139,9 +143,30 @@ private string ReadOperator(SourceText stream) stream.MoveNext(); } + string op = builder.ToString(); + if (IsOperator(op)) + { + return builder.ToString(); + } + + for (int i = 0; i < op.Length; i++) + { + var possibleOperator = builder.ToString(0, i); + if (IsOperator(possibleOperator)) + { + stream.Position -= op.Length - i; + return possibleOperator; + } + } + return builder.ToString(); } + private bool IsOperator(string text) + { + return _context.IsBinary(text) || _context.IsUnary(text); + } + private string ReadNumber(SourceText stream) { StringBuilder builder = new StringBuilder(8);