Skip to content

Fix some operators not parsing correctly #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
miroiu merged 1 commit into from
Jul 1, 2025
Merged

Fix some operators not parsing correctly #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions StringMath.Benchmarks/Benchmarks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ public class Benchmarks
[Benchmark]
public void Tokenize()
{
var tokenizer = new Tokenizer("1.23235456576878798 - ((3 + {b}) max .1) ^ sqrt(-999 / 2 * 3 max 5) + !5 - 0.00000000002 / {ahghghh}");
var context = MathContext.Default;

var tokenizer = new Tokenizer("1.23235456576878798 - ((3 + {b}) max .1) ^ sqrt(-999 / 2 * 3 max 5) + !5 - 0.00000000002 / {ahghghh}", context);

Token token;

Expand All @@ -24,8 +26,9 @@ public void Tokenize()
[Benchmark]
public void Parse()
{
var tokenizer = new Tokenizer("1.23235456576878798 - ((3 + {b}) max .1) ^ sqrt(-999 / 2 * 3 max 5) + !5 - 0.00000000002 / {ahghghh}");
var parser = new Parser(tokenizer, MathContext.Default);
var context = MathContext.Default;
var tokenizer = new Tokenizer("1.23235456576878798 - ((3 + {b}) max .1) ^ sqrt(-999 / 2 * 3 max 5) + !5 - 0.00000000002 / {ahghghh}", context);
var parser = new Parser(tokenizer, context);
_ = parser.Parse();
}

Expand Down
4 changes: 2 additions & 2 deletions StringMath.Tests/Extensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ namespace StringMath.Tests
{
static class Extensions
{
public static List<Token> ReadAllTokens(this string input)
public static List<Token> ReadAllTokens(this string input, IMathContext context)
{
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, context);
List<Token> tokens = new List<Token>();

Token t;
Expand Down
3 changes: 2 additions & 1 deletion StringMath.Tests/MathExprTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,11 @@ public void SetOperator_Unary_Should_Not_Overwrite_Global_Operator()
[TestCase("1 + sqrt 4", 3)]
[TestCase("sind(90) + sind 30", 1.5)]
[TestCase("((1 + 1) + ((1 + 1) + (((1) + 1)) + 1))", 7)]
[TestCase("719.04+sin(60)", 718.735d)]
public void Evaluate(string input, double expected)
{
double result = input.Eval();
Assert.AreEqual(expected, result);
Assert.AreEqual(expected, result, 0.001);
}

[TestCase("{b}+3*{a}", 3, 2, 11)]
Expand Down
32 changes: 18 additions & 14 deletions StringMath.Tests/ParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,12 @@ public void Setup()
[TestCase("1.15215345346", "1.15215345346")]
[TestCase("0", "0")]
[TestCase("!2", "2!")]
[TestCase("--1", "-(-1)")]
[TestCase("1+sin(3)", "1 + sin(3)")]
[TestCase("1+sin 3", "1 + sin(3)")]
public void ParseMathExpression(string input, string expected)
{
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);
Parser parser = new Parser(tokenizer, _context);

IExpression result = parser.Parse();
Expand Down Expand Up @@ -56,8 +59,9 @@ public void ParseMathExpression(string input, string expected)
[TestCase("1+")]
[TestCase("1.")]
[TestCase("1..1")]
[TestCase("--1")]
[TestCase("-*1")]
[TestCase("-+1")]
[TestCase("+-1")]
[TestCase("{")]
[TestCase("}")]
[TestCase("asd")]
Expand All @@ -67,7 +71,7 @@ public void ParseMathExpression(string input, string expected)
[TestCase("1 + 2 1")]
public void ParseBadExpression_Exception(string input)
{
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);
Parser parser = new Parser(tokenizer, _context);

MathException exception = Assert.Throws<MathException>(() => parser.Parse());
Expand All @@ -84,7 +88,7 @@ public void ParseExpression_CustomOperators(string input, string expected)
context.RegisterBinary("pow", (a, b) => a);
context.RegisterUnary("rand", (a) => a);

Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);
Parser parser = new Parser(tokenizer, context);

IExpression result = parser.Parse();
Expand All @@ -100,7 +104,7 @@ public void ParseExpression_CustomOperators_Exception(string expected)
{
MathContext context = new MathContext();

Tokenizer tokenizer = new Tokenizer(expected);
Tokenizer tokenizer = new Tokenizer(expected, _context);
Parser parser = new Parser(tokenizer, context);

MathException exception = Assert.Throws<MathException>(() => parser.Parse());
Expand All @@ -119,7 +123,7 @@ public void ParseExpression_CustomOperators_Exception(string expected)
[TestCase("{a13}", "a13")]
public void ParseVariableExpression(string expected, string name)
{
Tokenizer tokenizer = new Tokenizer(expected);
Tokenizer tokenizer = new Tokenizer(expected, _context);
Parser parser = new Parser(tokenizer, _context);

IExpression result = parser.Parse();
Expand All @@ -139,7 +143,7 @@ public void ParseVariableExpression(string expected, string name)
[TestCase("{-a}")]
public void ParseVariableExpression_Exception(string expected)
{
Tokenizer tokenizer = new Tokenizer(expected);
Tokenizer tokenizer = new Tokenizer(expected, _context);
Parser parser = new Parser(tokenizer, _context);

MathException exception = Assert.Throws<MathException>(() => parser.Parse());
Expand All @@ -159,7 +163,7 @@ public void ParseVariableExpression_Exception(string expected)
[TestCase("1 / 2 / 3", "1 / 2 / 3")]
public void ParseBinaryExpression(string input, string expected)
{
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);
Parser parser = new Parser(tokenizer, _context);

IExpression result = parser.Parse();
Expand All @@ -183,7 +187,7 @@ public void ParseBinaryExpression(string input, string expected)
[TestCase("sqrt{a}", "sqrt({a})")]
public void ParseUnaryExpression(string input, string expected)
{
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);
Parser parser = new Parser(tokenizer, _context);

IExpression result = parser.Parse();
Expand All @@ -199,7 +203,7 @@ public void ParseUnaryExpression(string input, string expected)
[TestCase("+5")]
public void ParseUnaryExpression_Exception(string input)
{
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);
Parser parser = new Parser(tokenizer, _context);

MathException exception = Assert.Throws<MathException>(() => parser.Parse());
Expand All @@ -213,7 +217,7 @@ public void ParseUnaryExpression_Exception(string input)
[TestCase("9999999", "9999999")]
public void ParseConstantExpression(string input, string expected)
{
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);
Parser parser = new Parser(tokenizer, _context);

IExpression result = parser.Parse();
Expand All @@ -231,7 +235,7 @@ public void ParseConstantExpression(string input, string expected)
[TestCase("9.01+")]
public void ParseConstantExpression_Exception(string expected)
{
Tokenizer tokenizer = new Tokenizer(expected);
Tokenizer tokenizer = new Tokenizer(expected, _context);
Parser parser = new Parser(tokenizer, _context);

MathException exception = Assert.Throws<MathException>(() => parser.Parse());
Expand All @@ -250,7 +254,7 @@ public void ParseConstantExpression_Exception(string expected)
[TestCase("((5 - 2) + ((-1 + 2) * 3))", "5 - 2 + (-1 + 2) * 3")]
public void ParseGroupingExpression(string input, string expected)
{
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);
Parser parser = new Parser(tokenizer, _context);

IExpression result = parser.Parse();
Expand All @@ -271,7 +275,7 @@ public void ParseGroupingExpression(string input, string expected)
[TestCase("({a} + (1 + 2)")]
public void ParseGroupingExpression_Fail(string expected)
{
Tokenizer tokenizer = new Tokenizer(expected);
Tokenizer tokenizer = new Tokenizer(expected, _context);
Parser parser = new Parser(tokenizer, _context);

MathException exception = Assert.Throws<MathException>(() => parser.Parse());
Expand Down
26 changes: 19 additions & 7 deletions StringMath.Tests/TokenizerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,25 @@ namespace StringMath.Tests
[TestFixture]
internal class TokenizerTests
{
private IMathContext _context;

[OneTimeSetUp]
public void Setup()
{
_context = MathContext.Default;
}

[Test]
[TestCase("-1 * 3.5", new[] { TokenType.Operator, TokenType.Number, TokenType.Operator, TokenType.Number })]
[TestCase("2 pow 3", new[] { TokenType.Number, TokenType.Operator, TokenType.Number })]
[TestCase("{a} + 2", new[] { TokenType.Identifier, TokenType.Operator, TokenType.Number })]
[TestCase("(-1) + 2", new[] { TokenType.OpenParen, TokenType.Operator, TokenType.Number, TokenType.CloseParen, TokenType.Operator, TokenType.Number })]
[TestCase("5!", new[] { TokenType.Number, TokenType.Exclamation })]
[TestCase("1+sin(3)", new[] { TokenType.Number, TokenType.Operator, TokenType.Operator, TokenType.OpenParen, TokenType.Number, TokenType.CloseParen })]
[TestCase("1+sin 3", new[] { TokenType.Number, TokenType.Operator, TokenType.Operator, TokenType.Number })]
public void ReadToken(string input, TokenType[] expected)
{
IEnumerable<TokenType> actualTokens = input.ReadAllTokens()
IEnumerable<TokenType> actualTokens = input.ReadAllTokens(_context)
.Where(token => token.Type != TokenType.EndOfCode)
.Select(t => t.Type);
Assert.That(actualTokens, Is.EquivalentTo(expected));
Expand All @@ -29,7 +39,7 @@ public void ReadToken(string input, TokenType[] expected)
public void ReadToken_IgnoresWhitespace(string input)
{
// Arrange
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);

// Act
Token token1 = tokenizer.ReadToken();
Expand All @@ -53,7 +63,7 @@ public void ReadToken_IgnoresWhitespace(string input)
public void ReadIdentifier(string input)
{
// Arrange
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);

// Act
Token token = tokenizer.ReadToken();
Expand All @@ -75,7 +85,7 @@ public void ReadIdentifier(string input)
public void ReadIdentifier_Exception(string input)
{
// Arrange
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);

// Act & Assert
MathException exception = Assert.Throws<MathException>(() => tokenizer.ReadToken());
Expand All @@ -91,8 +101,10 @@ public void ReadIdentifier_Exception(string input)
[TestCase("a@a")]
public void ReadOperator(string input)
{
_context.RegisterBinary("**", (a, b) => a * b, Precedence.Multiplication);

// Arrange
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);

// Act
Token token = tokenizer.ReadToken();
Expand All @@ -113,7 +125,7 @@ public void ReadOperator(string input)
public void ReadNumber(string input)
{
// Arrange
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);

// Act
Token token = tokenizer.ReadToken();
Expand All @@ -133,7 +145,7 @@ public void ReadNumber(string input)
public void ReadNumber_Exception(string input)
{
// Arrange
Tokenizer tokenizer = new Tokenizer(input);
Tokenizer tokenizer = new Tokenizer(input, _context);

// Act & Assert
MathException exception = Assert.Throws<MathException>(() => tokenizer.ReadToken());
Expand Down
2 changes: 1 addition & 1 deletion StringMath/Extensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public static IExpression Parse(this string text, IMathContext context)
{
text.EnsureNotNull(nameof(text));

Tokenizer tokenizer = new Tokenizer(text);
Tokenizer tokenizer = new Tokenizer(text, context);
Parser parser = new Parser(tokenizer, context);
return parser.Parse();
}
Expand Down
2 changes: 1 addition & 1 deletion StringMath/Parser/SourceText.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ namespace StringMath
internal sealed class SourceText : IEnumerator<char>
{
public string Text { get; }
public int Position { get; private set; }
public int Position { get; set; }
public char Current => Text[Position];
object IEnumerator.Current => Current;

Expand Down
29 changes: 27 additions & 2 deletions StringMath/Parser/Tokenizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ namespace StringMath
internal sealed partial class Tokenizer
{
private readonly SourceText _text;
private readonly IMathContext _context;

// Excluded characters for custom operators
private static readonly HashSet<char> _invalidOperatorCharacters = new HashSet<char>
Expand All @@ -16,14 +17,17 @@ internal sealed partial class Tokenizer

/// <summary>Creates a new instance of the tokenizer.</summary>
/// <param name="text">The text to tokenize.</param>
public Tokenizer(SourceText text)
/// <param name="context">The math context.</param>
public Tokenizer(SourceText text, IMathContext context)
{
_text = text;
_context = context;
}

/// <summary>Creates a new instance of the tokenizer.</summary>
/// <param name="text">The text to tokenize.</param>
public Tokenizer(string text) : this(new SourceText(text))
/// <param name="context">The math context.</param>
public Tokenizer(string text, IMathContext context) : this(new SourceText(text), context)
{
}

Expand Down Expand Up @@ -139,9 +143,30 @@ private string ReadOperator(SourceText stream)
stream.MoveNext();
}

string op = builder.ToString();
if (IsOperator(op))
{
return builder.ToString();
}

for (int i = 0; i < op.Length; i++)
{
var possibleOperator = builder.ToString(0, i);
if (IsOperator(possibleOperator))
{
stream.Position -= op.Length - i;
return possibleOperator;
}
}

return builder.ToString();
}

private bool IsOperator(string text)
{
return _context.IsBinary(text) || _context.IsUnary(text);
}

private string ReadNumber(SourceText stream)
{
StringBuilder builder = new StringBuilder(8);
Expand Down
Loading