Update backend to System.Linq.Dynamic.Core

Bonsai.Scripting.Expressions.sln

@@ -7,6 +7,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.Scripting.Expression
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.Scripting.Expressions.Design", "src\Bonsai.Scripting.Expressions.Design\Bonsai.Scripting.Expressions.Design.csproj", "{A6ADFFAA-CA25-545C-AC3F-4BC3D819B8CB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.Scripting.Expressions.Tests", "src\Bonsai.Scripting.Expressions.Tests\Bonsai.Scripting.Expressions.Tests.csproj", "{CE4BD71F-2CCD-4ED5-8F4E-EAC123EEF518}"
+EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{DEE5DD87-39C1-BF34-B639-A387DCCF972B}"
 	ProjectSection(SolutionItems) = preProject
 		build\Common.csproj.props = build\Common.csproj.props
@@ -31,6 +33,10 @@ Global
 		{A6ADFFAA-CA25-545C-AC3F-4BC3D819B8CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{A6ADFFAA-CA25-545C-AC3F-4BC3D819B8CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A6ADFFAA-CA25-545C-AC3F-4BC3D819B8CB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CE4BD71F-2CCD-4ED5-8F4E-EAC123EEF518}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CE4BD71F-2CCD-4ED5-8F4E-EAC123EEF518}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CE4BD71F-2CCD-4ED5-8F4E-EAC123EEF518}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CE4BD71F-2CCD-4ED5-8F4E-EAC123EEF518}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/Bonsai.Scripting.Expressions.Design/Bonsai.Scripting.Expressions.Design.csproj

@@ -3,7 +3,7 @@
     <Description>This package provides editors for expression scripting in the Bonsai programming language.</Description>
     <PackageTags>$(PackageTags) Design</PackageTags>
     <UseWindowsForms>true</UseWindowsForms>
-    <TargetFramework>net472</TargetFramework>
+    <TargetFrameworks>net472;net8.0-windows</TargetFrameworks>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Bonsai.Design" Version="2.9.0" />

src/Bonsai.Scripting.Expressions.Tests/AssemblyInfo.cs

@@ -0,0 +1 @@
+[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)]

src/Bonsai.Scripting.Expressions.Tests/Bonsai.Scripting.Expressions.Tests.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>net472;net8.0</TargetFrameworks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.8.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.8.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bonsai.Scripting.Expressions\Bonsai.Scripting.Expressions.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="Microsoft.VisualStudio.TestTools.UnitTesting" />
+  </ItemGroup>
+
+</Project>

src/Bonsai.Scripting.Expressions.Tests/ExpressionScriptingTests.cs

@@ -0,0 +1,87 @@
+using Bonsai.Expressions;
+using System;
+using System.Collections;
+using System.Reactive.Linq;
+using System.Threading.Tasks;
+
+namespace Bonsai.Scripting.Expressions.Tests
+{
+    [TestClass]
+    public sealed class ExpressionScriptingTests
+    {
+        async Task AssertExpressionTransform<TSource, TResult>(string expression, TSource value, TResult expected)
+        {
+            var workflowBuilder = new WorkflowBuilder();
+            var source = workflowBuilder.Workflow.Add(new CombinatorBuilder { Combinator = new Return<TSource>(value) });
+            var transform = workflowBuilder.Workflow.Add(new ExpressionTransform { Expression = expression });
+            var output = workflowBuilder.Workflow.Add(new WorkflowOutputBuilder());
+            workflowBuilder.Workflow.AddEdge(source, transform, new());
+            workflowBuilder.Workflow.AddEdge(transform, output, new());
+            var result = await workflowBuilder.Workflow.BuildObservable<TResult>();
+            if (expected is ICollection expectedCollection && result is ICollection resultCollection)
+                CollectionAssert.AreEqual(expectedCollection, resultCollection);
+            else
+                Assert.AreEqual(expected, result);
+        }
+
+        [DataTestMethod]
+        [DataRow("it", 42, 42)]
+        [DataRow("it * 2", 21, 42)]
+        [DataRow("Single(it)", 42, 42f)]
+        [DataRow("Math.PI", 42, Math.PI)]
+        [DataRow("Convert.ToInt16(it)", 42, (short)42)]
+        [DataRow("new(it as Data).Data", 42, 42)]
+        // modern Dynamic LINQ parser
+        [DataRow("float(it)", 42, 42f)]
+        [DataRow("long?(it).HasValue", 42, true)]
+        [DataRow("bool.TrueString", 42, "True")]
+        [DataRow("new[] { it }", 42, new[] { 42 })]
+        [DataRow("new[] { it }.Select(x => x * 2).ToArray()", 21, new[] { 42 })]
+        [DataRow("np(string(null).Length) ?? it", 42, 42)]
+        public Task TestExpressionTransform<TSource, TResult>(string expression, TSource value, TResult expected)
+        {
+            return AssertExpressionTransform(expression, value, expected);
+        }
+
+        [TestMethod]
+        public Task TestNullExpression() => AssertExpressionTransform("null", 0, (object)null);
+
+        [TestMethod]
+        public Task TestNullString() => AssertExpressionTransform("string(null)", 0, (string)null);
+
+        [TestMethod]
+        public Task TestObjectExpression() => AssertExpressionTransform("object(it)", 42, (object)42);
+
+        [DataTestMethod]
+        [DataRow("single(it)", 42, 42f)]
+        [DataRow("int64?(it).hasvalue", 42, true)]
+        [DataRow("math.pi", 42, Math.PI)]
+        [DataRow("boolean.truestring", 42, "True")]
+        [DataRow("convert.toint16(it)", 42, (short)42)]
+        [DataRow("datetime.minvalue.second", 42, 0)]
+        [DataRow("datetimeoffset.minvalue.second", 42, 0)]
+        [DataRow("guid.empty.tobytearray()[0]", 42, 0)]
+        [DataRow("timespan.tickspermillisecond", 0, TimeSpan.TicksPerMillisecond)]
+        [DataRow("it > 0 ? convert.toint16(it) : int16.minvalue", 42, (short)42)]
+        public Task TestCasingCompatibility<TSource, TResult>(string expression, TSource value, TResult expected)
+        {
+            return AssertExpressionTransform(expression, value, expected);
+        }
+
+        [DataTestMethod]
+        [DataRow("")]
+        [DataRow("string(it)")]
+        public Task TestInvalidExpression(string expression)
+        {
+            return Assert.ThrowsExactlyAsync<WorkflowBuildException>(() =>
+                AssertExpressionTransform(expression, 42, (object)null));
+        }
+
+        class Return<TValue>(TValue value) : Source<TValue>
+        {
+            public TValue Value { get; } = value;
+
+            public override IObservable<TValue> Generate() => Observable.Return(Value);
+        }
+    }
+}

src/Bonsai.Scripting.Expressions/Bonsai.Scripting.Expressions.csproj

@@ -1,10 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <Description>This package provides operators implementing expression scripting infrastructure.</Description>
-    <TargetFramework>net472</TargetFramework>
+    <TargetFrameworks>net472;net8.0</TargetFrameworks>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Bonsai.Core" Version="2.9.0" />
-    <PackageReference Include="System.Linq.Dynamic" Version="1.0.7" />
+    <PackageReference Include="System.Linq.Dynamic.Core" Version="1.6.7" />
   </ItemGroup>
 </Project>

src/Bonsai.Scripting.Expressions/CompatibilityAnalyzer.cs

@@ -0,0 +1,69 @@
+using System.Collections.Generic;
+using System.Linq.Dynamic.Core;
+using System.Linq.Dynamic.Core.Tokenizer;
+using System.Text;
+
+namespace Bonsai.Scripting.Expressions
+{
+    internal static class CompatibilityAnalyzer
+    {
+        internal static readonly Dictionary<string, string> LegacyKeywords = new()
+        {
+            { "boolean", "bool" },
+            { "datetime", "DateTime" },
+            { "datetimeoffset", "DateTimeOffset" },
+            { "guid", "Guid" },
+            { "int16", "short" },
+            { "int32", "int" },
+            { "int64", "long" },
+            { "single", "float" },
+            { "timespan", "TimeSpan" },
+            { "uint32", "uint" },
+            { "uint64", "ulong" },
+            { "uint16", "ushort" },
+            { "math", "Math" },
+            { "convert", "Convert" }
+        };
+
+        public static bool ReplaceLegacyKeywords(ParsingConfig? parsingConfig, string text, out string result)
+        {
+            result = text;
+            if (string.IsNullOrEmpty(text))
+                return false;
+
+
+            List<(Token, string)> replacements = null;
+            var previousTokenId = TokenId.Unknown;
+            var textParser = new TextParser(parsingConfig, text);
+            while (textParser.CurrentToken.Id != TokenId.End)
+            {
+                if (textParser.CurrentToken.Id == TokenId.Identifier &&
+                    previousTokenId != TokenId.Dot &&
+                    LegacyKeywords.TryGetValue(textParser.CurrentToken.Text, out var keyword))
+                {
+                    replacements ??= new();
+                    replacements.Add((textParser.CurrentToken, keyword));
+                }
+
+                previousTokenId = textParser.CurrentToken.Id;
+                textParser.NextToken();
+            }
+
+            if (replacements?.Count > 0)
+            {
+                var sb = new StringBuilder(text);
+                for (int i = 0; i < replacements.Count; i++)
+                {
+                    var (token, keyword) = replacements[i];
+                    sb.Remove(token.Pos, token.Text.Length);
+                    sb.Insert(token.Pos, keyword);
+                }
+
+                result = sb.ToString();
+                return true;
+            }
+
+            return false;
+        }
+    }
+}

src/Bonsai.Scripting.Expressions/DynamicExpressionHelper.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Linq.Dynamic.Core;
+using System.Linq.Dynamic.Core.Exceptions;
+using System.Linq.Expressions;
+
+namespace Bonsai.Scripting.Expressions
+{
+    internal class DynamicExpressionHelper
+    {
+        public static LambdaExpression ParseLambda(Type delegateType, ParsingConfig? parsingConfig, ParameterExpression[] parameters, Type? resultType, string expression, params object?[] values)
+        {
+            return ParseLambda(delegateType, parsingConfig, true, parameters, resultType, expression, values);
+        }
+
+        public static LambdaExpression ParseLambda(ParsingConfig? parsingConfig, Type itType, Type? resultType, string expression, params object?[] values)
+        {
+            return ParseLambda(null, parsingConfig, true, new[] { Expression.Parameter(itType, "it") }, resultType, expression, values);
+        }
+
+        public static LambdaExpression ParseLambda(Type? delegateType, ParsingConfig? parsingConfig, bool createParameterCtor, ParameterExpression[] parameters, Type? resultType, string expression, params object?[] values)
+        {
+            try
+            {
+                return DynamicExpressionParser.ParseLambda(delegateType, parsingConfig, createParameterCtor, parameters, resultType, expression, values);
+            }
+            catch (ParseException)
+            {
+                if (!CompatibilityAnalyzer.ReplaceLegacyKeywords(parsingConfig, expression, out expression))
+                    throw;
+
+                return DynamicExpressionParser.ParseLambda(delegateType, parsingConfig, createParameterCtor, parameters, resultType, expression, values);
+            }
+        }
+    }
+}

src/Bonsai.Scripting.Expressions/ExpressionCondition.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Linq;
+using System.Linq.Dynamic.Core;
 using System.Linq.Expressions;
 using System.Reactive.Linq;
 using System.Reflection;
@@ -52,7 +53,8 @@ public override Expression Build(IEnumerable<Expression> arguments)
         {
             var source = arguments.First();
             var sourceType = source.Type.GetGenericArguments()[0];
-            var predicate = System.Linq.Dynamic.DynamicExpression.ParseLambda(sourceType, typeof(bool), Expression);
+            var config = ParsingConfigHelper.CreateParsingConfig(sourceType);
+            var predicate = DynamicExpressionHelper.ParseLambda(config, sourceType, typeof(bool), Expression);
             return System.Linq.Expressions.Expression.Call(whereMethod.MakeGenericMethod(sourceType), source, predicate);
         }
 

src/Bonsai.Scripting.Expressions/ExpressionSink.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Linq;
+using System.Linq.Dynamic.Core;
 using System.Linq.Expressions;
 using System.Reactive.Linq;
 using System.Reflection;
@@ -59,7 +60,8 @@ public override Expression Build(IEnumerable<Expression> arguments)
                 var sourceType = source.Type.GetGenericArguments()[0];
                 var actionType = System.Linq.Expressions.Expression.GetActionType(sourceType);
                 var itParameter = new[] { System.Linq.Expressions.Expression.Parameter(sourceType, string.Empty) };
-                var onNext = System.Linq.Dynamic.DynamicExpression.ParseLambda(actionType, itParameter, null, Expression);
+                var config = ParsingConfigHelper.CreateParsingConfig(sourceType);
+                var onNext = DynamicExpressionHelper.ParseLambda(actionType, config, itParameter, null, Expression);
                 return System.Linq.Expressions.Expression.Call(doMethod.MakeGenericMethod(sourceType), source, onNext);
             }
             else return source;

src/Bonsai.Scripting.Expressions/ExpressionTransform.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Linq;
+using System.Linq.Dynamic.Core;
 using System.Linq.Expressions;
 using System.Reactive.Linq;
 using System.Reflection;
@@ -53,7 +54,8 @@ public override Expression Build(IEnumerable<Expression> arguments)
         {
             var source = arguments.First();
             var sourceType = source.Type.GetGenericArguments()[0];
-            var selector = System.Linq.Dynamic.DynamicExpression.ParseLambda(sourceType, null, Expression);
+            var config = ParsingConfigHelper.CreateParsingConfig(sourceType);
+            var selector = DynamicExpressionHelper.ParseLambda(config, sourceType, null, Expression);
             return System.Linq.Expressions.Expression.Call(selectMethod.MakeGenericMethod(sourceType, selector.ReturnType), source, selector);
         }
 

src/Bonsai.Scripting.Expressions/ParsingConfigHelper.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Dynamic.Core;
+using System.Linq.Dynamic.Core.CustomTypeProviders;
+
+namespace Bonsai.Scripting.Expressions
+{
+    internal static class ParsingConfigHelper
+    {
+        public static ParsingConfig CreateParsingConfig(params Type[] additionalTypes)
+        {
+            var config = new ParsingConfig();
+            config.CustomTypeProvider = CreateCustomTypeProvider(config, additionalTypes);
+            return config;
+        }
+
+        static IDynamicLinqCustomTypeProvider CreateCustomTypeProvider(ParsingConfig config, params Type[] additionalTypes)
+        {
+            return new SimpleDynamicLinqCustomTypeProvider(
+                config,
+                additionalTypes.SelectMany(EnumerateTypeHierarchy).ToList());
+        }
+
+        static IEnumerable<Type> EnumerateTypeHierarchy(Type type)
+        {
+            var interfaces = type.GetInterfaces();
+            for (int i = 0; i < interfaces.Length; i++)
+            {
+                yield return interfaces[i];
+            }
+
+            while (type is not null)
+            {
+                yield return type;
+                type = type.BaseType;
+            }
+        }
+
+        class SimpleDynamicLinqCustomTypeProvider : DefaultDynamicLinqCustomTypeProvider
+        {
+            readonly HashSet<Type> customTypes;
+
+            public SimpleDynamicLinqCustomTypeProvider(ParsingConfig config, IList<Type> additionalTypes)
+                : base(config, additionalTypes, cacheCustomTypes: false)
+            {
+                customTypes = new(AdditionalTypes);
+            }
+
+            public override HashSet<Type> GetCustomTypes()
+            {
+                return customTypes;
+            }
+        }
+    }
+}