From 30e5cdfaf5b87286068f1b127ece57753f9fe77a Mon Sep 17 00:00:00 2001 From: RomeCore <62770895+RomeCore@users.noreply.github.com> Date: Tue, 21 Oct 2025 15:54:32 +0500 Subject: [PATCH] Add while statement, fix indentation processing, fix metadata --- src/LLTSharp/LLTParser.cs | 70 ++++-- src/LLTSharp/LLTSharp.csproj | 4 +- src/LLTSharp/Metadata/AdditionalMetadata.cs | 14 ++ .../Metadata/ImmutableAdditionalMetadata.cs | 14 ++ src/LLTSharp/Metadata/VersionMetadata.cs | 43 ++++ .../MessagesTemplateWhileNode.cs | 69 ++++++ .../TemplateNodes/TextTemplateForeachNode.cs | 1 + .../TextTemplateSequentialNode.cs | 21 +- .../TemplateNodes/TextTemplateWhileNode.cs | 72 ++++++ src/LLTSharp/TypeDictionary.cs | 5 +- src/LLTSharp/Utils/DictionaryUtils.cs | 59 +++++ .../BasicTemplateParsingTests.cs | 26 ++ .../LLTSharp.Tests/TemplateFormattingTests.cs | 222 ++++++++++++++++++ 13 files changed, 590 insertions(+), 30 deletions(-) create mode 100644 src/LLTSharp/Metadata/VersionMetadata.cs create mode 100644 src/LLTSharp/TemplateNodes/MessagesTemplateWhileNode.cs create mode 100644 src/LLTSharp/TemplateNodes/TextTemplateWhileNode.cs create mode 100644 src/LLTSharp/Utils/DictionaryUtils.cs diff --git a/src/LLTSharp/LLTParser.cs b/src/LLTSharp/LLTParser.cs index 7ca9ddf..dc70b22 100644 --- a/src/LLTSharp/LLTParser.cs +++ b/src/LLTSharp/LLTParser.cs @@ -227,7 +227,7 @@ private static void DeclareExpressions(ParserBuilder builder) return target; }); - builder.CreateRule("nop_expression") + builder.CreateRule("simple_expression") .Rule("prefix_operator"); // Operators // @@ -372,12 +372,11 @@ private static void DeclareMessagesTemplates(ParserBuilder builder) builder.CreateRule("message_statements") .ZeroOrMore(b => b .Literal('@') - .ConfigureLast(c => c.SkippingStrategy(ParserSkippingStrategy.SkipBeforeParsingGreedy)) .Choice( c => c.Rule("message_block"), c => c.Rule("messages_if"), c => c.Rule("messages_foreach"), - // c => c.Rule("messages_while"), + c => c.Rule("messages_while"), c => c.Rule("messages_render"), c => c.Rule("messages_var_assignment")) .Transform(v => v.GetValue(1))) @@ -409,7 +408,7 @@ private static void DeclareMessagesTemplates(ParserBuilder builder) builder.CreateRule("message_block_variable_role") .Keyword("message") .Literal('{') - .Literal('@').Keyword("role").Rule("nop_expression") // 4 + .Literal('@').Keyword("role").Rule("simple_expression") // 4 .Rule("text_statements") // 5 .Literal('}') .Transform(v => @@ -451,18 +450,23 @@ private static void DeclareMessagesTemplates(ParserBuilder builder) return new MessagesTemplateForeachNode(collection, block, variable); }); - // TEMPORARY EXCLUDED: messages while loop builder.CreateRule("messages_while") .Keyword("while") .Rule("expression") - .Rule("messages_template_block"); + .Rule("messages_template_block") + .Transform(v => + { + var expression = v.GetValue(1); + var block = v.GetValue(2); + return new MessagesTemplateWhileNode(expression, block); + }); builder.CreateRule("messages_render") .Keyword("render") - .Rule("nop_expression") // 1 + .Rule("simple_expression") // 1 .Optional(b => b .Keyword("with") - .Rule("nop_expression") + .Rule("simple_expression") .Transform(v => v.GetValue(1))) .Transform(v => { @@ -557,14 +561,14 @@ private static void DeclareTextTemplates(ParserBuilder builder) b => b.Rule("text_if"), b => b.Rule("text_foreach"), b => b.Rule("text_render"), - // b => b.Rule("text_while"), + b => b.Rule("text_while"), b => b.Rule("text_var_assignment"), b => b.Rule("text_expression") ) .Transform(v => v.GetValue(1)); - builder.CreateRule("text_expression") - .Rule("nop_expression") // We don't want to use binary expressions in text statements + builder.CreateRule("text_expression_inner") + .Rule("simple_expression") // We don't want to use binary expressions in text statements .Optional(b => b .Literal(':') .Token("raw_string")) @@ -573,6 +577,21 @@ private static void DeclareTextTemplates(ParserBuilder builder) var format = v.Children[1].Children.Count > 0 ? v.Children[1].Children[0].GetValue(1) : null; return new TextTemplateExpressionNode(v.GetValue(0), format); }); + + builder.CreateRule("text_expression") + .Custom( + (self, ctx, sett, childSett, children, childrenIds) => + { + var result = self.ParseRule(childrenIds[0], ctx, childSett); + if (!result.success) + return result; + var modifierChild = result.children[1]; + if (modifierChild.length == 0) + result.length = result.children[0].length; + return result; + }, + b => b.Rule("text_expression_inner") + ); builder.CreateRule("text_if") .Keyword("if") @@ -607,11 +626,16 @@ private static void DeclareTextTemplates(ParserBuilder builder) return new TextTemplateForeachNode(collection, block, variable); }); - // TEMPORARY EXCLUDED: text while loop builder.CreateRule("text_while") .Keyword("while") .Rule("expression") - .Rule("text_template_block"); + .Rule("text_template_block") + .Transform(v => + { + var expression = v.GetValue(1); + var block = v.GetValue(2); + return new TextTemplateWhileNode(expression, block); + }); builder.CreateRule("text_render") .Keyword("render") @@ -632,7 +656,7 @@ private static void DeclareTextTemplates(ParserBuilder builder) .Keyword("let") .Token("identifier") .Literal("=") - .Rule("nop_expression") + .Rule("simple_expression") .Transform(v => { var name = v.GetValue(1); @@ -641,7 +665,7 @@ private static void DeclareTextTemplates(ParserBuilder builder) }), b => b .Token("identifier") .Literal("=") - .Rule("nop_expression") + .Rule("simple_expression") .Transform(v => { var name = v.GetValue(0); @@ -683,6 +707,12 @@ private static void DeclareMainRules(ParserBuilder builder) case "model_family": metadata.Add(new TargetModelFamilyMetadata(pair.Value.ToString())); break; + case "version": + metadata.Add(new VersionMetadata((int)((double)pair.Value.GetValue()))); + break; + default: + metadata.Add(new AdditionalMetadata(pair.Key, pair.Value.GetValue())); + break; } } @@ -703,11 +733,11 @@ static LLTParser() builder.Settings .Skip(s => s.Choice( c => c.Whitespaces(), - c => c.Literal("@//").TextUntil('\n', '\r'), // @// C#-like comments + c => c.Literal("@/").TextUntil('\n', '\r'), // @/ C#-like comments c => c.Literal("@*").TextUntil("*@").Literal("*@")) // @*...*@ comments .ConfigureForSkip(), // Ignore all errors when parsing comments and unnecessary whitespace ParserSkippingStrategy.TryParseThenSkipLazy) // Allows rules to capture skip-rules contents if can, such as whitespaces - .UseCaching(); // If caching is disabled, prepare to wait for a long time (seconds) when encountering an error :P (you will also get a million of errors, seriously) + .UseCaching().RecordWalkTrace(); // If caching is disabled, prepare to wait for a long time (seconds) when encountering an error :P (you will also get a million of errors, seriously) // ---- Values ---- // DeclareValues(builder); @@ -728,6 +758,12 @@ static LLTParser() } public ParsedRuleResultBase ParseAST(string templateString) + { + var ctx = new LLTParsingContext { LocalLibrary = new TemplateLibrary() }; + return _parser.Parse(templateString, ctx); + } + + public ParsedRuleResultBase ParseOptimizedAST(string templateString) { var ctx = new LLTParsingContext { LocalLibrary = new TemplateLibrary() }; return _parser.Parse(templateString, ctx).Optimized(ParseTreeOptimization.Default); diff --git a/src/LLTSharp/LLTSharp.csproj b/src/LLTSharp/LLTSharp.csproj index e05c1e9..3152ca0 100644 --- a/src/LLTSharp/LLTSharp.csproj +++ b/src/LLTSharp/LLTSharp.csproj @@ -8,7 +8,7 @@ LLTSharp - 1.0.0 + 1.1.0 Roman K. RomeCore LLTSharp @@ -26,7 +26,7 @@ - + \ No newline at end of file diff --git a/src/LLTSharp/Metadata/AdditionalMetadata.cs b/src/LLTSharp/Metadata/AdditionalMetadata.cs index bd6fd40..67e59a3 100644 --- a/src/LLTSharp/Metadata/AdditionalMetadata.cs +++ b/src/LLTSharp/Metadata/AdditionalMetadata.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Dynamic; using System.Text; +using LLTSharp.Utils; namespace LLTSharp.Metadata { @@ -199,5 +200,18 @@ public override IEnumerable GetDynamicMemberNames() { return _metadata.Keys; } + + public override bool Equals(object? obj) + { + return obj is AdditionalMetadata other && + DictionaryUtils.DictionariesEqual(_metadata, other._metadata); + } + + public override int GetHashCode() + { + int hash = 17; + hash = hash * 397 + DictionaryUtils.GetDictionaryHashCode(_metadata); + return hash; + } } } \ No newline at end of file diff --git a/src/LLTSharp/Metadata/ImmutableAdditionalMetadata.cs b/src/LLTSharp/Metadata/ImmutableAdditionalMetadata.cs index d09b57f..2212adb 100644 --- a/src/LLTSharp/Metadata/ImmutableAdditionalMetadata.cs +++ b/src/LLTSharp/Metadata/ImmutableAdditionalMetadata.cs @@ -3,6 +3,7 @@ using System.Dynamic; using System.Linq; using System.Text; +using LLTSharp.Utils; namespace LLTSharp.Metadata { @@ -118,5 +119,18 @@ public override IEnumerable GetDynamicMemberNames() { return _metadata.Keys; } + + public override bool Equals(object? obj) + { + return obj is ImmutableAdditionalMetadata other && + DictionaryUtils.DictionariesEqual(_metadata, other._metadata); + } + + public override int GetHashCode() + { + int hash = 17; + hash = hash * 397 + DictionaryUtils.GetDictionaryHashCode(_metadata); + return hash; + } } } \ No newline at end of file diff --git a/src/LLTSharp/Metadata/VersionMetadata.cs b/src/LLTSharp/Metadata/VersionMetadata.cs new file mode 100644 index 0000000..8db8cab --- /dev/null +++ b/src/LLTSharp/Metadata/VersionMetadata.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLTSharp.Metadata +{ + /// + /// Represents the version metadata. + /// + public class VersionMetadata : IMetadata + { + /// + /// Gets the version code associated with this metadata. + /// + public int Version { get; } + + /// + /// Initializes a new instance of the class with the specified version code. + /// + /// The version code associated with this metadata. + public VersionMetadata(int version) + { + Version = version; + } + + public override string ToString() + { + return $"Version: {Version}"; + } + + public override bool Equals(object? obj) + { + return obj is VersionMetadata other && Version == other.Version; + } + + public override int GetHashCode() + { + int hash = 17; + hash *= 397 + Version.GetHashCode(); + return hash; + } + } +} \ No newline at end of file diff --git a/src/LLTSharp/TemplateNodes/MessagesTemplateWhileNode.cs b/src/LLTSharp/TemplateNodes/MessagesTemplateWhileNode.cs new file mode 100644 index 0000000..ba0634d --- /dev/null +++ b/src/LLTSharp/TemplateNodes/MessagesTemplateWhileNode.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace LLTSharp.TemplateNodes +{ + /// + /// Represents a node in the messages template that executes a child node while a condition is true. + /// + public class MessagesTemplateWhileNode : MessagesTemplateNode + { + /// + /// Gets the condition expression controlling the loop execution. + /// + public TemplateExpressionNode Condition { get; } + + /// + /// Gets the child node that will be executed each iteration. + /// + public MessagesTemplateNode Child { get; } + + /// + /// Creates a new instance of the class. + /// + public MessagesTemplateWhileNode(TemplateExpressionNode condition, MessagesTemplateNode child) + { + Condition = condition ?? throw new ArgumentNullException(nameof(condition)); + Child = child ?? throw new ArgumentNullException(nameof(child)); + } + + public override IEnumerable Render(TemplateContextAccessor context) + { + List messages = new List(); + + context.PushFrame(); + try + { + while (EvaluateCondition(context)) + { + var childResult = Child.Render(context); + messages.AddRange(childResult); + } + } + finally + { + context.PopFrame(); + } + + return messages; + } + + private bool EvaluateCondition(TemplateContextAccessor context) + { + var value = Condition.Evaluate(context); + return value.AsBoolean(); + } + + public override void Refine(int depth) + { + Child.Refine(depth + 1); + } + + public override string ToString() + { + return $"@messages while {Condition} {{\n{Child}\n}}"; + } + } + +} \ No newline at end of file diff --git a/src/LLTSharp/TemplateNodes/TextTemplateForeachNode.cs b/src/LLTSharp/TemplateNodes/TextTemplateForeachNode.cs index 35c97c4..d86e971 100644 --- a/src/LLTSharp/TemplateNodes/TextTemplateForeachNode.cs +++ b/src/LLTSharp/TemplateNodes/TextTemplateForeachNode.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using LLTSharp.DataAccessors; +using Microsoft.Extensions.AI; namespace LLTSharp.TemplateNodes { diff --git a/src/LLTSharp/TemplateNodes/TextTemplateSequentialNode.cs b/src/LLTSharp/TemplateNodes/TextTemplateSequentialNode.cs index aa94d0c..ef09328 100644 --- a/src/LLTSharp/TemplateNodes/TextTemplateSequentialNode.cs +++ b/src/LLTSharp/TemplateNodes/TextTemplateSequentialNode.cs @@ -88,16 +88,17 @@ public override void Refine(int depth) int startIndex = 0; int indent = 0; - while (startIndex < line.Length && indent < maxIndent) - { - if (line[startIndex] == '\t') - indent += 4; - else if (line[startIndex] == ' ') - indent++; - else - break; - startIndex++; - } + if (li > 0) + while (startIndex < line.Length && indent < maxIndent) + { + if (line[startIndex] == '\t') + indent += 4; + else if (line[startIndex] == ' ') + indent++; + else + break; + startIndex++; + } if (li == endLine - 1) sb.Append(line.Substring(startIndex)); diff --git a/src/LLTSharp/TemplateNodes/TextTemplateWhileNode.cs b/src/LLTSharp/TemplateNodes/TextTemplateWhileNode.cs new file mode 100644 index 0000000..d3bc743 --- /dev/null +++ b/src/LLTSharp/TemplateNodes/TextTemplateWhileNode.cs @@ -0,0 +1,72 @@ +using System; +using System.Text; + +namespace LLTSharp.TemplateNodes +{ + /// + /// Represents a node in the text template that executes a child node while a condition is true. + /// + public class TextTemplateWhileNode : TextTemplateNode + { + /// + /// Gets the condition expression controlling the loop execution. + /// + public TemplateExpressionNode Condition { get; } + + /// + /// Gets the child node that will be executed each iteration. + /// + public TextTemplateNode Child { get; } + + /// + /// Creates a new instance of the class. + /// + public TextTemplateWhileNode(TemplateExpressionNode condition, TextTemplateNode child) + { + Condition = condition ?? throw new ArgumentNullException(nameof(condition)); + Child = child ?? throw new ArgumentNullException(nameof(child)); + } + + public override string Render(TemplateContextAccessor context) + { + StringBuilder result = new StringBuilder(); + + context.PushFrame(); + try + { + while (EvaluateCondition(context)) + { + var childResult = Child.Render(context); + if (!string.IsNullOrEmpty(childResult)) + result.AppendLine(childResult); + } + } + finally + { + context.PopFrame(); + } + + if (result.Length > 0) + result.Length -= Environment.NewLine.Length; // Remove trailing newline + + return result.ToString(); + } + + private bool EvaluateCondition(TemplateContextAccessor context) + { + var value = Condition.Evaluate(context); + return value.AsBoolean(); + } + + public override void Refine(int depth) + { + Child.Refine(depth + 1); + } + + public override string ToString() + { + return $"@while {Condition} {{\n{Child}\n}}"; + } + } + +} \ No newline at end of file diff --git a/src/LLTSharp/TypeDictionary.cs b/src/LLTSharp/TypeDictionary.cs index 7c43674..a39f42b 100644 --- a/src/LLTSharp/TypeDictionary.cs +++ b/src/LLTSharp/TypeDictionary.cs @@ -71,7 +71,10 @@ private static HashSet GetHierarchySetInternal(Type type) var types = new List(); types.Add(type); - types.AddRange(GetHierarchySet(type.BaseType)); + + var baseType = type.BaseType; + if (baseType != null) + types.AddRange(GetHierarchySet(baseType)); foreach (var iface in type.GetInterfaces()) types.AddRange(GetHierarchySet(iface)); diff --git a/src/LLTSharp/Utils/DictionaryUtils.cs b/src/LLTSharp/Utils/DictionaryUtils.cs new file mode 100644 index 0000000..8a745d1 --- /dev/null +++ b/src/LLTSharp/Utils/DictionaryUtils.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLTSharp.Utils +{ + public static class DictionaryUtils + { + public static bool DictionariesEqual( + IDictionary left, + IDictionary right) + { + if (ReferenceEquals(left, right)) + return true; + + if (left == null || right == null) + return false; + + if (left.Count != right.Count) + return false; + + foreach (var kvp in left) + { + if (!right.TryGetValue(kvp.Key, out var value)) + return false; + + if (!Equals(kvp.Value, value)) + return false; + } + + return true; + } + + public static int GetDictionaryHashCode( + IDictionary dictionary) + { + if (dictionary == null) + return 0; + + int hash = 0; + + foreach (var kvp in dictionary) + { + int keyHash = EqualityComparer.Default.GetHashCode(kvp.Key); + int valueHash = kvp.Value != null + ? EqualityComparer.Default.GetHashCode(kvp.Value) + : 0; + + unchecked + { + hash ^= keyHash * 397 + valueHash; + } + } + + return hash; + } + + } +} \ No newline at end of file diff --git a/tests/LLTSharp.Tests/BasicTemplateParsingTests.cs b/tests/LLTSharp.Tests/BasicTemplateParsingTests.cs index 563f002..5c10d58 100644 --- a/tests/LLTSharp.Tests/BasicTemplateParsingTests.cs +++ b/tests/LLTSharp.Tests/BasicTemplateParsingTests.cs @@ -5,11 +5,19 @@ using System.Threading.Tasks; using LLTSharp; using RCParsing; +using Xunit.Abstractions; namespace LLTSharp.Tests { public class BasicTemplateParsingTests { + private ITestOutputHelper output; + + public BasicTemplateParsingTests(ITestOutputHelper output) + { + this.output = output; + } + [Fact] public void ExpressionsASTPasing() { @@ -154,5 +162,23 @@ @assistant message { var parser = new LLTParser(); parser.Parse(templateStr); } + + [Fact] + public void MultipleInlineVariables() + { + string templateStr = + """ + @template { + Here is some variable: @var @number + } + """; + + var parser = new LLTParser(); + var ast = parser.ParseAST(templateStr); + var ctx = ast.Context; + var value = ast.GetValue>(); + + output.WriteLine(ctx.walkTrace.Render(400)); + } } } \ No newline at end of file diff --git a/tests/LLTSharp.Tests/TemplateFormattingTests.cs b/tests/LLTSharp.Tests/TemplateFormattingTests.cs index c917aa4..9f17f55 100644 --- a/tests/LLTSharp.Tests/TemplateFormattingTests.cs +++ b/tests/LLTSharp.Tests/TemplateFormattingTests.cs @@ -107,6 +107,183 @@ End of list. Assert.Equal(expected, rendered); } + [Fact] + public void ForeachInlineListTemplateFormatting() + { + var parser = new LLTParser(); + + var templateStr = + """ + @template foreach_format + { + Grocery list: + + @foreach item in [ + { name: 'Fish', quantity: 5 }, + { name: 'Meat', quantity: 10 }, + { name: 'Eggs', quantity: 0 } @/ Why eggs is zero??? + ] + { + @if item.quantity > 0 + { + - @item.name: @item.quantity + } + } + + End of list. + } + """; + + var template = parser.Parse(templateStr).First(); + + var rendered = template.Render().ToString(); + + var expected = + """ + Grocery list: + + - Fish: 5 + - Meat: 10 + + End of list. + """; + + Assert.Equal(expected, rendered); + } + + [Fact] + public void ComplexTemplateScenarioFormatting() + { + var parser = new LLTParser(); + + var templateStr = + """ + @template complex_demo + { + @let attempt = 1 + Processing datasets: + + @foreach dataset in [ + { name: 'set-A', items: [1, 3, 5] }, + { name: 'set-B', items: [2, 4, 0] }, + { name: 'set-C', items: [] } + ] + { + @if attempt == 1 { + --- + } + @let sum = 0 + Dataset: @dataset.name + @foreach value in dataset.items + { + @if value > 0 + { + Value: @value + @sum = (sum + value) + } + else + { + Skipping zero. + } + } + @if sum > 0 + { + Sum: @sum + @let retry = 2 + @while retry > 0 + { + Retry iteration @retry for @dataset.name + @retry = (retry - 1) + } + } + else + { + No positive values! + } + @attempt = (attempt + 1) + --- + } + + All done. Total attempts: @attempt + } + """; + + var template = parser.Parse(templateStr).First(); + + var rendered = template.Render(new { }).ToString(); + + var expected = + """ + Processing datasets: + + --- + Dataset: set-A + Value: 1 + Value: 3 + Value: 5 + Sum: 9 + Retry iteration 2 for set-A + Retry iteration 1 for set-A + --- + Dataset: set-B + Value: 2 + Value: 4 + Skipping zero. + Sum: 6 + Retry iteration 2 for set-B + Retry iteration 1 for set-B + --- + Dataset: set-C + No positive values! + --- + + All done. Total attempts: 4 + """; + + Assert.Equal(expected, rendered); + } + + [Fact] + public void WhileTemplateFormatting() + { + var parser = new LLTParser(); + + var templateStr = + """ + @template while_format + { + Counter demo: + + @let counter = 3 + @while counter > 0 + { + Current counter: @counter + @counter = (counter - 1) + } + + Done. + } + """; + + var template = parser.Parse(templateStr).First(); + + var rendered = template.Render(new { }).ToString(); + + var expected = + """ + Counter demo: + + Current counter: 3 + Current counter: 2 + Current counter: 1 + + Done. + """; + + Assert.Equal(expected, rendered); + } + + [Fact] public void NestedIfFormatting() { @@ -168,6 +345,51 @@ You are too young! Assert.Equal(expectedYoung, renderedYoung); } + [Fact] + public void NestedTemplateRendering() + { + var parser = new LLTParser(); + + var templateStr = + """ + @template nested_host + { + Here is groceries list: + @render 'nested_template' + } + + @template nested_template + { + Here is nested list: + @foreach item in ctx + { + Item: @item + } + } + """; + + var template = parser.Parse(templateStr).First(); + + var groceries = new[] { + "Apples", + "Bananas", + "Oranges" + }; + + var rendered = template.Render(groceries).ToString(); + + var expected = + """ + Here is groceries list: + Here is nested list: + Item: Apples + Item: Bananas + Item: Oranges + """; + + Assert.Equal(expected, rendered); + } + [Fact] public void ForeachVariableShadowing() {