From 0f50657ab1a0f28dfb4fa0e0e8be47095e5ae280 Mon Sep 17 00:00:00 2001 From: lice Date: Fri, 21 Mar 2025 15:59:45 +0100 Subject: [PATCH 1/5] Fixed issues with escaped strings inside the phrase We've had issues with phrases that looks like this "Byt artikel" listar artiklar direkt since the quotes will be escaped in the source. But the backslash escaping the quote should not be a part of the phrase to translate in and of itself. This is very similar to the issue with newlines we fixed here https://github.com/MultinetInteractive/MN.L10n/pull/176 I could add more cases (for instance \t) but this is the only one I know we actually use --- MN.L10n.Tests/ParserTests.cs | 27 +++++++++++++++++++++++++++ MN.L10n/L10nParser.cs | 11 +++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/MN.L10n.Tests/ParserTests.cs b/MN.L10n.Tests/ParserTests.cs index 9a5ef98..642e351 100644 --- a/MN.L10n.Tests/ParserTests.cs +++ b/MN.L10n.Tests/ParserTests.cs @@ -115,5 +115,32 @@ public void LineBreakCharInCall() Assert.Collection(result, x => Assert.Equal("Hello\nBrother", x.Phrase)); } + + [Fact] + public void TestEscapedStringContainerCharacter() + { + var parser = new L10nParser(); + var result = parser.Parse(@"_s(""Hello \""friend\"". How it do?"")"); + + Assert.Collection(result, x => Assert.Equal(@"Hello ""friend"". How it do?", x.Phrase)); + } + + [Fact] + public void TestEscapedStringContainerCharacter2() + { + var parser = new L10nParser(); + var result = parser.Parse(@"_s('Hello \""friend\"". How it do?')"); + + Assert.Collection(result, x => Assert.Equal(@"Hello \""friend\"". How it do?", x.Phrase)); + } + + [Fact] + public void TestEscapedStringContainerCharacter3() + { + var parser = new L10nParser(); + var result = parser.Parse(@"_s('Hello \'friend\'. How it do?')"); + + Assert.Collection(result, x => Assert.Equal(@"Hello 'friend'. How it do?", x.Phrase)); + } } } diff --git a/MN.L10n/L10nParser.cs b/MN.L10n/L10nParser.cs index d021e77..2b942ed 100644 --- a/MN.L10n/L10nParser.cs +++ b/MN.L10n/L10nParser.cs @@ -181,10 +181,17 @@ bool TryPeek(int forward) } } - if (inToken && !isVerbatim && source[_pos] == '\\' && TryPeek(1) && source[_pos + 1] == 'n') + if (inToken && !isVerbatim && source[_pos] == '\\' && TryPeek(1) && (source[_pos + 1] == 'n' || source[_pos + 1] == _stringContainer)) { + if (source[_pos + 1] == 'n') + { + _tokenContent.Append('\n'); + } + else if (source[_pos + 1] == _stringContainer) + { + _tokenContent.Append(_stringContainer); + } _pos++; - _tokenContent.Append('\n'); } else { From 18860eb2f04f3c7a778a4a7ef4829ccca4611e67 Mon Sep 17 00:00:00 2001 From: lice Date: Fri, 21 Mar 2025 16:07:47 +0100 Subject: [PATCH 2/5] bump versions --- MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj | 2 +- MN.L10n/MN.L10n.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj b/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj index a4625e6..c85346e 100644 --- a/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj +++ b/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj @@ -6,7 +6,7 @@ Exe MN.L10n.BuildTasks.Program true - 4.0.1 + 4.0.2 Chris Gårdenberg MultiNet Interactive AB diff --git a/MN.L10n/MN.L10n.csproj b/MN.L10n/MN.L10n.csproj index 2cfe70b..ee8bb4f 100644 --- a/MN.L10n/MN.L10n.csproj +++ b/MN.L10n/MN.L10n.csproj @@ -19,7 +19,7 @@ Translation package Library - 4.1.1 + 4.1.2 From eba8f4707f03395d532c987a016191424d8f1f7c Mon Sep 17 00:00:00 2001 From: lice Date: Fri, 21 Mar 2025 16:12:42 +0100 Subject: [PATCH 3/5] Actually bump the version --- MN.L10n/MN.L10n.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MN.L10n/MN.L10n.csproj b/MN.L10n/MN.L10n.csproj index ee8bb4f..342e781 100644 --- a/MN.L10n/MN.L10n.csproj +++ b/MN.L10n/MN.L10n.csproj @@ -12,7 +12,7 @@ Translation package https://github.com/MultinetInteractive/MN.L10n git © 20XX MultiNet Interactive AB - 4.1.1 + 4.1.2 latest True Now includes analyzer From dda57916e29e555a7eccb424db3e357bedeccffa Mon Sep 17 00:00:00 2001 From: lice Date: Fri, 21 Mar 2025 16:14:02 +0100 Subject: [PATCH 4/5] I screwed up the publish of the build task package. Need to bump again to get the correct dependency --- MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj b/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj index c85346e..ccc3d2e 100644 --- a/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj +++ b/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj @@ -6,7 +6,7 @@ Exe MN.L10n.BuildTasks.Program true - 4.0.2 + 4.0.3 Chris Gårdenberg MultiNet Interactive AB From dc15771800de03777fb4613c4d71a00050744e33 Mon Sep 17 00:00:00 2001 From: lice Date: Fri, 21 Mar 2025 17:17:30 +0100 Subject: [PATCH 5/5] Postprocess the string instead. This fixes a bug where the first character in the string is an escaped stringcontainer char --- MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj | 2 +- MN.L10n.Tests/ParserTests.cs | 24 ++++++ MN.L10n/L10nParser.cs | 78 ++++++++++++++------ MN.L10n/MN.L10n.csproj | 4 +- 4 files changed, 84 insertions(+), 24 deletions(-) diff --git a/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj b/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj index ccc3d2e..759b992 100644 --- a/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj +++ b/MN.L10n.BuildTasks/MN.L10n.BuildTasks.csproj @@ -6,7 +6,7 @@ Exe MN.L10n.BuildTasks.Program true - 4.0.3 + 4.0.4 Chris Gårdenberg MultiNet Interactive AB diff --git a/MN.L10n.Tests/ParserTests.cs b/MN.L10n.Tests/ParserTests.cs index 642e351..297d2a4 100644 --- a/MN.L10n.Tests/ParserTests.cs +++ b/MN.L10n.Tests/ParserTests.cs @@ -142,5 +142,29 @@ public void TestEscapedStringContainerCharacter3() Assert.Collection(result, x => Assert.Equal(@"Hello 'friend'. How it do?", x.Phrase)); } + + [Fact] + public void TestEscapedStringContainerCharacterFirstChar() + { + var parser = new L10nParser(); + var result = parser.Parse(@"_s(""\""friend\""!"")"); + + Assert.Collection(result, x => Assert.Equal(@"""friend""!", x.Phrase)); + } + + [Fact] + public void TestEscapedStringContainerCharacterVerbatim() + { + var src = @" + _s( + @""Hej """"bror"""" +Nej"" + ) + "; + var parser = new L10nParser(); + var result = parser.Parse(src).ToList(); + Assert.Single(result); + Assert.Equal(@"Hej ""bror""\nNej", result[0].Phrase.Trim()); + } } } diff --git a/MN.L10n/L10nParser.cs b/MN.L10n/L10nParser.cs index 2b942ed..889cc2b 100644 --- a/MN.L10n/L10nParser.cs +++ b/MN.L10n/L10nParser.cs @@ -86,6 +86,7 @@ bool TryPeek(int forward) { break; } + isVerbatim = true; } else @@ -115,6 +116,7 @@ bool TryPeek(int forward) continue; } } + if (inToken) { _tokenContent.Append(source[_pos]); @@ -143,7 +145,7 @@ bool TryPeek(int forward) phrase = phrase.Substring(0, phrase.Length - 1); } - yield return new PhraseInvocation + yield return Unescape(new PhraseInvocation { Phrase = phrase, Row = row, @@ -151,7 +153,7 @@ bool TryPeek(int forward) EndChar = _pos, IsEscaped = isEscaped, StringContainer = _stringContainer - }; + }, isVerbatim); inToken = false; } } @@ -159,9 +161,11 @@ bool TryPeek(int forward) { var _tail = source[_pos - 1]; var _peek = source[_pos + 1]; - if (source[_pos] == _stringContainer && _peek != _stringContainer && _tail != _stringContainer) + if (source[_pos] == _stringContainer && _peek != _stringContainer && + _tail != _stringContainer) { - var phrase = _tokenContent.ToString().Replace("\n", "\\n").Replace("\r", "").Replace("\"\"", "\\\""); + var phrase = _tokenContent.ToString().Replace("\n", "\\n").Replace("\r", "") + .Replace("\"\"", "\\\""); // Hoppa över sista \ om den är escape:ad if (isEscaped) @@ -169,38 +173,70 @@ bool TryPeek(int forward) phrase = phrase.Substring(0, phrase.Length - 1); } - yield return new PhraseInvocation + yield return Unescape(new PhraseInvocation { Phrase = phrase, Row = row, StartChar = startChar, EndChar = _pos, StringContainer = _stringContainer - }; + }, isVerbatim); inToken = false; } } + _tokenContent.Append(source[_pos]); + } - if (inToken && !isVerbatim && source[_pos] == '\\' && TryPeek(1) && (source[_pos + 1] == 'n' || source[_pos + 1] == _stringContainer)) - { - if (source[_pos + 1] == 'n') - { - _tokenContent.Append('\n'); - } - else if (source[_pos + 1] == _stringContainer) + break; + } + } + } + + private PhraseInvocation Unescape(PhraseInvocation phraseInvocation, bool isVerbatim) + { + if (isVerbatim) + { + for (var i = 0; i < phraseInvocation.Phrase.Length; i++) + { + if (phraseInvocation.Phrase[i] == '\\' && i + 1 < phraseInvocation.Phrase.Length) + { + if (phraseInvocation.Phrase[i + 1] == phraseInvocation.StringContainer) + { + phraseInvocation.Phrase = phraseInvocation.Phrase.Remove(i, 1); + } + } + } + } + else + { + for (var i = 0; i < phraseInvocation.Phrase.Length; i++) + { + if (phraseInvocation.Phrase[i] == '\\' && i + 1 < phraseInvocation.Phrase.Length) + { + switch (phraseInvocation.Phrase[i + 1]) + { + case 'n': + phraseInvocation.Phrase = phraseInvocation.Phrase.Remove(i, 2).Insert(i, "\n"); + break; + case 't': + phraseInvocation.Phrase = phraseInvocation.Phrase.Remove(i, 2).Insert(i, "\t"); + break; + case '\\': + phraseInvocation.Phrase = phraseInvocation.Phrase.Remove(i, 2).Insert(i, "\\"); + break; + default: + if (phraseInvocation.Phrase[i + 1] == phraseInvocation.StringContainer) { - _tokenContent.Append(_stringContainer); + phraseInvocation.Phrase = phraseInvocation.Phrase.Remove(i, 1); } - _pos++; - } - else - { - _tokenContent.Append(source[_pos]); - } + + break; } - break; + } } } + + return phraseInvocation; } } } diff --git a/MN.L10n/MN.L10n.csproj b/MN.L10n/MN.L10n.csproj index 342e781..f528924 100644 --- a/MN.L10n/MN.L10n.csproj +++ b/MN.L10n/MN.L10n.csproj @@ -12,14 +12,14 @@ Translation package https://github.com/MultinetInteractive/MN.L10n git © 20XX MultiNet Interactive AB - 4.1.2 + 4.1.3 latest True Now includes analyzer Library - 4.1.2 + 4.1.3