Merge pull request #183 from opendocx/fix/whitespace-handling

Fix whitespace handling in UnicodeMapper and DocumentAssembler

Author: stesee

OpenXmlPowerTools.Tests/DocumentAssemblerTests.cs

@@ -157,6 +157,24 @@ public void DA259(string name, string data, bool err)
             Assert.Equal(4, brCount);
         }
 
+        [Fact]
+        public void DA240()
+        {
+            string name = "DA240-Whitespace.docx";
+            DA101(name, "DA240-Whitespace.xml", false);
+            var assembledDocx = new FileInfo(Path.Combine(TestUtil.TempDir.FullName, name.Replace(".docx", "-processed-by-DocumentAssembler.docx")));
+            WmlDocument afterAssembling = new WmlDocument(assembledDocx.FullName);
+
+            // when elements are inserted that begin or end with white space, make sure white space is preserved
+            string firstParaTextIncorrect = afterAssembling.MainDocumentPart.Element(W.body).Elements(W.p).First().Value;
+            Assert.Equal("Content may or may not have spaces: he/she; he, she; he and she.", firstParaTextIncorrect);
+            // warning: XElement.Value returns the string resulting from direct concatenation of all W.t elements. This is fast but ignores
+            // proper handling of xml:space="preserve" attributes, which Word honors when rendering content. Below we also check
+            // the result of UnicodeMapper.RunToString, which has been enhanced to take xml:space="preserve" into account.
+            string firstParaTextCorrect = InnerText(afterAssembling.MainDocumentPart.Element(W.body).Elements(W.p).First());
+            Assert.Equal("Content may or may not have spaces: he/she; he, she; he and she.", firstParaTextCorrect);
+        }
+
         [Theory]
         [InlineData("DA024-TrackedRevisions.docx", "DA-Data.xml")]
         public void DA102_Throws(string name, string data)
@@ -488,6 +506,15 @@ private static string GetDocumentText(WmlDocument document)
         private const string WidePngBase64 = "iVBORw0KGgoAAAANSUhEUgAAAZAAAADICAIAAABJdyC1AAACuUlEQVR4nO3UMQ7CQBAEwT3EvxEvXz/BZKalqniCifrs7gAUvGfmnO/TNwBu7H5edxuAfyFYQIZgARmCBWQIFpAhWECGYAEZggVkCBaQIVhAhmABGYIFZAgWkCFYQIZgARmCBWQIFpAhWECGYAEZggVkCBaQIVhAhmABGYIFZAgWkCFYQIZgARmCBWQIFpAhWECGYAEZggVkCBaQIVhAhmABGYIFZAgWkCFYQIZgARmCBWQIFpAhWECGYAEZggVkCBaQIVhAhmABGYIFZAgWkCFYQIZgARmCBWQIFpAhWECGYAEZggVkCBaQIVhAhmABGYIFFzMzu2y8A5u4PQZkIj89BEMEAAAAASUVORK5CYII=";
         private const string TallPngBase64 = "iVBORw0KGgoAAAANSUhEUgAAAMgAAAGQCAIAAABkkLjnAAAEF0lEQVR4nO3S0QkCURAEwX1i3mLke0lcI3hVAQzz0Wd3B+72npnzPbfv8mT72devP/CfhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkhEVCWCSERUJYJIRFQlgkzu42y8yTXVnDDBu1Y983AAAAAElFTkSuQmCC";
         private const string TruncatedGifBase64 = "R0lGODlhyABQAA==";
+
+        private static string InnerText(XContainer e)
+        {
+            return e.Descendants(W.r)
+                .Where(r => r.Parent.Name != W.del)
+                .Select(UnicodeMapper.RunToString)
+                .StringConcatenate();
+        }
+
         private static readonly List<string> s_ExpectedErrors = new List<string>()
         {
             "The 'http://schemas.openxmlformats.org/wordprocessingml/2006/main:evenHBand' attribute is not declared.",

OpenXmlPowerTools.Tests/UnicodeMapperTests.cs

@@ -153,5 +153,125 @@ public void IgnoresTemporaryLayoutMarkers()
             // characters) should exactly match the output of UnicodeMapper:
             Assert.Equal(p.Value, actual);
         }
+
+        private const string PreserveSpacingXmlString =
+@"<w:document xmlns:w=""http://schemas.openxmlformats.org/wordprocessingml/2006/main"">
+  <w:body>
+    <w:p>
+      <w:r>
+        <w:t xml:space=""preserve"">The following space is retained: </w:t>
+      </w:r>
+      <w:r>
+        <w:t>but this one is not: </w:t>
+      </w:r>
+      <w:r>
+        <w:t xml:space=""preserve"">. Similarly these two lines should have only a space between them: </w:t>
+      </w:r>
+      <w:r>
+        <w:t>
+          Line 1!
+Line 2!
+        </w:t>
+      </w:r>
+    </w:p>
+  </w:body>
+</w:document>";
+
+        [Fact]
+        public void HonorsXmlSpace()
+        {
+            // This somewhat rudimentary test is superceded by TreatsXmlSpaceLikeWord() below,
+            // but it has been left in to provide a simple/direct illustration of a couple of
+            // the specific test cases covered by that more extensive suite.
+            XDocument partDocument = XDocument.Parse(PreserveSpacingXmlString);
+            XElement p = partDocument.Descendants(W.p).Last();
+            string innerText = p.Descendants(W.r)
+                .Select(UnicodeMapper.RunToString)
+                .StringConcatenate();
+            Assert.Equal(@"The following space is retained: but this one is not:. Similarly these two lines should have only a space between them: Line 1! Line 2!", innerText);
+        }
+
+        // Verifies that UnicodeMapper.RunToString interprets whitespace in <w:t> elements
+        // exactly the way Microsoft Word does, including honoring xml:space="preserve".
+        // This is essential because RunToString is used by higher‑level features
+        // (OpenXmlRegex, DocumentAssembler, etc.) that rely on its output to reflect the
+        // text an end‑user would actually see and edit in Word.
+        //
+        // Word accepts a wide range of “valid” DOCX input, but it normalizes that input
+        // into a canonical form when displaying or saving the document. These tests
+        // compare RunToString’s output against Word’s canonicalized output to ensure
+        // that whitespace is treated as semantic content in the same way Word treats it.
+        [Fact]
+        public void TreatsXmlSpaceLikeWord()
+        {
+            var sourceDir = new System.IO.DirectoryInfo("../../../../TestFiles/");
+            // Test document: crafted to include many whitespace patterns that Word accepts as valid input
+            var testDoc = new System.IO.FileInfo(System.IO.Path.Combine(sourceDir.FullName, "UM-Whitespace-test.docx"));
+            var testWmlDoc = new WmlDocument(testDoc.FullName);
+            var testParagraphs = testWmlDoc.MainDocumentPart
+                            .Element(W.body)
+                            .Elements(W.p).ToList();
+            // Canonical document: the same test document after being opened and saved by Word,
+            // representing Word's own normalized interpretation of that whitespace
+            var expectedDoc = new System.IO.FileInfo(System.IO.Path.Combine(sourceDir.FullName, "UM-Whitespace-Word-saved.docx"));
+            var expectedWmlDoc = new WmlDocument(expectedDoc.FullName);
+            var expectedParagraphs = expectedWmlDoc.MainDocumentPart
+                            .Element(W.body)
+                            .Elements(W.p).ToList();
+            // Iterate through pairs of paragraphs (test name, test content, expected result)
+            for (int i = 0; i < testParagraphs.Count - 1; i += 2)
+            {
+                var testNameParagraph = testParagraphs[i];
+                var testContentParagraph = testParagraphs[i + 1];
+                // Get the test name from the first paragraph
+                var testName = testNameParagraph.Descendants(W.t)
+                    .Select(t => (string)t)
+                    .StringConcatenate();
+                // Get the actual result by calling UnicodeMapper.RunToString on the test content runs
+                var actualResult = testContentParagraph.Descendants(W.r)
+                    .Select(UnicodeMapper.RunToString)
+                    .StringConcatenate();
+                // Find corresponding expected result paragraph (same index in expected document)
+                var expectedResult = ExtractExpectedFromWord(expectedParagraphs[i + 1]);
+                Assert.True(
+                    expectedResult == actualResult,
+                    $"Test '{testName}' failed. Expected: [{expectedResult}] Actual: [{actualResult}]"
+                );
+            }
+        }
+
+        // Extracts the expected text from Word’s canonicalized output for the whitespace tests.
+        // This helper intentionally handles *only* the constructs that Word emits in the saved
+        // version of UM-whitespace-test.docx:
+        //   • <w:t>      → literal text
+        //   • <w:tab/>   → '\t'
+        //   • <w:lastRenderedPageBreak/> (intentionally ignored)
+        // If any other run-level element appears, it means Word has emitted something this test
+        // was not designed to handle, and the test fails loudly. This prevents the helper
+        // from drifting toward reimplementing UnicodeMapper.RunToString.
+        private static string ExtractExpectedFromWord(XElement p)
+        {
+            var sb = new System.Text.StringBuilder();
+            foreach (var run in p.Elements(W.r))
+            {
+                foreach (var child in run.Elements())
+                {
+                    if (child.Name == W.t)
+                    {
+                        sb.Append((string)child);
+                    }
+                    else if (child.Name == W.tab)
+                    {
+                        sb.Append('\t');
+                    }
+                    else if (child.Name != W.lastRenderedPageBreak)
+                    {
+                        throw new System.InvalidOperationException(
+                            $"Unexpected element <{child.Name.LocalName}> encountered in expected Word output.");
+                    }
+                }
+            }
+            return sb.ToString();
+        }
     }
 }

OpenXmlPowerTools/DocumentAssembler/DocumentAssembler.cs

@@ -654,7 +654,7 @@ private class RunReplacementInfo
                             p.Add(new XElement(W.r,
                                     para.Elements(W.r).Elements(W.rPr).FirstOrDefault(),
                                 (p.Elements().Count() > 1) ? new XElement(W.br) : null,
-                                new XElement(W.t, line)));
+                                new XElement(W.t, GetXmlSpaceAttribute(line), line)));
                         }
                         return p;
                     }
@@ -666,7 +666,7 @@ private class RunReplacementInfo
                             list.Add(new XElement(W.r,
                                 run.Elements().Where(e => e.Name != W.t),
                                 (list.Count > 0) ? new XElement(W.br) : null,
-                                new XElement(W.t, line)));
+                                new XElement(W.t, GetXmlSpaceAttribute(line), line)));
                         }
                         return list;
                     }
@@ -1415,5 +1415,18 @@ private static string EvaluateXPathToString(XElement element, string xPath, bool
 
             return xPathSelectResult.ToString();
         }
+
+        private static XAttribute GetXmlSpaceAttribute(string textOfTextElement)
+        {
+            if (!string.IsNullOrEmpty(textOfTextElement))
+            {
+                if (char.IsWhiteSpace(textOfTextElement[0]) ||
+                    char.IsWhiteSpace(textOfTextElement[textOfTextElement.Length - 1]))
+                {
+                    return new XAttribute(XNamespace.Xml + "space", "preserve");
+                }
+            }
+            return null;
+        }
     }
 }

OpenXmlPowerTools/UnicodeMapper.cs

@@ -60,7 +60,10 @@ public static string RunToString(XElement element)
             // For w:t elements, we obviously want the element's value.
             if (element.Name == W.t)
             {
-                return (string)element;
+                // Emulate Word's handling of the xml:space attribute on text elements
+                XAttribute? spaceAttribute = element.Attribute(XNamespace.Xml + "space");
+                string? space = spaceAttribute?.Value;
+                return NormalizeWhitespace((string) element, space == "preserve");
             }
 
             // Turn elements representing special characters into their corresponding
@@ -141,6 +144,50 @@ public static string RunToString(XElement element)
             return StartOfHeading.ToString();
         }
 
+        /// <summary>
+        /// Emulate the way Word interprets the content of text elements
+        /// depending on whether the xml:space="preserve" attribute is present.
+        /// </summary>
+        /// <param name="text">The entire content of the w:t element.</param>
+        /// <returns>The corresponding text string Word would display, print, save,
+        /// and allow to be edited.</returns>
+        private static string NormalizeWhitespace(string text, bool preserve)
+        {
+            if (string.IsNullOrEmpty(text))
+                return string.Empty;
+            // Trim leading & trailing whitespace when NOT preserving
+            ReadOnlySpan<char> span = preserve
+                ? text.AsSpan()
+                : text.AsSpan().Trim();
+            if (span.Length == 0)
+                return string.Empty;
+            var sb = new System.Text.StringBuilder(span.Length);
+            int i = 0;
+            while (i < span.Length)
+            {
+                char c = span[i];
+                switch (c)
+                {
+                    case '\r': // CR or CRLF → space
+                        sb.Append(' ');
+                        if (i + 1 < span.Length && span[i + 1] == '\n')
+                            i++; // skip LF so CRLF becomes one space
+                        break;
+                    case '\n': // LF → space
+                        sb.Append(' ');
+                        break;
+                    case '\t': // TAB preserved or converted to space, depending on mode
+                        sb.Append(preserve ? c : ' ');
+                        break;
+                    default: // SPACE or any other character → preserved exactly
+                        sb.Append(c);
+                        break;
+                }
+                i++;
+            }
+            return sb.ToString();
+        }
+
         /// <summary>
         /// Translate a symbol into a Unicode character, using the specified w:font attribute
         /// value and unicode value (represented by the w:sym element's w:char attribute),

TestFiles/DA240-Whitespace.docx

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<Test240>
+  <MayOrMayNot>may or may not</MayOrMayNot>
+  <Join1>/</Join1>
+  <Join2>, </Join2>
+  <Join3> and </Join3>
+</Test240>