From 684df8a6f5e8bebaccf86ae9167800a8942077b4 Mon Sep 17 00:00:00 2001
From: XingY <xyang@labkey.com>
Date: Thu, 13 Nov 2025 15:43:50 -0800
Subject: [PATCH 1/2] Handle surrogate pair characters substring

---
 flow/src/org/labkey/flow/ScriptParser.java    | 273 +++++++++---------
 .../labkey/api/protein/ProteinManager.java    |   3 +-
 .../org/labkey/api/protein/ProteinPlus.java   |   3 +-
 .../api/protein/fasta/FastaDbLoader.java      |   3 +-
 .../labkey/api/protein/fasta/IdPattern.java   |   3 +-
 .../labkey/api/protein/uniprot/uniprot.java   |   8 +-
 6 files changed, 149 insertions(+), 144 deletions(-)

diff --git a/flow/src/org/labkey/flow/ScriptParser.java b/flow/src/org/labkey/flow/ScriptParser.java
index 74f1507de7..83fda0bece 100644
--- a/flow/src/org/labkey/flow/ScriptParser.java
+++ b/flow/src/org/labkey/flow/ScriptParser.java
@@ -1,136 +1,137 @@
-/*
- * Copyright (c) 2007-2019 LabKey Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.labkey.flow;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.xmlbeans.XmlError;
-import org.apache.xmlbeans.XmlException;
-import org.apache.xmlbeans.XmlOptions;
-import org.fhcrc.cpas.flow.script.xml.ScriptDef;
-import org.fhcrc.cpas.flow.script.xml.ScriptDocument;
-import org.xml.sax.SAXParseException;
-
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-
-public class ScriptParser
-{
-    List<Error> _errors;
-    ScriptDef _script;
-
-    public ScriptParser()
-    {
-    }
-
-    static public class Error
-    {
-        String _message;
-        int _line;
-        int _column;
-
-        public Error(String message)
-        {
-            this(message, 0, 0);
-        }
-
-        public Error(String message, int line, int column)
-        {
-            _message = message;
-            _line = line;
-            _column = column;
-        }
-
-        public Error(SAXParseException spe)
-        {
-            this(spe.getLocalizedMessage(), spe.getLineNumber(), spe.getColumnNumber());
-        }
-
-        public String getMessage()
-        {
-            return _message;
-        }
-
-        public int getLine()
-        {
-            return _line;
-        }
-
-        public int getColumn()
-        {
-            return _column;
-        }
-    }
-
-    public void parse(String script)
-    {
-        try
-        {
-            XmlOptions options = new XmlOptions();
-            List<XmlError> errors = new ArrayList<>();
-            options.setDocumentType(ScriptDocument.type);
-            script = StringUtils.replace(script, "<script>", "<script xmlns=\"" + ScriptDocument.type.getContentModel().getName().getNamespaceURI() + "\">");
-            ScriptDocument doc = ScriptDocument.Factory.parse(new StringReader(script), options);
-            options.setErrorListener(errors);
-
-            if (!doc.validate(options))
-            {
-                for (XmlError xmlError : errors)
-                {
-                    String message = xmlError.getMessage();
-                    message = StringUtils.replace(message, "@" + ScriptDocument.type.getContentModel().getName().getNamespaceURI(), "");
-                    String location = xmlError.getCursorLocation().xmlText();
-                    if (location.length() > 100)
-                        location = location.substring(0, 100);
-                    addError(new Error("Schema Validation Error: " + message + "\nLocation of invalid XML: " + location));
-                }
-            }
-            _script = doc.getScript();
-        }
-        catch (XmlException xmlException)
-        {
-            if (xmlException.getErrors().isEmpty())
-            {
-                addError(new Error(xmlException.toString()));
-            }
-            else
-            {
-                for (XmlError xmlError : xmlException.getErrors())
-                {
-                    addError(new Error(xmlError.getMessage(), xmlError.getLine(), xmlError.getColumn() > 0 ? xmlError.getColumn() : 1));
-                }
-            }
-        }
-        catch (Exception e)
-        {
-            addError(new Error(e.toString()));
-        }
-    }
-
-    void addError(Error error)
-    {
-        if (_errors == null)
-            _errors = new ArrayList<>();
-        _errors.add(error);
-    }
-
-    public Error[] getErrors()
-    {
-        if (_errors == null || _errors.isEmpty())
-            return null;
-        return _errors.toArray(new Error[0]);
-    }
-}
+/*
+ * Copyright (c) 2007-2019 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.flow;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.xmlbeans.XmlError;
+import org.apache.xmlbeans.XmlException;
+import org.apache.xmlbeans.XmlOptions;
+import org.fhcrc.cpas.flow.script.xml.ScriptDef;
+import org.fhcrc.cpas.flow.script.xml.ScriptDocument;
+import org.labkey.api.util.StringUtilsLabKey;
+import org.xml.sax.SAXParseException;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ScriptParser
+{
+    List<Error> _errors;
+    ScriptDef _script;
+
+    public ScriptParser()
+    {
+    }
+
+    static public class Error
+    {
+        String _message;
+        int _line;
+        int _column;
+
+        public Error(String message)
+        {
+            this(message, 0, 0);
+        }
+
+        public Error(String message, int line, int column)
+        {
+            _message = message;
+            _line = line;
+            _column = column;
+        }
+
+        public Error(SAXParseException spe)
+        {
+            this(spe.getLocalizedMessage(), spe.getLineNumber(), spe.getColumnNumber());
+        }
+
+        public String getMessage()
+        {
+            return _message;
+        }
+
+        public int getLine()
+        {
+            return _line;
+        }
+
+        public int getColumn()
+        {
+            return _column;
+        }
+    }
+
+    public void parse(String script)
+    {
+        try
+        {
+            XmlOptions options = new XmlOptions();
+            List<XmlError> errors = new ArrayList<>();
+            options.setDocumentType(ScriptDocument.type);
+            script = StringUtils.replace(script, "<script>", "<script xmlns=\"" + ScriptDocument.type.getContentModel().getName().getNamespaceURI() + "\">");
+            ScriptDocument doc = ScriptDocument.Factory.parse(new StringReader(script), options);
+            options.setErrorListener(errors);
+
+            if (!doc.validate(options))
+            {
+                for (XmlError xmlError : errors)
+                {
+                    String message = xmlError.getMessage();
+                    message = StringUtils.replace(message, "@" + ScriptDocument.type.getContentModel().getName().getNamespaceURI(), "");
+                    String location = xmlError.getCursorLocation().xmlText();
+                    if (location.length() > 100)
+                        location = StringUtilsLabKey.leftSurrogatePairFriendly(location, 100);
+                    addError(new Error("Schema Validation Error: " + message + "\nLocation of invalid XML: " + location));
+                }
+            }
+            _script = doc.getScript();
+        }
+        catch (XmlException xmlException)
+        {
+            if (xmlException.getErrors().isEmpty())
+            {
+                addError(new Error(xmlException.toString()));
+            }
+            else
+            {
+                for (XmlError xmlError : xmlException.getErrors())
+                {
+                    addError(new Error(xmlError.getMessage(), xmlError.getLine(), xmlError.getColumn() > 0 ? xmlError.getColumn() : 1));
+                }
+            }
+        }
+        catch (Exception e)
+        {
+            addError(new Error(e.toString()));
+        }
+    }
+
+    void addError(Error error)
+    {
+        if (_errors == null)
+            _errors = new ArrayList<>();
+        _errors.add(error);
+    }
+
+    public Error[] getErrors()
+    {
+        if (_errors == null || _errors.isEmpty())
+            return null;
+        return _errors.toArray(new Error[0]);
+    }
+}
diff --git a/protein/api-src/org/labkey/api/protein/ProteinManager.java b/protein/api-src/org/labkey/api/protein/ProteinManager.java
index 38ffa0480e..71710d2e00 100644
--- a/protein/api-src/org/labkey/api/protein/ProteinManager.java
+++ b/protein/api-src/org/labkey/api/protein/ProteinManager.java
@@ -20,6 +20,7 @@
 import org.labkey.api.util.HashHelpers;
 import org.labkey.api.util.HtmlString;
 import org.labkey.api.util.LinkBuilder;
+import org.labkey.api.util.StringUtilsLabKey;
 import org.labkey.api.view.NotFoundException;
 
 import java.io.ByteArrayOutputStream;
@@ -100,7 +101,7 @@ private static SimpleProtein ensureProteinInDatabase(String sequence, Organism o
             map.put("Mass", PeptideHelpers.computeMass(sequenceBytes, 0, sequenceBytes.length, PeptideHelpers.AMINO_ACID_AVERAGE_MASSES));
             map.put("OrgId", organism.getOrgId());
             map.put("Hash", hashSequence(sequence));
-            map.put("Description", description == null ? null : (description.length() > 200 ? description.substring(0, 196) + "..." : description));
+            map.put("Description", description == null ? null : (description.length() > 200 ? StringUtilsLabKey.leftSurrogatePairFriendly(description, 196) + "..." : description));
             map.put("BestName", name);
             map.put("Length", sequence.length());
             map.put("InsertDate", new Date());
diff --git a/protein/api-src/org/labkey/api/protein/ProteinPlus.java b/protein/api-src/org/labkey/api/protein/ProteinPlus.java
index 673b63de42..8718d17cc2 100644
--- a/protein/api-src/org/labkey/api/protein/ProteinPlus.java
+++ b/protein/api-src/org/labkey/api/protein/ProteinPlus.java
@@ -18,6 +18,7 @@
 
 import org.labkey.api.protein.fasta.FastaProtein;
 import org.labkey.api.util.HashHelpers;
+import org.labkey.api.util.StringUtilsLabKey;
 
 public class ProteinPlus
 {
@@ -111,7 +112,7 @@ public String getBestName()
         {
             result = getProtein().getHeader();
         }
-        if (result.length() > 500) result = result.substring(0, 499);
+        if (result.length() > 500) result = StringUtilsLabKey.leftSurrogatePairFriendly(result, 499);
         return result;
     }
 }
diff --git a/protein/api-src/org/labkey/api/protein/fasta/FastaDbLoader.java b/protein/api-src/org/labkey/api/protein/fasta/FastaDbLoader.java
index 721809eaa0..befb46042b 100644
--- a/protein/api-src/org/labkey/api/protein/fasta/FastaDbLoader.java
+++ b/protein/api-src/org/labkey/api/protein/fasta/FastaDbLoader.java
@@ -37,6 +37,7 @@
 import org.labkey.api.protein.organism.OrganismGuessStrategy;
 import org.labkey.api.util.HashHelpers;
 import org.labkey.api.util.NetworkDrive;
+import org.labkey.api.util.StringUtilsLabKey;
 import org.labkey.api.view.ViewBackgroundInfo;
 
 import java.io.File;
@@ -259,7 +260,7 @@ protected void preProcessSequences(List<ProteinPlus> mouthful, Connection c, Log
             }
             else
             {
-                if (desc.length() >= 200) desc = desc.substring(0, 195) + "...";
+                if (desc.length() >= 200) desc = StringUtilsLabKey.leftSurrogatePairFriendly(desc, 195) + "...";
                 fdbu._addSeqStmt.setString(3, desc);
             }
             fdbu._addSeqStmt.setDouble(4, curSeq.getProtein().getMass());
diff --git a/protein/api-src/org/labkey/api/protein/fasta/IdPattern.java b/protein/api-src/org/labkey/api/protein/fasta/IdPattern.java
index a8dffeb6f6..257449ab3e 100644
--- a/protein/api-src/org/labkey/api/protein/fasta/IdPattern.java
+++ b/protein/api-src/org/labkey/api/protein/fasta/IdPattern.java
@@ -27,6 +27,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
+import org.labkey.api.util.StringUtilsLabKey;
 
 /**
  * this class implements a regular expression-based recognition of identifiers parsed from the fasta files.
@@ -178,7 +179,7 @@ public static Map<String, Set<String>> createIdMap(String key, String value)
             {
                 v = v.trim();
                 if (v.length() > 50)
-                    v = v.substring(0, 50);
+                    v = StringUtilsLabKey.leftSurrogatePairFriendly(v, 50);
                 if (!v.isEmpty())
                     vals.add(v);
             }
diff --git a/protein/api-src/org/labkey/api/protein/uniprot/uniprot.java b/protein/api-src/org/labkey/api/protein/uniprot/uniprot.java
index 8c8c7c8a5b..fa16a5cea9 100644
--- a/protein/api-src/org/labkey/api/protein/uniprot/uniprot.java
+++ b/protein/api-src/org/labkey/api/protein/uniprot/uniprot.java
@@ -605,7 +605,7 @@ public int insertSequences(ParseContext context, Connection conn) throws SQLExce
             else
             {
                 String tmp = curSeq.getDescription();
-                if (tmp.length() >= 200) tmp = tmp.substring(0, 190) + "...";
+                if (tmp.length() >= 200) tmp = StringUtilsLabKey.leftSurrogatePairFriendly(tmp, 190) + "...";
                 _addSeq.setString(3, tmp);
             }
             if (curSeq.getSourceChangeDate() == null)
@@ -657,7 +657,7 @@ public int insertSequences(ParseContext context, Connection conn) throws SQLExce
             else
             {
                 String tmp = curSeq.getBestName();
-                if (tmp.length() >= 50) tmp = tmp.substring(0, 45) + "...";
+                if (tmp.length() >= 50) tmp = StringUtilsLabKey.leftSurrogatePairFriendly(tmp, 45) + "...";
                 _addSeq.setString(11, tmp);
             }
             if (curSeq.getBestGeneName() == null)
@@ -667,7 +667,7 @@ public int insertSequences(ParseContext context, Connection conn) throws SQLExce
             else
             {
                 String tmp = curSeq.getBestGeneName();
-                if (tmp.length() >= 50) tmp = tmp.substring(0, 45) + "...";
+                if (tmp.length() >= 50) tmp = StringUtilsLabKey.leftSurrogatePairFriendly(tmp, 45) + "...";
                 _addSeq.setString(12, tmp);
             }
             // Timestamp at index 13 is set once for the whole prepared statement
@@ -709,7 +709,7 @@ public int insertIdentifiers(ParseContext context, Connection conn) throws SQLEx
         {
             transactionCount++;
             String curIdentVal = curIdent.getIdentifier();
-            if (curIdentVal.length() > 50) curIdentVal = curIdentVal.substring(0, 45) + "...";
+            if (curIdentVal.length() > 50) curIdentVal = StringUtilsLabKey.leftSurrogatePairFriendly(curIdentVal, 45) + "...";
             _addIdent.setString(1, curIdentVal);
             _addIdent.setString(2, curIdent.getIdentType());
             UniprotSequence curSeq = curIdent.getSequence();

From 7fb793aeea837bd90149cb7ccf76c982fa500d88 Mon Sep 17 00:00:00 2001
From: XingY <xyang@labkey.com>
Date: Tue, 18 Nov 2025 14:44:54 -0800
Subject: [PATCH 2/2] crlf

---
 flow/src/org/labkey/flow/ScriptParser.java | 274 ++++++++++-----------
 1 file changed, 137 insertions(+), 137 deletions(-)

diff --git a/flow/src/org/labkey/flow/ScriptParser.java b/flow/src/org/labkey/flow/ScriptParser.java
index 83fda0bece..908bb131b5 100644
--- a/flow/src/org/labkey/flow/ScriptParser.java
+++ b/flow/src/org/labkey/flow/ScriptParser.java
@@ -1,137 +1,137 @@
-/*
- * Copyright (c) 2007-2019 LabKey Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.labkey.flow;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.xmlbeans.XmlError;
-import org.apache.xmlbeans.XmlException;
-import org.apache.xmlbeans.XmlOptions;
-import org.fhcrc.cpas.flow.script.xml.ScriptDef;
-import org.fhcrc.cpas.flow.script.xml.ScriptDocument;
-import org.labkey.api.util.StringUtilsLabKey;
-import org.xml.sax.SAXParseException;
-
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-
-public class ScriptParser
-{
-    List<Error> _errors;
-    ScriptDef _script;
-
-    public ScriptParser()
-    {
-    }
-
-    static public class Error
-    {
-        String _message;
-        int _line;
-        int _column;
-
-        public Error(String message)
-        {
-            this(message, 0, 0);
-        }
-
-        public Error(String message, int line, int column)
-        {
-            _message = message;
-            _line = line;
-            _column = column;
-        }
-
-        public Error(SAXParseException spe)
-        {
-            this(spe.getLocalizedMessage(), spe.getLineNumber(), spe.getColumnNumber());
-        }
-
-        public String getMessage()
-        {
-            return _message;
-        }
-
-        public int getLine()
-        {
-            return _line;
-        }
-
-        public int getColumn()
-        {
-            return _column;
-        }
-    }
-
-    public void parse(String script)
-    {
-        try
-        {
-            XmlOptions options = new XmlOptions();
-            List<XmlError> errors = new ArrayList<>();
-            options.setDocumentType(ScriptDocument.type);
-            script = StringUtils.replace(script, "<script>", "<script xmlns=\"" + ScriptDocument.type.getContentModel().getName().getNamespaceURI() + "\">");
-            ScriptDocument doc = ScriptDocument.Factory.parse(new StringReader(script), options);
-            options.setErrorListener(errors);
-
-            if (!doc.validate(options))
-            {
-                for (XmlError xmlError : errors)
-                {
-                    String message = xmlError.getMessage();
-                    message = StringUtils.replace(message, "@" + ScriptDocument.type.getContentModel().getName().getNamespaceURI(), "");
-                    String location = xmlError.getCursorLocation().xmlText();
-                    if (location.length() > 100)
-                        location = StringUtilsLabKey.leftSurrogatePairFriendly(location, 100);
-                    addError(new Error("Schema Validation Error: " + message + "\nLocation of invalid XML: " + location));
-                }
-            }
-            _script = doc.getScript();
-        }
-        catch (XmlException xmlException)
-        {
-            if (xmlException.getErrors().isEmpty())
-            {
-                addError(new Error(xmlException.toString()));
-            }
-            else
-            {
-                for (XmlError xmlError : xmlException.getErrors())
-                {
-                    addError(new Error(xmlError.getMessage(), xmlError.getLine(), xmlError.getColumn() > 0 ? xmlError.getColumn() : 1));
-                }
-            }
-        }
-        catch (Exception e)
-        {
-            addError(new Error(e.toString()));
-        }
-    }
-
-    void addError(Error error)
-    {
-        if (_errors == null)
-            _errors = new ArrayList<>();
-        _errors.add(error);
-    }
-
-    public Error[] getErrors()
-    {
-        if (_errors == null || _errors.isEmpty())
-            return null;
-        return _errors.toArray(new Error[0]);
-    }
-}
+/*
+ * Copyright (c) 2007-2019 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.flow;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.xmlbeans.XmlError;
+import org.apache.xmlbeans.XmlException;
+import org.apache.xmlbeans.XmlOptions;
+import org.fhcrc.cpas.flow.script.xml.ScriptDef;
+import org.fhcrc.cpas.flow.script.xml.ScriptDocument;
+import org.labkey.api.util.StringUtilsLabKey;
+import org.xml.sax.SAXParseException;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ScriptParser
+{
+    List<Error> _errors;
+    ScriptDef _script;
+
+    public ScriptParser()
+    {
+    }
+
+    static public class Error
+    {
+        String _message;
+        int _line;
+        int _column;
+
+        public Error(String message)
+        {
+            this(message, 0, 0);
+        }
+
+        public Error(String message, int line, int column)
+        {
+            _message = message;
+            _line = line;
+            _column = column;
+        }
+
+        public Error(SAXParseException spe)
+        {
+            this(spe.getLocalizedMessage(), spe.getLineNumber(), spe.getColumnNumber());
+        }
+
+        public String getMessage()
+        {
+            return _message;
+        }
+
+        public int getLine()
+        {
+            return _line;
+        }
+
+        public int getColumn()
+        {
+            return _column;
+        }
+    }
+
+    public void parse(String script)
+    {
+        try
+        {
+            XmlOptions options = new XmlOptions();
+            List<XmlError> errors = new ArrayList<>();
+            options.setDocumentType(ScriptDocument.type);
+            script = StringUtils.replace(script, "<script>", "<script xmlns=\"" + ScriptDocument.type.getContentModel().getName().getNamespaceURI() + "\">");
+            ScriptDocument doc = ScriptDocument.Factory.parse(new StringReader(script), options);
+            options.setErrorListener(errors);
+
+            if (!doc.validate(options))
+            {
+                for (XmlError xmlError : errors)
+                {
+                    String message = xmlError.getMessage();
+                    message = StringUtils.replace(message, "@" + ScriptDocument.type.getContentModel().getName().getNamespaceURI(), "");
+                    String location = xmlError.getCursorLocation().xmlText();
+                    if (location.length() > 100)
+                        location = StringUtilsLabKey.leftSurrogatePairFriendly(location, 100);
+                    addError(new Error("Schema Validation Error: " + message + "\nLocation of invalid XML: " + location));
+                }
+            }
+            _script = doc.getScript();
+        }
+        catch (XmlException xmlException)
+        {
+            if (xmlException.getErrors().isEmpty())
+            {
+                addError(new Error(xmlException.toString()));
+            }
+            else
+            {
+                for (XmlError xmlError : xmlException.getErrors())
+                {
+                    addError(new Error(xmlError.getMessage(), xmlError.getLine(), xmlError.getColumn() > 0 ? xmlError.getColumn() : 1));
+                }
+            }
+        }
+        catch (Exception e)
+        {
+            addError(new Error(e.toString()));
+        }
+    }
+
+    void addError(Error error)
+    {
+        if (_errors == null)
+            _errors = new ArrayList<>();
+        _errors.add(error);
+    }
+
+    public Error[] getErrors()
+    {
+        if (_errors == null || _errors.isEmpty())
+            return null;
+        return _errors.toArray(new Error[0]);
+    }
+}