Merge pull request #297 from boxrhcp/fix/new-line-escape

Add escape new line option for lookups

Author: Worvast

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.0.2] - 2025-03-07
+
+### Added
+ - Added new --escapenewline or -enl option in lookup creation, edition and deletion. It will escape with
+double backslash any new line char found inside a csv field.
+
 ## [6.0.1] - 2025-02-05
 
 ### Added

devo/sender/lookup.py

@@ -46,7 +46,7 @@ class Lookup:
     DATA_TABLE = "my.lookup.data"
     CONTROL_TABLE = "my.lookup.control"
 
-    def __init__(self, name="example", historic_tag=None, con=None, delay=5, escape_quotes=False):
+    def __init__(self, name="example", historic_tag=None, con=None, delay=5, escape_quotes=False, escape_newline=False):
         if not re.match(r"^[A-Za-z0-9_]+$", name):
             raise Exception(
                 "Devo Lookup: The name of the lookup is incorrect,"
@@ -63,6 +63,7 @@ def __init__(self, name="example", historic_tag=None, con=None, delay=5, escape_
         self.name = name.replace(" ", "_")
         self.delay = delay
         self.escape_quotes = escape_quotes
+        self.escape_newline = escape_newline
 
     # Helper methods
     # --------------------------------------------------------------------------
@@ -106,7 +107,7 @@ def send_data_line(self, fields=None, delete=False, key_index=None):
         :return:
         """
         p_fields = Lookup.process_fields(
-            fields=fields, key_index=key_index, escape_quotes=self.escape_quotes
+            fields=fields, key_index=key_index, escape_quotes=self.escape_quotes, escape_newline=self.escape_newline
         )
         self.send_data(row=p_fields, delete=delete)
 
@@ -242,6 +243,7 @@ def send_csv(
                             fields=fields,
                             key_index=key_index,
                             escape_quotes=self.escape_quotes,
+                            escape_newline=self.escape_newline
                         )
                         self.send_data(
                             row=p_fields,
@@ -258,6 +260,7 @@ def send_csv(
                             fields=fields,
                             key_index=key_index,
                             escape_quotes=self.escape_quotes,
+                            escape_newline=self.escape_newline
                         )
                         self.send_data(row=p_fields)
 
@@ -388,38 +391,41 @@ def list_to_headers(headers=None, key=None, type_of_key="str", key_index=None, t
         return out
 
     @staticmethod
-    def field_to_str(field, escape_quotes=False):
+    def field_to_str(field, escape_quotes=False, escape_newline=False):
         """
         Convert one value to STR, cleaning it
         :param field: field to clean
         :param escape_quotes: whether to escape quotes in response
+        :param escape_newline: whether to escape new line char in response
         :return:
         """
-        return ",%s" % Lookup.clean_field(field, escape_quotes)
+        return ",%s" % Lookup.clean_field(field, escape_quotes, escape_newline)
 
     @staticmethod
-    def process_fields(fields=None, key_index=None, escape_quotes=False):
+    def process_fields(fields=None, key_index=None, escape_quotes=False, escape_newline=False):
         """
         Method to convert list with one row/fields to STR to send
         :param fields: fields list
         :param key_index: index of key in fields
         :param escape_quotes: whether to escape quotes in response
+        :param escape_newline: whether to escape new line char in response
         :return:
         """
         # First the key
-        out = "%s" % Lookup.clean_field(fields[key_index], escape_quotes)
+        out = "%s" % Lookup.clean_field(fields[key_index], escape_quotes, escape_newline)
 
         # The rest of the fields
         for item in fields[:key_index] + fields[key_index + 1 :]:
-            out += Lookup.field_to_str(item, escape_quotes)
+            out += Lookup.field_to_str(item, escape_quotes, escape_newline)
         return out
 
     @staticmethod
-    def clean_field(field=None, escape_quotes=False):
+    def clean_field(field=None, escape_quotes=False, escape_newline=False):
         """
         Strip and quotechar the fields
         :param str field: field for clean
         :param escape_quotes: whether to escape quotes in response
+        :param escape_newline: whether to escape new line char in response
         :return str: cleaned field
         """
         if not isinstance(field, (str, bytes)):
@@ -432,6 +438,9 @@ def clean_field(field=None, escape_quotes=False):
         if escape_quotes:
             field = field.replace('"', '""')
 
+        if escape_newline:
+            field = field.replace('\n', '\\n')
+
         return '"%s"' % field
 
     @staticmethod

devo/sender/scripts/sender_cli.py

@@ -254,6 +254,13 @@ def data(**kwargs):
     help="Escape Quotes. Default: False",
     default=False,
 )
+@click.option(
+    "--escapenewline",
+    "-enl",
+    is_flag=True,
+    help="Escape New Line char. Default: False",
+    default=False,
+)
 @click.option(
     "--no-verify-certificates",
     help="Do not Verify certificates credentials before connection",
@@ -279,6 +286,7 @@ def lookup(**kwargs):
         historic_tag=None,
         con=con,
         escape_quotes=config["escapequotes"],
+        escape_newline=config["escapenewline"]
     )
 
     if lookup.check_quotes(config["file"]):

docs/sender/lookup.md

@@ -257,3 +257,24 @@ To see an example for the lookup configuration please refer to the one shown at
 This will escape ALL double quotes in the field value by adding a second double quote to it.
 
 The default behavior is to NOT escape any double quotes.
+
+#### New line chars in the field value
+
+Any new line char inside the field value can cause trouble when creating the lookup if this is not escaped.
+An example of this case can be seen below:
+
+```python
+lookup.send_data_line(key_index=0, fields=["11", 'new lines\n must be escaped'])
+```
+
+That lookup creation will fail because that new line char quote may create null fields. To avoid this, add `"escape_newline": true` to the lookup configuration file and `escape_newline=True` to the `Lookup` constructor. Below, an example for the constructor is shown:
+
+```python
+lookup = Lookup(name=self.lookup_name, historic_tag=None, con=con, escape_newline=True)
+```
+
+To see an example for the lookup configuration please refer to the one shown at the start of the document.
+
+This will escape ALL new line chars in the field value by adding a second backslash to it.
+
+The default behavior is to NOT escape any new line chars.

tests/integration/resources/testfile_lookup_with_quotes.csv

@@ -2,4 +2,4 @@ KEY,DATA
 Key_one,Message_one
 Key_two,Message_two
 Key_three,Message_thre
-Key_four,Monitor 24" and 19
+Key_four,Monitor 24" \nand 19

tests/integration/test_sender_cli.py

@@ -386,6 +386,7 @@ def test_cli_escape_quotes(setup):
             "-lk",
             "KEY",
             "-eq",
+            "-enl",
             "--no-verify-certificates",
         ],
     )

tests/unit/test_sender_number_lookup.py

@@ -23,6 +23,13 @@ def test_process_fields_does_not_modify_arguments():
     assert fields == ["a", "b", "c"]
     assert processed_fields == '"b","a","c"'
 
+def test_process_fields_does_modify_newline():
+    fields = ["a\ns", "b", "c"]
+    processed_fields = Lookup.process_fields(fields, key_index=1, escape_newline=True)
+
+    assert fields == ["a\ns", "b", "c"]
+    assert processed_fields == '"b","a\\ns","c"'
+
 
 # Clean field
 @pytest.mark.parametrize(