feat: Added new check to base reader for sense check of whether processing a text file - applied for all reads via read_to_entity_type

Author: stevenhsd

src/dve/core_engine/backends/base/reader.py

@@ -8,9 +8,11 @@
 from pydantic import BaseModel
 from typing_extensions import Protocol
 
-from dve.core_engine.backends.exceptions import ReaderLacksEntityTypeSupport
+from dve.core_engine.backends.exceptions import MessageBearingError, ReaderLacksEntityTypeSupport
 from dve.core_engine.backends.types import EntityName, EntityType
+from dve.core_engine.message import FeedbackMessage
 from dve.core_engine.type_hints import URI, ArbitraryFunction, WrapDecorator
+from dve.parser.file_handling.service import open_stream
 
 T = TypeVar("T")
 ET_co = TypeVar("ET_co", covariant=True)
@@ -115,6 +117,8 @@ def read_to_entity_type(
         """
         if entity_name == Iterator[dict[str, Any]]:
             return self.read_to_py_iterator(resource, entity_name, schema)  # type: ignore
+        
+        self.raise_if_not_sensible_file(resource, entity_name)
 
         try:
             reader_func = self.__read_methods__[entity_type]
@@ -137,3 +141,34 @@ def write_parquet(
 
         """
         raise NotImplementedError(f"write_parquet not implemented in {self.__class__}")
+    
+    @staticmethod
+    def _check_likely_text_file(resource: URI) -> bool:
+        """Quick sense check of file to see if it looks like text
+           - not 100% full proof, but hopefully enough to weed out most
+           non-text files"""
+        with open_stream(resource, "rb") as fle:
+            start_chunk = fle.read(4096)
+        # check for BOM character - utf-16 can contain NULL bytes
+        if start_chunk.startswith((b"\xff\xfe", b"\xfe\xff")):
+            return True
+        # if null byte in - unlikely text
+        if b"\x00" in start_chunk:
+            return False
+        return True
+    
+    def raise_if_not_sensible_file(self, resource: URI, entity_name:str):
+        if not self._check_likely_text_file(resource):
+            raise MessageBearingError(
+                "The submitted file doesn't appear to be text",
+                messages=[
+                    FeedbackMessage(
+                        entity=entity_name,
+                        record=None,
+                        failure_type="submission",
+                        error_location="Whole File",
+                        error_code="MalformedFile",
+                        error_message=f"The submitted resource doesn't seem to be a valid text file",
+                    )
+                ],
+            )

src/dve/core_engine/backends/implementations/duckdb/utilities.py

@@ -54,7 +54,7 @@ def check_csv_header_expected(
     quote_char: str = '"') -> set[str]:
     """Check the header of a CSV matches the expected fields"""
     with open_stream(resource) as fle:
-        header_fields = fle.readline().replace(quote_char,"").split(delimiter)
+        header_fields = fle.readline().rstrip().replace(quote_char,"").split(delimiter)
     expected_fields = expected_schema.__fields__.keys()
     return set(expected_fields).difference(header_fields)