refactor: merging develop_v04 branch

Author: stevenhsd

pyproject.toml

@@ -39,7 +39,7 @@ requests = "2.32.4"  # Mitigates security vuln in < 2.31.0
 schedula = "1.2.19"
 sqlalchemy = "2.0.19"
 typing_extensions = "4.6.2"
-urllib3 = "2.5.0"  # Mitigates security vuln in < 1.26.19
+urllib3 = "2.6.0"  # Mitigates security vuln in < 2.5.0
 xmltodict = "0.13.0"
 
 [tool.poetry.group.dev]

src/dve/core_engine/backends/metadata/reporting.py

@@ -28,7 +28,7 @@ class BaseReportingConfig(BaseModel):
 
     """
 
-    UNTEMPLATED_FIELDS: ClassVar[set[str]] = {"message"}
+    UNTEMPLATED_FIELDS: ClassVar[set[str]] = set()
     """Fields that should not be templated."""
 
     emit: Optional[str] = None

src/dve/core_engine/templating.py

@@ -11,6 +11,16 @@
 from dve.core_engine.type_hints import JSONable, TemplateVariables
 
 
+class PreserveTemplateUndefined(jinja2.Undefined):
+    """
+    Preserve the original template in instances where the value cannot be populated. Whilst this 
+    may result in templates coming back in the FeedbackMessage object, it's more useful to know
+    exactly what should have been populated rather than just returning blank values.
+    """
+    def __str__(self):
+        return "{{" + self._undefined_name + "}}"
+
+
 class RuleTemplateError(ValueError):
     """A rule template error."""
 
@@ -21,7 +31,10 @@ def _raise_rule_templating_error(message: str) -> NoReturn:
 
 
 T = TypeVar("T", bound=JSONable)
-ENVIRONMENT = jinja2.Environment(autoescape=False)
+ENVIRONMENT = jinja2.Environment(
+    autoescape=jinja2.select_autoescape(default_for_string=False),
+    undefined=PreserveTemplateUndefined,
+)
 ENVIRONMENT.globals["repr"] = repr
 ENVIRONMENT.globals["str"] = str
 ENVIRONMENT.globals["raise"] = _raise_rule_templating_error

src/dve/pipeline/foundry_ddb_pipeline.py

@@ -50,7 +50,7 @@ def file_transformation(
             self._audit_tables.mark_failed(submissions=[submission_info.submission_id])
             return submission_info, SubmissionStatus(processing_failed=True)
 
-    def apply_data_contract(self, submission_info: SubmissionInfo, submission_status: SubmissionStatus) -> tuple[SubmissionInfo | SubmissionStatus]:
+    def apply_data_contract(self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None) -> tuple[SubmissionInfo | SubmissionStatus]:
         try:
             return super().apply_data_contract(submission_info, submission_status)
         except Exception as exc: # pylint: disable=W0718
@@ -64,7 +64,7 @@ def apply_data_contract(self, submission_info: SubmissionInfo, submission_status
             self._audit_tables.mark_failed(submissions=[submission_info.submission_id])
             return submission_info, SubmissionStatus(processing_failed=True)
 
-    def apply_business_rules(self, submission_info: SubmissionInfo, submission_status: SubmissionStatus):
+    def apply_business_rules(self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None):
         try:
             return super().apply_business_rules(submission_info, submission_status)
         except Exception as exc: # pylint: disable=W0718
@@ -78,7 +78,7 @@ def apply_business_rules(self, submission_info: SubmissionInfo, submission_statu
             self._audit_tables.mark_failed(submissions=[submission_info.submission_id])
             return submission_info, SubmissionStatus(processing_failed=True)
 
-    def error_report(self, submission_info: SubmissionInfo, submission_status: SubmissionStatus):
+    def error_report(self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None):
         try:
             return super().error_report(submission_info, submission_status)
         except Exception as exc: # pylint: disable=W0718

src/dve/reporting/utils.py

@@ -47,6 +47,10 @@ def dump_processing_errors(
     """Write out critical processing errors"""
     if not working_folder:
         raise AttributeError("processed files path not passed")
+    if not step_name:
+        raise AttributeError("step name not passed")
+    if not errors:
+        raise AttributeError("errors list not passed")
 
     error_file: URI = fh.joinuri(working_folder, "errors", f"processing_errors.json")
     processed = []