In local baserow integrations, allow to convert field values before they are returned (baserow#4231)

Author: jrmi

backend/src/baserow/contrib/automation/data_providers/data_provider_types.py

@@ -93,10 +93,15 @@ def get_data_chunk(
         parent_node_id, *rest = path
 
         parent_node_id = int(parent_node_id)
+        try:
+            parent_node = AutomationNodeHandler().get_node(parent_node_id)
+        except AutomationNodeDoesNotExist as exc:
+            message = "The parent node doesn't exist"
+            raise InvalidFormulaContext(message) from exc
 
         try:
             parent_node_results = dispatch_context.previous_nodes_results[
-                parent_node_id
+                parent_node.id
             ]
         except KeyError as exc:
             message = (

backend/src/baserow/contrib/builder/data_providers/data_provider_types.py

@@ -490,13 +490,15 @@ def get_data_chunk(self, dispatch_context: DispatchContext, path: List[str]):
             cache_key = self.get_dispatch_action_cache_key(
                 dispatch_id, workflow_action.id
             )
-            prepared_path = workflow_action.service.get_type().prepare_value_path(
-                workflow_action.service.specific, rest
-            )
-            return get_value_at_path(cache.get(cache_key), prepared_path)
+            dispatch_result = cache.get(cache_key)
+            service = workflow_action.service.specific
+            prepared_path = service.get_type().prepare_value_path(service, rest)
         else:
             # Frontend actions
-            return get_value_at_path(previous_action_results[previous_action_id], rest)
+            dispatch_result = previous_action_results[previous_action_id]
+            prepared_path = rest
+
+        return get_value_at_path(dispatch_result, prepared_path)
 
     def post_dispatch(
         self,

backend/src/baserow/contrib/database/fields/field_types.py

@@ -867,6 +867,19 @@ def parse_filter_value(self, field, model_field, value):
             raise ValueError(f"Invalid value for number field: {value}")
         return value
 
+    def to_runtime_formula_value(self, field, value):
+        """
+        Transform the value to be usable in runtime formula land.
+        """
+
+        if value is None or value == "":
+            return None
+
+        if field.number_decimal_places == 0:
+            return int(Decimal(value))
+
+        return float(Decimal(value))
+
 
 class RatingFieldType(FieldType):
     type = "rating"

backend/src/baserow/contrib/database/fields/registries.py

@@ -2024,6 +2024,13 @@ def get_distribution_group_by_value(self, field_name: str):
 
         return field_name
 
+    def to_runtime_formula_value(self, field, value):
+        """
+        Transform the value to be usable in runtime formula land.
+        """
+
+        return value
+
 
 class ReadOnlyFieldType(FieldType):
     read_only = True

backend/src/baserow/contrib/integrations/local_baserow/service_types.py

@@ -241,6 +241,34 @@ def prepare_value_path(self, service: Service, path: List[str]):
 
         return [human_name, *rest]
 
+    def _prepare_one_row(self, mapping, row):
+        def convert(key, value):
+            if key in mapping:
+                return mapping[key]["type"].to_runtime_formula_value(
+                    mapping[key]["field"], value
+                )
+            return value
+
+        return {key: convert(key, value) for key, value in row.items()}
+
+    def _prepare_result(self, table_model, dispatch_result):
+        mapping = {
+            field_obj["field"].name: field_obj
+            for field_obj in table_model.get_field_objects() or []
+        }
+
+        if self.returns_list:
+            return {
+                **dispatch_result,
+                "results": [
+                    self._prepare_one_row(mapping, r)
+                    for r in dispatch_result["results"]
+                ],
+            }
+
+        else:
+            return self._prepare_one_row(mapping, dispatch_result)
+
     def build_queryset(
         self,
         service: LocalBaserowTableService,
@@ -1093,13 +1121,16 @@ def dispatch_transform(self, dispatch_data: Dict[str, Any]) -> DispatchResult:
             user_field_names=True,
         )
 
-        return DispatchResult(
-            data={
+        result = self._prepare_result(
+            dispatch_data["baserow_table_model"],
+            {
                 "results": serializer(dispatch_data["results"], many=True).data,
                 "has_next_page": dispatch_data["has_next_page"],
-            }
+            },
         )
 
+        return DispatchResult(data=result)
+
     def get_record_names(
         self,
         service: LocalBaserowListRows,
@@ -1611,32 +1642,6 @@ def import_context_path(
 
         return self.import_path(path, id_mapping)
 
-    def dispatch_transform(self, dispatch_data: Dict[str, Any]) -> DispatchResult:
-        """
-        Responsible for serializing the `dispatch_data` row.
-
-        :param dispatch_data: The `dispatch_data` result.
-        :return:
-        """
-
-        field_ids = (
-            extract_field_ids_from_list(dispatch_data["public_allowed_properties"])
-            if isinstance(dispatch_data["public_allowed_properties"], list)
-            else None
-        )
-
-        serializer = get_row_serializer_class(
-            dispatch_data["baserow_table_model"],
-            RowSerializer,
-            is_response=True,
-            field_ids=field_ids,
-            user_field_names=True,
-        )
-
-        serialized_row = serializer(dispatch_data["data"]).data
-
-        return DispatchResult(data=serialized_row)
-
     def dispatch_data(
         self,
         service: LocalBaserowGetRow,
@@ -1683,6 +1688,34 @@ def dispatch_data(
         except table_model.DoesNotExist:
             raise DoesNotExist()
 
+    def dispatch_transform(self, dispatch_data: Dict[str, Any]) -> DispatchResult:
+        """
+        Responsible for serializing the `dispatch_data` row.
+
+        :param dispatch_data: The `dispatch_data` result.
+        :return:
+        """
+
+        field_ids = (
+            extract_field_ids_from_list(dispatch_data["public_allowed_properties"])
+            if isinstance(dispatch_data["public_allowed_properties"], list)
+            else None
+        )
+
+        serializer = get_row_serializer_class(
+            dispatch_data["baserow_table_model"],
+            RowSerializer,
+            is_response=True,
+            field_ids=field_ids,
+            user_field_names=True,
+        )
+
+        serialized_row = self._prepare_result(
+            dispatch_data["baserow_table_model"], serializer(dispatch_data["data"]).data
+        )
+
+        return DispatchResult(data=serialized_row)
+
 
 class LocalBaserowUpsertRowServiceType(
     LocalBaserowTableServiceSpecificRowMixin, LocalBaserowTableServiceType
@@ -2123,7 +2156,10 @@ def dispatch_transform(self, dispatch_data: Dict[str, Any]) -> DispatchResult:
             field_ids=field_ids,
             user_field_names=True,
         )
-        serialized_row = serializer(dispatch_data["data"]).data
+
+        serialized_row = self._prepare_result(
+            dispatch_data["baserow_table_model"], serializer(dispatch_data["data"]).data
+        )
 
         return DispatchResult(data=serialized_row)
 
@@ -2296,6 +2332,8 @@ def get_data():
                 "has_next_page": False,
             }
 
+        data_to_process = self._prepare_result(table.get_model(), data_to_process)
+
         self._process_event(
             self.model_class.objects.filter(table=table),
             get_data,

backend/src/baserow/core/formula/parser/python_executor.py

@@ -1,5 +1,3 @@
-from decimal import Decimal
-
 from baserow.core.formula import BaserowFormula, BaserowFormulaVisitor
 from baserow.core.formula.parser.exceptions import (
     BaserowFormulaSyntaxError,
@@ -31,7 +29,7 @@ def visitStringLiteral(self, ctx: BaserowFormula.StringLiteralContext):
         return self.process_string(ctx)
 
     def visitDecimalLiteral(self, ctx: BaserowFormula.DecimalLiteralContext):
-        return Decimal(ctx.getText())
+        return float(ctx.getText())
 
     def visitBooleanLiteral(self, ctx: BaserowFormula.BooleanLiteralContext):
         return ctx.TRUE() is not None

backend/src/baserow/core/formula/validator.py

@@ -122,6 +122,9 @@ def ensure_string(value: Any, allow_empty: bool = True) -> str:
             raise ValidationError("A valid String is required.")
         return ""
 
+    if isinstance(value, bool):
+        # To match the frontend
+        return "true" if value else "false"
     if isinstance(value, list):
         results = [ensure_string(item) for item in value if item]
         return ",".join(results)

backend/src/baserow/core/services/registries.py

@@ -312,6 +312,10 @@ def resolve_service_formulas(
         return resolved_values
 
     def prepare_value_path(self, service: Service, path: List[str]):
+        """
+        Allow to change the path inside a service before it's used.
+        """
+
         return path
 
     def dispatch_transform(

backend/tests/baserow/contrib/automation/nodes/test_node_dispatch.py

@@ -5,6 +5,7 @@
 )
 from baserow.contrib.automation.nodes.handler import AutomationNodeHandler
 from baserow.contrib.automation.workflows.constants import WorkflowState
+from baserow.contrib.database.rows.handler import RowHandler
 
 
 @pytest.mark.django_db
@@ -42,6 +43,71 @@ def test_run_workflow_with_create_row_action(data_fixture):
     assert dispatch_context.dispatch_history == [trigger.id, action_node.id]
 
 
+@pytest.mark.django_db(transaction=True)
+def test_run_workflow_with_create_row_action_and_advanced_formula(data_fixture):
+    user = data_fixture.create_user()
+    workspace = data_fixture.create_workspace(user=user)
+    integration = data_fixture.create_local_baserow_integration(user=user)
+    database = data_fixture.create_database_application(workspace=workspace)
+
+    trigger_table, trigger_table_fields, _ = data_fixture.build_table(
+        user=user,
+        columns=[
+            ("Food", "text"),
+            ("Spiciness", "number"),
+        ],
+        rows=[
+            ["Paneer Tikka", 5],
+            ["Gobi Manchurian", 8],
+        ],
+    )
+
+    action_table, action_table_fields, action_rows = data_fixture.build_table(
+        database=database,
+        user=user,
+        columns=[("Name", "text")],
+        rows=[],
+    )
+    workflow = data_fixture.create_automation_workflow(user, state="live")
+    trigger = workflow.get_trigger()
+    trigger_service = trigger.service.specific
+    trigger_service.table = trigger_table
+    trigger_service.integration = integration
+    trigger_service.save()
+    action_node = data_fixture.create_local_baserow_create_row_action_node(
+        workflow=workflow,
+        service=data_fixture.create_local_baserow_upsert_row_service(
+            table=action_table,
+            integration=integration,
+        ),
+    )
+    action_node.service.field_mappings.create(
+        field=action_table_fields[0],
+        value=f"concat('The comparaison is ', "
+        f"get('previous_node.{trigger.id}.0.{trigger_table_fields[1].db_column}') > 7)",
+    )
+
+    action_table_model = action_table.get_model()
+    assert action_table_model.objects.count() == 0
+
+    # Triggers a row creation
+    RowHandler().create_rows(
+        user=user,
+        table=trigger_table,
+        model=trigger_table.get_model(),
+        rows_values=[
+            {
+                trigger_table_fields[0].db_column: "Spice",
+                trigger_table_fields[1].db_column: "4.14",
+            },
+        ],
+        skip_search_update=True,
+    )
+
+    row = action_table_model.objects.first()
+    assert getattr(row, action_table_fields[0].db_column) == "The comparaison is false"
+
+
 @pytest.mark.django_db
 def test_run_workflow_with_update_row_action(data_fixture):
     user = data_fixture.create_user()

backend/tests/baserow/contrib/builder/api/data_sources/test_data_source_views.py

@@ -2385,13 +2385,13 @@ def test_private_dispatch_data_source_view_returns_all_fields(api_client, data_f
                 # Although only field_1 is explicitly used by an element in this
                 # page, field_2 is still returned because the Editor page needs
                 # access to all data source fields.
-                fields[1].name: "5",
+                fields[1].name: 5,
                 "id": AnyInt(),
                 "order": AnyStr(),
             },
             {
                 fields[0].name: "Gobi Manchurian",
-                fields[1].name: "8",
+                fields[1].name: 8,
                 "id": AnyInt(),
                 "order": AnyStr(),
             },