[pull] develop from baserow:develop

.env.example

@@ -100,6 +100,7 @@ DATABASE_NAME=baserow
 # BASEROW_WEBHOOK_ROWS_ENTER_VIEW_BATCH_SIZE=
 
 # BASEROW_INTEGRATIONS_ALLOW_PRIVATE_ADDRESS=
+# BASEROW_INTEGRATIONS_PERIODIC_MINUTE_MIN=
 
 # BASEROW_AIRTABLE_IMPORT_SOFT_TIME_LIMIT=
 # HOURS_UNTIL_TRASH_PERMANENTLY_DELETED=

backend/src/baserow/config/settings/base.py

@@ -1046,6 +1046,11 @@ def __setitem__(self, key, value):
     os.getenv("BASEROW_INTEGRATIONS_ALLOW_PRIVATE_ADDRESS", False)
 )
 INTEGRATIONS_PERIODIC_TASK_CRONTAB = crontab(minute="*")
+# The minimum amount of minutes the periodic task's "minute" interval
+# supports. Self-hosters can run every minute, if they choose to.
+INTEGRATIONS_PERIODIC_MINUTE_MIN = int(
+    os.getenv("BASEROW_INTEGRATIONS_PERIODIC_MINUTE_MIN", 1)
+)
 
 TOTP_ISSUER_NAME = os.getenv("BASEROW_TOTP_ISSUER_NAME", "Baserow")
 

backend/src/baserow/contrib/automation/data_providers/data_provider_types.py

@@ -37,17 +37,27 @@ def get_data_chunk(
             previous_node_results = dispatch_context.previous_nodes_results[
                 int(previous_node.id)
             ]
-            if previous_node.service.get_type().returns_list:
-                previous_node_results = previous_node_results["results"]
         except KeyError as exc:
             message = (
                 "The previous node id is not present in the dispatch context results"
             )
             raise InvalidFormulaContext(message) from exc
+
+        service = previous_node.service.specific
+
+        if service.get_type().returns_list:
+            previous_node_results = previous_node_results["results"]
+            if len(rest) >= 2:
+                prepared_path = [
+                    rest[0],
+                    *service.get_type().prepare_value_path(service, rest[1:]),
+                ]
+            else:
+                prepared_path = rest
         else:
-            return previous_node.service.get_type().get_value_at_path(
-                previous_node.service.specific, previous_node_results, rest
-            )
+            prepared_path = service.get_type().prepare_value_path(service, rest)
+
+        return get_value_at_path(previous_node_results, prepared_path)
 
     def import_path(self, path, id_mapping, **kwargs):
         """

backend/src/baserow/contrib/automation/formula_importer.py

@@ -4,6 +4,7 @@
     automation_data_provider_type_registry,
 )
 from baserow.core.formula import BaserowFormulaObject, get_parse_tree_for_formula
+from baserow.core.formula.types import BASEROW_FORMULA_MODE_RAW
 from baserow.core.services.formula_importer import BaserowFormulaImporter
 
 
@@ -20,7 +21,7 @@ def get_data_provider_type_registry(self):
 
 def import_formula(
     formula: Union[str, BaserowFormulaObject], id_mapping: Dict[str, str], **kwargs
-) -> str:
+) -> BaserowFormulaObject:
     """
     When a formula is used in an automation, it must be migrated when we import it
     because it could contain IDs referencing other objects.
@@ -32,11 +33,17 @@ def import_formula(
     :return: The updated path.
     """
 
-    # Figure out what our formula string is.
-    formula_str = formula if isinstance(formula, str) else formula["formula"]
+    formula = BaserowFormulaObject.to_formula(formula)
 
-    if not formula_str:
-        return formula_str
+    if formula["mode"] == BASEROW_FORMULA_MODE_RAW or not formula["formula"]:
+        return formula
 
-    tree = get_parse_tree_for_formula(formula_str)
-    return AutomationFormulaImporter(id_mapping, **kwargs).visit(tree)
+    tree = get_parse_tree_for_formula(formula["formula"])
+    new_formula = AutomationFormulaImporter(id_mapping, **kwargs).visit(tree)
+
+    if new_formula != formula["formula"]:
+        # We create a new instance to show it's a different formula
+        formula = dict(formula)
+        formula["formula"] = new_formula
+
+    return formula

backend/src/baserow/contrib/automation/nodes/handler.py

@@ -7,6 +7,8 @@
 from baserow.contrib.automation.automation_dispatch_context import (
     AutomationDispatchContext,
 )
+from baserow.contrib.automation.constants import IMPORT_SERIALIZED_IMPORTING
+from baserow.contrib.automation.formula_importer import import_formula
 from baserow.contrib.automation.models import AutomationWorkflow
 from baserow.contrib.automation.nodes.exceptions import (
     AutomationNodeDoesNotExist,
@@ -28,9 +30,7 @@
 from baserow.core.services.handler import ServiceHandler
 from baserow.core.services.models import Service
 from baserow.core.storage import ExportZipFile
-from baserow.core.utils import MirrorDict, extract_allowed
-
-from .signals import automation_node_updated
+from baserow.core.utils import ChildProgressBuilder, MirrorDict, extract_allowed
 
 
 class AutomationNodeHandler:
@@ -275,6 +275,7 @@ def import_nodes(
         serialized_nodes: List[AutomationNodeDict],
         id_mapping: Dict[str, Dict[int, int]],
         cache: Optional[Dict] = None,
+        progress: Optional[ChildProgressBuilder] = None,
         *args,
         **kwargs,
     ):
@@ -303,9 +304,22 @@ def import_nodes(
                 *args,
                 **kwargs,
             )
-            imported_nodes.append([node_instance, serialized_node])
+            imported_nodes.append(node_instance)
+
+            if progress:
+                progress.increment(state=IMPORT_SERIALIZED_IMPORTING)
+
+        # We migrate service formulas here to make sure all nodes are imported before
+        # we migrate them
+        for imported_node in imported_nodes:
+            service = imported_node.service.specific
+            updated_models = service.get_type().import_formulas(
+                service, id_mapping, import_formula, **kwargs
+            )
+
+            [u.save() for u in updated_models]
 
-        return [i[0] for i in imported_nodes]
+        return imported_nodes
 
     def import_node_only(
         self,
@@ -362,10 +376,6 @@ def dispatch_node(
             # Return early if this is a simulated dispatch
             if until_node := dispatch_context.simulate_until_node:
                 if until_node.id == node.id:
-                    # sample_data was updated as it's a simulation we should tell to
-                    # the frontend
-                    node.service.refresh_from_db(fields=["sample_data"])
-                    automation_node_updated.send(self, user=None, node=node)
                     return
 
             if children := node.get_children():

backend/src/baserow/contrib/automation/nodes/registries.py

@@ -5,7 +5,6 @@
 from baserow.contrib.automation.automation_dispatch_context import (
     AutomationDispatchContext,
 )
-from baserow.contrib.automation.formula_importer import import_formula
 from baserow.contrib.automation.nodes.exceptions import AutomationNodeNotReplaceable
 from baserow.contrib.automation.nodes.models import AutomationNode
 from baserow.contrib.automation.nodes.types import AutomationNodeDict, NodePositionType
@@ -204,7 +203,7 @@ def deserialize_property(
                 storage=storage,
                 cache=cache,
                 files_zip=files_zip,
-                import_formula=import_formula,
+                # We don't migrate formulas here but later after the node import
                 import_export_config=kwargs.get("import_export_config"),
             )
         return super().deserialize_property(
@@ -259,6 +258,7 @@ def prepare_values(
             # as part of creating a new node. If this happens, we need
             # to create a new service.
             service = ServiceHandler().create_service(service_type)
+
         else:
             service = instance.service.specific
 

backend/src/baserow/contrib/automation/nodes/service.py

@@ -36,6 +36,7 @@
 )
 from baserow.contrib.automation.workflows.signals import automation_workflow_updated
 from baserow.core.handler import CoreHandler
+from baserow.core.integrations.handler import IntegrationHandler
 from baserow.core.trash.handler import TrashHandler
 
 
@@ -173,6 +174,18 @@ def create_node(
 
         prepared_values = node_type.prepare_values(kwargs, user)
 
+        # Preselect first integration if exactly one exists
+        if node_type.get_service_type().integration_type:
+            integrations_of_type = [
+                i
+                for i in IntegrationHandler().get_integrations(workflow.automation)
+                if i.get_type().type == node_type.get_service_type().integration_type
+            ]
+
+            if len(integrations_of_type) == 1:
+                prepared_values["service"].integration = integrations_of_type[0]
+                prepared_values["service"].save()
+
         new_node = self.handler.create_node(
             node_type,
             workflow=workflow,

backend/src/baserow/contrib/automation/workflows/graph_handler.py

@@ -340,13 +340,19 @@ def remove(self, node_to_delete: AutomationNode, keep_info=False):
                 node_to_delete.id,
                 next_node_ids,
             )
+            if not graph[node_position_id]["next"][output]:
+                del graph[node_position_id]["next"][output]
+            if not graph[node_position_id]["next"]:
+                del graph[node_position_id]["next"]
         elif position == "child":
             next_nodes = self._get_all_next_nodes(node_to_delete)
             graph[node_position_id]["children"] = _replace(
                 graph[node_position_id]["children"],
                 node_to_delete.id,
                 next_nodes,
             )
+            if not graph[node_position_id]["children"]:
+                del graph[node_position_id]["children"]
 
         if not keep_info:
             del graph[str(node_to_delete.id)]

backend/src/baserow/contrib/automation/workflows/handler.py

@@ -440,23 +440,16 @@ def import_nodes(
 
         imported_nodes = []
 
-        for serialized_node in serialized_nodes:
-            # check that the node has not already been imported in a
-            # previous pass or if the parent doesn't exist yet.
-            imported_node = AutomationNodeHandler().import_node(
-                workflow,
-                serialized_node,
-                id_mapping,
-                import_export_config=import_export_config,
-                files_zip=files_zip,
-                storage=storage,
-                cache=cache,
-            )
-
-            imported_nodes.append(imported_node)
-
-            if progress:
-                progress.increment(state=IMPORT_SERIALIZED_IMPORTING)
+        imported_nodes = AutomationNodeHandler().import_nodes(
+            workflow,
+            serialized_nodes,
+            id_mapping,
+            import_export_config=import_export_config,
+            files_zip=files_zip,
+            storage=storage,
+            cache=cache,
+            progress=progress,
+        )
 
         return imported_nodes
 
@@ -957,3 +950,10 @@ def start_workflow(
                 history.message = history_message
                 history.status = history_status
                 history.save()
+            else:
+                # sample_data was updated as it's a simulation we should tell to
+                # the frontend
+                simulate_until_node.service.specific.refresh_from_db(
+                    fields=["sample_data"]
+                )
+                automation_node_updated.send(self, user=None, node=simulate_until_node)

backend/src/baserow/contrib/builder/api/elements/serializers.py

@@ -508,7 +508,7 @@ def to_representation(self, value):
         if not is_formula:
             # We force the type to raw as it's not a formula
             # For compat with unmigrated values.
-            value["mode"] = "raw"
+            value["mode"] = BASEROW_FORMULA_MODE_RAW
 
         return value
 

backend/src/baserow/contrib/builder/data_providers/data_provider_types.py

@@ -177,12 +177,21 @@ def get_data_chunk(self, dispatch_context: BuilderDispatchContext, path: List[st
             data_source, dispatch_context
         )
 
-        if data_source.service.get_type().returns_list:
+        service = data_source.service.specific
+
+        if service.get_type().returns_list:
             dispatch_result = dispatch_result["results"]
+            if len(rest) >= 2:
+                prepared_path = [
+                    rest[0],
+                    *service.get_type().prepare_value_path(service, rest[1:]),
+                ]
+            else:
+                prepared_path = rest
+        else:
+            prepared_path = service.get_type().prepare_value_path(service, rest)
 
-        return data_source.service.get_type().get_value_at_path(
-            data_source.service.specific, dispatch_result, rest
-        )
+        return get_value_at_path(dispatch_result, prepared_path)
 
     def import_path(self, path, id_mapping, **kwargs):
         """
@@ -484,9 +493,10 @@ def get_data_chunk(self, dispatch_context: DispatchContext, path: List[str]):
             cache_key = self.get_dispatch_action_cache_key(
                 dispatch_id, workflow_action.id
             )
-            return workflow_action.service.get_type().get_value_at_path(
-                workflow_action.service.specific, cache.get(cache_key), rest
+            prepared_path = workflow_action.service.get_type().prepare_value_path(
+                workflow_action.service.specific, rest
             )
+            return get_value_at_path(cache.get(cache_key), prepared_path)
         else:
             # Frontend actions
             return get_value_at_path(previous_action_results[previous_action_id], rest)

backend/src/baserow/contrib/builder/elements/collection_field_types.py

@@ -10,7 +10,7 @@
 from baserow.contrib.builder.workflow_actions.models import BuilderWorkflowAction
 from baserow.core.constants import RatingStyleChoices
 from baserow.core.formula.serializers import FormulaSerializerField
-from baserow.core.formula.types import BaserowFormulaObject
+from baserow.core.formula.types import BASEROW_FORMULA_MODE_RAW, BaserowFormulaObject
 from baserow.core.registry import Instance
 
 
@@ -186,15 +186,19 @@ def formula_generator(
         for index, page_parameter in enumerate(
             collection_field.config.get("page_parameters") or []
         ):
-            new_formula = yield page_parameter.get("value")
+            new_formula = yield BaserowFormulaObject.to_formula(
+                page_parameter.get("value")
+            )
             if new_formula is not None:
                 collection_field.config["page_parameters"][index]["value"] = new_formula
                 yield collection_field
 
         for index, query_parameter in enumerate(
             collection_field.config.get("query_parameters") or []
         ):
-            new_formula = yield query_parameter.get("value")
+            new_formula = yield BaserowFormulaObject.to_formula(
+                query_parameter.get("value")
+            )
             if new_formula is not None:
                 collection_field.config["query_parameters"][index][
                     "value"
@@ -266,11 +270,18 @@ def formula_generator(
 
         yield from super().formula_generator(collection_field)
 
-        if collection_field.config.get("colors_is_formula"):
-            new_formula = yield collection_field.config.get("colors", "")
-            if new_formula is not None:
-                collection_field.config["colors"] = new_formula
-                yield collection_field
+        is_formula = collection_field.config.get("colors_is_formula", False)
+        colors = BaserowFormulaObject.to_formula(
+            collection_field.config.get("colors", "")
+        )
+
+        if not is_formula:
+            colors["mode"] = BASEROW_FORMULA_MODE_RAW
+
+        new_formula = yield colors
+        if new_formula is not None:
+            collection_field.config["colors"] = new_formula
+            yield collection_field
 
 
 class ButtonCollectionFieldType(CollectionFieldType):