Implement datetime_format() formula function (baserow#4148)

* Implement datetime_format and update examples.

* Implement the moment to python datetime format converter

* Add tests for moment.js to Python datetime formatter.

* Refactor to support timezone in the user data provider.

* Use the new Timezone argument type and validate the timezone name

Author: paljort

backend/src/baserow/contrib/automation/automation_dispatch_context.py

@@ -69,6 +69,14 @@ def clone(self, **kwargs):
     def data_provider_registry(self):
         return automation_data_provider_type_registry
 
+    def get_timezone_name(self) -> str:
+        """
+        TODO: Get the timezone from the application settings. For now, returns
+            the default of "UTC". See: https://github.com/baserow/baserow/issues/4157
+        """
+
+        return super().get_timezone_name()
+
     def _register_node_result(
         self, node: AutomationNode, dispatch_data: Dict[str, Any]
     ):

backend/src/baserow/contrib/builder/api/data_providers/serializers.py

@@ -1,8 +1,11 @@
+import pytz
 from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 
 from baserow.contrib.builder.elements.models import Element
 
+IANA_TIMEZONES = [(tz, tz) for tz in pytz.all_timezones]
+
 
 class DispatchDataSourceDataSourceContextSerializer(serializers.Serializer):
     element = serializers.PrimaryKeyRelatedField(
@@ -34,3 +37,15 @@ def validate(self, data):
                 )
 
         return data
+
+
+class DispatchDataSourceUserContextSerializer(serializers.Serializer):
+    id = serializers.IntegerField(
+        help_text="Current user id.", required=False, allow_null=True
+    )
+    timezone = serializers.ChoiceField(
+        help_text="An IANA timezone name.",
+        required=False,
+        allow_null=True,
+        choices=IANA_TIMEZONES,
+    )

backend/src/baserow/contrib/builder/data_providers/data_provider_types.py

@@ -593,10 +593,12 @@ def get_request_serializer(self):
         Returns the serializer used to parse data for this data provider.
         """
 
-        return serializers.IntegerField(
-            help_text="Current user id.", required=False, allow_null=True
+        from baserow.contrib.builder.api.data_providers.serializers import (
+            DispatchDataSourceUserContextSerializer,
         )
 
+        return DispatchDataSourceUserContextSerializer(required=False, allow_null=True)
+
     def translate_default_user_role(self, user: UserSourceUser) -> str:
         """
         Returns the translated version of the user role if it is a default role,

backend/src/baserow/contrib/builder/data_sources/builder_dispatch_context.py

@@ -127,6 +127,15 @@ def element_type(self) -> Optional[Type["ElementType"]]:
 
         return self.element.get_type()  # type: ignore
 
+    def get_timezone_name(self) -> str:
+        """
+        Returns the timezone from the user data provider.
+        """
+
+        return self.request_data.get("user", {}).get(
+            "timezone", super().get_timezone_name()
+        )
+
     def range(self, service):
         """
         Return page range from the `offset`, `count` kwargs,

backend/src/baserow/core/formula/argument_types.py

@@ -1,5 +1,9 @@
+from typing import Optional
+
 from django.core.exceptions import ValidationError
 
+import pytz
+
 from baserow.core.formula.validator import (
     ensure_boolean,
     ensure_datetime,
@@ -10,6 +14,9 @@
 
 
 class BaserowRuntimeFormulaArgumentType:
+    def __init__(self, optional: Optional[bool] = False):
+        self.optional = optional
+
     def test(self, value):
         return True
 
@@ -87,3 +94,14 @@ def test(self, value):
 
     def parse(self, value):
         return ensure_boolean(value)
+
+
+class TimezoneBaserowRuntimeFormulaArgumentType(BaserowRuntimeFormulaArgumentType):
+    def test(self, value):
+        if not isinstance(value, str):
+            return False
+
+        return value in pytz.all_timezones
+
+    def parse(self, value):
+        return ensure_string(value)

backend/src/baserow/core/formula/registries.py

@@ -84,7 +84,13 @@ def validate_number_of_args(self, args: FormulaArgs) -> bool:
         :return: If the number of arguments is correct
         """
 
-        return self.num_args is None or len(args) <= self.num_args
+        if self.num_args is None:
+            return True
+
+        required_args = len([arg for arg in self.args if not arg.optional])
+        total_args = len(self.args)
+
+        return len(args) >= required_args and len(args) <= total_args
 
     def validate_type_of_args(self, args: FormulaArgs) -> Optional[FormulaArg]:
         """

backend/src/baserow/core/formula/runtime_formula_context.py

@@ -1,5 +1,7 @@
 from typing import TYPE_CHECKING, Any
 
+from django.utils import timezone
+
 from baserow.core.formula.types import FormulaContext
 from baserow.core.utils import to_path
 
@@ -38,3 +40,12 @@ def __getitem__(self, key: str) -> Any:
             self,
             rest,
         )
+
+    def get_timezone_name(self) -> str:
+        """
+        Returns the current IANA timezone name, e.g. "Europe/Amsterdam".
+
+        Defaults to "UTC".
+        """
+
+        return timezone.get_current_timezone_name()

backend/src/baserow/core/formula/runtime_formula_types.py

@@ -15,9 +15,11 @@
     NumberBaserowRuntimeFormulaArgumentType,
     SubtractableBaserowRuntimeFormulaArgumentType,
     TextBaserowRuntimeFormulaArgumentType,
+    TimezoneBaserowRuntimeFormulaArgumentType,
 )
 from baserow.core.formula.registries import RuntimeFormulaFunction
 from baserow.core.formula.types import FormulaArg, FormulaArgs, FormulaContext
+from baserow.core.formula.utils.date import convert_date_format_moment_to_python
 from baserow.core.formula.validator import ensure_string
 
 
@@ -262,23 +264,32 @@ class RuntimeDateTimeFormat(RuntimeFormulaFunction):
     args = [
         DateTimeBaserowRuntimeFormulaArgumentType(),
         TextBaserowRuntimeFormulaArgumentType(),
-        TextBaserowRuntimeFormulaArgumentType(),
+        TimezoneBaserowRuntimeFormulaArgumentType(optional=True),
     ]
 
     def execute(self, context: FormulaContext, args: FormulaArgs):
-        tz_name = None
-        tz_format = "%Y-%m-%d %H:%M:%S"
+        datetime_obj = args[0]
+        moment_format = args[1]
+
+        if (len(args)) == 2:
+            timezone_name = context.get_timezone_name()
+        else:
+            timezone_name = args[2]
 
-        if len(args) == 3:
-            tz_name = args[2]
-            tz_format = args[1]
-        elif len(args) == 2:
-            tz_format = args[1]
+        python_format = convert_date_format_moment_to_python(moment_format)
+        result = datetime_obj.astimezone(ZoneInfo(timezone_name)).strftime(
+            python_format
+        )
 
-        if tz_name:
-            return args[0].replace(tzinfo=ZoneInfo(tz_name)).strftime(tz_format)
+        if "SSS" in moment_format:
+            # When Moment's SSS is milliseconds (3 digits), but Python's %f
+            # is microseconds (6 digits). We need to replace the microseconds
+            # with milliseconds.
+            microseconds_str = f"{datetime_obj.microsecond:06d}"
+            milliseconds_str = f"{datetime_obj.microsecond // 1000:03d}"
+            result = result.replace(microseconds_str, milliseconds_str)
 
-        return args[0].strftime(tz_format)
+        return result
 
 
 class RuntimeDay(RuntimeFormulaFunction):

backend/src/baserow/core/formula/utils/__init__.py

@@ -0,0 +1,49 @@
+import re
+
+
+def convert_date_format_moment_to_python(moment_format: str) -> str:
+    """
+    Convert a Moment.js datetime string to the Python strftime equivalent.
+
+    :param moment_format: The Moment.js format, e.g. 'YYYY-MM-DD'
+    :return: The Python datetime equivalent, e.g. '%Y-%m-%d'
+    """
+
+    format_map = {
+        "YYYY": "%Y",
+        "YY": "%y",
+        "MMMM": "%B",
+        "MMM": "%b",
+        "MM": "%m",
+        "M": "%-m",
+        "DD": "%d",
+        "D": "%-d",
+        "dddd": "%A",
+        "ddd": "%a",
+        "HH": "%H",
+        "H": "%-H",
+        "hh": "%I",
+        "h": "%-I",
+        "mm": "%M",
+        "m": "%-M",
+        "ss": "%S",
+        "s": "%-S",
+        "A": "%p",
+        "a": "%p",
+        # Moment's SSS is milliseconds, whereas Python's %f is microseconds
+        "SSS": "%f",
+    }
+
+    # Sort longest tokens first so 'MMMM' matches before 'MM', etc.
+    tokens = sorted(format_map.keys(), key=len, reverse=True)
+
+    # Build a regex, e.g. re.compile("YYYY|YY")
+    pattern = re.compile("|".join(token for token in tokens))
+
+    def replace_token(match: re.Match) -> str:
+        """Replace a matched Moment token with its Python equivalent."""
+
+        token = match.group(0)
+        return format_map[token]
+
+    return pattern.sub(replace_token, moment_format)