Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions backend/src/baserow/contrib/database/fields/field_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -5734,19 +5734,27 @@ def run_periodic_update(
# LinkRowFields might depends on FormulaFields, but we can't update
# them here because this is only valid for FormulaFields.
continue
if dependant_field in updated_fields:
continue
if table_id not in update_collectors:
update_collectors[table_id] = FieldUpdateCollector(
dependant_field.table, update_changes_only=True
)
self._update_field_values(
dependant_field,
update_collectors[table_id],
field_cache,
via_path_to_starting_table,
)
updated_fields |= set(
update_collector.apply_updates_and_get_updated_fields(
field_cache, skip_search_updates=skip_search_updates
for collector in update_collectors.values():
updated_fields |= set(
collector.apply_updates_and_get_updated_fields(
field_cache, skip_search_updates=skip_search_updates
)
)
)

update_collector.send_force_refresh_signals_for_all_updated_tables()
for collector in update_collectors.values():
collector.send_force_refresh_signals_for_all_updated_tables()

return list(updated_fields)

Expand Down
26 changes: 14 additions & 12 deletions backend/src/baserow/contrib/database/fields/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -730,18 +730,17 @@ def cached_formula_type(self):
return FormulaHandler.get_formula_type_from_field(self)

def clear_cached_properties(self):
try:
# noinspection PyPropertyAccess
del self.cached_untyped_expression
except AttributeError:
# It has not been cached yet so nothing to deleted.
pass
try:
# noinspection PyPropertyAccess
del self.cached_formula_type
except AttributeError:
# It has not been cached yet so nothing to deleted.
pass
for attr in (
"cached_untyped_expression",
"cached_typed_internal_expression",
"cached_formula_type",
):
try:
# noinspection PyPropertyAccess
delattr(self, attr)
except AttributeError:
# It has not been cached yet so nothing to delete.
pass

def recalculate_internal_fields(self, raise_if_invalid=False, field_cache=None):
self.clear_cached_properties()
Expand Down Expand Up @@ -781,6 +780,9 @@ def save(self, *args, **kwargs):
field_cache=field_cache, raise_if_invalid=raise_if_invalid
)
super().save(*args, **kwargs)
# Keep field_cache consistent to avoid stale type info downstream. See GH #5371.
if field_cache is not None:
field_cache.cache_field(self)

def refresh_from_db(self, *args, **kwargs) -> None:
super().refresh_from_db(*args, **kwargs)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3107,8 +3107,12 @@ def to_django_expression_given_args(
args: List["WrappedExpressionWithMetadata"],
context: BaserowExpressionContext,
) -> "WrappedExpressionWithMetadata":
mode = _unwrap_literal_value(args[2].expression) or "text"
value_sql = _unwrap_literal_value(args[3].expression) or "{elem} ->> 'value'"
# Fall back to text defaults if args weren't augmented at type time.
mode = "text"
value_sql = "{elem} ->> 'value'"
if len(args) >= 4:
mode = _unwrap_literal_value(args[2].expression) or mode
value_sql = _unwrap_literal_value(args[3].expression) or value_sql
safe_index = handle_arg_being_nan(
args[1].expression,
Value(None, output_field=fields.IntegerField()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -875,6 +875,14 @@ class BaserowFormulaDateType(
"date_force_timezone",
]
nullable_option_fields = ["date_force_timezone"]
# Only `date_force_timezone` is semantically nullable; others have defaults
# applied by `construct_type_from_formula_field` for legacy/corrupt rows.
non_nullable_option_defaults = {
"date_format": "ISO",
"date_include_time": False,
"date_time_format": "24",
"date_show_tzinfo": False,
}
can_represent_date = True
can_order_by_in_array = True
can_group_by = True
Expand All @@ -897,6 +905,16 @@ def __init__(
self.date_show_tzinfo = date_show_tzinfo
self.date_force_timezone = date_force_timezone

@classmethod
def construct_type_from_formula_field(cls, formula_field):
kwargs = {}
for field_name in cls.all_fields():
value = getattr(formula_field, field_name)
if value is None and field_name in cls.non_nullable_option_defaults:
value = cls.non_nullable_option_defaults[field_name]
kwargs[field_name] = value
return cls(**kwargs)

@property
def array_index_sql(self) -> str:
cast = "::timestamptz" if self.date_include_time else "::date"
Expand Down
5 changes: 4 additions & 1 deletion backend/src/baserow/core/user/handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -938,7 +938,10 @@ def verify_email_address(self, token: str) -> User:
"""

signer = self._get_email_verification_signer()
token_data = signer.loads(token)
try:
token_data = signer.loads(token)
except BadSignature as ex:
raise InvalidVerificationToken() from ex

if datetime.fromisoformat(token_data["expires_at"]) < datetime.now(
tz=timezone.utc
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ def clean_registry_cache():

view_type_registry.get_for_class.cache_clear()
yield
view_type_registry.get_for_class.cache_clear()


@pytest.mark.django_db
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,12 @@

from baserow.contrib.database.fields.field_types import DateFieldType
from baserow.contrib.database.fields.handler import FieldHandler
from baserow.contrib.database.fields.models import DateField, TextField
from baserow.contrib.database.fields.models import DateField, FormulaField, TextField
from baserow.contrib.database.fields.registries import field_type_registry
from baserow.contrib.database.fields.utils import DeferredForeignKeyUpdater
from baserow.contrib.database.formula.types.formula_types import (
BaserowFormulaDateType,
)
from baserow.contrib.database.rows.handler import RowHandler
from baserow.contrib.database.views.handler import ViewHandler
from baserow.core.psycopg import is_psycopg3
Expand Down Expand Up @@ -840,3 +843,54 @@ def test_datetime_field_overflow(on_db_connection, data_fixture):
assert len(out) == 1

assert getattr(out[0], date_field.db_column, None) is None


def test_baserow_formula_date_type_normalizes_null_non_nullable_options():
# `FormulaField` allows every date option to be NULL. When existing rows
# violate invariants, `construct_type_from_formula_field` normalizes NULLs
# to prevent downstream crashes.
corrupt = FormulaField(
formula="TODAY()",
formula_type="date",
date_format=None,
date_include_time=None,
date_time_format=None,
date_show_tzinfo=None,
date_force_timezone=None,
version=0,
requires_refresh_after_insert=False,
nullable=True,
)

formula_type = BaserowFormulaDateType.construct_type_from_formula_field(corrupt)

assert formula_type.date_format == "ISO"
assert formula_type.date_include_time is False
assert formula_type.date_time_format == "24"
assert formula_type.date_show_tzinfo is False
assert formula_type.date_force_timezone is None


def test_baserow_formula_date_type_preserves_user_set_options():
formula_field = FormulaField(
formula="TODAY()",
formula_type="date",
date_format="EU",
date_include_time=True,
date_time_format="12",
date_show_tzinfo=True,
date_force_timezone="Europe/Amsterdam",
version=0,
requires_refresh_after_insert=False,
nullable=True,
)

formula_type = BaserowFormulaDateType.construct_type_from_formula_field(
formula_field
)

assert formula_type.date_format == "EU"
assert formula_type.date_include_time is True
assert formula_type.date_time_format == "12"
assert formula_type.date_show_tzinfo is True
assert formula_type.date_force_timezone == "Europe/Amsterdam"
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ def clean_registry_cache():

field_type_registry.get_for_class.cache_clear()
yield
field_type_registry.get_for_class.cache_clear()


def _test_can_convert_between_fields(data_fixture, field_type_to_test):
Expand Down
43 changes: 42 additions & 1 deletion backend/tests/baserow/contrib/database/field/test_field_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -363,16 +363,18 @@ def test_all_formula_that_needs_updates_are_periodically_updated(data_fixture):
table = data_fixture.create_database_table(database=database)
with freeze_time("2023-02-27 10:15"):
now_field = data_fixture.create_formula_field(
table=table, formula="now()", date_include_time=True
name="now", table=table, formula="now()", date_include_time=True
)
data_fixture.create_formula_field(
name="ref_now",
table=table,
formula=f"field('{now_field.name}')",
date_include_time=True,
)

date_field = data_fixture.create_date_field(table=table, date_include_time=True)
data_fixture.create_formula_field(
name="now_vs_date",
table=table,
formula=f"now() > field('{date_field.name}')",
date_include_time=True,
Expand Down Expand Up @@ -605,3 +607,42 @@ def test_invalid_formula_is_skipped_by_periodic_update(data_fixture):
assert bool_formula.needs_periodic_update is True

assert FormulaFieldType().get_fields_needing_periodic_update().exists() is False


@pytest.mark.django_db
def test_cross_table_dependent_formulas_update_when_multiple_tables_have_now(
data_fixture,
):
with freeze_time("2023-01-01"):
database = data_fixture.create_database_application()

table_b = data_fixture.create_database_table(database=database)
primary_b = data_fixture.create_formula_field(
table=table_b, primary=True, formula="now()"
)

table_a = data_fixture.create_database_table(database=database)
link_a_to_b = data_fixture.create_link_row_field(
table=table_a, link_row_table=table_b
)
formula_a = data_fixture.create_formula_field(
table=table_a,
formula=(f"join(datetime_format(field('{link_a_to_b.name}'), 'DD'), ',')"),
)

# Second formula with now()
now_in_a = data_fixture.create_formula_field(
table=table_a,
formula="datetime_format(now(), 'YYYY-MM-DD')",
)

row_b = RowHandler().force_create_row(None, table_b, {})
row_a = RowHandler().force_create_row(
None, table_a, {link_a_to_b.db_column: [row_b.id]}
)

with freeze_time("2023-01-02"), local_cache.context():
run_periodic_fields_updates()

row_a.refresh_from_db()
assert getattr(row_a, formula_a.db_column) == "02"
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from django.urls import reverse

import pytest
from freezegun import freeze_time
from rest_framework.status import HTTP_200_OK, HTTP_204_NO_CONTENT

from baserow.contrib.database.fields.dependencies.update_collector import (
Expand Down Expand Up @@ -2453,3 +2454,84 @@ def test_count_formula_for_link_row_field_with_file_primary_field(data_fixture):
).created_rows[0]

assert getattr(row_a, count_formula.db_column) == 2


@pytest.mark.django_db
def test_periodic_update_does_not_crash_on_outer_if_after_inner_invalidated(
data_fixture,
):
"""
Regression test for Sentry issue BASEROW-SAAS-BACKEND-5 / GH #5371.

When deleting a field referenced by a chain of formulas, the propagation
used to leave the FieldCache populated with the pre-mutation instance of
an inner formula. A later same-cascade re-type of an outer IF formula
then read the stale 'boolean' type from the cache, stayed valid as
'text', and the next periodic update crashed in BaserowIf with
"When() supports a Q object, a boolean expression, or lookups as a
condition." because the underlying column is generated as TextField
for invalid formulas.
"""
user = data_fixture.create_user()
table = data_fixture.create_database_table(user=user)

primary = data_fixture.create_text_field(
table=table, name="identifier", primary=True
)
seed = data_fixture.create_text_field(table=table, name="seed")
FieldHandler().update_field(
user=user,
field=primary,
new_type_name="formula",
formula=f"field('{seed.name}')",
)

inner_flag = data_fixture.create_formula_field(
table=table,
name="inner_flag",
formula=f"field('{primary.name}') = 'yes'",
)
inner_flag.refresh_from_db()
assert inner_flag.formula_type == "boolean"

with freeze_time("2026-01-01"):
ticker = data_fixture.create_formula_field(
table=table,
name="ticker",
formula="now()",
date_include_time=True,
)

outer_icon = data_fixture.create_formula_field(
table=table,
name="outer_icon",
formula=(
f"if(field('{inner_flag.name}'), "
f"datetime_format(field('{ticker.name}'), 'YYYY'), "
f"'no')"
),
)
outer_icon.refresh_from_db()
assert outer_icon.formula_type == "text"

FieldHandler().delete_field(user=user, field=seed)

fields_for_periodic = list(
FormulaFieldType()
.get_fields_needing_periodic_update()
.filter(table__database__workspace_id=table.database.workspace_id)
)
assert ticker in fields_for_periodic

FormulaFieldType().run_periodic_update(
fields_for_periodic,
skip_search_updates=True,
database_id=table.database_id,
)

outer_icon_after = FormulaField.objects.get(pk=outer_icon.pk)
assert outer_icon_after.formula_type == "invalid", (
f"outer IF formula was not re-typed to invalid when inner_flag "
f"became invalid. formula_type={outer_icon_after.formula_type!r}, "
f"internal_formula={outer_icon_after.internal_formula!r}"
)
Loading
Loading