diff --git a/backend/src/baserow/api/authentication.py b/backend/src/baserow/api/authentication.py index d0ac255d8f..6166abccb1 100644 --- a/backend/src/baserow/api/authentication.py +++ b/backend/src/baserow/api/authentication.py @@ -9,6 +9,7 @@ from baserow.api.user.errors import ERROR_INVALID_ACCESS_TOKEN from baserow.core.sentry import setup_user_in_sentry from baserow.core.telemetry.utils import setup_user_in_baggage_and_spans +from baserow.core.user.cache import get_cached_user, set_cached_user from baserow.core.user.exceptions import DeactivatedUserException from .sessions import set_user_session_data_from_request @@ -25,14 +26,23 @@ def get_user(self, validated_token): except KeyError: raise InvalidToken(_("Token contained no recognizable user identification")) + cached = get_cached_user(user_id) + if cached is not None: + return cached + try: - user = self.user_model.objects.select_related("profile").get( - **{jwt_settings.USER_ID_FIELD: user_id} + # defer the password so we don't cache it + user = ( + self.user_model.objects.select_related("profile") + .defer("password") + .get(**{jwt_settings.USER_ID_FIELD: user_id}) ) except self.user_model.DoesNotExist: raise exceptions.AuthenticationFailed( _("User not found"), code="user_not_found" ) + + set_cached_user(user) return user def authenticate(self, request): diff --git a/backend/src/baserow/api/exceptions.py b/backend/src/baserow/api/exceptions.py index e7f96f78eb..c70cc8dcd8 100644 --- a/backend/src/baserow/api/exceptions.py +++ b/backend/src/baserow/api/exceptions.py @@ -1,7 +1,25 @@ from django.conf import settings +from django.http import JsonResponse from rest_framework import status -from rest_framework.exceptions import APIException, ValidationError +from rest_framework.exceptions import APIException, Throttled, ValidationError + + +def api_exception_to_json_response(exc: APIException) -> JsonResponse: + """ + Serialize a DRF ``APIException`` the same way DRF's default + ``exception_handler`` does, for use in Django middleware that runs before + the DRF view and therefore outside the handler's reach. + """ + + detail = exc.detail + data = detail if isinstance(detail, (list, dict)) else {"detail": str(detail)} + response = JsonResponse(data, status=exc.status_code, safe=False) + if getattr(exc, "auth_header", None): + response["WWW-Authenticate"] = exc.auth_header + if getattr(exc, "wait", None): + response["Retry-After"] = "%d" % exc.wait + return response class RequestBodyValidationException(APIException): @@ -26,6 +44,10 @@ def __init__(self, detail=None, code=None): self.status_code = 400 +class ThrottledAPIException(Throttled): + pass + + class InvalidClientSessionIdAPIException(APIException): status_code = status.HTTP_400_BAD_REQUEST default_code = "ERROR_INVALID_CLIENT_SESSION_ID" diff --git a/backend/src/baserow/api/two_factor_auth/views.py b/backend/src/baserow/api/two_factor_auth/views.py index 8446a97323..ad85e282a2 100644 --- a/backend/src/baserow/api/two_factor_auth/views.py +++ b/backend/src/baserow/api/two_factor_auth/views.py @@ -49,8 +49,9 @@ TOTPAuthProviderType, two_factor_auth_type_registry, ) -from baserow.throttling import RateLimitExceededException, rate_limit -from baserow.throttling_types import RateLimit +from baserow.throttling.exceptions import RateLimitExceededException +from baserow.throttling.handler import rate_limit +from baserow.throttling.types import RateLimit class ConfigureTwoFactorAuthView(APIView): diff --git a/backend/src/baserow/api/workspaces/invitations/errors.py b/backend/src/baserow/api/workspaces/invitations/errors.py index ea38c57e3e..93767e2c2b 100644 --- a/backend/src/baserow/api/workspaces/invitations/errors.py +++ b/backend/src/baserow/api/workspaces/invitations/errors.py @@ -10,9 +10,3 @@ HTTP_400_BAD_REQUEST, "Your email address does not match with the invitation.", ) -ERROR_MAX_NUMBER_OF_PENDING_WORKSPACE_INVITES_REACHED = ( - "ERROR_MAX_NUMBER_OF_PENDING_WORKSPACE_INVITES_REACHED", - HTTP_400_BAD_REQUEST, - "The maximum number of pending invites for this workspace has been reached. " - "Please wait for some invitees to accept the invite or cancel the existing ones.", -) diff --git a/backend/src/baserow/api/workspaces/invitations/serializers.py b/backend/src/baserow/api/workspaces/invitations/serializers.py index e2a7f54e05..86227b9738 100644 --- a/backend/src/baserow/api/workspaces/invitations/serializers.py +++ b/backend/src/baserow/api/workspaces/invitations/serializers.py @@ -13,7 +13,6 @@ class Meta: "workspace", "email", "permissions", - "message", "created_on", ) extra_kwargs = {"id": {"read_only": True}} @@ -28,7 +27,7 @@ class CreateWorkspaceInvitationSerializer(serializers.ModelSerializer): class Meta: model = WorkspaceInvitation - fields = ("email", "permissions", "message", "base_url") + fields = ("email", "permissions", "base_url") class UpdateWorkspaceInvitationSerializer(serializers.ModelSerializer): @@ -54,13 +53,11 @@ class Meta: "invited_by", "workspace", "email", - "message", "created_on", "email_exists", ) extra_kwargs = { "id": {"read_only": True}, - "message": {"read_only": True}, "created_on": {"read_only": True}, } diff --git a/backend/src/baserow/api/workspaces/invitations/views.py b/backend/src/baserow/api/workspaces/invitations/views.py index 4a3f4e74d5..7ab9c21c49 100755 --- a/backend/src/baserow/api/workspaces/invitations/views.py +++ b/backend/src/baserow/api/workspaces/invitations/views.py @@ -26,7 +26,6 @@ from baserow.api.workspaces.invitations.errors import ( ERROR_GROUP_INVITATION_DOES_NOT_EXIST, ERROR_GROUP_INVITATION_EMAIL_MISMATCH, - ERROR_MAX_NUMBER_OF_PENDING_WORKSPACE_INVITES_REACHED, ) from baserow.api.workspaces.serializers import WorkspaceUserWorkspaceSerializer from baserow.api.workspaces.users.errors import ERROR_GROUP_USER_ALREADY_EXISTS @@ -40,7 +39,6 @@ ) from baserow.core.exceptions import ( BaseURLHostnameNotAllowed, - MaxNumberOfPendingWorkspaceInvitesReached, UserInvalidWorkspacePermissionsError, UserNotInWorkspace, WorkspaceDoesNotExist, @@ -68,10 +66,9 @@ class WorkspaceInvitationsView(APIView, SortableViewMixin, SearchableViewMixin): permission_classes = (IsAuthenticated,) - search_fields = ["email", "message"] + search_fields = ["email"] sort_field_mapping = { "email": "email", - "message": "message", } @extend_schema( @@ -157,7 +154,6 @@ def get(self, request, workspace_id, query_params): "ERROR_USER_NOT_IN_GROUP", "ERROR_USER_INVALID_GROUP_PERMISSIONS", "ERROR_REQUEST_BODY_VALIDATION", - "ERROR_MAX_NUMBER_OF_PENDING_WORKSPACE_INVITES_REACHED", ] ), 404: get_error_schema(["ERROR_GROUP_DOES_NOT_EXIST"]), @@ -172,7 +168,6 @@ def get(self, request, workspace_id, query_params): UserInvalidWorkspacePermissionsError: ERROR_USER_INVALID_GROUP_PERMISSIONS, WorkspaceUserAlreadyExists: ERROR_GROUP_USER_ALREADY_EXISTS, BaseURLHostnameNotAllowed: ERROR_HOSTNAME_IS_NOT_ALLOWED, - MaxNumberOfPendingWorkspaceInvitesReached: ERROR_MAX_NUMBER_OF_PENDING_WORKSPACE_INVITES_REACHED, } ) def post(self, request, data, workspace_id): @@ -187,7 +182,6 @@ def post(self, request, data, workspace_id): data["email"], data["permissions"], data["base_url"], - data.get("message", ""), ) return Response(WorkspaceInvitationSerializer(workspace_invitation).data) diff --git a/backend/src/baserow/config/settings/base.py b/backend/src/baserow/config/settings/base.py index 3cc8755629..1f36577853 100644 --- a/backend/src/baserow/config/settings/base.py +++ b/backend/src/baserow/config/settings/base.py @@ -24,7 +24,7 @@ try_int, ) from baserow.core.telemetry.utils import otel_is_enabled -from baserow.throttling_types import RateLimit +from baserow.throttling.types import RateLimit from baserow.version import VERSION # A comma separated list of feature flags used to enable in-progress or not ready @@ -283,11 +283,18 @@ DATABASE_READ_REPLICAS.append(db_key) -# Enable connection health checks for all database connections. This makes Django -# verify that a database connection is still usable before each request/task, which -# prevents "connection already closed" errors when connections are dropped by the -# server, a load balancer, or a connection pooler. +# Default 0 = new connection per request; each runs a locale-setting query. +# Increase in WSGI to save those round-trips. In ASGI be careful: async tasks +# open their own connections and persistent ones can exhaust the pool. +BASEROW_CONN_MAX_AGE = int(os.getenv("BASEROW_CONN_MAX_AGE", 0)) + +# Apply the configured connection reuse timeout consistently to every database. +# Also enable connection health checks by default so Django verifies that a +# connection is still usable before each request/task, which prevents +# "connection already closed" errors when connections are dropped by the server, +# a load balancer, or a connection pooler. for _db_key in DATABASES: + DATABASES[_db_key]["CONN_MAX_AGE"] = BASEROW_CONN_MAX_AGE DATABASES[_db_key].setdefault("CONN_HEALTH_CHECKS", True) DATABASE_ROUTERS = ["baserow.config.db_routers.ReadReplicaRouter"] @@ -401,30 +408,42 @@ "DEFAULT_SCHEMA_CLASS": "baserow.api.openapi.AutoSchema", } -# Limits the number of concurrent requests per user. -# If BASEROW_MAX_CONCURRENT_USER_REQUESTS is not set, then the default value of -1 -# will be used which means the throttling is disabled. +# Throttling / rate-limiting — see docs/installation/configuration.md BASEROW_MAX_CONCURRENT_USER_REQUESTS = int( os.getenv("BASEROW_MAX_CONCURRENT_USER_REQUESTS", "") or -1 ) +BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT = int( + os.getenv("BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT", 180) +) +BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS = int( + os.getenv("BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS", "") or -1 +) +BASEROW_THROTTLE_IP_ENABLED = str_to_bool(os.getenv("BASEROW_THROTTLE_IP_ENABLED", "")) if BASEROW_MAX_CONCURRENT_USER_REQUESTS > 0: REST_FRAMEWORK["DEFAULT_THROTTLE_CLASSES"] = [ - "baserow.throttling.ConcurrentUserRequestsThrottle", + "baserow.throttling.handler.ConcurrentUserRequestsThrottle", ] REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"] = { "concurrent_user_requests": BASEROW_MAX_CONCURRENT_USER_REQUESTS } + if BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS > 0: + # Insert after SecurityMiddleware so 429s still get security/CORS headers. + _security_idx = MIDDLEWARE.index( + "django.middleware.security.SecurityMiddleware" + ) + MIDDLEWARE.insert( + _security_idx + 1, + "baserow.throttling.middleware.ThrottleBlacklistMiddleware", + ) + MIDDLEWARE += [ - "baserow.middleware.ConcurrentUserRequestsMiddleware", + "baserow.throttling.middleware.ConcurrentUserRequestsMiddleware", ] -# The maximum number of seconds that a request can be throttled for. -BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT = int( - os.getenv("BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT", 30) -) +BASEROW_CACHE_TTL_SECONDS = int(os.getenv("BASEROW_CACHE_TTL_SECONDS", 0)) PUBLIC_VIEW_AUTHORIZATION_HEADER = "Baserow-View-Authorization" @@ -1186,12 +1205,6 @@ def __setitem__(self, key, value): BASEROW_USER_LOG_ENTRY_RETENTION_DAYS = int( os.getenv("BASEROW_USER_LOG_ENTRY_RETENTION_DAYS", 61) ) -# The maximum number of pending invites that a workspace can have. If `0` then -# unlimited invites are allowed, which is the default value. -BASEROW_MAX_PENDING_WORKSPACE_INVITES = int( - os.getenv("BASEROW_MAX_PENDING_WORKSPACE_INVITES", 0) -) - BASEROW_IMPORT_EXPORT_RESOURCE_CLEANUP_INTERVAL_MINUTES = int( os.getenv("BASEROW_IMPORT_EXPORT_RESOURCE_CLEANUP_INTERVAL_MINUTES", 5) ) @@ -1255,6 +1268,12 @@ def __setitem__(self, key, value): "level": BASEROW_BACKEND_DATABASE_LOG_LEVEL, "propagate": True, }, + # Default to ERROR to suppress 429 spam under heavy throttling. + "django.request": { + "handlers": ["console"], + "level": os.getenv("BASEROW_DJANGO_REQUEST_LOG_LEVEL", "ERROR"), + "propagate": False, + }, }, "root": { "handlers": ["console"], diff --git a/backend/src/baserow/config/settings/test.py b/backend/src/baserow/config/settings/test.py index efce073b52..0b04644780 100644 --- a/backend/src/baserow/config/settings/test.py +++ b/backend/src/baserow/config/settings/test.py @@ -112,6 +112,17 @@ def getenv_for_tests(key: str, default: str = "") -> str: # Look into tests.baserow.api.test_api_utils.py if you need to test the throttle REST_FRAMEWORK["DEFAULT_THROTTLE_CLASSES"] = [] +# Disable object caches (users, tokens, settings, licenses) so every request +# hits the DB. This ensures query-count assertions remain stable and predictable. +BASEROW_CACHE_TTL_SECONDS = 0 + +# Tests should not inherit the anonymous IP throttle from any local env. +BASEROW_THROTTLE_IP_ENABLED = False + +# Default TTL used by tests that call blacklist_token / blacklist_ip without +# specifying one explicitly. Individual tests can still override this. +BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS = 10 + BUILDER_PUBLICLY_USED_PROPERTIES_CACHE_TTL_SECONDS = 10 BUILDER_DISPATCH_ACTION_CACHE_TTL_SECONDS = 300 diff --git a/backend/src/baserow/contrib/database/apps.py b/backend/src/baserow/contrib/database/apps.py index 942f403fda..c4d07b90bd 100755 --- a/backend/src/baserow/contrib/database/apps.py +++ b/backend/src/baserow/contrib/database/apps.py @@ -1181,6 +1181,7 @@ def ready(self): import baserow.contrib.database.search.receivers # noqa: F403, F401 import baserow.contrib.database.search.tasks # noqa: F401 import baserow.contrib.database.table.receivers # noqa: F401 + import baserow.contrib.database.tokens.receivers # noqa: F401 import baserow.contrib.database.views.receivers # noqa: F401 import baserow.contrib.database.views.tasks # noqa: F401 from baserow.contrib.database.fields.models import SelectOption diff --git a/backend/src/baserow/contrib/database/fields/models.py b/backend/src/baserow/contrib/database/fields/models.py index e6dda57587..f311f6a136 100644 --- a/backend/src/baserow/contrib/database/fields/models.py +++ b/backend/src/baserow/contrib/database/fields/models.py @@ -3,6 +3,7 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.postgres.fields import ArrayField +from django.core.exceptions import ObjectDoesNotExist from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from django.utils.functional import cached_property @@ -809,10 +810,15 @@ def save(self, *args, **kwargs): from baserow.contrib.database.formula.ast.function_defs import BaserowCount from baserow.contrib.database.formula.ast.tree import BaserowFieldReference - field_reference = BaserowFieldReference( - getattr(self.through_field, "name", ""), None, None - ) - self.formula = f"{BaserowCount.type}({field_reference})" + try: + through_field = getattr(self, "through_field") + field_name = getattr(through_field, "name", "") + except ObjectDoesNotExist: + field_name = "'invalid through'" + + field_ref = BaserowFieldReference(field_name, None, None) + self.formula = f"{BaserowCount.type}({field_ref})" + super().save(*args, **kwargs) def __str__(self): @@ -852,13 +858,23 @@ def save(self, *args, **kwargs): formula_function_registry, ) + try: + through_field = getattr(self, "through_field") + through_name = getattr(through_field, "name", "") + except ObjectDoesNotExist: + self.through_field = None + through_name = "'invalid through'" + + try: + target_field = getattr(self, "target_field") + target_name = getattr(target_field, "name", "") + except ObjectDoesNotExist: + self.target_field = None + target_name = "'invalid target'" + formula_function = formula_function_registry.get(self.rollup_function) - field_reference = BaserowFieldReference( - getattr(self.through_field, "name", ""), - getattr(self.target_field, "name", ""), - None, - ) - self.formula = f"{formula_function.type}({field_reference})" + field_ref = BaserowFieldReference(through_name, target_name, None) + self.formula = f"{formula_function.type}({field_ref})" super().save(*args, **kwargs) def __str__(self): diff --git a/backend/src/baserow/contrib/database/migrations/0206_rowhistory_database_ro_action__6ea699_idx.py b/backend/src/baserow/contrib/database/migrations/0206_rowhistory_database_ro_action__6ea699_idx.py index f0a4ba6a3d..f5877033aa 100644 --- a/backend/src/baserow/contrib/database/migrations/0206_rowhistory_database_ro_action__6ea699_idx.py +++ b/backend/src/baserow/contrib/database/migrations/0206_rowhistory_database_ro_action__6ea699_idx.py @@ -1,7 +1,6 @@ # Generated by Django 5.2.12 on 2026-03-17 09:16 from django.db import migrations, models -from django.contrib.postgres.operations import AddIndexConcurrently class Migration(migrations.Migration): @@ -12,8 +11,14 @@ class Migration(migrations.Migration): ] operations = [ - AddIndexConcurrently( - model_name='rowhistory', - index=models.Index(fields=['action_timestamp'], name='database_ro_action__6ea699_idx'), + migrations.RunSQL( + sql='CREATE INDEX CONCURRENTLY IF NOT EXISTS "database_ro_action__6ea699_idx" ON "database_rowhistory" ("action_timestamp")', + reverse_sql='DROP INDEX IF EXISTS "database_ro_action__6ea699_idx"', + state_operations=[ + migrations.AddIndex( + model_name='rowhistory', + index=models.Index(fields=['action_timestamp'], name='database_ro_action__6ea699_idx'), + ), + ], ), ] diff --git a/backend/src/baserow/contrib/database/tokens/cache.py b/backend/src/baserow/contrib/database/tokens/cache.py new file mode 100644 index 0000000000..46a1e8cf6f --- /dev/null +++ b/backend/src/baserow/contrib/database/tokens/cache.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING + +from django.conf import settings +from django.core.cache import cache + +if TYPE_CHECKING: + from baserow.contrib.database.tokens.models import Token + +_KEY_PREFIX = "db_token:" + + +def _cache_key(token_key: str) -> str: + # Hash the token key so raw API tokens don't sit in Redis cache keys. + digest = hashlib.sha256(token_key.encode("utf-8")).hexdigest() + return f"{_KEY_PREFIX}{digest}" + + +def get_cached_token(token_key: str) -> Token | None: + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return None + + token = cache.get(_cache_key(token_key)) + if token is not None: + # The key is not stored for security reasons, so we set it here. + token.key = token_key + return token + + +def set_cached_token(token: Token, ttl: int | None = None) -> None: + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return + + # Don't store the key in the cache for security reasons. + token_key = token.key + token.key = "" + + cache.set( + _cache_key(token_key), + token, + timeout=ttl or settings.BASEROW_CACHE_TTL_SECONDS, + ) + + # Restore the key on the token instance after caching. + token.key = token_key + + +def invalidate_cached_token(token_key: str) -> None: + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return + cache.delete(_cache_key(token_key)) diff --git a/backend/src/baserow/contrib/database/tokens/handler.py b/backend/src/baserow/contrib/database/tokens/handler.py index 5faa32aa0c..e28a0ba014 100644 --- a/backend/src/baserow/contrib/database/tokens/handler.py +++ b/backend/src/baserow/contrib/database/tokens/handler.py @@ -1,10 +1,17 @@ from typing import List, Union +from django.db import transaction + from rest_framework.request import Request from baserow.contrib.database.exceptions import DatabaseDoesNotBelongToGroup from baserow.contrib.database.models import Database, Table from baserow.contrib.database.table.exceptions import TableDoesNotBelongToGroup +from baserow.contrib.database.tokens.cache import ( + get_cached_token, + invalidate_cached_token, + set_cached_token, +) from baserow.contrib.database.tokens.constants import ( TOKEN_OPERATION_TYPES, TOKEN_TO_OPERATION_MAP, @@ -41,11 +48,20 @@ def get_by_key(self, key): :rtype: Token """ + cached = get_cached_token(key) + if cached is not None: + return cached + try: - token = Token.objects.select_related("workspace", "user").get(key=key) + token = ( + Token.objects.select_related("workspace", "user__profile") + .defer("user__password") + .get(key=key) + ) except Token.DoesNotExist: raise TokenDoesNotExist(f"The token with key {key} does not exist.") + set_cached_token(token) return token def get_token(self, user, token_id, base_queryset=None): @@ -161,9 +177,15 @@ def rotate_token_key(self, user, token): "The user is not authorized to rotate the key." ) + old_key = token.key + token.key = self.generate_unique_key() token.save() + # The post_save signal invalidates the new key; the old one still points + # to the now-stale cached token and must be evicted separately. + transaction.on_commit(lambda: invalidate_cached_token(old_key)) + return token def update_token(self, user, token, name): diff --git a/backend/src/baserow/contrib/database/tokens/receivers.py b/backend/src/baserow/contrib/database/tokens/receivers.py new file mode 100644 index 0000000000..3198491311 --- /dev/null +++ b/backend/src/baserow/contrib/database/tokens/receivers.py @@ -0,0 +1,38 @@ +from django.conf import settings +from django.db import transaction +from django.db.models.signals import post_delete, post_save +from django.dispatch import receiver + +from baserow.contrib.database.tokens.cache import invalidate_cached_token +from baserow.contrib.database.tokens.models import Token +from baserow.core.models import UserProfile + + +@receiver(post_save, sender=Token, dispatch_uid="db_token_cache_save") +def invalidate_db_token_cache_on_save(sender, instance, **kwargs): + transaction.on_commit(lambda: invalidate_cached_token(instance.key)) + + +@receiver(post_delete, sender=Token, dispatch_uid="db_token_cache_delete") +def invalidate_db_token_cache_on_delete(sender, instance, **kwargs): + transaction.on_commit(lambda: invalidate_cached_token(instance.key)) + + +def _invalidate_tokens_of_user(user_id: int) -> None: + keys = Token.objects.filter(user_id=user_id).values_list("key", flat=True) + for key in keys: + invalidate_cached_token(key) + + +@receiver( + post_save, + sender=settings.AUTH_USER_MODEL, + dispatch_uid="db_token_cache_user_save", +) +def invalidate_db_token_cache_on_user_save(sender, instance, **kwargs): + transaction.on_commit(lambda: _invalidate_tokens_of_user(instance.id)) + + +@receiver(post_save, sender=UserProfile, dispatch_uid="db_token_cache_profile_save") +def invalidate_db_token_cache_on_profile_save(sender, instance, **kwargs): + transaction.on_commit(lambda: _invalidate_tokens_of_user(instance.user_id)) diff --git a/backend/src/baserow/core/actions.py b/backend/src/baserow/core/actions.py index 172f763d7f..67af4753dd 100755 --- a/backend/src/baserow/core/actions.py +++ b/backend/src/baserow/core/actions.py @@ -817,7 +817,6 @@ def do( email: str, permissions: str, base_url: str, - message: str = "", ) -> WorkspaceInvitation: """ Creates a new workspace invitation for the given email address and sends out an @@ -829,7 +828,7 @@ def do( """ workspace_invitation = CoreHandler().create_workspace_invitation( - user, workspace, email, permissions, base_url, message + user, workspace, email, permissions, base_url ) cls.register_action( diff --git a/backend/src/baserow/core/apps.py b/backend/src/baserow/core/apps.py index caa9bd2187..6ef98cbc3e 100755 --- a/backend/src/baserow/core/apps.py +++ b/backend/src/baserow/core/apps.py @@ -491,6 +491,7 @@ def ready(self): if settings.SENTRY_DSN: patch_user_model_str() + import baserow.core.receivers # noqa: F401 from baserow.core.telemetry.telemetry import setup_logging setup_logging() diff --git a/backend/src/baserow/core/cache.py b/backend/src/baserow/core/cache.py index 9bf0ff4d40..4a73799c9a 100644 --- a/backend/src/baserow/core/cache.py +++ b/backend/src/baserow/core/cache.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from contextlib import contextmanager -from typing import Callable, TypeVar +from typing import TYPE_CHECKING, Callable, TypeVar from django.conf import settings from django.core.cache import cache @@ -10,6 +12,9 @@ from baserow.version import VERSION as BASEROW_VERSION +if TYPE_CHECKING: + from baserow.core.models import Settings + T = TypeVar("T") # This var is to invalidate global cache when we can't bump the Baserow version for @@ -332,3 +337,24 @@ def invalidate(self, key: None | str = None, invalidate_key: None | str = None): global_cache = GlobalCache() + + +_SETTINGS_CACHE_KEY = "core:settings" + + +def get_cached_settings() -> Settings | None: + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return None + return cache.get(_SETTINGS_CACHE_KEY) + + +def set_cached_settings(instance: Settings) -> None: + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return + cache.set(_SETTINGS_CACHE_KEY, instance, timeout=settings.BASEROW_CACHE_TTL_SECONDS) + + +def invalidate_cached_settings() -> None: + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return + cache.delete(_SETTINGS_CACHE_KEY) diff --git a/backend/src/baserow/core/exceptions.py b/backend/src/baserow/core/exceptions.py index 9f6fa2134f..b7db92734f 100644 --- a/backend/src/baserow/core/exceptions.py +++ b/backend/src/baserow/core/exceptions.py @@ -76,13 +76,6 @@ class WorkspaceUserAlreadyExists(Exception): """ -class MaxNumberOfPendingWorkspaceInvitesReached(Exception): - """ - Raised when the maximum number of pending workspace invites has been reached. - This value is configurable via the `BASEROW_MAX_PENDING_WORKSPACE_INVITES` setting. - """ - - class WorkspaceUserIsLastAdmin(Exception): """ Raised when the last admin of the workspace tries to leave it. This will leave the diff --git a/backend/src/baserow/core/handler.py b/backend/src/baserow/core/handler.py index f528a4093b..d0a7b07382 100755 --- a/backend/src/baserow/core/handler.py +++ b/backend/src/baserow/core/handler.py @@ -24,6 +24,7 @@ from opentelemetry import trace from tqdm import tqdm +from baserow.core.cache import get_cached_settings, set_cached_settings from baserow.core.db import specific_queryset from baserow.core.registries import plugin_registry from baserow.core.user.utils import normalize_email_address @@ -38,7 +39,6 @@ DuplicateApplicationMaxLocksExceededException, InvalidPermissionContext, LastAdminOfWorkspace, - MaxNumberOfPendingWorkspaceInvitesReached, PermissionDenied, PermissionException, TemplateDoesNotExist, @@ -140,7 +140,7 @@ class ApplicationUpdatedResult: updated_app_allowed_values: Dict[str, Any] -class CoreHandler(metaclass=baserow_trace_methods(tracer)): +class CoreHandler(metaclass=baserow_trace_methods(tracer, exclude="clear_context")): default_create_allowed_fields = ["name", "init_with_data"] default_update_allowed_fields = ["name"] @@ -153,32 +153,40 @@ def clear_context(self): clear_current_workspace_id() - def get_settings(self): + def get_settings(self) -> Settings: """ Returns a settings model instance containing all the admin configured settings. + The result is cached in Redis and invalidated by a ``post_save`` receiver + on the ``Settings`` model (see ``baserow.core.receivers``). :return: The settings instance. - :rtype: Settings """ + cached = get_cached_settings() + if cached is not None: + return cached + try: - return Settings.objects.all().select_related("co_branding_logo")[:1].get() + instance = ( + Settings.objects.all().select_related("co_branding_logo")[:1].get() + ) except Settings.DoesNotExist: - return Settings.objects.create() + instance = Settings.objects.create() + + set_cached_settings(instance) + return instance - def update_settings(self, user, settings_instance=None, **kwargs): + def update_settings( + self, user: User, settings_instance: Optional[Settings] = None, **kwargs + ) -> Settings: """ Updates one or more setting values if the user has staff permissions. :param user: The user on whose behalf the settings are updated. - :type user: User :param settings_instance: If already fetched, the settings instance can be provided to avoid fetching the values for a second time. - :type settings_instance: Settings :param kwargs: An dict containing the settings that need to be updated. - :type kwargs: dict :return: The update settings instance. - :rtype: Settings """ CoreHandler().check_permissions( @@ -1080,35 +1088,30 @@ def get_workspace_invitation(self, workspace_invitation_id, base_queryset=None): return workspace_invitation def create_workspace_invitation( - self, user, workspace, email, permissions, base_url, message="" - ): + self, + user: AbstractUser, + workspace: Workspace, + email: str, + permissions: str, + base_url: str, + ) -> WorkspaceInvitation: """ Creates a new workspace invitation for the given email address and sends out an email containing the invitation. :param user: The user on whose behalf the invitation is created. - :type user: User :param workspace: The workspace for which the user is invited. - :type workspace: Workspace :param email: The email address of the person that is invited to the workspace. Can be an existing or not existing user. - :type email: str :param permissions: The workspace permissions that the user will get once they have accepted the invitation. - :type permissions: str :param base_url: The base url of the frontend, where the user can accept his invitation. The signed invitation id is appended to the URL (base_url + '/TOKEN'). Only the PUBLIC_WEB_FRONTEND_HOSTNAME is allowed as domain name. - :type base_url: str - :param message: A custom message that will be included in the invitation email. - :type message: Optional[str] :raises ValueError: If the provided permissions are not allowed. :raises UserInvalidWorkspacePermissionsError: If the user does not belong to the workspace or doesn't have right permissions in the workspace. - :raises MaxNumberOfPendingWorkspaceInvitesReached: When the maximum number of - pending invites have been reached. :return: The created workspace invitation. - :rtype: WorkspaceInvitation """ CoreHandler().check_permissions( @@ -1127,23 +1130,10 @@ def create_workspace_invitation( f"The user {email} is already part of the workspace." ) - max_invites = settings.BASEROW_MAX_PENDING_WORKSPACE_INVITES - if max_invites > 0 and ( - WorkspaceInvitation.objects.filter(workspace=workspace) - .exclude(email=email) - .count() - >= max_invites - ): - raise MaxNumberOfPendingWorkspaceInvitesReached( - f"The maximum number of pending workspaces invites {max_invites} has " - f"been reached." - ) - invitation, created = WorkspaceInvitation.objects.update_or_create( workspace=workspace, email=email, defaults={ - "message": message, "permissions": permissions, "invited_by": user, }, @@ -1167,23 +1157,21 @@ def create_workspace_invitation( return invitation - def update_workspace_invitation(self, user, invitation, permissions): + def update_workspace_invitation( + self, user: AbstractUser, invitation: WorkspaceInvitation, permissions: str + ) -> WorkspaceInvitation: """ Updates the permissions of an existing invitation if the user has ADMIN permissions to the related workspace. :param user: The user on whose behalf the invitation is updated. - :type user: User :param invitation: The invitation that must be updated. - :type invitation: WorkspaceInvitation :param permissions: The new permissions of the invitation that the user must has after accepting. - :type permissions: str :raises ValueError: If the provided permissions is not allowed. :raises UserInvalidWorkspacePermissionsError: If the user does not belong to the workspace or doesn't have right permissions in the workspace. :return: The updated workspace permissions instance. - :rtype: WorkspaceInvitation """ CoreHandler().check_permissions( @@ -1198,15 +1186,15 @@ def update_workspace_invitation(self, user, invitation, permissions): return invitation - def delete_workspace_invitation(self, user, invitation): + def delete_workspace_invitation( + self, user: AbstractUser, invitation: WorkspaceInvitation + ) -> None: """ Deletes an existing workspace invitation if the user has ADMIN permissions to the related workspace. :param user: The user on whose behalf the invitation is deleted. - :type user: User :param invitation: The invitation that must be deleted. - :type invitation: WorkspaceInvitation :raises UserInvalidWorkspacePermissionsError: If the user does not belong to the workspace or doesn't have right permissions in the workspace. """ @@ -1406,10 +1394,8 @@ def filter_specific_applications( ] = None, ) -> QuerySet[Application]: if per_content_type_queryset_hook is None: - per_content_type_queryset_hook = ( - lambda model, qs: application_type_registry.get_by_model( - model - ).enhance_queryset(qs) + per_content_type_queryset_hook = lambda model, qs: ( + application_type_registry.get_by_model(model).enhance_queryset(qs) ) return specific_queryset(queryset, per_content_type_queryset_hook) diff --git a/backend/src/baserow/core/migrations/0113_alter_notification_options_and_more.py b/backend/src/baserow/core/migrations/0113_alter_notification_options_and_more.py index 0d34ba2d0e..0a43360d15 100644 --- a/backend/src/baserow/core/migrations/0113_alter_notification_options_and_more.py +++ b/backend/src/baserow/core/migrations/0113_alter_notification_options_and_more.py @@ -31,7 +31,7 @@ class Migration(migrations.Migration): name='core_notifi_created_06e5e6_idx', ), migrations.RunSQL( - sql='CREATE INDEX CONCURRENTLY "core_notifi_created_7f4b88_idx" ON "core_notification" ("created_on" DESC, "id" DESC)', + sql='CREATE INDEX CONCURRENTLY IF NOT EXISTS "core_notifi_created_7f4b88_idx" ON "core_notification" ("created_on" DESC, "id" DESC)', reverse_sql='DROP INDEX IF EXISTS "core_notifi_created_7f4b88_idx"', state_operations=[ migrations.AddIndex( @@ -41,7 +41,7 @@ class Migration(migrations.Migration): ], ), migrations.RunSQL( - sql='CREATE INDEX CONCURRENTLY "core_notifi_created_4b2233_idx" ON "core_notificationrecipient" ("created_on" DESC, "id" DESC)', + sql='CREATE INDEX CONCURRENTLY IF NOT EXISTS "core_notifi_created_4b2233_idx" ON "core_notificationrecipient" ("created_on" DESC, "id" DESC)', reverse_sql='DROP INDEX IF EXISTS "core_notifi_created_4b2233_idx"', state_operations=[ migrations.AddIndex( diff --git a/backend/src/baserow/core/migrations/0114_alter_workspaceinvitation_message.py b/backend/src/baserow/core/migrations/0114_alter_workspaceinvitation_message.py new file mode 100644 index 0000000000..e510625242 --- /dev/null +++ b/backend/src/baserow/core/migrations/0114_alter_workspaceinvitation_message.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.13 on 2026-04-20 16:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0113_alter_notification_options_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='workspaceinvitation', + name='message', + field=models.TextField(default='', help_text='Deprecated legacy field retained for compatibility. This message is not exposed to invitation recipients.', max_length=250), + ), + ] diff --git a/backend/src/baserow/core/models.py b/backend/src/baserow/core/models.py index 3ab427fc24..98bae9256c 100755 --- a/backend/src/baserow/core/models.py +++ b/backend/src/baserow/core/models.py @@ -385,11 +385,11 @@ class WorkspaceInvitation( help_text="The permissions that the user is going to get within the workspace " "after accepting the invitation.", ) + # TODO ZDM: Remove this field in a future migration (no longer used) message = models.TextField( - blank=True, + default="", max_length=250, - help_text="An optional message that the invitor can provide. This will be " - "visible to the receiver of the invitation.", + help_text="Deprecated legacy field retained for compatibility. This message is not exposed to invitation recipients.", ) def get_parent(self): diff --git a/backend/src/baserow/core/receivers.py b/backend/src/baserow/core/receivers.py new file mode 100644 index 0000000000..a509b72f31 --- /dev/null +++ b/backend/src/baserow/core/receivers.py @@ -0,0 +1,35 @@ +from django.conf import settings +from django.db import transaction +from django.db.models.signals import post_delete, post_save +from django.dispatch import receiver + +from baserow.core.cache import invalidate_cached_settings +from baserow.core.models import Settings, UserProfile +from baserow.core.user.cache import invalidate_cached_user + + +@receiver(post_save, sender=settings.AUTH_USER_MODEL, dispatch_uid="cache_user_save") +def invalidate_user_cache_on_user_save(sender, instance, **kwargs): + transaction.on_commit(lambda: invalidate_cached_user(instance.id)) + + +@receiver(post_save, sender=UserProfile, dispatch_uid="cache_profile_save") +def invalidate_user_cache_on_profile_save(sender, instance, **kwargs): + transaction.on_commit(lambda: invalidate_cached_user(instance.user_id)) + + +@receiver( + post_delete, sender=settings.AUTH_USER_MODEL, dispatch_uid="cache_user_delete" +) +def invalidate_user_cache_on_user_delete(sender, instance, **kwargs): + transaction.on_commit(lambda: invalidate_cached_user(instance.id)) + + +@receiver(post_delete, sender=UserProfile, dispatch_uid="cache_profile_delete") +def invalidate_user_cache_on_profile_delete(sender, instance, **kwargs): + transaction.on_commit(lambda: invalidate_cached_user(instance.user_id)) + + +@receiver(post_save, sender=Settings, dispatch_uid="cache_settings_save") +def invalidate_settings_cache_on_save(sender, **kwargs): + transaction.on_commit(invalidate_cached_settings) diff --git a/backend/src/baserow/core/telemetry/telemetry.py b/backend/src/baserow/core/telemetry/telemetry.py index 456bb2b005..04e7c43c5a 100644 --- a/backend/src/baserow/core/telemetry/telemetry.py +++ b/backend/src/baserow/core/telemetry/telemetry.py @@ -159,9 +159,9 @@ def _setup_standard_backend_instrumentation(): BotocoreInstrumentor().instrument() PsycopgInstrumentor().instrument() - RedisInstrumentor().instrument() RequestsInstrumentor().instrument() CeleryInstrumentor().instrument() + RedisInstrumentor().instrument() def _setup_django_process_instrumentation(): diff --git a/backend/src/baserow/core/templates/baserow/core/workspace_invitation.html b/backend/src/baserow/core/templates/baserow/core/workspace_invitation.html index 0b4106c71e..75e45c7628 100644 --- a/backend/src/baserow/core/templates/baserow/core/workspace_invitation.html +++ b/backend/src/baserow/core/templates/baserow/core/workspace_invitation.html @@ -178,13 +178,6 @@
{% blocktrans trimmed with invitation.invited_by.first_name as first_name and invitation.workspace.name as workspace_name %} {{ first_name }} has invited you to collaborate on {{ workspace_name }}. {% endblocktrans %}
- {% if invitation.message %} - - -
"{{ invitation.message }}"
- - - {% endif %} diff --git a/backend/src/baserow/core/templates/baserow/core/workspace_invitation.mjml.eta b/backend/src/baserow/core/templates/baserow/core/workspace_invitation.mjml.eta index 92ece47883..dcb81921b8 100644 --- a/backend/src/baserow/core/templates/baserow/core/workspace_invitation.mjml.eta +++ b/backend/src/baserow/core/templates/baserow/core/workspace_invitation.mjml.eta @@ -9,11 +9,6 @@ {{ workspace_name }}. {% endblocktrans %} - {% if invitation.message %} - - "{{ invitation.message }}" - - {% endif %} {% trans "Accept invitation" %} diff --git a/backend/src/baserow/core/user/actions.py b/backend/src/baserow/core/user/actions.py index 80654900cd..1c4a0f46af 100644 --- a/backend/src/baserow/core/user/actions.py +++ b/backend/src/baserow/core/user/actions.py @@ -16,7 +16,7 @@ from baserow.core.models import Template, User from baserow.core.registries import auth_provider_type_registry from baserow.core.user.handler import UserHandler -from baserow.throttling import rate_limit +from baserow.throttling.handler import rate_limit class CreateUserActionType(ActionType): diff --git a/backend/src/baserow/core/user/cache.py b/backend/src/baserow/core/user/cache.py new file mode 100644 index 0000000000..5d5f270e90 --- /dev/null +++ b/backend/src/baserow/core/user/cache.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from django.conf import settings +from django.core.cache import cache + +if TYPE_CHECKING: + from django.contrib.auth.models import AbstractUser + +_KEY_PREFIX = "user:" + + +def _cache_key(user_id: int) -> str: + return f"{_KEY_PREFIX}{user_id}" + + +def get_cached_user(user_id: int) -> AbstractUser | None: + """ + Return a cached User instance (with profile pre-loaded) or ``None`` on + cache miss or when caching is disabled. + """ + + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return None + return cache.get(_cache_key(user_id)) + + +def set_cached_user(user: AbstractUser) -> None: + """ + Store *user* (with its pre-loaded profile) in Redis. No-op when the + cache TTL is 0 or negative. + """ + + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return + cache.set( + _cache_key(user.id), + user, + timeout=settings.BASEROW_CACHE_TTL_SECONDS, + ) + + +def invalidate_cached_user(user_id: int) -> None: + """ + Invalidate the cached User instance for the given user ID. No-op when the + cache TTL is 0 or negative. + """ + + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return + cache.delete(_cache_key(user_id)) diff --git a/backend/src/baserow/core/user/handler.py b/backend/src/baserow/core/user/handler.py index 7d4903287c..eccff2eb02 100755 --- a/backend/src/baserow/core/user/handler.py +++ b/backend/src/baserow/core/user/handler.py @@ -49,7 +49,7 @@ ) from baserow.core.trash.handler import TrashHandler from baserow.core.utils import generate_hash, get_baserow_saas_base_url -from baserow.throttling import rate_limit +from baserow.throttling.handler import rate_limit from ..telemetry.utils import baserow_trace_methods from .emails import ( diff --git a/backend/src/baserow/middleware.py b/backend/src/baserow/middleware.py index e763a69167..6300238943 100644 --- a/backend/src/baserow/middleware.py +++ b/backend/src/baserow/middleware.py @@ -8,7 +8,6 @@ from baserow.config.db_routers import clear_db_state from baserow.core.handler import CoreHandler -from baserow.throttling import ConcurrentUserRequestsThrottle def json_error_404_add_trailing_slash(path: str) -> HttpResponse: @@ -66,23 +65,6 @@ def __call__(self, request: HttpRequest) -> HttpResponse: return response -class ConcurrentUserRequestsMiddleware: - """ - This middleware is used as counterpart of the - `ConcurrentUserRequestsThrottle` to remove the request id from the throttle - cache once processed. This is needed because the throttle is - not aware of the request lifecycle and therefore can't remove it by itself. - """ - - def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]): - self.get_response = get_response - - def __call__(self, request: HttpRequest) -> HttpResponse: - response = self.get_response(request) - ConcurrentUserRequestsThrottle.on_request_processed(request) - return response - - class ClearContextMiddleware: """ This middleware is used to clear the context after the response has been returned. diff --git a/backend/src/baserow/throttling/__init__.py b/backend/src/baserow/throttling/__init__.py new file mode 100644 index 0000000000..98c583df2d --- /dev/null +++ b/backend/src/baserow/throttling/__init__.py @@ -0,0 +1 @@ +"""Throttling package.""" diff --git a/backend/src/baserow/throttling/blacklist.py b/backend/src/baserow/throttling/blacklist.py new file mode 100644 index 0000000000..4b7cf326bb --- /dev/null +++ b/backend/src/baserow/throttling/blacklist.py @@ -0,0 +1,69 @@ +""" +Redis blacklist for throttled tokens and IPs. + +When the ``ConcurrentUserRequestsThrottle`` denies a request, the bearer +token's SHA-256 hash (or the client IP) is written here. The +``ThrottleBlacklistMiddleware`` checks this blacklist *before* authentication +so that repeat offenders are rejected with zero DB or DRF overhead. +""" + +import hashlib +import math +import time + +from django.conf import settings +from django.core.cache import cache + +_TOKEN_PREFIX = "throttle_bl:" +_IP_PREFIX = "throttle_ip_bl:" + + +def _token_key(raw_token: str) -> str: + return _TOKEN_PREFIX + hashlib.sha256(raw_token.encode()).hexdigest() + + +def _ip_key(ip: str) -> str: + return _IP_PREFIX + ip + + +def _remaining_ttl(expires_at: float) -> int | None: + remaining = math.ceil(expires_at - time.time()) + return remaining if remaining > 0 else None + + +def _resolve_ttl(ttl: int | None) -> int | None: + if ttl is None: + ttl = settings.BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS + return ttl if ttl > 0 else None + + +def blacklist_token(raw_token: str, ttl: int | None = None) -> None: + ttl = _resolve_ttl(ttl) + if ttl is None: + return + cache.set(_token_key(raw_token), time.time() + ttl, timeout=ttl) + + +def blacklist_ip(ip: str, ttl: int | None = None) -> None: + ttl = _resolve_ttl(ttl) + if ttl is None: + return + cache.set(_ip_key(ip), time.time() + ttl, timeout=ttl) + + +def get_token_cooldown_time(raw_token: str) -> int | None: + """Return the remaining blacklist TTL if blacklisted, else ``None``.""" + + expires_at = cache.get(_token_key(raw_token)) + if expires_at is None: + return None + return _remaining_ttl(expires_at) + + +def is_ip_blacklisted(ip: str) -> int | None: + """Return the remaining blacklist TTL if blacklisted, else ``None``.""" + + expires_at = cache.get(_ip_key(ip)) + if expires_at is None: + return None + return _remaining_ttl(expires_at) diff --git a/backend/src/baserow/throttling/exceptions.py b/backend/src/baserow/throttling/exceptions.py new file mode 100644 index 0000000000..bfce5dba1b --- /dev/null +++ b/backend/src/baserow/throttling/exceptions.py @@ -0,0 +1,4 @@ +class RateLimitExceededException(Exception): + """Raised when rate limit is exceeded.""" + + pass diff --git a/backend/src/baserow/throttling.py b/backend/src/baserow/throttling/handler.py similarity index 50% rename from backend/src/baserow/throttling.py rename to backend/src/baserow/throttling/handler.py index dfb3545d54..8b99d84baf 100644 --- a/backend/src/baserow/throttling.py +++ b/backend/src/baserow/throttling/handler.py @@ -8,15 +8,17 @@ from django_redis import get_redis_connection from loguru import logger -from opentelemetry import trace from rest_framework.throttling import SimpleRateThrottle -from baserow.core.telemetry.utils import baserow_trace_methods -from baserow.throttling_types import RateLimit +from baserow.api.exceptions import ThrottledAPIException +from baserow.api.sessions import get_user_remote_ip_address_from_request -BASEROW_CONCURRENCY_THROTTLE_REQUEST_ID = "baserow_concurrency_throttle_request_id" +from .blacklist import blacklist_ip, blacklist_token +from .exceptions import RateLimitExceededException +from .types import RateLimit +from .utils import get_auth_token -tracer = trace.get_tracer(__name__) +BASEROW_CONCURRENCY_THROTTLE_REQUEST_ID = "baserow_concurrency_throttle_request_id" # Slightly modified version of # https://gist.github.com/ptarjan/e38f45f2dfe601419ca3af937fff574d @@ -28,7 +30,6 @@ local request_id = ARGV[3] local timeout = tonumber(ARGV[4]) local old_request_cutoff = timestamp - timeout -local wait = 0 local count = redis.call("zcard", key) local allowed = count < max_concurrent_requests @@ -43,12 +44,9 @@ if allowed then redis.call("zadd", key, timestamp, request_id) -else - local first = redis.call("zrange", key, 0, 0, "WITHSCORES") - wait = tonumber(first[2]) - old_request_cutoff end -return { allowed, count, wait } +return { allowed, count } """ @@ -56,11 +54,12 @@ def _get_redis_cli(): return get_redis_connection("default") -class ConcurrentUserRequestsThrottle( - SimpleRateThrottle, metaclass=baserow_trace_methods(tracer) -): +class ConcurrentUserRequestsThrottle(SimpleRateThrottle): """ - Limits the number of concurrent requests made by a given user. + Limits the number of concurrent requests made by a given user or IP address. When + the limit is exceeded and the blacklist is enabled, the token or IP is blacklisted + for a short time to prevent further abuse and reduce load on the system. See + ``ThrottleBlacklistMiddleware`` and ``baserow.throttling.blacklist``. """ scope = "concurrent_user_requests" @@ -74,15 +73,18 @@ def __new__(cls, *args, **kwargs): @classmethod def _init_redis_cli(cls): cls.redis_cli = _get_redis_cli() - cls.incr_concurrent_requests_count_if_allowed = cls.redis_cli.register_script( + cls.is_allowed = cls.redis_cli.register_script( incr_concurrent_requests_count_if_allowed_lua_script ) @classmethod - def _log(cls, request, log_msg, request_id=None, *args, **kwargs): + def _get_ip(cls, request) -> str: + return get_user_remote_ip_address_from_request(request) + + @classmethod + def _debug(cls, request, log_msg, request_id=None, **kwargs): logger.debug( - "{{path={path},user_id={user_id},req_id={request_id}}} %s" % log_msg, - *args, + "{{path={path},user_id={user_id},req_id={request_id}}} " + log_msg, path=request.path, user_id=request.user.id if request.user.is_authenticated else None, request_id=str(request_id), @@ -94,78 +96,121 @@ def parse_rate(self, rate): return int(rate), duration @classmethod - def get_cache_key(cls, request, view=None): + def get_cache_key(cls, request, view=None) -> str | None: + """ + Return a unique cache key for the given request, or ``None`` if the request + should be exempt from throttling: + + - Staff users are always exempt. + + - if the user is authenticated, the key is base on the user ID, so all tokens + for the same user share the same concurrency limit. + + - If the user is anonymous and IP-based throttling is enabled, the key is based + on the client IP. + + - If the user is anonymous and IP-based throttling is disabled, ``None`` is + returned to skip throttling. + """ + user = request.user - if user.is_authenticated and not user.is_staff: - return cls.cache_format % { - "scope": cls.scope, - "ident": request.user.id, - } - if not user.is_authenticated: - cls._log(request, "ALLOWING: not throttling anonymous users") - elif user.is_staff: - cls._log(request, "ALLOWING: not throttling staff users") + if user.is_authenticated: + if user.is_staff: # Don't throttle staff users + return None + + ident = str(user.id) + elif settings.BASEROW_THROTTLE_IP_ENABLED: + ident = cls._get_ip(request) + else: + return None - return None + return cls.cache_format % {"scope": cls.scope, "ident": ident} def allow_request(self, request, view): profile = getattr(request.user, "profile", None) - if profile is not None and profile.concurrency_limit: - limit = profile.concurrency_limit - else: - limit = self.num_requests - if limit <= 0: - self._log( - request, - "ALLOWING: throttling disabled as configured rate <= 0", - ) - return True + limit = ( + profile and getattr(profile, "concurrency_limit", None) or self.num_requests + ) - if (key := self.get_cache_key(request)) is None: + if limit <= 0 or (cache_key := self.get_cache_key(request)) is None: + self._debug(request, "ALLOWING: throttling skipped") return True - self.key = key + self.cache_key = cache_key self.timestamp = timestamp = self.timer() request_id = str(uuid4()) args = [limit, timestamp, request_id, self.duration] - allowed, count, wait = self.incr_concurrent_requests_count_if_allowed( - [key], args + allowed, count = self.is_allowed([cache_key], args) + + if not allowed: + self._raise_deny_exc(request, request_id, count, limit) + + return self._allow(request, request_id, count, limit) + + def _allow(self, request, request_id, count, limit): + django_request = request._request + # Needed to remove request from sorted set in on_request_processed when done. + setattr(django_request, BASEROW_CONCURRENCY_THROTTLE_REQUEST_ID, request_id) + self._debug( + request, + "ALLOWING: as count={count} < limit={limit}", + request_id=request_id, + count=count, + limit=limit, ) - - if allowed: - django_request = request._request - setattr(django_request, BASEROW_CONCURRENCY_THROTTLE_REQUEST_ID, request_id) - log_msg = "ALLOWING: as count={count} < limit={limit}" + return True + + def _raise_deny_exc(self, request, request_id, count, limit): + """ + Raise ThrottledAPIException to reject the request. When the blacklist + is enabled (BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS > 0) the caller is + also blacklisted for that cooldown, and the cooldown is surfaced as + the Retry-After hint; otherwise no Retry-After is emitted since a + concurrency slot may free up at any moment. + """ + + cooldown = settings.BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS + if cooldown > 0: + self._blacklist(request, ttl=cooldown) else: - self._wait = wait - log_msg = "DENYING: as count={count} >= limit={limit}. Wait {wait} secs" - - self._log( - request, log_msg, request_id=request_id, count=count, limit=limit, wait=wait + cooldown = None + + self._wait = cooldown + self._debug( + request, + "DENYING: as count={count} >= limit={limit}. Cooldown {wait} secs", + request_id=request_id, + count=count, + limit=limit, + wait=cooldown, ) - return bool(allowed) + raise ThrottledAPIException(wait=cooldown) + + @classmethod + def _blacklist(cls, request, ttl: int | None = None) -> None: + token = get_auth_token(request) + if token: + blacklist_token(token, ttl=ttl) + else: + ip = cls._get_ip(request) + blacklist_ip(ip, ttl=ttl) @classmethod def on_request_processed(cls, request): request_id = getattr(request, BASEROW_CONCURRENCY_THROTTLE_REQUEST_ID, None) - - if request_id is not None and (key := cls.get_cache_key(request)): - cls._log(request, "UNTRACKING: request has finished", request_id=request_id) - cls.redis_cli.zrem(key, request_id) + if request_id and (cache_key := cls.get_cache_key(request)): + cls._debug( + request, "UNTRACKING: request has finished", request_id=request_id + ) + cls.redis_cli.zrem(cache_key, request_id) def wait(self): return self._wait -class RateLimitExceededException(Exception): - """Raised when rate limit is exceeded.""" - - pass - - def rate_limit(rate: RateLimit, key: str, raise_exception: bool = True): """ A general purpose throttling function decorator. diff --git a/backend/src/baserow/throttling/middleware.py b/backend/src/baserow/throttling/middleware.py new file mode 100644 index 0000000000..36b783c64c --- /dev/null +++ b/backend/src/baserow/throttling/middleware.py @@ -0,0 +1,68 @@ +from typing import Callable + +from django.conf import settings +from django.http import HttpRequest, HttpResponse + +from baserow.api.exceptions import ( + ThrottledAPIException, + api_exception_to_json_response, +) +from baserow.api.sessions import get_user_remote_ip_address_from_request +from baserow.throttling.handler import ConcurrentUserRequestsThrottle + +from .blacklist import get_token_cooldown_time, is_ip_blacklisted +from .utils import get_auth_token + + +class ThrottleBlacklistMiddleware: + """ + Fast-path rejection for recently throttled tokens and, optionally, IPs. + + When ``ConcurrentUserRequestsThrottle`` denies a request it writes the + SHA-256 hash of the bearer token to Redis with a short TTL. This + middleware — placed *before* authentication — checks that blacklist on + every request. A hit returns 429 immediately, skipping JWT validation, + DB/cache lookups, DRF view initialisation, permissions, and serializers. + + When ``BASEROW_THROTTLE_IP_ENABLED`` is ``True``, anonymous requests + (no ``Authorization`` header) are also checked against an IP-based + blacklist. + """ + + def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]): + self.get_response = get_response + if settings.BASEROW_THROTTLE_IP_ENABLED: + self._check_anonymous = lambda request: is_ip_blacklisted( + get_user_remote_ip_address_from_request(request) + ) + else: + self._check_anonymous = lambda request: None + + def __call__(self, request: HttpRequest) -> HttpResponse: + if token := get_auth_token(request): + cooldown = get_token_cooldown_time(token) + else: + cooldown = self._check_anonymous(request) + + if cooldown is not None: + # Use the same response format returned by ConcurrentUserRequestsThrottle + return api_exception_to_json_response(ThrottledAPIException(wait=cooldown)) + + return self.get_response(request) + + +class ConcurrentUserRequestsMiddleware: + """ + Counterpart of ``ConcurrentUserRequestsThrottle``. Removes the request + id from the Redis sorted set once the response has been generated, freeing + the concurrency slot. + """ + + def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]): + self.get_response = get_response + + def __call__(self, request: HttpRequest) -> HttpResponse: + try: + return self.get_response(request) + finally: + ConcurrentUserRequestsThrottle.on_request_processed(request) diff --git a/backend/src/baserow/throttling_types.py b/backend/src/baserow/throttling/types.py similarity index 100% rename from backend/src/baserow/throttling_types.py rename to backend/src/baserow/throttling/types.py diff --git a/backend/src/baserow/throttling/utils.py b/backend/src/baserow/throttling/utils.py new file mode 100644 index 0000000000..949dca96e4 --- /dev/null +++ b/backend/src/baserow/throttling/utils.py @@ -0,0 +1,10 @@ +def get_auth_token(request) -> str | None: + """Extract a JWT or database-Token bearer value from the Authorization header.""" + + auth = request.META.get("HTTP_AUTHORIZATION", "") + head = auth[:6].lower() + if head.startswith("jwt "): + return auth[4:] + if head.startswith("token "): + return auth[6:] + return None diff --git a/backend/tests/baserow/api/groups/test_workspace_invitation_views.py b/backend/tests/baserow/api/groups/test_workspace_invitation_views.py index 716e20e25a..693e0bcd56 100644 --- a/backend/tests/baserow/api/groups/test_workspace_invitation_views.py +++ b/backend/tests/baserow/api/groups/test_workspace_invitation_views.py @@ -1,5 +1,4 @@ from django.shortcuts import reverse -from django.test.utils import override_settings import pytest from freezegun import freeze_time @@ -30,14 +29,12 @@ def test_list_workspace_invitations(api_client, data_fixture): invited_by=user_1, email="test3@test.nl", permissions="MEMBER", - message="Test bericht 1", ) invitation_2 = data_fixture.create_workspace_invitation( workspace=workspace_1, invited_by=user_1, email="test4@test.nl", permissions="ADMIN", - message="Test bericht 2", ) response = api_client.get( @@ -81,7 +78,6 @@ def test_list_workspace_invitations(api_client, data_fixture): assert response_json[0]["workspace"] == invitation_1.workspace_id assert response_json[0]["email"] == "test3@test.nl" assert response_json[0]["permissions"] == "MEMBER" - assert response_json[0]["message"] == "Test bericht 1" assert response_json[0]["created_on"] == "2020-01-02T12:00:00Z" assert response_json[1]["id"] == invitation_2.id @@ -101,7 +97,6 @@ def test_create_workspace_invitation(api_client, data_fixture): { "email": "test@test.nl", "permissions": "ADMIN", - "message": "Test", "base_url": "http://localhost:3000/invite", }, format="json", @@ -118,7 +113,6 @@ def test_create_workspace_invitation(api_client, data_fixture): { "email": "test@test.nl", "permissions": "ADMIN", - "message": "Test", "base_url": "http://localhost:3000/invite", }, format="json", @@ -135,7 +129,6 @@ def test_create_workspace_invitation(api_client, data_fixture): { "email": user_3.email, "permissions": "ADMIN", - "message": "Test", "base_url": "http://localhost:3000/invite", }, format="json", @@ -152,7 +145,6 @@ def test_create_workspace_invitation(api_client, data_fixture): { "email": "test@test.nl", "permissions": "ADMIN", - "message": "Test", "base_url": "http://localhost:3000/invite", }, format="json", @@ -169,7 +161,6 @@ def test_create_workspace_invitation(api_client, data_fixture): { "email": "test@test.nl", "permissions": "ADMIN", - "message": "Test", "base_url": "http://test.nl:3000/invite", }, format="json", @@ -179,6 +170,8 @@ def test_create_workspace_invitation(api_client, data_fixture): assert response.status_code == HTTP_400_BAD_REQUEST assert response_json["error"] == "ERROR_HOSTNAME_IS_NOT_ALLOWED" + # Custom messages are no longer supported: any `message` in the request body is + # silently ignored and neither persisted nor returned. response = api_client.post( reverse( "api:workspaces:invitations:list", kwargs={"workspace_id": workspace_1.id} @@ -199,7 +192,8 @@ def test_create_workspace_invitation(api_client, data_fixture): assert response_json["workspace"] == invitation.workspace_id assert response_json["email"] == "test0@test.nl" assert response_json["permissions"] == "ADMIN" - assert response_json["message"] == "Test" + assert invitation.message == "" + assert "message" not in response_json assert "created_on" in response_json response = api_client.post( @@ -209,67 +203,6 @@ def test_create_workspace_invitation(api_client, data_fixture): { "email": "test2@test.nl", "permissions": "ADMIN", - "message": "", - "base_url": "http://localhost:3000/invite", - }, - format="json", - HTTP_AUTHORIZATION=f"JWT {token_1}", - ) - response_json = response.json() - assert response.status_code == HTTP_200_OK - assert response_json["message"] == "" - - response = api_client.post( - reverse( - "api:workspaces:invitations:list", kwargs={"workspace_id": workspace_1.id} - ), - { - "email": "test2@test.nl", - "permissions": "ADMIN", - "base_url": "http://localhost:3000/invite", - }, - format="json", - HTTP_AUTHORIZATION=f"JWT {token_1}", - ) - response_json = response.json() - assert response.status_code == HTTP_200_OK - assert response_json["message"] == "" - - message = "" - for i in range(WorkspaceInvitation._meta.get_field("message").max_length + 1): - message += str(i % 10) - - response = api_client.post( - reverse( - "api:workspaces:invitations:list", kwargs={"workspace_id": workspace_1.id} - ), - { - "email": "test2@test.nl", - "permissions": "ADMIN", - "base_url": "http://localhost:3000/invite", - "message": message, - }, - format="json", - HTTP_AUTHORIZATION=f"JWT {token_1}", - ) - assert response.status_code == HTTP_400_BAD_REQUEST - assert response.json()["error"] == "ERROR_REQUEST_BODY_VALIDATION" - - -@pytest.mark.django_db -@override_settings(BASEROW_MAX_PENDING_WORKSPACE_INVITES=1) -def test_create_workspace_invitation_max_pending(api_client, data_fixture): - user_1, token_1 = data_fixture.create_user_and_token(email="test1@test.nl") - workspace_1 = data_fixture.create_workspace(user=user_1) - - response = api_client.post( - reverse( - "api:workspaces:invitations:list", kwargs={"workspace_id": workspace_1.id} - ), - { - "email": "test@test.nl", - "permissions": "ADMIN", - "message": "Test", "base_url": "http://localhost:3000/invite", }, format="json", @@ -277,26 +210,6 @@ def test_create_workspace_invitation_max_pending(api_client, data_fixture): ) assert response.status_code == HTTP_200_OK - response = api_client.post( - reverse( - "api:workspaces:invitations:list", kwargs={"workspace_id": workspace_1.id} - ), - { - "email": "test2@test.nl", - "permissions": "ADMIN", - "message": "Test", - "base_url": "http://localhost:3000/invite", - }, - format="json", - HTTP_AUTHORIZATION=f"JWT {token_1}", - ) - response_json = response.json() - assert response.status_code == HTTP_400_BAD_REQUEST - assert ( - response_json["error"] - == "ERROR_MAX_NUMBER_OF_PENDING_WORKSPACE_INVITES_REACHED" - ) - @pytest.mark.django_db def test_get_workspace_invitation(api_client, data_fixture): @@ -315,7 +228,6 @@ def test_get_workspace_invitation(api_client, data_fixture): workspace=workspace_1, email="test0@test.nl", permissions="ADMIN", - message="TEst", ) response = api_client.get( @@ -363,7 +275,6 @@ def test_get_workspace_invitation(api_client, data_fixture): assert response_json["workspace"] == invitation.workspace_id assert response_json["email"] == invitation.email assert response_json["permissions"] == invitation.permissions - assert response_json["message"] == invitation.message assert "created_on" in response_json @@ -384,7 +295,6 @@ def test_update_workspace_invitation(api_client, data_fixture): workspace=workspace_1, email="test0@test.nl", permissions="ADMIN", - message="TEst", ) response = api_client.patch( @@ -436,7 +346,6 @@ def test_update_workspace_invitation(api_client, data_fixture): assert response_json["workspace"] == invitation.workspace_id assert response_json["email"] == invitation.email assert response_json["permissions"] == "MEMBER" - assert response_json["message"] == invitation.message assert "created_on" in response_json response = api_client.patch( @@ -447,7 +356,6 @@ def test_update_workspace_invitation(api_client, data_fixture): { "email": "should.be@ignored.nl", "permissions": "ADMIN", - "message": "Should be ignored", "base_url": "http://should.be.ignored:3000/invite", }, HTTP_AUTHORIZATION=f"JWT {token_1}", @@ -458,7 +366,6 @@ def test_update_workspace_invitation(api_client, data_fixture): assert response_json["workspace"] == invitation.workspace_id assert response_json["email"] == invitation.email assert response_json["permissions"] == "ADMIN" - assert response_json["message"] == invitation.message @pytest.mark.django_db @@ -478,7 +385,6 @@ def test_delete_workspace_invitation(api_client, data_fixture): workspace=workspace_1, email="test0@test.nl", permissions="ADMIN", - message="TEst", ) response = api_client.delete( @@ -534,7 +440,6 @@ def test_accept_workspace_invitation(api_client, data_fixture): workspace=workspace_1, email="test1@test.nl", permissions="ADMIN", - message="Test", ) response = api_client.post( @@ -586,7 +491,6 @@ def test_reject_workspace_invitation(api_client, data_fixture): workspace=workspace_1, email="test1@test.nl", permissions="ADMIN", - message="Test", ) response = api_client.post( @@ -626,8 +530,10 @@ def test_reject_workspace_invitation(api_client, data_fixture): @pytest.mark.django_db def test_get_workspace_invitation_by_token(api_client, data_fixture): data_fixture.create_user(email="test1@test.nl") + # `message` is a deprecated column kept on the model; even when a legacy row + # still has a value, the API must not expose it. invitation = data_fixture.create_workspace_invitation( - email="test0@test.nl", permissions="ADMIN", message="TEst" + email="test0@test.nl", permissions="ADMIN", message="Legacy message" ) invitation_2 = data_fixture.create_workspace_invitation( email="test1@test.nl", @@ -665,7 +571,7 @@ def test_get_workspace_invitation_by_token(api_client, data_fixture): assert response_json["invited_by"] == invitation.invited_by.first_name assert response_json["workspace"] == invitation.workspace.name assert response_json["email"] == invitation.email - assert response_json["message"] == invitation.message + assert "message" not in response_json assert response_json["email_exists"] is False assert "created_on" in response_json @@ -691,14 +597,12 @@ def test_when_workspace_is_trashed_so_is_invitation(data_fixture, api_client): invited_by=user_1, email="test4@test.nl", permissions="ADMIN", - message="Test bericht 2", ) trashed_invitation = data_fixture.create_workspace_invitation( workspace=trashed_workspace, invited_by=user_1, email="test4@test.nl", permissions="ADMIN", - message="Test bericht 2", ) # Put the trashed_workspace in the trash CoreHandler().delete_workspace_by_id(user_1, trashed_workspace.id) diff --git a/backend/tests/baserow/api/test_api_utils.py b/backend/tests/baserow/api/test_api_utils.py index 8a98139a67..52e1281eac 100644 --- a/backend/tests/baserow/api/test_api_utils.py +++ b/backend/tests/baserow/api/test_api_utils.py @@ -3,6 +3,7 @@ from django.conf import settings from django.contrib.auth.models import AnonymousUser +from django.http import HttpResponse from django.test import override_settings import pytest @@ -13,7 +14,10 @@ from rest_framework.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND from rest_framework.test import APIRequestFactory -from baserow.api.exceptions import QueryParameterValidationException +from baserow.api.exceptions import ( + QueryParameterValidationException, + ThrottledAPIException, +) from baserow.api.registries import RegisteredException, api_exception_registry from baserow.api.utils import ( get_serializer_class, @@ -29,7 +33,7 @@ ModelInstanceMixin, Registry, ) -from baserow.throttling import ( +from baserow.throttling.handler import ( BASEROW_CONCURRENCY_THROTTLE_REQUEST_ID, ConcurrentUserRequestsThrottle, ) @@ -123,9 +127,9 @@ def test_map_exceptions_context_manager(): with pytest.raises(APIException) as api_exception_3: with map_exceptions( { - TemporaryException: lambda ex: "CONDITIONAL_ERROR" - if "test" in str(ex) - else None + TemporaryException: lambda ex: ( + "CONDITIONAL_ERROR" if "test" in str(ex) else None + ) } ): raise TemporaryException("test") @@ -136,9 +140,9 @@ def test_map_exceptions_context_manager(): with pytest.raises(TemporaryException): with map_exceptions( { - TemporaryException: lambda ex: "CONDITIONAL_ERROR" - if "test" in str(ex) - else None + TemporaryException: lambda ex: ( + "CONDITIONAL_ERROR" if "test" in str(ex) else None + ) } ): raise TemporaryException("not matching lambda") @@ -147,9 +151,9 @@ def test_map_exceptions_context_manager(): with pytest.raises(APIException) as api_exception_5: with map_exceptions( { - TemporaryException: lambda ex: error_tuple - if "test" in ex.message - else None + TemporaryException: lambda ex: ( + error_tuple if "test" in ex.message else None + ) } ): exception = TemporaryException() @@ -443,7 +447,10 @@ def __init__(self, path, user): return request -@override_settings(BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT=30) +@override_settings( + BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT=30, + BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS=7, +) @pytest.mark.django_db def test_concurrent_user_requests_throttle_non_staff_authenticated_users(data_fixture): user = data_fixture.create_user() @@ -456,8 +463,10 @@ def test_concurrent_user_requests_throttle_non_staff_authenticated_users(data_fi with freeze_time("2023-03-30 00:00:01"): throttle = ConcurrentUserRequestsThrottle() - assert not throttle.allow_request(create_dummy_request(user), None) - assert throttle.wait() == 29 + with pytest.raises(ThrottledAPIException) as exc_info: + throttle.allow_request(create_dummy_request(user), None) + assert exc_info.value.wait == 7 + assert throttle.wait() == 7 # once the timeout is over, the user should be able to make a new request request = create_dummy_request(user) @@ -475,6 +484,37 @@ def test_concurrent_user_requests_throttle_non_staff_authenticated_users(data_fi assert throttle.allow_request(request, None) +@override_settings( + BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT=30, + BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS=0, +) +@pytest.mark.django_db +def test_concurrent_throttle_denies_without_retry_after_when_blacklist_disabled( + data_fixture, +): + """ + With the blacklist off the throttle still raises, but without a + Retry-After hint — a concurrency slot can free up at any moment, so any + time estimate would be a lie. + """ + + user = data_fixture.create_user() + ConcurrentUserRequestsThrottle.timer = lambda s: time.time() + ConcurrentUserRequestsThrottle.rate = 1 + + with freeze_time("2023-03-30 00:00:00"): + throttle = ConcurrentUserRequestsThrottle() + assert throttle.allow_request(create_dummy_request(user), None) + + with freeze_time("2023-03-30 00:00:01"): + throttle = ConcurrentUserRequestsThrottle() + with pytest.raises(ThrottledAPIException) as exc_info: + throttle.allow_request(create_dummy_request(user), None) + assert exc_info.value.wait is None + assert str(exc_info.value.detail) == "Request was throttled." + assert throttle.wait() is None + + @pytest.mark.django_db def test_concurrent_user_requests_does_not_throttle_staff_users(data_fixture): user = data_fixture.create_user(is_staff=True) @@ -497,28 +537,39 @@ def test_concurrent_user_requests_does_not_throttle_staff_users(data_fixture): @override_settings( MIDDLEWARE=[ *settings.MIDDLEWARE, - "baserow.middleware.ConcurrentUserRequestsMiddleware", + "baserow.throttling.middleware.ConcurrentUserRequestsMiddleware", ], ) -@patch("baserow.throttling.ConcurrentUserRequestsThrottle.on_request_processed") @pytest.mark.django_db def test_throttle_set_baserow_concurrency_throttle_request_id_and_middleware_can_get_it( - mock_on_request_processed, data_fixture, api_client + data_fixture, ): - # Looking at - # https://github.com/encode/django-rest-framework/blob/3.14.0/rest_framework/views.py#L110 - # it seems like the throttle_classes are set when the class is created so - # @override_settings does not work as expected. We need to set the - # throttle_classes on the class itself to be able to override the settings. - from baserow.api.user.views import DashboardView + from baserow.throttling.middleware import ConcurrentUserRequestsMiddleware ConcurrentUserRequestsThrottle.rate = 1 - DashboardView.throttle_classes = [ConcurrentUserRequestsThrottle] - _, token = data_fixture.create_user_and_token() + user = data_fixture.create_user() + django_request = APIRequestFactory().get("/api/user/dashboard") + django_request.user = user + + drf_request = APIRequestFactory().get("/api/user/dashboard") + drf_request.user = user + drf_request._request = django_request + + throttle = ConcurrentUserRequestsThrottle() + assert throttle.allow_request(drf_request, None) + + middleware = ConcurrentUserRequestsMiddleware( + lambda request: HttpResponse(status=200) + ) - api_client.get("/api/user/dashboard/", HTTP_AUTHORIZATION=f"JWT {token}") + with patch( + "baserow.throttling.handler.ConcurrentUserRequestsThrottle.on_request_processed", + wraps=ConcurrentUserRequestsThrottle.on_request_processed, + ) as mock_on_request_processed: + response = middleware(django_request) + assert response.status_code == 200 assert mock_on_request_processed.call_count == 1 request = mock_on_request_processed.call_args[0][0] assert getattr(request, BASEROW_CONCURRENCY_THROTTLE_REQUEST_ID, None) is not None @@ -556,11 +607,13 @@ def test_can_set_throttle_per_user_profile_custom_limit(data_fixture): with freeze_time("2023-03-30 00:00:01"): throttle = ConcurrentUserRequestsThrottle() - assert not throttle.allow_request(create_dummy_request(user), None) + with pytest.raises(ThrottledAPIException): + throttle.allow_request(create_dummy_request(user), None) with freeze_time("2023-03-30 00:00:02"): throttle = ConcurrentUserRequestsThrottle() - assert not throttle.allow_request(create_dummy_request(user), None) + with pytest.raises(ThrottledAPIException): + throttle.allow_request(create_dummy_request(user), None) @pytest.mark.django_db diff --git a/backend/tests/baserow/api/test_jwt_user_cache_perf.py b/backend/tests/baserow/api/test_jwt_user_cache_perf.py new file mode 100644 index 0000000000..4bff60d3e9 --- /dev/null +++ b/backend/tests/baserow/api/test_jwt_user_cache_perf.py @@ -0,0 +1,62 @@ +""" +Perf comparison: cached vs uncached JWT user lookup. + +Run with: + just b test tests/baserow/api/test_jwt_user_cache_perf.py -s +""" + +import time + +from django.test.utils import override_settings + +import pytest +from rest_framework_simplejwt.tokens import AccessToken + +from baserow.api.authentication import JSONWebTokenAuthentication +from baserow.core.user.cache import invalidate_cached_user + +ITERATIONS = 500 + + +def _bench(auth, token) -> float: + """Return average time in microseconds over ITERATIONS calls.""" + + auth.get_user(token) + + start = time.perf_counter_ns() + for _ in range(ITERATIONS): + auth.get_user(token) + elapsed_ns = time.perf_counter_ns() - start + + return elapsed_ns / ITERATIONS / 1_000 + + +@pytest.mark.disabled_in_ci +@pytest.mark.django_db +def test_perf_cached_vs_uncached(data_fixture): + user = data_fixture.create_user() + token = AccessToken.for_user(user) + auth = JSONWebTokenAuthentication() + + with override_settings(BASEROW_CACHE_TTL_SECONDS=30): + invalidate_cached_user(user.id) + avg_cached = _bench(auth, token) + + with override_settings(BASEROW_CACHE_TTL_SECONDS=0): + invalidate_cached_user(user.id) + avg_uncached = _bench(auth, token) + + speedup = avg_uncached / avg_cached if avg_cached > 0 else float("inf") + + print() + print(f" Benchmark: {ITERATIONS} iterations of get_user()") + print(f" ┌──────────────────────────────────────────┐") + print(f" │ Uncached (DB every call): {avg_uncached:>8.1f} µs/call │") + print(f" │ Cached (Redis hit): {avg_cached:>8.1f} µs/call │") + print(f" │ Speedup: {speedup:>8.1f}x │") + print(f" └──────────────────────────────────────────┘") + + assert avg_cached < avg_uncached, ( + f"Expected cache to be faster, got cached={avg_cached:.1f}µs " + f"vs uncached={avg_uncached:.1f}µs" + ) diff --git a/backend/tests/baserow/api/test_throttle_blacklist.py b/backend/tests/baserow/api/test_throttle_blacklist.py new file mode 100644 index 0000000000..4f53b766b1 --- /dev/null +++ b/backend/tests/baserow/api/test_throttle_blacklist.py @@ -0,0 +1,313 @@ +from unittest.mock import patch + +from django.http import HttpResponse +from django.test import RequestFactory +from django.test.utils import CaptureQueriesContext, override_settings +from django.utils.module_loading import import_string + +import pytest +from rest_framework.status import HTTP_429_TOO_MANY_REQUESTS + +from baserow.throttling.blacklist import ( + _token_key, + blacklist_ip, + blacklist_token, + get_token_cooldown_time, + is_ip_blacklisted, +) +from baserow.throttling.middleware import ThrottleBlacklistMiddleware + + +def test_blacklist_key_is_sha256_hex(): + key = _token_key("my-secret-token") + assert key.startswith("throttle_bl:") + # SHA-256 hex digest is 64 chars + assert len(key) == len("throttle_bl:") + 64 + # Token itself must not appear in the key + assert "my-secret-token" not in key + + +@override_settings(BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS=7) +def test_blacklist_and_check(): + assert not get_token_cooldown_time("token-abc") + + blacklist_token("token-abc") + assert get_token_cooldown_time("token-abc") == 7 + + assert not get_token_cooldown_time("token-xyz") + + +def test_blacklist_different_tokens_are_independent(): + blacklist_token("token-1") + assert get_token_cooldown_time("token-1") + assert not get_token_cooldown_time("token-2") + + +def test_blacklist_returns_remaining_ttl(): + with patch("baserow.throttling.blacklist.time.time", return_value=100): + blacklist_token("token-remaining", ttl=7) + + with patch("baserow.throttling.blacklist.time.time", return_value=103.1): + assert get_token_cooldown_time("token-remaining") == 4 + + +def test_blacklist_token_noops_when_ttl_is_zero(): + blacklist_token("token-zero", ttl=0) + + assert get_token_cooldown_time("token-zero") is None + + +def test_blacklist_ip_noops_when_ttl_is_negative(): + blacklist_ip("192.168.1.9", ttl=-1) + + assert is_ip_blacklisted("192.168.1.9") is None + + +def test_throttle_handler_import_path_is_valid(): + from baserow.throttling.handler import ConcurrentUserRequestsThrottle + + imported = import_string( + "baserow.throttling.handler.ConcurrentUserRequestsThrottle" + ) + + assert imported is ConcurrentUserRequestsThrottle + + +def _make_middleware(status_code=200): + """Build the middleware with a dummy downstream response.""" + + def ok_response(request): + return HttpResponse(status=status_code) + + return ThrottleBlacklistMiddleware(ok_response) + + +def test_middleware_rejects_blacklisted_token(): + middleware = _make_middleware() + blacklist_token("the-token") + + factory = RequestFactory() + request = factory.get("/api/workspaces/", HTTP_AUTHORIZATION="JWT the-token") + + response = middleware(request) + + assert response.status_code == HTTP_429_TOO_MANY_REQUESTS + assert b"Request was throttled." in response.content + + +def test_middleware_allows_non_blacklisted_token(): + middleware = _make_middleware() + + factory = RequestFactory() + request = factory.get("/api/workspaces/", HTTP_AUTHORIZATION="JWT clean-token") + + response = middleware(request) + assert response.status_code == 200 + + +def test_middleware_ignores_non_jwt_requests(): + middleware = _make_middleware() + + factory = RequestFactory() + request = factory.get("/api/workspaces/") + + response = middleware(request) + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_middleware_zero_db_queries_on_blacklist_hit(data_fixture): + """The whole point: a blacklisted token triggers zero DB queries.""" + + middleware = _make_middleware() + blacklist_token("db-test-token") + + factory = RequestFactory() + request = factory.get("/api/workspaces/", HTTP_AUTHORIZATION="JWT db-test-token") + + from django.db import connection + + with CaptureQueriesContext(connection) as ctx: + response = middleware(request) + + assert response.status_code == HTTP_429_TOO_MANY_REQUESTS + assert len(ctx.captured_queries) == 0 + + +@pytest.mark.django_db +@override_settings( + BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT=30, + BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS=7, +) +def test_throttle_populates_blacklist_on_deny(data_fixture): + """When the throttle denies a request, the token is blacklisted.""" + + from rest_framework.test import APIRequestFactory + + from baserow.api.exceptions import ThrottledAPIException + from baserow.throttling.handler import ConcurrentUserRequestsThrottle + + user = data_fixture.create_user() + token_str = f"fake-token-{user.id}" + + # Build a realistic DRF request (matching existing throttle test pattern) + factory = APIRequestFactory() + request = factory.get("/api/workspaces/", HTTP_AUTHORIZATION=f"JWT {token_str}") + request.user = user + + class DummyDjangoRequest: + def __init__(self): + self.path = "/api/workspaces/" + self.user = user + self.META = {"HTTP_AUTHORIZATION": f"JWT {token_str}"} + + request._request = DummyDjangoRequest() + + ConcurrentUserRequestsThrottle.timer = lambda s: 1000 + ConcurrentUserRequestsThrottle.rate = 1 + + throttle = ConcurrentUserRequestsThrottle() + + # First request is allowed + assert throttle.allow_request(request, None) + assert not get_token_cooldown_time(token_str) + + # Second concurrent request is denied → raises and blacklists the token + # with the fixed cooldown (BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS). + throttle2 = ConcurrentUserRequestsThrottle() + with pytest.raises(ThrottledAPIException) as exc_info: + throttle2.allow_request(request, None) + + assert exc_info.value.wait == 7 + assert throttle2.wait() == 7 + assert get_token_cooldown_time(token_str) == 7 + + ConcurrentUserRequestsThrottle.on_request_processed(request._request) + + +# --- IP blacklist tests --- + + +@override_settings(BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS=9) +def test_ip_blacklist_and_check(): + assert not is_ip_blacklisted("192.168.1.1") + + blacklist_ip("192.168.1.1") + assert is_ip_blacklisted("192.168.1.1") == 9 + assert not is_ip_blacklisted("192.168.1.2") + + +@override_settings(BASEROW_THROTTLE_IP_ENABLED=True) +def test_middleware_rejects_blacklisted_ip_for_anonymous_request(): + middleware = _make_middleware() + blacklist_ip("10.0.0.1") + + factory = RequestFactory() + # REMOTE_ADDR is how Django exposes the client IP + request = factory.get("/api/workspaces/", REMOTE_ADDR="10.0.0.1") + + response = middleware(request) + assert response.status_code == HTTP_429_TOO_MANY_REQUESTS + assert b"Request was throttled." in response.content + + +@override_settings(BASEROW_THROTTLE_IP_ENABLED=True) +def test_middleware_allows_non_blacklisted_ip(): + middleware = _make_middleware() + blacklist_ip("10.0.0.1") + + factory = RequestFactory() + request = factory.get("/api/workspaces/", REMOTE_ADDR="10.0.0.2") + + response = middleware(request) + assert response.status_code == 200 + + +@override_settings(BASEROW_THROTTLE_IP_ENABLED=False) +def test_middleware_skips_ip_check_when_disabled(): + middleware = _make_middleware() + blacklist_ip("10.0.0.1") + + factory = RequestFactory() + request = factory.get("/api/workspaces/", REMOTE_ADDR="10.0.0.1") + + response = middleware(request) + assert response.status_code == 200 + + +@override_settings(BASEROW_THROTTLE_IP_ENABLED=True) +def test_middleware_ip_check_does_not_apply_to_jwt_requests(): + """JWT requests use the token blacklist, not the IP blacklist.""" + + middleware = _make_middleware() + blacklist_ip("10.0.0.1") + + factory = RequestFactory() + request = factory.get( + "/api/workspaces/", + REMOTE_ADDR="10.0.0.1", + HTTP_AUTHORIZATION="JWT some-clean-token", + ) + + # IP is blacklisted but this is a JWT request — should pass + response = middleware(request) + assert response.status_code == 200 + + +@override_settings(BASEROW_THROTTLE_IP_ENABLED=True) +def test_middleware_uses_x_forwarded_for_header(): + middleware = _make_middleware() + blacklist_ip("203.0.113.50") + + factory = RequestFactory() + request = factory.get( + "/api/workspaces/", + REMOTE_ADDR="10.0.0.1", # proxy IP + HTTP_X_FORWARDED_FOR="203.0.113.50, 70.41.3.18", + ) + + response = middleware(request) + assert response.status_code == HTTP_429_TOO_MANY_REQUESTS + + +@override_settings(BASEROW_THROTTLE_IP_ENABLED=True) +def test_middleware_does_not_blacklist_anonymous_ip_after_non_429_response(): + middleware = _make_middleware(status_code=200) + + factory = RequestFactory() + request = factory.get("/api/workspaces/", REMOTE_ADDR="198.51.100.11") + + response = middleware(request) + + assert response.status_code == 200 + assert not is_ip_blacklisted("198.51.100.11") + + +@override_settings(BASEROW_THROTTLE_IP_ENABLED=True) +def test_middleware_does_not_blacklist_ip_for_jwt_429_response(): + middleware = _make_middleware(status_code=HTTP_429_TOO_MANY_REQUESTS) + + factory = RequestFactory() + request = factory.get( + "/api/workspaces/", + REMOTE_ADDR="198.51.100.12", + HTTP_AUTHORIZATION="JWT some-clean-token", + ) + + response = middleware(request) + + assert response.status_code == HTTP_429_TOO_MANY_REQUESTS + assert not is_ip_blacklisted("198.51.100.12") + + +@override_settings(BASEROW_THROTTLE_IP_ENABLED=False) +def test_middleware_does_not_blacklist_anonymous_ip_when_disabled(): + middleware = _make_middleware(status_code=HTTP_429_TOO_MANY_REQUESTS) + + factory = RequestFactory() + request = factory.get("/api/workspaces/", REMOTE_ADDR="198.51.100.13") + + response = middleware(request) + + assert response.status_code == HTTP_429_TOO_MANY_REQUESTS + assert not is_ip_blacklisted("198.51.100.13") diff --git a/backend/tests/baserow/api/test_user_cache.py b/backend/tests/baserow/api/test_user_cache.py new file mode 100644 index 0000000000..319dbdd6bd --- /dev/null +++ b/backend/tests/baserow/api/test_user_cache.py @@ -0,0 +1,263 @@ +from django.db import connection +from django.shortcuts import reverse +from django.test.utils import CaptureQueriesContext, override_settings + +import pytest +from freezegun import freeze_time +from rest_framework.exceptions import AuthenticationFailed +from rest_framework_simplejwt.tokens import AccessToken + +from baserow.api.authentication import JSONWebTokenAuthentication +from baserow.core.user.cache import ( + get_cached_user, + invalidate_cached_user, + set_cached_user, +) +from baserow.core.user.handler import UserHandler + +_CACHE_ON = override_settings(BASEROW_CACHE_TTL_SECONDS=30) +_CACHE_OFF = override_settings(BASEROW_CACHE_TTL_SECONDS=0) + + +@pytest.mark.django_db +@_CACHE_ON +def test_set_and_get_cached_user(data_fixture): + user = data_fixture.create_user() + + assert get_cached_user(user.id) is None + + set_cached_user(user) + + cached = get_cached_user(user.id) + assert cached is not None + assert cached.id == user.id + assert cached.is_active == user.is_active + assert cached.profile.concurrency_limit == user.profile.concurrency_limit + + +@pytest.mark.django_db +@_CACHE_OFF +def test_caching_disabled_when_ttl_is_zero(data_fixture): + user = data_fixture.create_user() + + set_cached_user(user) + assert get_cached_user(user.id) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_invalidate_cached_user(data_fixture): + user = data_fixture.create_user() + set_cached_user(user) + assert get_cached_user(user.id) is not None + + invalidate_cached_user(user.id) + assert get_cached_user(user.id) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_signal_invalidates_cache_on_user_save( + data_fixture, django_capture_on_commit_callbacks +): + user = data_fixture.create_user() + set_cached_user(user) + assert get_cached_user(user.id) is not None + + with django_capture_on_commit_callbacks(execute=True): + user.first_name = "Changed" + user.save() + + assert get_cached_user(user.id) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_signal_invalidates_cache_on_profile_save( + data_fixture, django_capture_on_commit_callbacks +): + user = data_fixture.create_user() + set_cached_user(user) + assert get_cached_user(user.id) is not None + + with django_capture_on_commit_callbacks(execute=True): + user.profile.concurrency_limit = 5 + user.profile.save() + + assert get_cached_user(user.id) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_signal_invalidates_cache_on_deactivation( + data_fixture, django_capture_on_commit_callbacks +): + user = data_fixture.create_user() + set_cached_user(user) + + with django_capture_on_commit_callbacks(execute=True): + user.is_active = False + user.save() + + assert get_cached_user(user.id) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_signal_invalidates_cache_on_user_delete( + data_fixture, django_capture_on_commit_callbacks +): + user = data_fixture.create_user() + set_cached_user(user) + assert get_cached_user(user.id) is not None + + with django_capture_on_commit_callbacks(execute=True): + user.delete() + + assert get_cached_user(user.id) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_cached_user_profile_accessible_without_extra_query(data_fixture): + user = data_fixture.create_user() + set_cached_user(user) + + cached = get_cached_user(user.id) + + with CaptureQueriesContext(connection) as ctx: + _ = cached.profile.concurrency_limit + _ = cached.profile.last_password_change + _ = cached.is_active + _ = cached.is_staff + + assert len(ctx.captured_queries) == 0 + + +@pytest.mark.django_db +@_CACHE_ON +def test_cached_user_omits_password_hash(data_fixture): + import pickle + + user = data_fixture.create_user(password="passwordhashcanary") + invalidate_cached_user(user.id) + token = AccessToken.for_user(user) + + JSONWebTokenAuthentication().get_user(token) + + cached = get_cached_user(user.id) + assert cached is not None + assert user.password.encode() not in pickle.dumps(cached) + + +@pytest.mark.django_db +@_CACHE_ON +def test_get_user_uses_cache_on_second_call(data_fixture): + user = data_fixture.create_user() + token = AccessToken.for_user(user) + auth = JSONWebTokenAuthentication() + + # First call — cache miss, hits DB + with CaptureQueriesContext(connection) as ctx1: + result1 = auth.get_user(token) + db_queries_first = len(ctx1.captured_queries) + assert db_queries_first >= 1 + assert result1.id == user.id + + # Second call — cache hit, no DB + with CaptureQueriesContext(connection) as ctx2: + result2 = auth.get_user(token) + assert len(ctx2.captured_queries) == 0 + assert result2.id == user.id + + +@pytest.mark.django_db +@_CACHE_OFF +def test_get_user_always_hits_db_when_cache_disabled(data_fixture): + user = data_fixture.create_user() + token = AccessToken.for_user(user) + auth = JSONWebTokenAuthentication() + + with CaptureQueriesContext(connection) as ctx1: + auth.get_user(token) + first_count = len(ctx1.captured_queries) + + with CaptureQueriesContext(connection) as ctx2: + auth.get_user(token) + second_count = len(ctx2.captured_queries) + + assert first_count >= 1 + assert second_count >= 1 + + +@pytest.mark.django_db +@_CACHE_ON +def test_password_change_invalidates_cache_and_rejects_old_token( + data_fixture, api_request_factory, django_capture_on_commit_callbacks +): + with freeze_time("2020-01-01 12:00:00"): + user, token = data_fixture.create_user_and_token(password="oldpass") + + auth = JSONWebTokenAuthentication() + + # Warm the cache by making an authenticated request + with freeze_time("2020-01-01 12:00:01"): + request = api_request_factory.get( + reverse("api:user:account"), + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + authenticated_user, _ = auth.authenticate(request) + assert authenticated_user.id == user.id + assert get_cached_user(user.id) is not None + + # Change password — should invalidate cache + with freeze_time("2020-01-01 12:00:02"): + with django_capture_on_commit_callbacks(execute=True): + UserHandler().change_password(user, "oldpass", "newpass123") + assert get_cached_user(user.id) is None + + # Old token should be rejected (even after cache is re-populated) + with freeze_time("2020-01-01 12:00:03"): + request = api_request_factory.get( + reverse("api:user:account"), + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + with pytest.raises(AuthenticationFailed): + auth.authenticate(request) + + +@pytest.mark.django_db +@_CACHE_ON +def test_user_delete_invalidates_cache_and_rejects_old_token( + data_fixture, api_request_factory, django_capture_on_commit_callbacks +): + with freeze_time("2020-01-01 12:00:00"): + user, token = data_fixture.create_user_and_token(password="oldpass") + + auth = JSONWebTokenAuthentication() + + with freeze_time("2020-01-01 12:00:01"): + request = api_request_factory.get( + reverse("api:user:account"), + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + authenticated_user, _ = auth.authenticate(request) + assert authenticated_user.id == user.id + assert get_cached_user(user.id) is not None + + with freeze_time("2020-01-01 12:00:02"): + with django_capture_on_commit_callbacks(execute=True): + user.delete() + assert get_cached_user(user.id) is None + + with freeze_time("2020-01-01 12:00:03"): + request = api_request_factory.get( + reverse("api:user:account"), + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + with pytest.raises(AuthenticationFailed): + auth.authenticate(request) diff --git a/backend/tests/baserow/api/users/test_user_views.py b/backend/tests/baserow/api/users/test_user_views.py index b9f09a851d..9ffa91f555 100755 --- a/backend/tests/baserow/api/users/test_user_views.py +++ b/backend/tests/baserow/api/users/test_user_views.py @@ -964,7 +964,7 @@ def test_dashboard(data_fixture, client): invitation_1.invited_by.first_name ) assert response_json["workspace_invitations"][0]["workspace"] == "Test1" - assert response_json["workspace_invitations"][0]["message"] == invitation_1.message + assert "message" not in response_json["workspace_invitations"][0] assert "created_on" in response_json["workspace_invitations"][0] diff --git a/backend/tests/baserow/contrib/automation/workflows/test_workflow_handler.py b/backend/tests/baserow/contrib/automation/workflows/test_workflow_handler.py index 5c7aa169c4..3b2b8d9b0c 100644 --- a/backend/tests/baserow/contrib/automation/workflows/test_workflow_handler.py +++ b/backend/tests/baserow/contrib/automation/workflows/test_workflow_handler.py @@ -1260,6 +1260,8 @@ def test_async_start_workflow_queues_celery_task_on_commit( ): workflow = data_fixture.create_automation_workflow() + mock_on_commit.reset_mock() + AutomationWorkflowHandler().async_start_workflow(workflow) history = workflow.workflow_histories.get() diff --git a/backend/tests/baserow/contrib/database/tokens/test_token_cache.py b/backend/tests/baserow/contrib/database/tokens/test_token_cache.py new file mode 100644 index 0000000000..8631e348f6 --- /dev/null +++ b/backend/tests/baserow/contrib/database/tokens/test_token_cache.py @@ -0,0 +1,212 @@ +from django.db import connection +from django.test.utils import CaptureQueriesContext, override_settings + +import pytest + +from baserow.contrib.database.api.tokens.authentications import TokenAuthentication +from baserow.contrib.database.tokens.cache import ( + _cache_key, + cache, + get_cached_token, + invalidate_cached_token, + set_cached_token, +) +from baserow.contrib.database.tokens.handler import TokenHandler + +_CACHE_ON = override_settings(BASEROW_CACHE_TTL_SECONDS=30) +_CACHE_OFF = override_settings(BASEROW_CACHE_TTL_SECONDS=0) + + +@pytest.mark.django_db +@_CACHE_ON +def test_set_and_get_cached_token(data_fixture): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + + assert get_cached_token(token.key) is None + + set_cached_token(token) + + cached = get_cached_token(token.key) + assert cached is not None + assert cached.id == token.id + assert cached.user_id == user.id + assert cached.workspace_id == workspace.id + + +@pytest.mark.django_db +@_CACHE_ON +def test_cached_token_omits_auth_secrets(data_fixture): + import pickle + + user = data_fixture.create_user(password="passwordhashcanary") + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + invalidate_cached_token(token.key) + + TokenHandler().get_by_key(key=token.key) + + cached = cache.get(_cache_key(token.key)) + assert cached is not None + payload = pickle.dumps(cached) + assert token.key.encode() not in payload + assert user.password.encode() not in payload + + +@pytest.mark.django_db +@_CACHE_OFF +def test_db_token_caching_disabled_when_ttl_is_zero(data_fixture): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + + set_cached_token(token) + assert get_cached_token(token.key) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_get_by_key_populates_cache_on_first_call(data_fixture): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + invalidate_cached_token(token.key) + + handler = TokenHandler() + + # First call — cache miss, hits DB + with CaptureQueriesContext(connection) as ctx1: + first = handler.get_by_key(key=token.key) + assert first.id == token.id + assert len(ctx1.captured_queries) >= 1 + + # Second call — cache hit, no DB + with CaptureQueriesContext(connection) as ctx2: + second = handler.get_by_key(key=token.key) + assert second.id == token.id + assert len(ctx2.captured_queries) == 0 + + +@pytest.mark.django_db +@_CACHE_ON +def test_token_http_request_runs_no_queries_after_cache_warmup( + data_fixture, api_request_factory +): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + invalidate_cached_token(token.key) + + auth = TokenAuthentication() + + def _make_request(): + return api_request_factory.get( + "/api/database/rows/table/1/", + HTTP_AUTHORIZATION=f"Token {token.key}", + ) + + # First request — cache miss, hits DB to resolve token + user + profile + with CaptureQueriesContext(connection) as ctx1: + authenticated_user, _ = auth.authenticate(_make_request()) + assert authenticated_user.id == user.id + assert len(ctx1.captured_queries) >= 1 + + # Subsequent requests — cache hit, full auth flow runs no queries + for _ in range(3): + with CaptureQueriesContext(connection) as ctx: + authenticated_user, _ = auth.authenticate(_make_request()) + assert authenticated_user.id == user.id + assert len(ctx.captured_queries) == 0, [q["sql"] for q in ctx.captured_queries] + + +@pytest.mark.django_db +@_CACHE_ON +def test_token_save_invalidates_cache(data_fixture, django_capture_on_commit_callbacks): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + set_cached_token(token) + assert get_cached_token(token.key) is not None + + with django_capture_on_commit_callbacks(execute=True): + token.name = "Renamed" + token.save() + + assert get_cached_token(token.key) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_token_delete_invalidates_cache( + data_fixture, django_capture_on_commit_callbacks +): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + set_cached_token(token) + assert get_cached_token(token.key) is not None + + cached_key = token.key + with django_capture_on_commit_callbacks(execute=True): + token.delete() + + assert get_cached_token(cached_key) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_rotate_token_key_invalidates_old_and_new_key( + data_fixture, django_capture_on_commit_callbacks +): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + old_key = token.key + set_cached_token(token) + assert get_cached_token(old_key) is not None + + with django_capture_on_commit_callbacks(execute=True): + rotated = TokenHandler().rotate_token_key(user, token) + + assert rotated.key != old_key + # The old key must no longer point to the stale cached token. + assert get_cached_token(old_key) is None + # The new key starts uncached; next get_by_key will re-populate it. + assert get_cached_token(rotated.key) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_user_save_invalidates_token_cache( + data_fixture, django_capture_on_commit_callbacks +): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + set_cached_token(token) + assert get_cached_token(token.key) is not None + + with django_capture_on_commit_callbacks(execute=True): + user.first_name = "Changed" + user.save() + + assert get_cached_token(token.key) is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_user_profile_save_invalidates_token_cache( + data_fixture, django_capture_on_commit_callbacks +): + user = data_fixture.create_user() + workspace = data_fixture.create_workspace(user=user) + token = data_fixture.create_token(user=user, workspace=workspace) + set_cached_token(token) + assert get_cached_token(token.key) is not None + + with django_capture_on_commit_callbacks(execute=True): + user.profile.concurrency_limit = 5 + user.profile.save() + + assert get_cached_token(token.key) is None diff --git a/backend/tests/baserow/core/actions/test_group_invitation_actions.py b/backend/tests/baserow/core/actions/test_group_invitation_actions.py index c581e88c13..2719bd3d14 100755 --- a/backend/tests/baserow/core/actions/test_group_invitation_actions.py +++ b/backend/tests/baserow/core/actions/test_group_invitation_actions.py @@ -21,7 +21,6 @@ def test_create_workspace_invitation_action_type(data_fixture): email="user@test.com", permissions="ADMIN", base_url="http://localhost:3000/", - message="hello!", ) assert invitation.workspace == workspace diff --git a/backend/tests/baserow/core/test_core_handler.py b/backend/tests/baserow/core/test_core_handler.py index 5a4ff427c3..d77fa6ce09 100755 --- a/backend/tests/baserow/core/test_core_handler.py +++ b/backend/tests/baserow/core/test_core_handler.py @@ -7,7 +7,6 @@ from django.conf import settings from django.core.files.storage import FileSystemStorage from django.db import OperationalError, transaction -from django.test.utils import override_settings import pytest from itsdangerous.exc import BadSignature @@ -29,7 +28,6 @@ DuplicateApplicationMaxLocksExceededException, IsNotAdminError, LastAdminOfWorkspace, - MaxNumberOfPendingWorkspaceInvitesReached, TemplateDoesNotExist, TemplateFileDoesNotExist, UserInvalidWorkspacePermissionsError, @@ -614,7 +612,6 @@ def test_create_workspace_invitation(mock_send_email, data_fixture): workspace=workspace, email="test@test.nl", permissions="ADMIN", - message="Test", base_url="http://localhost:3000/invite", ) @@ -624,7 +621,6 @@ def test_create_workspace_invitation(mock_send_email, data_fixture): workspace=workspace, email="test@test.nl", permissions="ADMIN", - message="Test", base_url="http://localhost:3000/invite", ) @@ -634,7 +630,6 @@ def test_create_workspace_invitation(mock_send_email, data_fixture): workspace=workspace, email=user_3.email, permissions="ADMIN", - message="Test", base_url="http://localhost:3000/invite", ) @@ -643,14 +638,12 @@ def test_create_workspace_invitation(mock_send_email, data_fixture): workspace=workspace, email="test@test.nl", permissions="ADMIN", - message="Test", base_url="http://localhost:3000/invite", ) assert invitation.invited_by_id == user.id assert invitation.workspace_id == workspace.id assert invitation.email == "test@test.nl" assert invitation.permissions == "ADMIN" - assert invitation.message == "Test" assert WorkspaceInvitation.objects.all().count() == 1 mock_send_email.assert_called_once() @@ -664,14 +657,12 @@ def test_create_workspace_invitation(mock_send_email, data_fixture): workspace=workspace, email="test@test.nl", permissions="MEMBER", - message="New message", base_url="http://localhost:3000/invite", ) assert invitation.invited_by_id == user.id assert invitation.workspace_id == workspace.id assert invitation.email == "test@test.nl" assert invitation.permissions == "MEMBER" - assert invitation.message == "New message" assert WorkspaceInvitation.objects.all().count() == 1 invitation = handler.create_workspace_invitation( @@ -679,14 +670,12 @@ def test_create_workspace_invitation(mock_send_email, data_fixture): workspace=workspace, email="test2@test.nl", permissions="ADMIN", - message="", base_url="http://localhost:3000/invite", ) assert invitation.invited_by_id == user.id assert invitation.workspace_id == workspace.id assert invitation.email == "test2@test.nl" assert invitation.permissions == "ADMIN" - assert invitation.message == "" assert WorkspaceInvitation.objects.all().count() == 2 invitation = handler.create_workspace_invitation( @@ -700,51 +689,9 @@ def test_create_workspace_invitation(mock_send_email, data_fixture): assert invitation.workspace_id == workspace.id assert invitation.email == "test3@test.nl" assert invitation.permissions == "ADMIN" - assert invitation.message == "" assert WorkspaceInvitation.objects.all().count() == 3 -@pytest.mark.django_db -@patch("baserow.core.handler.CoreHandler.send_workspace_invitation_email") -@override_settings(BASEROW_MAX_PENDING_WORKSPACE_INVITES=1) -def test_create_workspace_invitation_max_pending(mock_send_email, data_fixture): - user_workspace = data_fixture.create_user_workspace() - user = user_workspace.user - workspace = user_workspace.workspace - - handler = CoreHandler() - - handler.create_workspace_invitation( - user=user, - workspace=workspace, - email="test@test.nl", - permissions="ADMIN", - message="Test", - base_url="http://localhost:3000/invite", - ) - - with pytest.raises(MaxNumberOfPendingWorkspaceInvitesReached): - handler.create_workspace_invitation( - user=user, - workspace=workspace, - email="test2@test.nl", - permissions="ADMIN", - message="Test", - base_url="http://localhost:3000/invite", - ) - - # This email address already exists, so it should just update the invite without - # failing. - handler.create_workspace_invitation( - user=user, - workspace=workspace, - email="test@test.nl", - permissions="MEMBER", - message="Test", - base_url="http://localhost:3000/invite", - ) - - @pytest.mark.django_db def test_update_workspace_invitation(data_fixture): workspace_invitation = data_fixture.create_workspace_invitation() diff --git a/backend/tests/baserow/core/test_settings_cache.py b/backend/tests/baserow/core/test_settings_cache.py new file mode 100644 index 0000000000..47deae2ba5 --- /dev/null +++ b/backend/tests/baserow/core/test_settings_cache.py @@ -0,0 +1,93 @@ +from django.db import connection +from django.test.utils import CaptureQueriesContext, override_settings + +import pytest + +from baserow.core.cache import ( + get_cached_settings, + invalidate_cached_settings, + set_cached_settings, +) +from baserow.core.handler import CoreHandler + +_CACHE_ON = override_settings(BASEROW_CACHE_TTL_SECONDS=30) +_CACHE_OFF = override_settings(BASEROW_CACHE_TTL_SECONDS=0) + + +@pytest.mark.django_db +@_CACHE_ON +def test_set_and_get_cached_settings(): + instance = CoreHandler().get_settings() + invalidate_cached_settings() + + assert get_cached_settings() is None + + set_cached_settings(instance) + + cached = get_cached_settings() + assert cached is not None + assert cached.pk == instance.pk + assert cached.allow_new_signups == instance.allow_new_signups + + +@pytest.mark.django_db +@_CACHE_OFF +def test_caching_disabled_when_ttl_is_zero(): + instance = CoreHandler().get_settings() + + set_cached_settings(instance) + assert get_cached_settings() is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_invalidate_cached_settings(): + instance = CoreHandler().get_settings() + set_cached_settings(instance) + assert get_cached_settings() is not None + + invalidate_cached_settings() + assert get_cached_settings() is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_signal_invalidates_cache_on_settings_save(django_capture_on_commit_callbacks): + instance = CoreHandler().get_settings() + set_cached_settings(instance) + assert get_cached_settings() is not None + + with django_capture_on_commit_callbacks(execute=True): + instance.allow_new_signups = not instance.allow_new_signups + instance.save() + + assert get_cached_settings() is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_get_settings_uses_cache_on_second_call(): + CoreHandler().get_settings() + invalidate_cached_settings() + + with CaptureQueriesContext(connection) as ctx1: + CoreHandler().get_settings() + assert len(ctx1.captured_queries) >= 1 + + with CaptureQueriesContext(connection) as ctx2: + CoreHandler().get_settings() + assert len(ctx2.captured_queries) == 0 + + +@pytest.mark.django_db +@_CACHE_OFF +def test_get_settings_always_hits_db_when_cache_disabled(): + CoreHandler().get_settings() + + with CaptureQueriesContext(connection) as ctx1: + CoreHandler().get_settings() + with CaptureQueriesContext(connection) as ctx2: + CoreHandler().get_settings() + + assert len(ctx1.captured_queries) >= 1 + assert len(ctx2.captured_queries) >= 1 diff --git a/backend/tests/baserow/core/test_throttling.py b/backend/tests/baserow/core/test_throttling.py index 821a08f190..bb6107f4d5 100644 --- a/backend/tests/baserow/core/test_throttling.py +++ b/backend/tests/baserow/core/test_throttling.py @@ -5,7 +5,9 @@ import pytest from freezegun import freeze_time -from baserow.throttling import RateLimit, RateLimitExceededException, rate_limit +from baserow.throttling.exceptions import RateLimitExceededException +from baserow.throttling.handler import rate_limit +from baserow.throttling.types import RateLimit throttled_fn_name = "throttled_function" throttled_fn_2_name = "throttled_function_2" diff --git a/backend/tests/baserow/core/user/test_user_actions.py b/backend/tests/baserow/core/user/test_user_actions.py index 3a302ec596..f7afe8ce6e 100755 --- a/backend/tests/baserow/core/user/test_user_actions.py +++ b/backend/tests/baserow/core/user/test_user_actions.py @@ -18,7 +18,7 @@ UpdateUserActionType, ) from baserow.core.user.handler import UserHandler -from baserow.throttling_types import RateLimit +from baserow.throttling.types import RateLimit from baserow_enterprise.audit_log.models import AuditLogEntry diff --git a/changelog/entries/unreleased/breaking_change/remove_workspace_invitation_messages_and_pending_invite_en.json b/changelog/entries/unreleased/breaking_change/remove_workspace_invitation_messages_and_pending_invite_en.json new file mode 100644 index 0000000000..b3c79601b1 --- /dev/null +++ b/changelog/entries/unreleased/breaking_change/remove_workspace_invitation_messages_and_pending_invite_en.json @@ -0,0 +1,9 @@ +{ + "type": "breaking_change", + "message": "Workspace invitations no longer support custom messages, and the `BASEROW_MAX_PENDING_WORKSPACE_INVITES` env var has been removed.", + "issue_origin": "github", + "issue_number": null, + "domain": "core", + "bullet_points": [], + "created_at": "2026-04-17" +} diff --git a/changelog/entries/unreleased/bug/fix_grid_field_quick_edit_without_update_permission.json b/changelog/entries/unreleased/bug/fix_grid_field_quick_edit_without_update_permission.json new file mode 100644 index 0000000000..b37cb8fa2c --- /dev/null +++ b/changelog/entries/unreleased/bug/fix_grid_field_quick_edit_without_update_permission.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Fix grid field quick edit crashing when the field update context is unavailable because the user lacks field update permission.", + "issue_origin": "github", + "issue_number": 5216, + "domain": "database", + "bullet_points": [], + "created_at": "2026-04-16" +} diff --git a/changelog/entries/unreleased/bug/fix_rollup_field_missing_relation_crash.json b/changelog/entries/unreleased/bug/fix_rollup_field_missing_relation_crash.json new file mode 100644 index 0000000000..fc4bc52eb4 --- /dev/null +++ b/changelog/entries/unreleased/bug/fix_rollup_field_missing_relation_crash.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Fix rollup and count fields crashing when a stale in-memory relation points to a deleted field during formula recalculation.", + "issue_origin": "github", + "issue_number": 5214, + "domain": "database", + "bullet_points": [], + "created_at": "2026-04-17" +} diff --git a/changelog/entries/unreleased/bug/fix_rowhistory_index_migration_concurrent.json b/changelog/entries/unreleased/bug/fix_rowhistory_index_migration_concurrent.json new file mode 100644 index 0000000000..86f5c2d622 --- /dev/null +++ b/changelog/entries/unreleased/bug/fix_rowhistory_index_migration_concurrent.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Make concurrent index migrations idempotent so they can be re-run after a partial failure.", + "issue_origin": "github", + "issue_number": null, + "domain": "core", + "bullet_points": [], + "created_at": "2026-04-20" +} diff --git a/changelog/entries/unreleased/refactor/throttle_performance_optimizations.json b/changelog/entries/unreleased/refactor/throttle_performance_optimizations.json new file mode 100644 index 0000000000..ce8aa5a8c8 --- /dev/null +++ b/changelog/entries/unreleased/refactor/throttle_performance_optimizations.json @@ -0,0 +1,8 @@ +{ + "type": "refactor", + "message": "Optimize rate limiting: cache user and settings lookups and reorganize throttling code.", + "issue_origin": "github", + "issue_number": null, + "domain": "core", + "created_at": "2026-04-15" +} diff --git a/docker-compose.no-caddy.yml b/docker-compose.no-caddy.yml index 327a8db151..b03b7c7418 100644 --- a/docker-compose.no-caddy.yml +++ b/docker-compose.no-caddy.yml @@ -45,6 +45,7 @@ x-backend-variables: DATABASE_PORT: DATABASE_OPTIONS: DATABASE_URL: + BASEROW_CONN_MAX_AGE: # Set these if you want to use an external redis instead of the redis service below. REDIS_HOST: @@ -109,6 +110,7 @@ x-backend-variables: BASEROW_BACKEND_DEBUG: BASEROW_BACKEND_LOG_LEVEL: + BASEROW_DJANGO_REQUEST_LOG_LEVEL: FEATURE_FLAGS: BASEROW_ENABLE_OTEL: BASEROW_DEPLOYMENT_ENV: @@ -166,6 +168,9 @@ x-backend-variables: BASEROW_PERIODIC_FIELD_UPDATE_QUEUE_NAME: BASEROW_MAX_CONCURRENT_USER_REQUESTS: BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT: + BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS: + BASEROW_CACHE_TTL_SECONDS: + BASEROW_THROTTLE_IP_ENABLED: BASEROW_SEND_VERIFY_EMAIL_RATE_LIMIT: BASEROW_LOGIN_ACTION_LOG_LIMIT: BASEROW_OSS_ONLY: diff --git a/docker-compose.yml b/docker-compose.yml index d4d4a64dd1..25ae46adfd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,7 @@ x-backend-variables: DATABASE_PORT: DATABASE_OPTIONS: DATABASE_URL: + BASEROW_CONN_MAX_AGE: DATABASE_READ_1_USER: DATABASE_READ_1_NAME: @@ -123,6 +124,7 @@ x-backend-variables: BASEROW_BACKEND_DEBUG: BASEROW_BACKEND_LOG_LEVEL: + BASEROW_DJANGO_REQUEST_LOG_LEVEL: FEATURE_FLAGS: BASEROW_ENABLE_OTEL: BASEROW_DEPLOYMENT_ENV: @@ -184,6 +186,9 @@ x-backend-variables: BASEROW_PERIODIC_FIELD_UPDATE_QUEUE_NAME: BASEROW_MAX_CONCURRENT_USER_REQUESTS: BASEROW_CONCURRENT_USER_REQUESTS_THROTTLE_TIMEOUT: + BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS: + BASEROW_CACHE_TTL_SECONDS: + BASEROW_THROTTLE_IP_ENABLED: BASEROW_SEND_VERIFY_EMAIL_RATE_LIMIT: BASEROW_LOGIN_ACTION_LOG_LIMIT: BASEROW_OSS_ONLY: diff --git a/docs/installation/configuration.md b/docs/installation/configuration.md index 43e283478d..8b7f7509e1 100644 --- a/docs/installation/configuration.md +++ b/docs/installation/configuration.md @@ -45,8 +45,10 @@ The installation methods referred to in the variable descriptions are: | BASEROW\_JWT\_SIGNING\_KEY | The signing key that is used to sign the content of generated tokens. For HMAC signing, this should be a random string with at least as many bits of data as is required by the signing protocol. See [https://django-rest-framework-simplejwt.readthedocs.io/en/latest/settings.html#signing-key](https://django-rest-framework-simplejwt.readthedocs.io/en/latest/settings.html#signing-key) for more details | Recommended to be set by you in the docker-compose and standalone installs (default to the SECRET\_KEY). Automatically generated by the baserow/baserow image if not provided and stored in /baserow/data/.jwt_signing_key. | | BASEROW\_ACCESS\_TOKEN\_LIFETIME\_MINUTES | The number of minutes which specifies how long access tokens are valid. This will be converted in a timedelta value and added to the current UTC time during token generation to obtain the token’s default “exp” claim value. | 10 minutes. | | BASEROW\_REFRESH\_TOKEN\_LIFETIME\_HOURS | The number of hours which specifies how long refresh tokens are valid. This will be converted in a timedelta value and added to the current UTC time during token generation to obtain the token’s default “exp” claim value. | 168 hours (7 days). | +| BASEROW\_CACHE\_TTL\_SECONDS | How long (in seconds) to cache lookups of authenticated users, database tokens, instance-wide settings, and active licenses in Redis, to speed up requests and reduce database load. Set to 0 to disable these caches entirely. | 0 (disabled) | | BASEROW\_BACKEND\_LOG\_LEVEL | The default log level used by the backend, supports ERROR, WARNING, INFO, DEBUG, TRACE | INFO | | BASEROW\_BACKEND\_DATABASE\_LOG\_LEVEL | The default log level used for database related logs in the backend. Supports the same values as the normal log level. If you also enable BASEROW\_BACKEND\_DEBUG and set this to DEBUG you will be able to see all SQL queries in the backend logs. | ERROR | +| BASEROW\_DJANGO\_REQUEST\_LOG\_LEVEL | The log level for the `django.request` logger. Default is ERROR to suppress noisy 429 responses under heavy throttling. Supports ERROR, WARNING, INFO, DEBUG. | ERROR | | BASEROW\_BACKEND\_DEBUG | If set to “on” then will enable the non production safe debug mode for the Baserow django backend. Only use this if you're using Baserow in development mode. Note that this causes any installed licenses to deactivate and prevent you from adding a new license. | off | | BASEROW\_AMOUNT\_OF\_GUNICORN\_WORKERS | The number of concurrent worker processes used by the Baserow backend gunicorn server to process incoming requests | | | BASEROW\_AIRTABLE\_IMPORT\_SOFT\_TIME\_LIMIT | The maximum amount of seconds an Airtable migration import job can run. | 1800 seconds - 30 minutes | @@ -65,6 +67,17 @@ The installation methods referred to in the variable descriptions are: | BASEROW\_ASGI\_HTTP\_MAX\_CONCURRENCY | Specifies a limit for concurrent requests handled by a single gunicorn worker. The default is: no limit. | | | BASEROW\_IMPORT\_EXPORT\_RESOURCE\_REMOVAL\_AFTER\_DAYS | Specifies the number of days after which an import/export resource will be automatically deleted. | 5 (days) | +### Rate Limiting + +Baserow can throttle the number of concurrent requests a single user (or, optionally, a single anonymous IP) is allowed to have in flight at the same time. When enabled, excess requests are rejected immediately with a `429 Too Many Requests` response — they do not wait for a slot to free up. Repeat offenders can additionally be added to a short-lived blacklist that is consulted by a lightweight middleware before authentication runs, so that further requests are rejected with zero DB / DRF overhead. + +| Name | Description | Defaults | +|--------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------| +| BASEROW\_MAX\_CONCURRENT\_USER\_REQUESTS | Limits the number of concurrent requests per authenticated user. When set to a value greater than 0, enables the concurrent-request throttle. Set to -1 or leave unset to disable throttling. Can be overridden on a per-user basis by setting the `concurrency_limit` field on the user's profile, which takes precedence over this value when greater than 0. | -1 (disabled) | +| BASEROW\_CONCURRENT\_USER\_REQUESTS\_THROTTLE\_TIMEOUT | How long (in seconds) an in-flight request stays counted toward the concurrent-request throttle before it is treated as stale and evicted automatically. | 180 | +| BASEROW\_THROTTLE\_BLACKLIST\_TTL\_SECONDS | Cooldown (in seconds) applied to denied requests. When greater than 0, a denied request blacklists the token (or IP, if enabled) for that many seconds; subsequent requests are rejected by the throttle-blacklist middleware with zero DB/DRF overhead, matching DRF's standard `Throttled` response shape (with `Retry-After`). When set to 0 or a negative value (default), no blacklist is kept and no `Retry-After` is emitted. Only takes effect when `BASEROW_MAX_CONCURRENT_USER_REQUESTS` is enabled. | -1 (disabled) | +| BASEROW\_THROTTLE\_IP\_ENABLED | When "true", anonymous requests are included in the concurrent-request throttle using the client IP as the key. Without this flag, anonymous requests bypass the throttle entirely. Independent of the blacklist: when the blacklist is also enabled (`BASEROW_THROTTLE_BLACKLIST_TTL_SECONDS` > 0) the IP-blacklist middleware will additionally reject repeat-offender IPs before they reach the rest of the stack. Only takes effect when `BASEROW_MAX_CONCURRENT_USER_REQUESTS` is enabled. | false | + ### Backend Database Configuration | Name | Description | Defaults | |--------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -94,6 +107,7 @@ The installation methods referred to in the variable descriptions are: | POSTGRES\_STARTUP\_CHECK\_ATTEMPTS | When Baserow's Backend service starts up it first checks to see if the postgres database is available. It checks 5 times by default, after which if it still has not connected it will crash. | 5 | | BASEROW\_PREVENT\_POSTGRESQL\_DATA\_SYNC\_CONNECTION\_TO\_DATABASE | If true, then it's impossible to connect to the Baserow PostgreSQL database using the PostgreSQL data sync. | true | | BASEROW\_POSTGRESQL\_DATA\_SYNC\_BLACKLIST | Optionally provide a comma separated list of hostnames that the Baserow PostgreSQL data sync can't connect to. (e.g. "localhost,baserow.io") | | +| BASEROW\_CONN\_MAX\_AGE | How long (in seconds) a database connection may be reused. Set to 0 to close connections after each request. With WSGI, values > 0 can improve performance. With ASGI, use caution as persistent connections can multiply across async coroutines. Applied to all configured databases (primary and read replicas). | 0 | ### Redis Configuration | Name | Description | Defaults | diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/crudTable/filters/DateFilter.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/crudTable/filters/DateFilter.vue index adbe8dc40f..b0dfb3dda9 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/components/crudTable/filters/DateFilter.vue +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/crudTable/filters/DateFilter.vue @@ -84,10 +84,6 @@ export default { copy: '', dateString: '', dateObject: '', - datePickerLang: { - en: {}, - fr: {}, - }, } }, created() { diff --git a/enterprise/web-frontend/modules/baserow_enterprise/module.js b/enterprise/web-frontend/modules/baserow_enterprise/module.js index 93bea7ad48..9ffdaa3452 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/module.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/module.js @@ -25,15 +25,11 @@ export default defineNuxtModule({ (route) => route.name === 'settings' ) - // TODO MIG do we still need that? - // Prevent for adding the route multiple times - if (!settingsRoute.children.find(({ path }) => path === 'teams')) { - settingsRoute.children.push({ - name: 'settings-teams', - path: 'teams', - file: path.resolve(__dirname, 'pages/settings/teams.vue'), - }) - } + settingsRoute.children.push({ + name: 'settings-teams', + path: 'teams', + file: path.resolve(__dirname, 'pages/settings/teams.vue'), + }) // Add enterprise routes as children of root (inherit layout and middlewares) rootChildRoutes.forEach((route) => { diff --git a/premium/backend/src/baserow_premium/apps.py b/premium/backend/src/baserow_premium/apps.py index b42a16ce10..2dd1002855 100644 --- a/premium/backend/src/baserow_premium/apps.py +++ b/premium/backend/src/baserow_premium/apps.py @@ -8,6 +8,7 @@ class BaserowPremiumConfig(AppConfig): def ready(self): # noinspection PyUnresolvedReferences + import baserow_premium.license.receivers # noqa: F401 import baserow_premium.row_comments.receivers # noqa: F401 from baserow.core.registries import application_type_registry from baserow_premium.api.user.user_data_types import ActiveLicensesDataType diff --git a/premium/backend/src/baserow_premium/license/cache.py b/premium/backend/src/baserow_premium/license/cache.py new file mode 100644 index 0000000000..fd148487dd --- /dev/null +++ b/premium/backend/src/baserow_premium/license/cache.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +from django.conf import settings +from django.core.cache import cache + +if TYPE_CHECKING: + from baserow_premium.license.models import License + +_CACHE_KEY = "license:instance_wide_active" + + +def get_cached_instance_wide_licenses() -> list[License] | None: + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return None + return cache.get(_CACHE_KEY) + + +def set_cached_instance_wide_licenses(licenses: list[License]) -> None: + """ + Cache the active instance-wide licenses for at most + ``BASEROW_CACHE_TTL_SECONDS`` and no longer than until the earliest license + expires, so the cache naturally self-heals when a license rolls over. + """ + + max_ttl = settings.BASEROW_CACHE_TTL_SECONDS + if max_ttl <= 0: + return + + if licenses: + now = datetime.now(tz=timezone.utc) + earliest = min( + int((lic.valid_through - now).total_seconds()) for lic in licenses + ) + ttl = max(1, min(max_ttl, earliest)) + else: + ttl = max_ttl + cache.set(_CACHE_KEY, licenses, timeout=ttl) + + +def invalidate_cached_instance_wide_licenses() -> None: + if settings.BASEROW_CACHE_TTL_SECONDS <= 0: + return + cache.delete(_CACHE_KEY) diff --git a/premium/backend/src/baserow_premium/license/plugin.py b/premium/backend/src/baserow_premium/license/plugin.py index 9bcff01e67..fc40d120f8 100644 --- a/premium/backend/src/baserow_premium/license/plugin.py +++ b/premium/backend/src/baserow_premium/license/plugin.py @@ -6,6 +6,10 @@ from baserow.core.cache import local_cache from baserow.core.models import Workspace +from baserow_premium.license.cache import ( + get_cached_instance_wide_licenses, + set_cached_instance_wide_licenses, +) from baserow_premium.license.exceptions import InvalidLicenseError from baserow_premium.license.models import License from baserow_premium.license.registries import LicenseType, SeatUsageSummary @@ -148,6 +152,15 @@ def get_active_instance_wide_licenses( def _get_active_instance_wide_licenses( self, user_id: Optional[int] ) -> Generator[License, None, None]: + # When no specific user is requested, the result is shared across + # requests and cached in Redis to skip the DB query and RSA verify. + use_shared_cache = user_id is None and not self.cache_queries + if use_shared_cache: + cached = get_cached_instance_wide_licenses() + if cached is not None: + yield from cached + return + if self.cache_queries and user_id in self.queried_licenses_per_user: available_licenses = self.queried_licenses_per_user[user_id] else: @@ -163,13 +176,23 @@ def _get_available_licenses(): _get_available_licenses, ) + active_licenses = [] for available_license in available_licenses: try: if available_license.is_active: - yield available_license + active_licenses.append(available_license) except InvalidLicenseError: pass + if use_shared_cache: + # Prime ``payload`` so consumers of the cached list don't re-run + # RSA signature verification after an unpickle. + for lic in active_licenses: + _ = lic.payload + set_cached_instance_wide_licenses(active_licenses) + + yield from active_licenses + if self.cache_queries: self.queried_licenses_per_user[user_id] = available_licenses diff --git a/premium/backend/src/baserow_premium/license/receivers.py b/premium/backend/src/baserow_premium/license/receivers.py new file mode 100644 index 0000000000..6252f8ff64 --- /dev/null +++ b/premium/backend/src/baserow_premium/license/receivers.py @@ -0,0 +1,16 @@ +from django.db import transaction +from django.db.models.signals import post_delete, post_save +from django.dispatch import receiver + +from baserow_premium.license.cache import invalidate_cached_instance_wide_licenses +from baserow_premium.license.models import License + + +@receiver(post_save, sender=License, dispatch_uid="cache_license_save") +def invalidate_license_cache_on_save(sender, **kwargs): + transaction.on_commit(invalidate_cached_instance_wide_licenses) + + +@receiver(post_delete, sender=License, dispatch_uid="cache_license_delete") +def invalidate_license_cache_on_delete(sender, **kwargs): + transaction.on_commit(invalidate_cached_instance_wide_licenses) diff --git a/premium/backend/tests/baserow_premium_tests/license/test_license_cache.py b/premium/backend/tests/baserow_premium_tests/license/test_license_cache.py new file mode 100644 index 0000000000..005961b635 --- /dev/null +++ b/premium/backend/tests/baserow_premium_tests/license/test_license_cache.py @@ -0,0 +1,136 @@ +from django.db import connection +from django.test.utils import CaptureQueriesContext, override_settings + +import pytest +from freezegun import freeze_time + +from baserow_premium.license.cache import ( + get_cached_instance_wide_licenses, + invalidate_cached_instance_wide_licenses, + set_cached_instance_wide_licenses, +) +from baserow_premium.license.models import License +from baserow_premium.license.plugin import LicensePlugin + +VALID_ONE_SEAT_LICENSE = ( + # valid_through: 2021-09-29T19:52:57.842696 UTC + b"eyJ2ZXJzaW9uIjogMSwgImlkIjogIjEiLCAidmFsaWRfZnJvbSI6ICIyMDIxLTA4LTI5VDE5OjUyOjU3" + b"Ljg0MjY5NiIsICJ2YWxpZF90aHJvdWdoIjogIjIwMjEtMDktMjlUMTk6NTI6NTcuODQyNjk2IiwgInBy" + b"b2R1Y3RfY29kZSI6ICJwcmVtaXVtIiwgInNlYXRzIjogMSwgImlzc3VlZF9vbiI6ICIyMDIxLTA4LTI5" + b"VDE5OjUyOjU3Ljg0MjY5NiIsICJpc3N1ZWRfdG9fZW1haWwiOiAiYnJhbUBiYXNlcm93LmlvIiwgImlz" + b"c3VlZF90b19uYW1lIjogIkJyYW0iLCAiaW5zdGFuY2VfaWQiOiAiMSJ9.e33Z4CxLSmD-R55Es24P3mR" + b"8Oqn3LpaXvgYLzF63oFHat3paon7IBjBmOX3eyd8KjirVf3empJds4uUw2Nn2m7TVvRAtJ8XzNl-8ytf" + b"2RLtmjMx1Xkgp5VZ8S7UqJ_cKLyl76eVRtGEA1DH2HdPKu1vBPJ4bzDfnhDPYl4k5z9XSSgqAbQ9WO0U" + b"5kiI3BYjVRZSKnZMeguAGZ47ezDj_WArGcHAB8Pa2v3HFp5Y34DMJ8r3_hD5hxCKgoNx4AHx1Q-hRDqp" + b"Aroj-4jl7KWvlP-OJNc1BgH2wnhFmeKHotv-Iumi83JQohyceUbG6j8rDDQvJfcn0W2_ebmUH3TKr-w=" + b"=" +) + +_CACHE_ON = override_settings(BASEROW_CACHE_TTL_SECONDS=30, DEBUG=True) +_CACHE_OFF = override_settings(BASEROW_CACHE_TTL_SECONDS=0, DEBUG=True) + + +@pytest.mark.django_db +@_CACHE_ON +def test_set_and_get_cached_instance_wide_licenses(): + license = License.objects.create(license=VALID_ONE_SEAT_LICENSE.decode()) + invalidate_cached_instance_wide_licenses() + assert get_cached_instance_wide_licenses() is None + + set_cached_instance_wide_licenses([license]) + + cached = get_cached_instance_wide_licenses() + assert cached is not None + assert len(cached) == 1 + assert cached[0].pk == license.pk + + +@pytest.mark.django_db +@_CACHE_OFF +def test_caching_disabled_when_ttl_is_zero(): + license = License.objects.create(license=VALID_ONE_SEAT_LICENSE.decode()) + + set_cached_instance_wide_licenses([license]) + assert get_cached_instance_wide_licenses() is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_invalidate_cached_instance_wide_licenses(): + license = License.objects.create(license=VALID_ONE_SEAT_LICENSE.decode()) + set_cached_instance_wide_licenses([license]) + assert get_cached_instance_wide_licenses() is not None + + invalidate_cached_instance_wide_licenses() + assert get_cached_instance_wide_licenses() is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_signal_invalidates_cache_on_license_save(django_capture_on_commit_callbacks): + license = License.objects.create(license=VALID_ONE_SEAT_LICENSE.decode()) + set_cached_instance_wide_licenses([license]) + assert get_cached_instance_wide_licenses() is not None + + with django_capture_on_commit_callbacks(execute=True): + license.save() + + assert get_cached_instance_wide_licenses() is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_signal_invalidates_cache_on_license_delete(django_capture_on_commit_callbacks): + license = License.objects.create(license=VALID_ONE_SEAT_LICENSE.decode()) + set_cached_instance_wide_licenses([license]) + assert get_cached_instance_wide_licenses() is not None + + with django_capture_on_commit_callbacks(execute=True): + license.delete() + + assert get_cached_instance_wide_licenses() is None + + +@pytest.mark.django_db +@_CACHE_ON +def test_cache_ttl_capped_at_earliest_license_expiry(mocker): + # VALID_ONE_SEAT_LICENSE expires at 2021-09-29T19:52:57 UTC. Freezing time + # 17s before expiry should cap the TTL below the 30s setting. + license = License(license=VALID_ONE_SEAT_LICENSE.decode()) + mock_set = mocker.patch("baserow_premium.license.cache.cache.set") + + with freeze_time("2021-09-29 19:52:40"): + set_cached_instance_wide_licenses([license]) + + assert mock_set.called + timeout = mock_set.call_args.kwargs["timeout"] + assert 0 < timeout <= 17 + + +@pytest.mark.django_db +@_CACHE_ON +def test_plugin_uses_shared_cache_for_user_none(): + # Seed the cache directly so the plugin must consult it and skip the DB. + set_cached_instance_wide_licenses([]) + plugin = LicensePlugin() + + with CaptureQueriesContext(connection) as ctx: + result = list(plugin._get_active_instance_wide_licenses(user_id=None)) + + assert result == [] + assert len(ctx.captured_queries) == 0 + + +@pytest.mark.django_db +@_CACHE_ON +def test_plugin_with_user_id_bypasses_shared_cache(data_fixture): + user = data_fixture.create_user() + set_cached_instance_wide_licenses([]) + plugin = LicensePlugin() + + # A specific user must not be served by the shared (user=None) cache — + # the per-user query should still fire. + with CaptureQueriesContext(connection) as ctx: + list(plugin._get_active_instance_wide_licenses(user_id=user.id)) + + assert len(ctx.captured_queries) >= 1 diff --git a/web-frontend/locales/en.json b/web-frontend/locales/en.json index b6f9ea7d34..5f2c6361e1 100644 --- a/web-frontend/locales/en.json +++ b/web-frontend/locales/en.json @@ -456,8 +456,6 @@ "modelDoesNotBelongToTypeDescription": "The selected model does not belong to the selected AI type.", "generateAIPromptTitle": "Prompt error", "generateAIPromptDescription": "Something was wrong with the constructed prompt.", - "maxNumberOfPendingWorkspaceInvitesReachedTitle": "Max pending invites reached", - "maxNumberOfPendingWorkspaceInvitesReachedDescription": "You've reached the maximum number of pending invites for this workspace. Please let invitees accept the invite or cancel existing ones to continue.", "fieldIsAlreadyPrimaryTitle": "Field is already primary", "fieldIsAlreadyPrimaryDescription": "The chose new primary field is already the primary field.", "incompatiblePrimaryFieldTypeTitle": "Field is not compatible", diff --git a/web-frontend/modules/core/applicationTypes.js b/web-frontend/modules/core/applicationTypes.js index eb136af930..3b4d5b85f7 100644 --- a/web-frontend/modules/core/applicationTypes.js +++ b/web-frontend/modules/core/applicationTypes.js @@ -176,11 +176,6 @@ export class ApplicationType extends Registerable { return true } - /** - * - */ - clearChildrenSelected(application) {} - /** * Before the application values are updated, they can be modified here. This * might be needed because providing certain values could break the update. diff --git a/web-frontend/modules/core/components/settings/members/MembersInvitesTable.vue b/web-frontend/modules/core/components/settings/members/MembersInvitesTable.vue index b52e4bbda2..a7f50df53b 100644 --- a/web-frontend/modules/core/components/settings/members/MembersInvitesTable.vue +++ b/web-frontend/modules/core/components/settings/members/MembersInvitesTable.vue @@ -111,12 +111,6 @@ export default { true, true ), - new CrudTableColumn( - 'message', - this.$t('membersSettings.invitesTable.columns.message'), - SimpleField, - true - ), new CrudTableColumn( 'permissions', this.$t('membersSettings.invitesTable.columns.role'), diff --git a/web-frontend/modules/core/components/workspace/WorkspaceInvitation.vue b/web-frontend/modules/core/components/workspace/WorkspaceInvitation.vue index ce68881c0b..f3681ab906 100644 --- a/web-frontend/modules/core/components/workspace/WorkspaceInvitation.vue +++ b/web-frontend/modules/core/components/workspace/WorkspaceInvitation.vue @@ -9,10 +9,6 @@ }} -

- "{{ invitation.message }}" -

-