refactor: use approximate count for admin pagination (baserow#5119)

Author: silvestrid

backend/src/baserow/api/admin/users/views.py

@@ -23,6 +23,7 @@
 )
 from baserow.api.admin.views import AdminListingView
 from baserow.api.decorators import map_exceptions, validate_body
+from baserow.api.pagination import PageNumberPaginationWithApproximateCount
 from baserow.api.schemas import get_error_schema
 from baserow.api.user.registries import member_data_registry
 from baserow.api.user.schemas import authenticate_user_schema
@@ -43,6 +44,7 @@
 
 class UsersAdminView(AdminListingView):
     serializer_class = UserAdminResponseSerializer
+    pagination_class = PageNumberPaginationWithApproximateCount
     search_fields = ["id", "username", "first_name"]
     sort_field_mapping = {
         "id": "id",

backend/src/baserow/api/admin/views.py

@@ -30,6 +30,7 @@ class APIListingView(
     APIView, SearchableViewMixin, SortableViewMixin, FilterableViewMixin
 ):
     serializer_class = None
+    pagination_class = PageNumberPagination
     search_fields: List[str] = ["id"]
     filters_field_mapping: Dict[str, str] = {}
     sort_field_mapping: Dict[str, str] = {}
@@ -56,7 +57,7 @@ def get(self, request):
         queryset = self.apply_sorts_or_default_sort(sorts, queryset)
         queryset = self.apply_ids_filter(ids_param, queryset)
 
-        paginator = PageNumberPagination(limit_page_size=100)
+        paginator = self.pagination_class(limit_page_size=100)
         page = paginator.paginate_queryset(queryset, request, self)
         serializer = self.get_serializer(request, page, many=True)
 

backend/src/baserow/api/admin/workspaces/views.py

@@ -10,6 +10,7 @@
 from baserow.api.admin.views import AdminListingView, APIListingView
 from baserow.api.decorators import map_exceptions
 from baserow.api.errors import ERROR_GROUP_DOES_NOT_EXIST
+from baserow.api.pagination import PageNumberPaginationWithApproximateCount
 from baserow.api.schemas import get_error_schema
 from baserow.core.admin.workspaces.exceptions import CannotDeleteATemplateGroupError
 from baserow.core.admin.workspaces.handler import WorkspacesAdminHandler
@@ -27,6 +28,7 @@
 
 class WorkspacesAdminView(AdminListingView):
     serializer_class = WorkspacesAdminResponseSerializer
+    pagination_class = PageNumberPaginationWithApproximateCount
     search_fields = ["id", "name"]
     sort_field_mapping = {
         "id": "id",

backend/src/baserow/api/pagination.py

@@ -1,3 +1,4 @@
+from functools import cached_property
 from typing import Protocol
 
 from django.core.paginator import Paginator as DjangoPaginator
@@ -12,6 +13,8 @@
 from rest_framework.response import Response
 from rest_framework.status import HTTP_400_BAD_REQUEST
 
+from baserow.core.db import get_approximate_row_count
+
 
 class Pageable(Protocol):
     def paginate_queryset(self, queryset, request, view=None):
@@ -150,3 +153,24 @@ def get_paginated_response_schema(self, schema):
                 "results": schema,
             },
         }
+
+
+class ApproximateCountPaginator(Paginator):
+    """
+    A paginator that uses Postgres EXPLAIN to estimate the total row count
+    instead of running an expensive COUNT(*) query.
+    """
+
+    @cached_property
+    def count(self):
+        return get_approximate_row_count(self.object_list)
+
+
+class PageNumberPaginationWithApproximateCount(PageNumberPagination):
+    """
+    Page number pagination that uses an approximate count from Postgres EXPLAIN
+    instead of COUNT(*). Suitable for large tables like audit logs where an
+    exact count is not required.
+    """
+
+    django_paginator_class = ApproximateCountPaginator

backend/src/baserow/core/db.py

@@ -1,4 +1,5 @@
 import contextlib
+import json
 import random
 import time
 from collections import defaultdict
@@ -20,7 +21,12 @@
 
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
-from django.db import DEFAULT_DB_ALIAS, OperationalError, connection, transaction
+from django.db import (
+    DEFAULT_DB_ALIAS,
+    OperationalError,
+    connection,
+    transaction,
+)
 from django.db.models import ForeignKey, ManyToManyField, Max, Model, Prefetch, QuerySet
 from django.db.models.functions import Collate
 from django.db.models.query import ModelIterable
@@ -37,6 +43,30 @@
 ModelInstance = TypeVar("ModelInstance", bound=object)
 
 
+APPROXIMATE_COUNT_THRESHOLD = 50_000
+
+
+def get_approximate_row_count(queryset: QuerySet) -> int:
+    """
+    Uses Postgres EXPLAIN to estimate the row count for the given queryset.
+    If the estimate is below APPROXIMATE_COUNT_THRESHOLD, falls back to an
+    exact COUNT(*) since the cost is negligible for small result sets and
+    the planner estimate is unreliable at that scale.
+
+    :param queryset: The queryset to estimate the row count for.
+    :return: An estimate of the row count for the queryset.
+    """
+
+    queryset = queryset.order_by()
+    plan = json.loads(queryset.explain(format="json"))
+    estimate = int(plan[0]["Plan"]["Plan Rows"])
+
+    if estimate < APPROXIMATE_COUNT_THRESHOLD:
+        return queryset.count()
+
+    return estimate
+
+
 def get_database_dsn() -> str:
     """
     Constructs the database DSN from the default database settings.

backend/tests/baserow/api/admin/groups/test_workspaces_admin_views.py

@@ -52,7 +52,7 @@ class as the list users endpoint which already has extensive tests. We only need
     )
     assert response.status_code == HTTP_403_FORBIDDEN
 
-    with django_assert_num_queries(5):
+    with django_assert_num_queries(6):
         response = api_client.get(
             reverse("api:admin:workspaces:list"),
             format="json",

backend/tests/baserow/api/admin/users/test_users_admin_views.py

@@ -771,7 +771,7 @@ def test_admin_getting_view_users_only_runs_two_queries_instead_of_n(
         first_name="Test1",
         is_staff=True,
     )
-    fixed_num_of_queries_unrelated_to_number_of_rows = 6
+    fixed_num_of_queries_unrelated_to_number_of_rows = 7
 
     for i in range(10):
         data_fixture.create_user_workspace()

backend/tests/baserow/contrib/database/api/rows/test_row_serializers.py

@@ -521,12 +521,10 @@ def test_get_row_serializer_with_user_field_names(
                         "value": "E",
                     },
                 ],
-                "formula_multiple_collaborators": unordered(
-                    [
-                        {"id": u2.id, "name": u2.first_name},
-                        {"id": u3.id, "name": u3.first_name},
-                    ]
-                ),
+                "formula_multiple_collaborators": [
+                    {"id": u2.id, "name": u2.first_name},
+                    {"id": u3.id, "name": u3.first_name},
+                ],
                 "formula_text": "test FORMULA",
                 "count": "3",
                 "rollup": "-122.222",
@@ -540,12 +538,10 @@ def test_get_row_serializer_with_user_field_names(
                 "multiple_collaborators_lookup": [
                     {
                         "id": 1,
-                        "value": unordered(
-                            [
-                                {"id": u2.id, "name": u2.first_name},
-                                {"id": u3.id, "name": u3.first_name},
-                            ]
-                        ),
+                        "value": [
+                            {"id": u2.id, "name": u2.first_name},
+                            {"id": u3.id, "name": u3.first_name},
+                        ],
                     },
                     {
                         "id": 2,
@@ -573,6 +569,12 @@ def test_get_row_serializer_with_user_field_names(
         )
     )
     test_result = json.loads(json.dumps(serializer_instance.data[0]))
+    expected_result["formula_multiple_collaborators"] = unordered(
+        expected_result["formula_multiple_collaborators"]
+    )
+    expected_result["multiple_collaborators_lookup"][0]["value"] = unordered(
+        expected_result["multiple_collaborators_lookup"][0]["value"]
+    )
     assert test_result == expected_result
 
 

backend/tests/baserow/core/test_core_db.py

@@ -28,6 +28,7 @@
     LockedAtomicTransaction,
     MultiFieldPrefetchQuerysetMixin,
     QuerySet,
+    get_approximate_row_count,
     specific_iterator,
     specific_queryset,
 )
@@ -701,3 +702,28 @@ def test_specific_iterator_skip_missing_specific_objects(data_fixture):
     mock_logger.error.assert_called_once_with(
         f"The specific object with id {field_without_specific.id} does not exist."
     )
+
+
+@pytest.mark.django_db
+def test_get_approximate_row_count_falls_back_to_exact_for_small_tables():
+    queryset = Workspace.objects.all()
+    count = get_approximate_row_count(queryset)
+    assert count == queryset.count()
+
+
+@pytest.mark.django_db
+def test_get_approximate_row_count_returns_estimate_above_threshold():
+    queryset = Workspace.objects.all()
+    with patch("baserow.core.db.APPROXIMATE_COUNT_THRESHOLD", 0):
+        count = get_approximate_row_count(queryset)
+    assert isinstance(count, int)
+    assert count >= 0
+
+
+@pytest.mark.django_db
+def test_get_approximate_row_count_works_with_filtered_queryset(data_fixture):
+    data_fixture.create_workspace(name="test_ws_1")
+    data_fixture.create_workspace(name="test_ws_2")
+    queryset = Workspace.objects.filter(name__startswith="test_ws_")
+    count = get_approximate_row_count(queryset)
+    assert count == 2

changelog/entries/unreleased/refactor/speedup_audit_log_pagination_with_approximate_count.json

@@ -0,0 +1,9 @@
+{
+    "type": "refactor",
+    "message": "Use Postgres EXPLAIN-based approximate count for audit log pagination to avoid expensive COUNT(*) on large tables.",
+    "issue_origin": "github",
+    "issue_number": null,
+    "domain": "core",
+    "bullet_points": [],
+    "created_at": "2026-04-02"
+}