Skip to content

Commit c69ce2f

Browse files
authored
feat: Data scanner (baserow#4955)
1 parent 067ed78 commit c69ce2f

81 files changed

Lines changed: 8736 additions & 216 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

backend/src/baserow/api/admin/views.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from baserow.api.exceptions import (
1414
InvalidSortAttributeException,
1515
InvalidSortDirectionException,
16+
QueryParameterValidationException,
1617
)
1718
from baserow.api.mixins import (
1819
FilterableViewMixin,
@@ -22,6 +23,7 @@
2223
from baserow.api.pagination import PageNumberPagination
2324
from baserow.api.schemas import get_error_schema
2425
from baserow.api.serializers import get_example_pagination_serializer_class
26+
from baserow.core.utils import split_comma_separated_string
2527

2628

2729
class APIListingView(
@@ -46,11 +48,13 @@ def get(self, request):
4648

4749
search = request.GET.get("search")
4850
sorts = request.GET.get("sorts")
51+
ids_param = request.GET.get("ids")
4952

5053
queryset = self.get_queryset(request)
5154
queryset = self.apply_filters(request.GET, queryset)
5255
queryset = self.apply_search(search, queryset)
5356
queryset = self.apply_sorts_or_default_sort(sorts, queryset)
57+
queryset = self.apply_ids_filter(ids_param, queryset)
5458

5559
paginator = PageNumberPagination(limit_page_size=100)
5660
page = paginator.paginate_queryset(queryset, request, self)
@@ -61,6 +65,30 @@ def get(self, request):
6165
def get_queryset(self, request):
6266
raise NotImplementedError("The get_queryset method must be set.")
6367

68+
def apply_ids_filter(self, ids_param, queryset):
69+
if not ids_param:
70+
return queryset
71+
72+
record_ids = split_comma_separated_string(ids_param)
73+
74+
invalid_id = next(
75+
(record for record in record_ids if not record.isdigit()), None
76+
)
77+
if invalid_id is not None:
78+
raise QueryParameterValidationException(
79+
{
80+
"ids": [
81+
{
82+
"code": "invalid",
83+
"error": f"'{invalid_id}' is not a valid ID. Only positive "
84+
f"integers are accepted.",
85+
}
86+
]
87+
}
88+
)
89+
90+
return queryset.filter(id__in=[int(record_id) for record_id in record_ids])
91+
6492
def get_serializer(self, request, *args, **kwargs):
6593
if not self.serializer_class:
6694
raise NotImplementedError(
@@ -134,6 +162,13 @@ def get_extend_schema_parameters(
134162
type=OpenApiTypes.INT,
135163
description=f"Defines how many {name} should be returned per page.",
136164
),
165+
OpenApiParameter(
166+
name="ids",
167+
location=OpenApiParameter.QUERY,
168+
type=OpenApiTypes.STR,
169+
description=f"A comma-separated list of {name} IDs to filter by. "
170+
f"When provided, only {name} with those IDs are returned.",
171+
),
137172
*(extra_parameters or []),
138173
],
139174
"responses": {

backend/src/baserow/api/admin/workspaces/serializers.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,11 @@ class Meta:
5353
"free_users",
5454
"created_on",
5555
)
56+
57+
58+
class AdminWorkspaceOptionsSerializer(serializers.ModelSerializer):
59+
value = serializers.CharField(source="name")
60+
61+
class Meta:
62+
model = Workspace
63+
fields = ("id", "value")
Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
from django.urls import re_path
22

3-
from baserow.api.admin.workspaces.views import WorkspaceAdminView, WorkspacesAdminView
3+
from baserow.api.admin.workspaces.views import (
4+
WorkspaceAdminView,
5+
WorkspaceOptionsAdminView,
6+
WorkspacesAdminView,
7+
)
48

59
app_name = "baserow.api.admin.workspaces"
610

711
urlpatterns = [
812
re_path(r"^$", WorkspacesAdminView.as_view(), name="list"),
13+
re_path(r"^options/$", WorkspaceOptionsAdminView.as_view(), name="options"),
914
re_path(r"^(?P<workspace_id>[0-9]+)/$", WorkspaceAdminView.as_view(), name="edit"),
1015
]

backend/src/baserow/api/admin/workspaces/views.py

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from rest_framework.response import Response
88
from rest_framework.views import APIView
99

10-
from baserow.api.admin.views import AdminListingView
10+
from baserow.api.admin.views import AdminListingView, APIListingView
1111
from baserow.api.decorators import map_exceptions
1212
from baserow.api.errors import ERROR_GROUP_DOES_NOT_EXIST
1313
from baserow.api.schemas import get_error_schema
@@ -19,7 +19,10 @@
1919
from baserow.core.usage.handler import UsageHandler
2020

2121
from .errors import ERROR_CANNOT_DELETE_A_TEMPLATE_GROUP
22-
from .serializers import WorkspacesAdminResponseSerializer
22+
from .serializers import (
23+
AdminWorkspaceOptionsSerializer,
24+
WorkspacesAdminResponseSerializer,
25+
)
2326

2427

2528
class WorkspacesAdminView(AdminListingView):
@@ -62,6 +65,30 @@ def get(self, request):
6265
return super().get(request)
6366

6467

68+
class WorkspaceOptionsAdminView(APIListingView):
69+
permission_classes = (IsAdminUser,)
70+
serializer_class = AdminWorkspaceOptionsSerializer
71+
search_fields = ["name"]
72+
default_order_by = "name"
73+
74+
def get_queryset(self, request):
75+
return Workspace.objects.filter(template__isnull=True)
76+
77+
@extend_schema(
78+
tags=["Admin"],
79+
operation_id="admin_list_workspaces_as_options",
80+
description=(
81+
"Lists all workspaces. This endpoint is intended for admin-level "
82+
"features that need a workspace dropdown."
83+
),
84+
**APIListingView.get_extend_schema_parameters(
85+
"workspaces", serializer_class, search_fields, {}
86+
),
87+
)
88+
def get(self, request):
89+
return super().get(request)
90+
91+
6592
class WorkspaceAdminView(APIView):
6693
permission_classes = (IsAdminUser,)
6794

backend/src/baserow/contrib/database/locale/en/LC_MESSAGES/django.po

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ msgid ""
88
msgstr ""
99
"Project-Id-Version: PACKAGE VERSION\n"
1010
"Report-Msgid-Bugs-To: \n"
11-
"POT-Creation-Date: 2026-03-16 14:50+0000\n"
11+
"POT-Creation-Date: 2026-03-16 21:52+0000\n"
1212
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
1313
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
1414
"Language-Team: LANGUAGE <LL@li.org>\n"

backend/src/baserow/core/locale/en/LC_MESSAGES/django.po

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ msgid ""
88
msgstr ""
99
"Project-Id-Version: PACKAGE VERSION\n"
1010
"Report-Msgid-Bugs-To: \n"
11-
"POT-Creation-Date: 2026-03-16 14:50+0000\n"
11+
"POT-Creation-Date: 2026-03-16 21:52+0000\n"
1212
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
1313
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
1414
"Language-Team: LANGUAGE <LL@li.org>\n"

backend/src/baserow/locale/en/LC_MESSAGES/django.po

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ msgid ""
88
msgstr ""
99
"Project-Id-Version: PACKAGE VERSION\n"
1010
"Report-Msgid-Bugs-To: \n"
11-
"POT-Creation-Date: 2026-03-16 14:50+0000\n"
11+
"POT-Creation-Date: 2026-03-16 21:52+0000\n"
1212
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
1313
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
1414
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -205,20 +205,24 @@ msgstr ""
205205
msgid "Widget \"%(widget_title)s\" (%(widget_id)s) deleted"
206206
msgstr ""
207207

208-
#: src/baserow/contrib/integrations/core/service_types.py:1067
208+
#: src/baserow/contrib/integrations/core/service_types.py:1062
209209
msgid "Branch taken"
210210
msgstr ""
211211

212-
#: src/baserow/contrib/integrations/core/service_types.py:1072
212+
#: src/baserow/contrib/integrations/core/service_types.py:1067
213213
msgid "Label"
214214
msgstr ""
215215

216-
#: src/baserow/contrib/integrations/core/service_types.py:1074
216+
#: src/baserow/contrib/integrations/core/service_types.py:1069
217217
msgid "The label of the branch that matched the condition."
218218
msgstr ""
219219

220-
#: src/baserow/contrib/integrations/core/service_types.py:1418
221-
msgid "Triggered at"
220+
#: src/baserow/contrib/integrations/core/service_types.py:1438
221+
msgid "Previous scheduled run"
222+
msgstr ""
223+
224+
#: src/baserow/contrib/integrations/core/service_types.py:1442
225+
msgid "Next scheduled run"
222226
msgstr ""
223227

224228
#: src/baserow/contrib/integrations/local_baserow/service_types.py:1688

backend/tests/baserow/api/admin/groups/test_workspaces_admin_views.py

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,3 +219,159 @@ def test_cant_delete_template_workspace(api_client, data_fixture):
219219
assert response.status_code == HTTP_400_BAD_REQUEST
220220
assert response.json()["error"] == "ERROR_CANNOT_DELETE_A_TEMPLATE_GROUP"
221221
assert Workspace.objects.all().count() == 1
222+
223+
224+
@pytest.mark.django_db
225+
@override_settings(DEBUG=True)
226+
def test_non_admin_list_workspaces_as_options(api_client, data_fixture):
227+
(
228+
admin_user,
229+
admin_token,
230+
) = data_fixture.create_user_and_token()
231+
232+
# no search query should return all workspaces
233+
response = api_client.get(
234+
reverse("api:admin:workspaces:options"),
235+
format="json",
236+
HTTP_AUTHORIZATION=f"JWT {admin_token}",
237+
)
238+
assert response.status_code == HTTP_403_FORBIDDEN
239+
240+
241+
@pytest.mark.django_db
242+
@override_settings(DEBUG=True)
243+
def test_admin_list_workspaces_as_options(api_client, data_fixture):
244+
(
245+
admin_user,
246+
admin_token,
247+
) = data_fixture.create_user_and_token(is_staff=True)
248+
workspace_1 = data_fixture.create_workspace(name="workspace 1", user=admin_user)
249+
workspace_2 = data_fixture.create_workspace(name="workspace 2", user=admin_user)
250+
251+
# no search query should return all workspaces
252+
response = api_client.get(
253+
reverse("api:admin:workspaces:options"),
254+
format="json",
255+
HTTP_AUTHORIZATION=f"JWT {admin_token}",
256+
)
257+
assert response.status_code == HTTP_200_OK
258+
assert response.json() == {
259+
"count": 2,
260+
"next": None,
261+
"previous": None,
262+
"results": [
263+
{"id": workspace_1.id, "value": workspace_1.name},
264+
{"id": workspace_2.id, "value": workspace_2.name},
265+
],
266+
}
267+
268+
# searching by name should return only the correct workspace
269+
response = api_client.get(
270+
reverse("api:admin:workspaces:options") + "?search=1",
271+
format="json",
272+
HTTP_AUTHORIZATION=f"JWT {admin_token}",
273+
)
274+
assert response.status_code == HTTP_200_OK
275+
assert response.json() == {
276+
"count": 1,
277+
"next": None,
278+
"previous": None,
279+
"results": [{"id": workspace_1.id, "value": workspace_1.name}],
280+
}
281+
282+
283+
@pytest.mark.django_db
284+
@override_settings(DEBUG=True)
285+
def test_admin_list_workspaces_as_options_filter_by_ids(api_client, data_fixture):
286+
(
287+
admin_user,
288+
admin_token,
289+
) = data_fixture.create_user_and_token(is_staff=True)
290+
workspace_1 = data_fixture.create_workspace(name="workspace 1", user=admin_user)
291+
workspace_2 = data_fixture.create_workspace(name="workspace 2", user=admin_user)
292+
data_fixture.create_workspace(name="workspace 3", user=admin_user)
293+
294+
# filtering by a single id should return only that workspace
295+
response = api_client.get(
296+
reverse("api:admin:workspaces:options") + f"?ids={workspace_1.id}",
297+
format="json",
298+
HTTP_AUTHORIZATION=f"JWT {admin_token}",
299+
)
300+
assert response.status_code == HTTP_200_OK
301+
assert response.json() == {
302+
"count": 1,
303+
"next": None,
304+
"previous": None,
305+
"results": [{"id": workspace_1.id, "value": workspace_1.name}],
306+
}
307+
308+
# filtering by multiple ids should return all matching workspaces
309+
response = api_client.get(
310+
reverse("api:admin:workspaces:options")
311+
+ f"?ids={workspace_1.id},{workspace_2.id}",
312+
format="json",
313+
HTTP_AUTHORIZATION=f"JWT {admin_token}",
314+
)
315+
assert response.status_code == HTTP_200_OK
316+
assert response.json() == {
317+
"count": 2,
318+
"next": None,
319+
"previous": None,
320+
"results": [
321+
{"id": workspace_1.id, "value": workspace_1.name},
322+
{"id": workspace_2.id, "value": workspace_2.name},
323+
],
324+
}
325+
326+
327+
@pytest.mark.django_db
328+
@override_settings(DEBUG=True)
329+
def test_admin_list_workspaces_as_options_filter_by_invalid_ids(
330+
api_client, data_fixture
331+
):
332+
_, admin_token = data_fixture.create_user_and_token(is_staff=True)
333+
334+
# Negative IDs should be rejected.
335+
response = api_client.get(
336+
reverse("api:admin:workspaces:options") + "?ids=-1",
337+
format="json",
338+
HTTP_AUTHORIZATION=f"JWT {admin_token}",
339+
)
340+
assert response.status_code == HTTP_400_BAD_REQUEST
341+
assert response.json()["error"] == "ERROR_QUERY_PARAMETER_VALIDATION"
342+
assert response.json()["detail"]["ids"] == [
343+
{
344+
"code": "invalid",
345+
"error": "'-1' is not a valid ID. Only positive integers are accepted.",
346+
}
347+
]
348+
349+
# Non-numeric values should be rejected.
350+
response = api_client.get(
351+
reverse("api:admin:workspaces:options") + "?ids=abc",
352+
format="json",
353+
HTTP_AUTHORIZATION=f"JWT {admin_token}",
354+
)
355+
assert response.status_code == HTTP_400_BAD_REQUEST
356+
assert response.json()["error"] == "ERROR_QUERY_PARAMETER_VALIDATION"
357+
assert response.json()["detail"]["ids"] == [
358+
{
359+
"code": "invalid",
360+
"error": "'abc' is not a valid ID. Only positive integers are accepted.",
361+
}
362+
]
363+
364+
# A mix of valid and invalid values should still be rejected.
365+
response = api_client.get(
366+
reverse("api:admin:workspaces:options") + "?ids=1,-2,3",
367+
format="json",
368+
HTTP_AUTHORIZATION=f"JWT {admin_token}",
369+
)
370+
assert response.status_code == HTTP_400_BAD_REQUEST
371+
assert response.json()["error"] == "ERROR_QUERY_PARAMETER_VALIDATION"
372+
assert response.json()["detail"]["ids"] == [
373+
{
374+
"code": "invalid",
375+
"error": "'-2' is not a valid ID. Only positive integers are accepted.",
376+
}
377+
]
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"type": "feature",
3+
"message": "Add instace wide data scanner.",
4+
"issue_origin": "github",
5+
"issue_number": null,
6+
"domain": "database",
7+
"bullet_points": [],
8+
"created_at": "2026-03-16"
9+
}

enterprise/backend/pytest.ini

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,6 @@ DJANGO_SETTINGS_MODULE = baserow.config.settings.test
33
python_files = test_*.py
44
markers =
55
eval: mark test as an eval test (requires LLM API key)
6+
data_scanner: mark test as a data scanner test
67
env =
78
DJANGO_SETTINGS_MODULE = baserow.config.settings.test

0 commit comments

Comments
 (0)