From 067ed78a6924144ad5c9160f5ec4c19f2b4c250f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Mar 2026 09:51:37 +0100 Subject: [PATCH 1/4] chore(deps): bump cbor2 from 5.8.0 to 5.9.0 in /backend (#5034) Bumps [cbor2](https://github.com/agronholm/cbor2) from 5.8.0 to 5.9.0. - [Release notes](https://github.com/agronholm/cbor2/releases) - [Commits](https://github.com/agronholm/cbor2/compare/5.8.0...5.9.0) --- updated-dependencies: - dependency-name: cbor2 dependency-version: 5.9.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- backend/uv.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/uv.lock b/backend/uv.lock index 6715e97cc4..ee367b6e29 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -534,16 +534,16 @@ wheels = [ [[package]] name = "cbor2" -version = "5.8.0" +version = "5.9.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d9/8e/8b4fdde28e42ffcd741a37f4ffa9fb59cd4fe01625b544dfcfd9ccb54f01/cbor2-5.8.0.tar.gz", hash = "sha256:b19c35fcae9688ac01ef75bad5db27300c2537eb4ee00ed07e05d8456a0d4931", size = 107825, upload-time = "2025-12-30T18:44:22.455Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/cb/09939728be094d155b5d4ac262e39877875f5f7e36eea66beb359f647bd0/cbor2-5.9.0.tar.gz", hash = "sha256:85c7a46279ac8f226e1059275221e6b3d0e370d2bb6bd0500f9780781615bcea", size = 111231, upload-time = "2026-03-22T15:56:50.638Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/0c/0654233d7543ac8a50f4785f172430ddc97538ba418eb305d6e529d1a120/cbor2-5.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ad72381477133046ce217617d839ea4e9454f8b77d9a6351b229e214102daeb7", size = 70710, upload-time = "2025-12-30T18:44:03.209Z" }, - { url = "https://files.pythonhosted.org/packages/84/62/4671d24e557d7f5a74a01b422c538925140c0495e57decde7e566f91d029/cbor2-5.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6da25190fad3434ce99876b11d4ca6b8828df6ca232cf7344cd14ae1166fb718", size = 285005, upload-time = "2025-12-30T18:44:05.109Z" }, - { url = "https://files.pythonhosted.org/packages/87/85/0c67d763a08e848c9a80d7e4723ba497cce676f41bc7ca1828ae90a0a872/cbor2-5.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c13919e3a24c5a6d286551fa288848a4cedc3e507c58a722ccd134e461217d99", size = 282435, upload-time = "2025-12-30T18:44:06.465Z" }, - { url = "https://files.pythonhosted.org/packages/b2/01/0650972b4dbfbebcfbe37cbba7fc3cd9019a8da6397ab3446e07175e342b/cbor2-5.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f8c40d32e5972047a777f9bf730870828f3cf1c43b3eb96fd0429c57a1d3b9e6", size = 277493, upload-time = "2025-12-30T18:44:07.609Z" }, - { url = "https://files.pythonhosted.org/packages/b3/6c/7704a4f32adc7f10f3b41ec067f500a4458f7606397af5e4cf2d368fd288/cbor2-5.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7627894bc0b3d5d0807f31e3107e11b996205470c4429dc2bb4ef8bfe7f64e1e", size = 276085, upload-time = "2025-12-30T18:44:09.021Z" }, - { url = "https://files.pythonhosted.org/packages/d6/4f/101071f880b4da05771128c0b89f41e334cff044dee05fb013c8f4be661c/cbor2-5.8.0-py3-none-any.whl", hash = "sha256:3727d80f539567b03a7aa11890e57798c67092c38df9e6c23abb059e0f65069c", size = 24374, upload-time = "2025-12-30T18:44:21.476Z" }, + { url = "https://files.pythonhosted.org/packages/08/7d/9ccc36d10ef96e6038e48046ebe1ce35a1e7814da0e1e204d09e6ef09b8d/cbor2-5.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23606d31ba1368bd1b6602e3020ee88fe9523ca80e8630faf6b2fc904fd84560", size = 71500, upload-time = "2026-03-22T15:56:31.876Z" }, + { url = "https://files.pythonhosted.org/packages/70/e1/a6cca2cc72e13f00030c6a649f57ae703eb2c620806ab70c40db8eab33fa/cbor2-5.9.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0322296b9d52f55880e300ba8ba09ecf644303b99b51138bbb1c0fb644fa7c3e", size = 286953, upload-time = "2026-03-22T15:56:33.292Z" }, + { url = "https://files.pythonhosted.org/packages/08/3c/24cd5ef488a957d90e016f200a3aad820e4c2f85edd61c9fe4523007a1ee/cbor2-5.9.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:422817286c1d0ce947fb2f7eca9212b39bddd7231e8b452e2d2cc52f15332dba", size = 285454, upload-time = "2026-03-22T15:56:34.703Z" }, + { url = "https://files.pythonhosted.org/packages/a4/35/dca96818494c0ba47cdd73e8d809b27fa91f8fa0ce32a068a09237687454/cbor2-5.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9a4907e0c3035bb8836116854ed8e56d8aef23909d601fa59706320897ec2551", size = 279441, upload-time = "2026-03-22T15:56:35.888Z" }, + { url = "https://files.pythonhosted.org/packages/a4/44/d3362378b16e53cf7e535a3f5aed8476e2109068154e24e31981ef5bde9e/cbor2-5.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fb7afe77f8d269e42d7c4b515c6fd14f1ccc0625379fb6829b269f493d16eddd", size = 279673, upload-time = "2026-03-22T15:56:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/42/ff/b83492b096fbef26e9cb62c1a4bf2d3cef579ea7b33138c6c37c4ae66f67/cbor2-5.9.0-py3-none-any.whl", hash = "sha256:27695cbd70c90b8de5c4a284642c2836449b14e2c2e07e3ffe0744cb7669a01b", size = 24627, upload-time = "2026-03-22T15:56:48.847Z" }, ] [[package]] From c69ce2f4feb8a6dad2c214351b949277bfbbcc94 Mon Sep 17 00:00:00 2001 From: Bram Date: Tue, 24 Mar 2026 11:30:54 +0100 Subject: [PATCH 2/4] feat: Data scanner (#4955) --- backend/src/baserow/api/admin/views.py | 35 + .../api/admin/workspaces/serializers.py | 8 + .../src/baserow/api/admin/workspaces/urls.py | 7 +- .../src/baserow/api/admin/workspaces/views.py | 31 +- .../database/locale/en/LC_MESSAGES/django.po | 2 +- .../core/locale/en/LC_MESSAGES/django.po | 2 +- .../baserow/locale/en/LC_MESSAGES/django.po | 16 +- .../groups/test_workspaces_admin_views.py | 156 ++ .../unreleased/feature/data_scanner.json | 9 + enterprise/backend/pytest.ini | 1 + .../api/admin/audit_log/urls.py | 2 - .../api/admin/data_scanner/__init__.py | 0 .../api/admin/data_scanner/errors.py | 19 + .../api/admin/data_scanner/serializers.py | 268 +++ .../api/admin/data_scanner/urls.py | 43 + .../api/admin/data_scanner/views.py | 370 ++++ .../src/baserow_enterprise/api/admin/urls.py | 2 + .../api/audit_log/serializers.py | 9 - .../baserow_enterprise/api/audit_log/urls.py | 2 - .../baserow_enterprise/api/audit_log/views.py | 34 +- .../backend/src/baserow_enterprise/apps.py | 17 + .../data_scanner/__init__.py | 0 .../data_scanner/actions.py | 94 + .../data_scanner/constants.py | 38 + .../data_scanner/exceptions.py | 10 + .../data_scanner/handler.py | 773 ++++++++ .../data_scanner/job_types.py | 279 +++ .../baserow_enterprise/data_scanner/models.py | 110 ++ .../data_scanner/notification_types.py | 97 + .../baserow_enterprise/data_scanner/tasks.py | 56 + .../src/baserow_enterprise/features.py | 1 + .../src/baserow_enterprise/license_types.py | 2 + .../locale/en/LC_MESSAGES/django.po | 83 +- ...tjob_datascan_datascanlistitem_and_more.py | 246 +++ .../data_scanner/test_data_scanner_views.py | 1703 +++++++++++++++++ .../audit_log/test_audit_log_admin_views.py | 50 +- .../data_scanner/__init__.py | 0 .../data_scanner/conftest.py | 36 + .../data_scanner/test_data_scanner_handler.py | 1655 ++++++++++++++++ .../test_data_scanner_notification_types.py | 277 +++ .../modules/baserow_enterprise/adminTypes.js | 35 + .../assets/scss/components/all.scss | 1 + .../assets/scss/components/data_scanner.scss | 52 + .../dataScanner/DataScanActionsContext.vue | 117 ++ .../admin/dataScanner/DataScanForm.vue | 514 +++++ .../dataScanner/DataScanFrequencyField.vue | 28 + .../dataScanner/DataScanLastRunField.vue | 28 + .../dataScanner/DataScanResolveField.vue | 56 + .../dataScanner/DataScanResultsCountField.vue | 31 + .../dataScanner/DataScanRowLinkField.vue | 46 + .../admin/dataScanner/DataScanStatusField.vue | 33 + .../admin/dataScanner/DataScanTypeField.vue | 27 + .../dataScanner/DataScannerResultsTab.vue | 183 ++ .../admin/dataScanner/DataScannerScansTab.vue | 234 +++ .../admin/dataScanner/DeleteDataScanModal.vue | 66 + .../admin/forms/DataScanExportForm.vue | 28 + .../admin/modals/AuditLogExportModal.vue | 3 +- .../admin/modals/DataScanExportModal.vue | 180 ++ .../components/admin/modals/DataScanModal.vue | 106 + .../DataScanNewResultsNotification.vue | 43 + .../modules/baserow_enterprise/features.js | 1 + .../modules/baserow_enterprise/jobTypes.js | 10 + .../baserow_enterprise/licenseTypes.js | 1 + .../baserow_enterprise/locales/en.json | 91 +- .../baserow_enterprise/notificationTypes.js | 25 + .../pages/admin/dataScanner.vue | 53 + .../pages/admin/dataScanner/results.vue | 22 + .../pages/admin/dataScanner/scans.vue | 20 + .../baserow_enterprise/paidFeatures.js | 26 + .../modules/baserow_enterprise/plugin.js | 20 +- .../modules/baserow_enterprise/routes.js | 18 + .../services/adminWorkspaces.js | 1 + .../baserow_enterprise/services/auditLog.js | 9 +- .../services/dataScanner.js | 48 + .../locale/en/LC_MESSAGES/django.po | 2 +- .../core/assets/scss/components/all.scss | 1 + .../assets/scss/components/tabs_body.scss | 15 + web-frontend/modules/core/components/Tabs.vue | 2 +- .../core/components/crudTable/CrudTable.vue | 223 ++- .../modules/core/crudTable/baseService.js | 10 + .../core/static/img/features/data_scanner.png | Bin 0 -> 125099 bytes 81 files changed, 8736 insertions(+), 216 deletions(-) create mode 100644 changelog/entries/unreleased/feature/data_scanner.json create mode 100644 enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/__init__.py create mode 100644 enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/errors.py create mode 100644 enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/serializers.py create mode 100644 enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/urls.py create mode 100644 enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/views.py create mode 100644 enterprise/backend/src/baserow_enterprise/data_scanner/__init__.py create mode 100644 enterprise/backend/src/baserow_enterprise/data_scanner/actions.py create mode 100644 enterprise/backend/src/baserow_enterprise/data_scanner/constants.py create mode 100644 enterprise/backend/src/baserow_enterprise/data_scanner/exceptions.py create mode 100644 enterprise/backend/src/baserow_enterprise/data_scanner/handler.py create mode 100644 enterprise/backend/src/baserow_enterprise/data_scanner/job_types.py create mode 100644 enterprise/backend/src/baserow_enterprise/data_scanner/models.py create mode 100644 enterprise/backend/src/baserow_enterprise/data_scanner/notification_types.py create mode 100644 enterprise/backend/src/baserow_enterprise/data_scanner/tasks.py create mode 100644 enterprise/backend/src/baserow_enterprise/migrations/0059_datascanresultexportjob_datascan_datascanlistitem_and_more.py create mode 100644 enterprise/backend/tests/baserow_enterprise_tests/api/admin/data_scanner/test_data_scanner_views.py create mode 100644 enterprise/backend/tests/baserow_enterprise_tests/data_scanner/__init__.py create mode 100644 enterprise/backend/tests/baserow_enterprise_tests/data_scanner/conftest.py create mode 100644 enterprise/backend/tests/baserow_enterprise_tests/data_scanner/test_data_scanner_handler.py create mode 100644 enterprise/backend/tests/baserow_enterprise_tests/data_scanner/test_data_scanner_notification_types.py create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/data_scanner.scss create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanActionsContext.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanForm.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanFrequencyField.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanLastRunField.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanResolveField.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanResultsCountField.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanRowLinkField.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanStatusField.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanTypeField.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScannerResultsTab.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScannerScansTab.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DeleteDataScanModal.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/forms/DataScanExportForm.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/DataScanExportModal.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/DataScanModal.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/components/notifications/DataScanNewResultsNotification.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner/results.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner/scans.vue create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/services/adminWorkspaces.js create mode 100644 enterprise/web-frontend/modules/baserow_enterprise/services/dataScanner.js create mode 100644 web-frontend/modules/core/assets/scss/components/tabs_body.scss create mode 100644 web-frontend/modules/core/static/img/features/data_scanner.png diff --git a/backend/src/baserow/api/admin/views.py b/backend/src/baserow/api/admin/views.py index 4ee8097977..002dc9a9b8 100755 --- a/backend/src/baserow/api/admin/views.py +++ b/backend/src/baserow/api/admin/views.py @@ -13,6 +13,7 @@ from baserow.api.exceptions import ( InvalidSortAttributeException, InvalidSortDirectionException, + QueryParameterValidationException, ) from baserow.api.mixins import ( FilterableViewMixin, @@ -22,6 +23,7 @@ from baserow.api.pagination import PageNumberPagination from baserow.api.schemas import get_error_schema from baserow.api.serializers import get_example_pagination_serializer_class +from baserow.core.utils import split_comma_separated_string class APIListingView( @@ -46,11 +48,13 @@ def get(self, request): search = request.GET.get("search") sorts = request.GET.get("sorts") + ids_param = request.GET.get("ids") queryset = self.get_queryset(request) queryset = self.apply_filters(request.GET, queryset) queryset = self.apply_search(search, queryset) queryset = self.apply_sorts_or_default_sort(sorts, queryset) + queryset = self.apply_ids_filter(ids_param, queryset) paginator = PageNumberPagination(limit_page_size=100) page = paginator.paginate_queryset(queryset, request, self) @@ -61,6 +65,30 @@ def get(self, request): def get_queryset(self, request): raise NotImplementedError("The get_queryset method must be set.") + def apply_ids_filter(self, ids_param, queryset): + if not ids_param: + return queryset + + record_ids = split_comma_separated_string(ids_param) + + invalid_id = next( + (record for record in record_ids if not record.isdigit()), None + ) + if invalid_id is not None: + raise QueryParameterValidationException( + { + "ids": [ + { + "code": "invalid", + "error": f"'{invalid_id}' is not a valid ID. Only positive " + f"integers are accepted.", + } + ] + } + ) + + return queryset.filter(id__in=[int(record_id) for record_id in record_ids]) + def get_serializer(self, request, *args, **kwargs): if not self.serializer_class: raise NotImplementedError( @@ -134,6 +162,13 @@ def get_extend_schema_parameters( type=OpenApiTypes.INT, description=f"Defines how many {name} should be returned per page.", ), + OpenApiParameter( + name="ids", + location=OpenApiParameter.QUERY, + type=OpenApiTypes.STR, + description=f"A comma-separated list of {name} IDs to filter by. " + f"When provided, only {name} with those IDs are returned.", + ), *(extra_parameters or []), ], "responses": { diff --git a/backend/src/baserow/api/admin/workspaces/serializers.py b/backend/src/baserow/api/admin/workspaces/serializers.py index 78bbe45816..fdbc2c64d6 100755 --- a/backend/src/baserow/api/admin/workspaces/serializers.py +++ b/backend/src/baserow/api/admin/workspaces/serializers.py @@ -53,3 +53,11 @@ class Meta: "free_users", "created_on", ) + + +class AdminWorkspaceOptionsSerializer(serializers.ModelSerializer): + value = serializers.CharField(source="name") + + class Meta: + model = Workspace + fields = ("id", "value") diff --git a/backend/src/baserow/api/admin/workspaces/urls.py b/backend/src/baserow/api/admin/workspaces/urls.py index 26247063b5..7802cce4fb 100755 --- a/backend/src/baserow/api/admin/workspaces/urls.py +++ b/backend/src/baserow/api/admin/workspaces/urls.py @@ -1,10 +1,15 @@ from django.urls import re_path -from baserow.api.admin.workspaces.views import WorkspaceAdminView, WorkspacesAdminView +from baserow.api.admin.workspaces.views import ( + WorkspaceAdminView, + WorkspaceOptionsAdminView, + WorkspacesAdminView, +) app_name = "baserow.api.admin.workspaces" urlpatterns = [ re_path(r"^$", WorkspacesAdminView.as_view(), name="list"), + re_path(r"^options/$", WorkspaceOptionsAdminView.as_view(), name="options"), re_path(r"^(?P[0-9]+)/$", WorkspaceAdminView.as_view(), name="edit"), ] diff --git a/backend/src/baserow/api/admin/workspaces/views.py b/backend/src/baserow/api/admin/workspaces/views.py index cad07b3de4..118bdc1728 100644 --- a/backend/src/baserow/api/admin/workspaces/views.py +++ b/backend/src/baserow/api/admin/workspaces/views.py @@ -7,7 +7,7 @@ from rest_framework.response import Response from rest_framework.views import APIView -from baserow.api.admin.views import AdminListingView +from baserow.api.admin.views import AdminListingView, APIListingView from baserow.api.decorators import map_exceptions from baserow.api.errors import ERROR_GROUP_DOES_NOT_EXIST from baserow.api.schemas import get_error_schema @@ -19,7 +19,10 @@ from baserow.core.usage.handler import UsageHandler from .errors import ERROR_CANNOT_DELETE_A_TEMPLATE_GROUP -from .serializers import WorkspacesAdminResponseSerializer +from .serializers import ( + AdminWorkspaceOptionsSerializer, + WorkspacesAdminResponseSerializer, +) class WorkspacesAdminView(AdminListingView): @@ -62,6 +65,30 @@ def get(self, request): return super().get(request) +class WorkspaceOptionsAdminView(APIListingView): + permission_classes = (IsAdminUser,) + serializer_class = AdminWorkspaceOptionsSerializer + search_fields = ["name"] + default_order_by = "name" + + def get_queryset(self, request): + return Workspace.objects.filter(template__isnull=True) + + @extend_schema( + tags=["Admin"], + operation_id="admin_list_workspaces_as_options", + description=( + "Lists all workspaces. This endpoint is intended for admin-level " + "features that need a workspace dropdown." + ), + **APIListingView.get_extend_schema_parameters( + "workspaces", serializer_class, search_fields, {} + ), + ) + def get(self, request): + return super().get(request) + + class WorkspaceAdminView(APIView): permission_classes = (IsAdminUser,) diff --git a/backend/src/baserow/contrib/database/locale/en/LC_MESSAGES/django.po b/backend/src/baserow/contrib/database/locale/en/LC_MESSAGES/django.po index 6ee1964a6f..418277a125 100644 --- a/backend/src/baserow/contrib/database/locale/en/LC_MESSAGES/django.po +++ b/backend/src/baserow/contrib/database/locale/en/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-03-16 14:50+0000\n" +"POT-Creation-Date: 2026-03-16 21:52+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" diff --git a/backend/src/baserow/core/locale/en/LC_MESSAGES/django.po b/backend/src/baserow/core/locale/en/LC_MESSAGES/django.po index 1e92428abd..7f4df3cba3 100644 --- a/backend/src/baserow/core/locale/en/LC_MESSAGES/django.po +++ b/backend/src/baserow/core/locale/en/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-03-16 14:50+0000\n" +"POT-Creation-Date: 2026-03-16 21:52+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" diff --git a/backend/src/baserow/locale/en/LC_MESSAGES/django.po b/backend/src/baserow/locale/en/LC_MESSAGES/django.po index f185ecf311..bf0c3be665 100755 --- a/backend/src/baserow/locale/en/LC_MESSAGES/django.po +++ b/backend/src/baserow/locale/en/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-03-16 14:50+0000\n" +"POT-Creation-Date: 2026-03-16 21:52+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -205,20 +205,24 @@ msgstr "" msgid "Widget \"%(widget_title)s\" (%(widget_id)s) deleted" msgstr "" -#: src/baserow/contrib/integrations/core/service_types.py:1067 +#: src/baserow/contrib/integrations/core/service_types.py:1062 msgid "Branch taken" msgstr "" -#: src/baserow/contrib/integrations/core/service_types.py:1072 +#: src/baserow/contrib/integrations/core/service_types.py:1067 msgid "Label" msgstr "" -#: src/baserow/contrib/integrations/core/service_types.py:1074 +#: src/baserow/contrib/integrations/core/service_types.py:1069 msgid "The label of the branch that matched the condition." msgstr "" -#: src/baserow/contrib/integrations/core/service_types.py:1418 -msgid "Triggered at" +#: src/baserow/contrib/integrations/core/service_types.py:1438 +msgid "Previous scheduled run" +msgstr "" + +#: src/baserow/contrib/integrations/core/service_types.py:1442 +msgid "Next scheduled run" msgstr "" #: src/baserow/contrib/integrations/local_baserow/service_types.py:1688 diff --git a/backend/tests/baserow/api/admin/groups/test_workspaces_admin_views.py b/backend/tests/baserow/api/admin/groups/test_workspaces_admin_views.py index f077b400a0..5fd5646a58 100644 --- a/backend/tests/baserow/api/admin/groups/test_workspaces_admin_views.py +++ b/backend/tests/baserow/api/admin/groups/test_workspaces_admin_views.py @@ -219,3 +219,159 @@ def test_cant_delete_template_workspace(api_client, data_fixture): assert response.status_code == HTTP_400_BAD_REQUEST assert response.json()["error"] == "ERROR_CANNOT_DELETE_A_TEMPLATE_GROUP" assert Workspace.objects.all().count() == 1 + + +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_non_admin_list_workspaces_as_options(api_client, data_fixture): + ( + admin_user, + admin_token, + ) = data_fixture.create_user_and_token() + + # no search query should return all workspaces + response = api_client.get( + reverse("api:admin:workspaces:options"), + format="json", + HTTP_AUTHORIZATION=f"JWT {admin_token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_admin_list_workspaces_as_options(api_client, data_fixture): + ( + admin_user, + admin_token, + ) = data_fixture.create_user_and_token(is_staff=True) + workspace_1 = data_fixture.create_workspace(name="workspace 1", user=admin_user) + workspace_2 = data_fixture.create_workspace(name="workspace 2", user=admin_user) + + # no search query should return all workspaces + response = api_client.get( + reverse("api:admin:workspaces:options"), + format="json", + HTTP_AUTHORIZATION=f"JWT {admin_token}", + ) + assert response.status_code == HTTP_200_OK + assert response.json() == { + "count": 2, + "next": None, + "previous": None, + "results": [ + {"id": workspace_1.id, "value": workspace_1.name}, + {"id": workspace_2.id, "value": workspace_2.name}, + ], + } + + # searching by name should return only the correct workspace + response = api_client.get( + reverse("api:admin:workspaces:options") + "?search=1", + format="json", + HTTP_AUTHORIZATION=f"JWT {admin_token}", + ) + assert response.status_code == HTTP_200_OK + assert response.json() == { + "count": 1, + "next": None, + "previous": None, + "results": [{"id": workspace_1.id, "value": workspace_1.name}], + } + + +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_admin_list_workspaces_as_options_filter_by_ids(api_client, data_fixture): + ( + admin_user, + admin_token, + ) = data_fixture.create_user_and_token(is_staff=True) + workspace_1 = data_fixture.create_workspace(name="workspace 1", user=admin_user) + workspace_2 = data_fixture.create_workspace(name="workspace 2", user=admin_user) + data_fixture.create_workspace(name="workspace 3", user=admin_user) + + # filtering by a single id should return only that workspace + response = api_client.get( + reverse("api:admin:workspaces:options") + f"?ids={workspace_1.id}", + format="json", + HTTP_AUTHORIZATION=f"JWT {admin_token}", + ) + assert response.status_code == HTTP_200_OK + assert response.json() == { + "count": 1, + "next": None, + "previous": None, + "results": [{"id": workspace_1.id, "value": workspace_1.name}], + } + + # filtering by multiple ids should return all matching workspaces + response = api_client.get( + reverse("api:admin:workspaces:options") + + f"?ids={workspace_1.id},{workspace_2.id}", + format="json", + HTTP_AUTHORIZATION=f"JWT {admin_token}", + ) + assert response.status_code == HTTP_200_OK + assert response.json() == { + "count": 2, + "next": None, + "previous": None, + "results": [ + {"id": workspace_1.id, "value": workspace_1.name}, + {"id": workspace_2.id, "value": workspace_2.name}, + ], + } + + +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_admin_list_workspaces_as_options_filter_by_invalid_ids( + api_client, data_fixture +): + _, admin_token = data_fixture.create_user_and_token(is_staff=True) + + # Negative IDs should be rejected. + response = api_client.get( + reverse("api:admin:workspaces:options") + "?ids=-1", + format="json", + HTTP_AUTHORIZATION=f"JWT {admin_token}", + ) + assert response.status_code == HTTP_400_BAD_REQUEST + assert response.json()["error"] == "ERROR_QUERY_PARAMETER_VALIDATION" + assert response.json()["detail"]["ids"] == [ + { + "code": "invalid", + "error": "'-1' is not a valid ID. Only positive integers are accepted.", + } + ] + + # Non-numeric values should be rejected. + response = api_client.get( + reverse("api:admin:workspaces:options") + "?ids=abc", + format="json", + HTTP_AUTHORIZATION=f"JWT {admin_token}", + ) + assert response.status_code == HTTP_400_BAD_REQUEST + assert response.json()["error"] == "ERROR_QUERY_PARAMETER_VALIDATION" + assert response.json()["detail"]["ids"] == [ + { + "code": "invalid", + "error": "'abc' is not a valid ID. Only positive integers are accepted.", + } + ] + + # A mix of valid and invalid values should still be rejected. + response = api_client.get( + reverse("api:admin:workspaces:options") + "?ids=1,-2,3", + format="json", + HTTP_AUTHORIZATION=f"JWT {admin_token}", + ) + assert response.status_code == HTTP_400_BAD_REQUEST + assert response.json()["error"] == "ERROR_QUERY_PARAMETER_VALIDATION" + assert response.json()["detail"]["ids"] == [ + { + "code": "invalid", + "error": "'-2' is not a valid ID. Only positive integers are accepted.", + } + ] diff --git a/changelog/entries/unreleased/feature/data_scanner.json b/changelog/entries/unreleased/feature/data_scanner.json new file mode 100644 index 0000000000..a2c9806adb --- /dev/null +++ b/changelog/entries/unreleased/feature/data_scanner.json @@ -0,0 +1,9 @@ +{ + "type": "feature", + "message": "Add instace wide data scanner.", + "issue_origin": "github", + "issue_number": null, + "domain": "database", + "bullet_points": [], + "created_at": "2026-03-16" +} diff --git a/enterprise/backend/pytest.ini b/enterprise/backend/pytest.ini index 28c968f590..37fe40b6ac 100644 --- a/enterprise/backend/pytest.ini +++ b/enterprise/backend/pytest.ini @@ -3,5 +3,6 @@ DJANGO_SETTINGS_MODULE = baserow.config.settings.test python_files = test_*.py markers = eval: mark test as an eval test (requires LLM API key) + data_scanner: mark test as a data scanner test env = DJANGO_SETTINGS_MODULE = baserow.config.settings.test diff --git a/enterprise/backend/src/baserow_enterprise/api/admin/audit_log/urls.py b/enterprise/backend/src/baserow_enterprise/api/admin/audit_log/urls.py index 259b5894ba..3144a96724 100755 --- a/enterprise/backend/src/baserow_enterprise/api/admin/audit_log/urls.py +++ b/enterprise/backend/src/baserow_enterprise/api/admin/audit_log/urls.py @@ -5,7 +5,6 @@ AuditLogActionTypeFilterView, AuditLogUserFilterView, AuditLogView, - AuditLogWorkspaceFilterView, ) app_name = "baserow_enterprise.api.audit_log" @@ -13,7 +12,6 @@ urlpatterns = [ re_path(r"^$", AuditLogView.as_view(), name="list"), re_path(r"users/$", AuditLogUserFilterView.as_view(), name="users"), - re_path(r"workspaces/$", AuditLogWorkspaceFilterView.as_view(), name="workspaces"), re_path( r"action-types/$", AuditLogActionTypeFilterView.as_view(), name="action_types" ), diff --git a/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/__init__.py b/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/errors.py b/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/errors.py new file mode 100644 index 0000000000..8f15f1e398 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/errors.py @@ -0,0 +1,19 @@ +from rest_framework.status import HTTP_404_NOT_FOUND, HTTP_409_CONFLICT + +ERROR_DATA_SCAN_DOES_NOT_EXIST = ( + "ERROR_DATA_SCAN_DOES_NOT_EXIST", + HTTP_404_NOT_FOUND, + "The requested data scan does not exist.", +) + +ERROR_DATA_SCAN_ALREADY_RUNNING = ( + "ERROR_DATA_SCAN_ALREADY_RUNNING", + HTTP_409_CONFLICT, + "The data scan is already running.", +) + +ERROR_DATA_SCAN_RESULT_DOES_NOT_EXIST = ( + "ERROR_DATA_SCAN_RESULT_DOES_NOT_EXIST", + HTTP_404_NOT_FOUND, + "The requested data scan result does not exist.", +) diff --git a/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/serializers.py b/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/serializers.py new file mode 100644 index 0000000000..f6ef8ce103 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/serializers.py @@ -0,0 +1,268 @@ +from rest_framework import serializers + +from baserow.contrib.database.fields.models import Field +from baserow.core.jobs.registries import job_type_registry +from baserow_enterprise.data_scanner.constants import ( + SCAN_TYPE_LIST_OF_VALUES, + SCAN_TYPE_LIST_TABLE, + SCAN_TYPE_PATTERN, + SCANNABLE_FIELD_CONTENT_TYPES, +) +from baserow_enterprise.data_scanner.job_types import DataScanResultExportJobType +from baserow_enterprise.data_scanner.models import DataScan, DataScanResult + + +class DataScanSerializer(serializers.ModelSerializer): + workspace_ids = serializers.SerializerMethodField() + list_items = serializers.SerializerMethodField() + source_table_id = serializers.SerializerMethodField() + source_field_id = serializers.SerializerMethodField() + source_workspace_id = serializers.SerializerMethodField() + source_database_id = serializers.SerializerMethodField() + results_count = serializers.SerializerMethodField() + + class Meta: + model = DataScan + fields = [ + "id", + "name", + "scan_type", + "pattern", + "frequency", + "scan_all_workspaces", + "workspace_ids", + "is_running", + "last_run_started_at", + "last_run_finished_at", + "last_error", + "list_items", + "results_count", + "source_table_id", + "source_field_id", + "source_workspace_id", + "source_database_id", + "created_on", + "updated_on", + ] + + def get_workspace_ids(self, obj): + return [ws.id for ws in obj.workspaces.all()] + + def get_list_items(self, obj): + return [item.value for item in obj.list_items.all()] + + def get_source_table_id(self, obj): + return obj.source_table_id + + def get_source_field_id(self, obj): + return obj.source_field_id + + def get_source_workspace_id(self, obj): + try: + return obj.source_table.database.workspace_id + except AttributeError: + return None + + def get_source_database_id(self, obj): + try: + return obj.source_table.database_id + except AttributeError: + return None + + def get_results_count(self, obj): + if hasattr(obj, "results_count"): + return obj.results_count + return obj.results.count() + + +class DataScanWriteSerializer(serializers.Serializer): + name = serializers.CharField(max_length=255, required=False) + scan_type = serializers.ChoiceField( + choices=DataScan.SCAN_TYPE_CHOICES, + required=False, + ) + pattern = serializers.CharField( + max_length=100, required=False, allow_blank=True, allow_null=True + ) + frequency = serializers.ChoiceField( + choices=DataScan.FREQUENCY_CHOICES, + required=False, + ) + scan_all_workspaces = serializers.BooleanField(required=False) + workspace_ids = serializers.ListField( + child=serializers.IntegerField(), + required=False, + ) + list_items = serializers.ListField( + child=serializers.CharField(), + required=False, + ) + source_table_id = serializers.IntegerField(required=False, allow_null=True) + source_field_id = serializers.IntegerField(required=False, allow_null=True) + + def validate_source_field_id(self, value): + if value is not None: + try: + field = Field.objects.get(id=value) + except Field.DoesNotExist: + raise serializers.ValidationError( + "The specified source field does not exist." + ) + if field.content_type.model not in SCANNABLE_FIELD_CONTENT_TYPES: + raise serializers.ValidationError( + "The specified source field type is not compatible with data " + "scanning." + ) + return value + + +class DataScanCreateSerializer(DataScanWriteSerializer): + name = serializers.CharField(max_length=255) + scan_type = serializers.ChoiceField( + choices=DataScan.SCAN_TYPE_CHOICES, + ) + workspace_ids = serializers.ListField( + child=serializers.IntegerField(), + default=list, + ) + list_items = serializers.ListField( + child=serializers.CharField(), + default=list, + ) + pattern = serializers.CharField( + max_length=100, required=False, allow_blank=True, default=None + ) + frequency = serializers.ChoiceField( + choices=DataScan.FREQUENCY_CHOICES, + default="manual", + ) + scan_all_workspaces = serializers.BooleanField(default=True) + + def validate(self, data): + scan_type = data.get("scan_type") + if scan_type == SCAN_TYPE_PATTERN and not data.get("pattern"): + raise serializers.ValidationError( + {"pattern": "Pattern is required for pattern scan type."} + ) + if scan_type == SCAN_TYPE_LIST_OF_VALUES and not data.get("list_items"): + raise serializers.ValidationError( + {"list_items": "List items are required for list of values scan type."} + ) + if scan_type == SCAN_TYPE_LIST_TABLE: + if not data.get("source_table_id") or not data.get("source_field_id"): + raise serializers.ValidationError( + { + "source_table_id": "Source table and field are required for list table scan type." + } + ) + return data + + +class DataScanUpdateSerializer(DataScanWriteSerializer): + pass + + +class DataScanResultSerializer(serializers.ModelSerializer): + scan_name = serializers.SerializerMethodField() + workspace_name = serializers.SerializerMethodField() + database_id = serializers.SerializerMethodField() + database_name = serializers.SerializerMethodField() + table_name = serializers.SerializerMethodField() + field_name = serializers.SerializerMethodField() + + class Meta: + model = DataScanResult + fields = [ + "id", + "scan_id", + "scan_name", + "workspace_name", + "database_id", + "database_name", + "table_id", + "table_name", + "field_name", + "row_id", + "matched_value", + "first_identified_on", + "last_identified_on", + ] + + def get_scan_name(self, obj): + return obj.scan.name + + def get_workspace_name(self, obj): + try: + return obj.field.table.database.workspace.name + except AttributeError: + return None + + def get_database_id(self, obj): + return obj.table.database_id + + def get_database_name(self, obj): + try: + return obj.field.table.database.name + except AttributeError: + return None + + def get_table_name(self, obj): + try: + return obj.field.table.name + except AttributeError: + return None + + def get_field_name(self, obj): + try: + return obj.field.name + except AttributeError: + return None + + +class WorkspaceStructureFieldSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + type = serializers.SerializerMethodField() + + def get_type(self, obj): + field_type = obj.content_type.model + if field_type.endswith("field"): + field_type = field_type[: -len("field")] + return field_type + + +class WorkspaceStructureTableSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + + def to_representation(self, instance): + data = super().to_representation(instance) + data["fields"] = WorkspaceStructureFieldSerializer( + instance.field_set.all(), many=True + ).data + return data + + +class WorkspaceStructureDatabaseSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + tables = serializers.SerializerMethodField() + + def get_tables(self, obj): + return WorkspaceStructureTableSerializer(obj.table_set.all(), many=True).data + + +DataScanResultExportJobRequestSerializer = job_type_registry.get( + DataScanResultExportJobType.type +).get_serializer_class( + base_class=serializers.Serializer, + request_serializer=True, + meta_ref_name="SingleDataScanResultExportJobRequestSerializer", +) + +DataScanResultExportJobResponseSerializer = job_type_registry.get( + DataScanResultExportJobType.type +).get_serializer_class( + base_class=serializers.Serializer, + meta_ref_name="SingleDataScanResultExportJobResponseSerializer", +) diff --git a/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/urls.py b/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/urls.py new file mode 100644 index 0000000000..738b6faad3 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/urls.py @@ -0,0 +1,43 @@ +from django.urls import re_path + +from .views import ( + DataScanDetailView, + DataScanListView, + DataScanResultDeleteView, + DataScanResultExportView, + DataScanResultListView, + DataScanTriggerView, + DataScanWorkspaceStructureView, +) + +app_name = "baserow_enterprise.api.admin.data_scanner" + +urlpatterns = [ + re_path(r"^scans/$", DataScanListView.as_view(), name="list"), + re_path( + r"^scans/(?P[0-9]+)/$", + DataScanDetailView.as_view(), + name="detail", + ), + re_path( + r"^scans/(?P[0-9]+)/trigger/$", + DataScanTriggerView.as_view(), + name="trigger", + ), + re_path(r"^results/$", DataScanResultListView.as_view(), name="results"), + re_path( + r"^results/export/$", + DataScanResultExportView.as_view(), + name="results_export", + ), + re_path( + r"^results/(?P[0-9]+)/$", + DataScanResultDeleteView.as_view(), + name="result_delete", + ), + re_path( + r"^workspace-structure/(?P[0-9]+)/$", + DataScanWorkspaceStructureView.as_view(), + name="workspace_structure", + ), +] diff --git a/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/views.py b/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/views.py new file mode 100644 index 0000000000..ea2989d154 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/api/admin/data_scanner/views.py @@ -0,0 +1,370 @@ +from django.db import transaction +from django.db.models import Prefetch + +from drf_spectacular.utils import extend_schema +from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response +from rest_framework.status import HTTP_202_ACCEPTED, HTTP_204_NO_CONTENT +from rest_framework.views import APIView + +from baserow.api.admin.views import APIListingView +from baserow.api.decorators import map_exceptions, validate_body +from baserow.api.errors import ERROR_GROUP_DOES_NOT_EXIST +from baserow.api.jobs.errors import ERROR_MAX_JOB_COUNT_EXCEEDED +from baserow.api.jobs.serializers import JobSerializer +from baserow.api.schemas import get_error_schema +from baserow.contrib.database.fields.models import Field +from baserow.contrib.database.models import Database +from baserow.contrib.database.table.models import Table +from baserow.core.action.registries import action_type_registry +from baserow.core.exceptions import WorkspaceDoesNotExist +from baserow.core.jobs.exceptions import MaxJobCountExceeded +from baserow.core.jobs.handler import JobHandler +from baserow.core.jobs.registries import job_type_registry +from baserow.core.models import Workspace +from baserow_enterprise.api.admin.data_scanner.errors import ( + ERROR_DATA_SCAN_ALREADY_RUNNING, + ERROR_DATA_SCAN_DOES_NOT_EXIST, + ERROR_DATA_SCAN_RESULT_DOES_NOT_EXIST, +) +from baserow_enterprise.api.admin.data_scanner.serializers import ( + DataScanCreateSerializer, + DataScanResultExportJobRequestSerializer, + DataScanResultExportJobResponseSerializer, + DataScanResultSerializer, + DataScanSerializer, + DataScanUpdateSerializer, + WorkspaceStructureDatabaseSerializer, +) +from baserow_enterprise.data_scanner.actions import ( + CreateDataScanActionType, + DeleteDataScanActionType, + UpdateDataScanActionType, +) +from baserow_enterprise.data_scanner.constants import SCANNABLE_FIELD_CONTENT_TYPES +from baserow_enterprise.data_scanner.exceptions import ( + DataScanDoesNotExist, + DataScanIsAlreadyRunning, + DataScanResultDoesNotExist, +) +from baserow_enterprise.data_scanner.handler import DataScannerHandler +from baserow_enterprise.data_scanner.job_types import DataScanResultExportJobType +from baserow_enterprise.data_scanner.models import DataScanResult +from baserow_enterprise.features import DATA_SCANNER +from baserow_premium.license.handler import LicenseHandler + + +class DataScanListView(APIListingView): + permission_classes = (IsAdminUser,) + serializer_class = DataScanSerializer + search_fields = ["name"] + sort_field_mapping = { + "name": "name", + "scan_type": "scan_type", + "frequency": "frequency", + "created_on": "created_on", + } + default_order_by = "created_on" + + def get_queryset(self, request): + return DataScannerHandler.list_scans(request.user) + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_list_scans", + description=( + "Lists all data scans configured for this Baserow instance. Data scans " + "allow administrators to search the entire instance for sensitive data " + "matching a pattern, a list of uploaded values, or values from another " + "Baserow table. **Enterprise feature.**" + ), + **APIListingView.get_extend_schema_parameters( + "data scans", + DataScanSerializer, + ["name"], + sort_field_mapping, + ), + ) + def get(self, request): + return super().get(request) + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_create_scan", + description=( + "Creates a new data scan. A data scan searches the Baserow instance " + "for sensitive data matching a pattern (e.g. credit card numbers), a " + "list of uploaded values, or values sourced from another Baserow table. " + "**Enterprise feature.**" + ), + request=DataScanCreateSerializer, + responses={200: DataScanSerializer}, + ) + @transaction.atomic + @validate_body(DataScanCreateSerializer) + def post(self, request, data): + LicenseHandler.raise_if_user_doesnt_have_feature_instance_wide( + DATA_SCANNER, request.user + ) + scan = action_type_registry.get_by_type(CreateDataScanActionType).do( + user=request.user, + name=data["name"], + scan_type=data["scan_type"], + pattern=data.get("pattern"), + frequency=data.get("frequency", "manual"), + scan_all_workspaces=data.get("scan_all_workspaces", True), + workspace_ids=data.get("workspace_ids", []), + list_items=data.get("list_items", []), + source_table_id=data.get("source_table_id"), + source_field_id=data.get("source_field_id"), + ) + return Response(DataScanSerializer(scan).data) + + +class DataScanDetailView(APIView): + permission_classes = (IsAdminUser,) + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_get_scan", + description=( + "Returns a single data scan configuration. **Enterprise feature.**" + ), + responses={ + 200: DataScanSerializer, + 404: get_error_schema(["ERROR_DATA_SCAN_DOES_NOT_EXIST"]), + }, + ) + @map_exceptions({DataScanDoesNotExist: ERROR_DATA_SCAN_DOES_NOT_EXIST}) + def get(self, request, scan_id): + scan = DataScannerHandler.get_scan(user=request.user, scan_id=scan_id) + return Response(DataScanSerializer(scan).data) + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_update_scan", + description=( + "Updates a data scan configuration. When the scan type, pattern, or " + "list items change, stale results are automatically cleaned up. " + "**Enterprise feature.**" + ), + request=DataScanUpdateSerializer, + responses={ + 200: DataScanSerializer, + 404: get_error_schema(["ERROR_DATA_SCAN_DOES_NOT_EXIST"]), + 409: get_error_schema(["ERROR_DATA_SCAN_ALREADY_RUNNING"]), + }, + ) + @transaction.atomic + @map_exceptions( + { + DataScanDoesNotExist: ERROR_DATA_SCAN_DOES_NOT_EXIST, + DataScanIsAlreadyRunning: ERROR_DATA_SCAN_ALREADY_RUNNING, + } + ) + @validate_body(DataScanUpdateSerializer) + def patch(self, request, scan_id, data): + scan = action_type_registry.get_by_type(UpdateDataScanActionType).do( + user=request.user, + scan_id=scan_id, + **data, + ) + return Response(DataScanSerializer(scan).data) + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_delete_scan", + description=( + "Deletes a data scan and all of its results. **Enterprise feature.**" + ), + responses={ + 204: None, + 404: get_error_schema(["ERROR_DATA_SCAN_DOES_NOT_EXIST"]), + 409: get_error_schema(["ERROR_DATA_SCAN_ALREADY_RUNNING"]), + }, + ) + @transaction.atomic + @map_exceptions( + { + DataScanDoesNotExist: ERROR_DATA_SCAN_DOES_NOT_EXIST, + DataScanIsAlreadyRunning: ERROR_DATA_SCAN_ALREADY_RUNNING, + } + ) + def delete(self, request, scan_id): + action_type_registry.get_by_type(DeleteDataScanActionType).do( + user=request.user, scan_id=scan_id + ) + return Response(status=HTTP_204_NO_CONTENT) + + +class DataScanTriggerView(APIView): + permission_classes = (IsAdminUser,) + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_trigger_scan", + description=( + "Triggers an immediate run of the given data scan. The scan executes " + "asynchronously and searches the configured workspaces for matches. " + "**Enterprise feature.**" + ), + responses={ + 202: DataScanSerializer, + 404: get_error_schema(["ERROR_DATA_SCAN_DOES_NOT_EXIST"]), + 409: get_error_schema(["ERROR_DATA_SCAN_ALREADY_RUNNING"]), + }, + ) + @map_exceptions( + { + DataScanDoesNotExist: ERROR_DATA_SCAN_DOES_NOT_EXIST, + DataScanIsAlreadyRunning: ERROR_DATA_SCAN_ALREADY_RUNNING, + } + ) + def post(self, request, scan_id): + scan = DataScannerHandler.trigger_scan(user=request.user, scan_id=scan_id) + return Response(DataScanSerializer(scan).data, status=HTTP_202_ACCEPTED) + + +class DataScanResultListView(APIListingView): + permission_classes = (IsAdminUser,) + serializer_class = DataScanResultSerializer + search_fields = ["matched_value"] + filters_field_mapping = { + "scan_id": "scan_id", + } + sort_field_mapping = { + "first_identified_on": "first_identified_on", + "last_identified_on": "last_identified_on", + } + default_order_by = "-first_identified_on" + + def get_queryset(self, request): + LicenseHandler.raise_if_user_doesnt_have_feature_instance_wide( + DATA_SCANNER, request.user + ) + return DataScanResult.objects.select_related( + "scan", "table__database", "field__table__database__workspace" + ).all() + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_list_results", + description=( + "Lists all data scan results across all scans. Results represent " + "individual matches found in database fields during scan execution. " + "Can be filtered by scan_id and searched by matched value. " + "**Enterprise feature.**" + ), + **APIListingView.get_extend_schema_parameters( + "data scan results", + DataScanResultSerializer, + ["matched_value"], + sort_field_mapping, + ), + ) + def get(self, request): + return super().get(request) + + +class DataScanWorkspaceStructureView(APIView): + permission_classes = (IsAdminUser,) + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_workspace_structure", + description=( + "Returns the database/table/field structure of a workspace for use " + "in data scan configuration. Only text-compatible fields are included. " + "**Enterprise feature.**" + ), + responses={ + 200: WorkspaceStructureDatabaseSerializer(many=True), + 404: get_error_schema(["ERROR_GROUP_DOES_NOT_EXIST"]), + }, + ) + @map_exceptions({WorkspaceDoesNotExist: ERROR_GROUP_DOES_NOT_EXIST}) + def get(self, request, workspace_id): + LicenseHandler.raise_if_user_doesnt_have_feature_instance_wide( + DATA_SCANNER, request.user + ) + + try: + workspace = Workspace.objects.get(id=workspace_id) + except Workspace.DoesNotExist: + raise WorkspaceDoesNotExist() + + databases = ( + Database.objects.filter(workspace=workspace) + .prefetch_related( + Prefetch( + "table_set", + queryset=Table.objects.prefetch_related( + Prefetch( + "field_set", + queryset=Field.objects.filter( + content_type__model__in=SCANNABLE_FIELD_CONTENT_TYPES, + ).select_related("content_type"), + ) + ), + ) + ) + .order_by("order", "id") + ) + + serializer = WorkspaceStructureDatabaseSerializer(databases, many=True) + return Response(serializer.data) + + +class DataScanResultExportView(APIView): + permission_classes = (IsAdminUser,) + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_export_results", + description=( + "Creates a job to export data scan results to CSV. The exported file " + "includes scan name, workspace, database, table, field, row ID, matched " + "value, and timestamps for each result. **Enterprise feature.**" + ), + request=DataScanResultExportJobRequestSerializer, + responses={ + 202: DataScanResultExportJobResponseSerializer, + 400: get_error_schema(["ERROR_MAX_JOB_COUNT_EXCEEDED"]), + }, + ) + @transaction.atomic + @map_exceptions({MaxJobCountExceeded: ERROR_MAX_JOB_COUNT_EXCEEDED}) + @validate_body(DataScanResultExportJobRequestSerializer, return_validated=True) + def post(self, request, data): + LicenseHandler.raise_if_user_doesnt_have_feature_instance_wide( + DATA_SCANNER, request.user + ) + job = JobHandler().create_and_start_job( + request.user, DataScanResultExportJobType.type, **data + ) + serializer = job_type_registry.get_serializer( + job, JobSerializer, context={"request": request} + ) + return Response(serializer.data, status=HTTP_202_ACCEPTED) + + +class DataScanResultDeleteView(APIView): + permission_classes = (IsAdminUser,) + + @extend_schema( + tags=["Admin data scanner"], + operation_id="admin_data_scanner_delete_result", + description=( + "Deletes (resolves) a single data scan result, marking it as reviewed. " + "**Enterprise feature.**" + ), + responses={ + 204: None, + 404: get_error_schema(["ERROR_DATA_SCAN_RESULT_DOES_NOT_EXIST"]), + }, + ) + @transaction.atomic + @map_exceptions({DataScanResultDoesNotExist: ERROR_DATA_SCAN_RESULT_DOES_NOT_EXIST}) + def delete(self, request, result_id): + DataScannerHandler.delete_result(user=request.user, result_id=result_id) + return Response(status=HTTP_204_NO_CONTENT) diff --git a/enterprise/backend/src/baserow_enterprise/api/admin/urls.py b/enterprise/backend/src/baserow_enterprise/api/admin/urls.py index 4b073f1adb..20efb8d2d6 100644 --- a/enterprise/backend/src/baserow_enterprise/api/admin/urls.py +++ b/enterprise/backend/src/baserow_enterprise/api/admin/urls.py @@ -2,10 +2,12 @@ from .audit_log import urls as audit_log_urls from .auth_provider import urls as auth_provider_urls +from .data_scanner import urls as data_scanner_urls app_name = "baserow_enterprise.api.admin" urlpatterns = [ path("auth-provider/", include(auth_provider_urls, namespace="auth_provider")), path("audit-log/", include(audit_log_urls, namespace="audit_log")), + path("data-scanner/", include(data_scanner_urls, namespace="data_scanner")), ] diff --git a/enterprise/backend/src/baserow_enterprise/api/audit_log/serializers.py b/enterprise/backend/src/baserow_enterprise/api/audit_log/serializers.py index a3ef44ab35..87c3caa0a3 100644 --- a/enterprise/backend/src/baserow_enterprise/api/audit_log/serializers.py +++ b/enterprise/backend/src/baserow_enterprise/api/audit_log/serializers.py @@ -8,7 +8,6 @@ from baserow.core.action.registries import action_type_registry from baserow.core.jobs.registries import job_type_registry -from baserow.core.models import Workspace from baserow_enterprise.audit_log.job_types import AuditLogExportJobType from baserow_enterprise.audit_log.models import AuditLogEntry @@ -92,14 +91,6 @@ class Meta: fields = ("id", "value") -class AuditLogWorkspaceSerializer(serializers.ModelSerializer): - value = serializers.CharField(source="name") - - class Meta: - model = Workspace - fields = ("id", "value") - - class AuditLogActionTypeSerializer(serializers.Serializer): id = serializers.ChoiceField( choices=lazy(action_type_registry.get_types, list)(), diff --git a/enterprise/backend/src/baserow_enterprise/api/audit_log/urls.py b/enterprise/backend/src/baserow_enterprise/api/audit_log/urls.py index e631b22510..a3a85d1f8b 100755 --- a/enterprise/backend/src/baserow_enterprise/api/audit_log/urls.py +++ b/enterprise/backend/src/baserow_enterprise/api/audit_log/urls.py @@ -5,7 +5,6 @@ AuditLogActionTypeFilterView, AuditLogUserFilterView, AuditLogView, - AuditLogWorkspaceFilterView, ) app_name = "baserow_enterprise.api.audit_log" @@ -13,7 +12,6 @@ urlpatterns = [ re_path(r"^$", AuditLogView.as_view(), name="list"), re_path(r"users/$", AuditLogUserFilterView.as_view(), name="users"), - re_path(r"workspaces/$", AuditLogWorkspaceFilterView.as_view(), name="workspaces"), re_path( r"action-types/$", AuditLogActionTypeFilterView.as_view(), name="action_types" ), diff --git a/enterprise/backend/src/baserow_enterprise/api/audit_log/views.py b/enterprise/backend/src/baserow_enterprise/api/audit_log/views.py index 13ff4a4bad..1410e1702f 100755 --- a/enterprise/backend/src/baserow_enterprise/api/audit_log/views.py +++ b/enterprise/backend/src/baserow_enterprise/api/audit_log/views.py @@ -3,7 +3,7 @@ from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiParameter, extend_schema -from rest_framework.permissions import IsAdminUser, IsAuthenticated +from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.status import HTTP_202_ACCEPTED from rest_framework.views import APIView @@ -23,14 +23,12 @@ from baserow.core.jobs.exceptions import MaxJobCountExceeded from baserow.core.jobs.handler import JobHandler from baserow.core.jobs.registries import job_type_registry -from baserow.core.models import User, Workspace +from baserow.core.models import User from baserow_enterprise.audit_log.job_types import AuditLogExportJobType from baserow_enterprise.audit_log.models import AuditLogEntry from baserow_enterprise.audit_log.utils import ( check_for_license_and_permissions_or_raise, ) -from baserow_enterprise.features import AUDIT_LOG -from baserow_premium.license.handler import LicenseHandler from .serializers import ( AuditLogActionTypeSerializer, @@ -40,7 +38,6 @@ AuditLogSerializer, AuditLogUserSerializer, AuditLogWorkspaceFilterQueryParamsSerializer, - AuditLogWorkspaceSerializer, serialize_filtered_action_types, ) @@ -244,33 +241,6 @@ def get(self, request, query_params): return super().get(request) -class AuditLogWorkspaceFilterView(APIListingView): - permission_classes = (IsAdminUser,) - serializer_class = AuditLogWorkspaceSerializer - search_fields = ["name"] - default_order_by = "name" - - def get_queryset(self, request): - return Workspace.objects.filter(template__isnull=True) - - @extend_schema( - tags=["Audit log"], - operation_id="audit_log_workspaces", - description=( - "List all distinct workspace names related to an audit log entry." - "\n\nThis is a **enterprise** feature." - ), - **APIListingView.get_extend_schema_parameters( - "workspaces", serializer_class, search_fields, {} - ), - ) - def get(self, request): - LicenseHandler.raise_if_user_doesnt_have_feature_instance_wide( - AUDIT_LOG, request.user - ) - return super().get(request) - - class AsyncAuditLogExportView(APIView): permission_classes = (IsAuthenticated,) diff --git a/enterprise/backend/src/baserow_enterprise/apps.py b/enterprise/backend/src/baserow_enterprise/apps.py index 9a482941bb..1c61528d94 100755 --- a/enterprise/backend/src/baserow_enterprise/apps.py +++ b/enterprise/backend/src/baserow_enterprise/apps.py @@ -14,8 +14,12 @@ def ready(self): from baserow_enterprise.audit_log.operations import ( ListWorkspaceAuditLogEntriesOperationType, ) + from baserow_enterprise.data_scanner.job_types import ( + DataScanResultExportJobType, + ) job_type_registry.register(AuditLogExportJobType()) + job_type_registry.register(DataScanResultExportJobType()) from baserow.api.user.registries import member_data_registry from baserow.core.action.registries import ( @@ -261,11 +265,19 @@ def ready(self): data_sync_type_registry.unregister(PostgreSQLDataSyncType.type) data_sync_type_registry.register(PostgreSQLDataSyncType()) + from baserow_enterprise.data_scanner.actions import ( + CreateDataScanActionType, + DeleteDataScanActionType, + UpdateDataScanActionType, + ) from baserow_enterprise.data_sync.actions import ( UpdatePeriodicDataSyncIntervalActionType, ) action_type_registry.register(UpdatePeriodicDataSyncIntervalActionType()) + action_type_registry.register(CreateDataScanActionType()) + action_type_registry.register(UpdateDataScanActionType()) + action_type_registry.register(DeleteDataScanActionType()) from baserow.contrib.database.webhooks.registries import ( webhook_event_type_registry, @@ -303,6 +315,9 @@ def ready(self): connect_to_post_delete_signals_to_cascade_deletion_to_role_assignments() from baserow.core.notifications.registries import notification_type_registry + from baserow_enterprise.data_scanner.notification_types import ( + DataScanNewResultsNotificationType, + ) from baserow_enterprise.data_sync.notification_types import ( PeriodicDataSyncDeactivatedNotificationType, TwoWaySyncDeactivatedNotificationType, @@ -314,6 +329,7 @@ def ready(self): ) notification_type_registry.register(TwoWaySyncUpdateFailedNotificationType()) notification_type_registry.register(TwoWaySyncDeactivatedNotificationType()) + notification_type_registry.register(DataScanNewResultsNotificationType()) from baserow_enterprise.views.operations import ( ListenToAllRestrictedViewEventsOperationType, @@ -368,6 +384,7 @@ def ready(self): # which need to be filled first. import baserow_enterprise.assistant.tasks # noqa: F401 import baserow_enterprise.audit_log.signals # noqa: F401 + import baserow_enterprise.data_scanner.tasks # noqa: F401 import baserow_enterprise.ws.signals # noqa: F401 diff --git a/enterprise/backend/src/baserow_enterprise/data_scanner/__init__.py b/enterprise/backend/src/baserow_enterprise/data_scanner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/enterprise/backend/src/baserow_enterprise/data_scanner/actions.py b/enterprise/backend/src/baserow_enterprise/data_scanner/actions.py new file mode 100644 index 0000000000..e7bf1407b1 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/data_scanner/actions.py @@ -0,0 +1,94 @@ +import dataclasses + +from django.contrib.auth.models import AbstractUser +from django.utils.translation import gettext_lazy as _ + +from baserow.core.action.registries import ( + ActionScopeStr, + ActionType, + ActionTypeDescription, +) +from baserow.core.action.scopes import RootActionScopeType +from baserow_enterprise.data_scanner.handler import DataScannerHandler + + +class CreateDataScanActionType(ActionType): + type = "create_data_scan" + description = ActionTypeDescription( + _("Create data scan"), + _('Data scan "%(scan_name)s" (%(scan_id)s) created'), + ) + analytics_params = ["scan_id"] + + @dataclasses.dataclass + class Params: + scan_id: int + scan_name: str + + @classmethod + def do(cls, user: AbstractUser, **kwargs): + scan = DataScannerHandler.create_scan(user=user, **kwargs) + + params = cls.Params(scan.id, scan.name) + cls.register_action(user, params, cls.scope()) + + return scan + + @classmethod + def scope(cls) -> ActionScopeStr: + return RootActionScopeType.value() + + +class UpdateDataScanActionType(ActionType): + type = "update_data_scan" + description = ActionTypeDescription( + _("Update data scan"), + _('Data scan "%(scan_name)s" (%(scan_id)s) updated'), + ) + analytics_params = ["scan_id"] + + @dataclasses.dataclass + class Params: + scan_id: int + scan_name: str + + @classmethod + def do(cls, user: AbstractUser, scan_id: int, **kwargs): + scan = DataScannerHandler.update_scan(user=user, scan_id=scan_id, **kwargs) + + params = cls.Params(scan.id, scan.name) + cls.register_action(user, params, cls.scope()) + + return scan + + @classmethod + def scope(cls) -> ActionScopeStr: + return RootActionScopeType.value() + + +class DeleteDataScanActionType(ActionType): + type = "delete_data_scan" + description = ActionTypeDescription( + _("Delete data scan"), + _('Data scan "%(scan_name)s" (%(scan_id)s) deleted'), + ) + analytics_params = ["scan_id"] + + @dataclasses.dataclass + class Params: + scan_id: int + scan_name: str + + @classmethod + def do(cls, user: AbstractUser, scan_id: int): + scan = DataScannerHandler.get_scan(user=user, scan_id=scan_id) + + scan_name = scan.name + DataScannerHandler.delete_scan(user=user, scan_id=scan_id) + + params = cls.Params(scan_id, scan_name) + cls.register_action(user, params, cls.scope()) + + @classmethod + def scope(cls) -> ActionScopeStr: + return RootActionScopeType.value() diff --git a/enterprise/backend/src/baserow_enterprise/data_scanner/constants.py b/enterprise/backend/src/baserow_enterprise/data_scanner/constants.py new file mode 100644 index 0000000000..d3ea77bdb7 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/data_scanner/constants.py @@ -0,0 +1,38 @@ +from datetime import timedelta + +from baserow.contrib.database.fields.models import ( + AutonumberField, + EmailField, + NumberField, + PhoneNumberField, + TextField, + URLField, + UUIDField, +) + +# Contains the field types that can be used in the Baserow source table. +SCANNABLE_FIELD_TYPES = [ + TextField, + URLField, + EmailField, + NumberField, + AutonumberField, + PhoneNumberField, + UUIDField, +] + +SCANNABLE_FIELD_CONTENT_TYPES = [ + field._meta.model_name for field in SCANNABLE_FIELD_TYPES +] + +SCAN_TYPE_PATTERN = "pattern" +SCAN_TYPE_LIST_OF_VALUES = "list_of_values" +SCAN_TYPE_LIST_TABLE = "list_table" + +STALE_SCAN_THRESHOLD_HOURS = 2 + +FREQUENCY_INTERVALS = { + "hourly": timedelta(hours=1), + "daily": timedelta(days=1), + "weekly": timedelta(weeks=1), +} diff --git a/enterprise/backend/src/baserow_enterprise/data_scanner/exceptions.py b/enterprise/backend/src/baserow_enterprise/data_scanner/exceptions.py new file mode 100644 index 0000000000..7b6526de32 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/data_scanner/exceptions.py @@ -0,0 +1,10 @@ +class DataScanDoesNotExist(Exception): + pass + + +class DataScanIsAlreadyRunning(Exception): + pass + + +class DataScanResultDoesNotExist(Exception): + pass diff --git a/enterprise/backend/src/baserow_enterprise/data_scanner/handler.py b/enterprise/backend/src/baserow_enterprise/data_scanner/handler.py new file mode 100644 index 0000000000..e276a7f244 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/data_scanner/handler.py @@ -0,0 +1,773 @@ +import re +import traceback +from datetime import datetime, timedelta +from typing import Optional + +from django.contrib.auth.models import AbstractUser +from django.contrib.postgres.search import SearchQuery +from django.core.exceptions import PermissionDenied +from django.db.models import Count, QuerySet, TextField +from django.db.models.functions import Cast +from django.utils import timezone + +from baserow.contrib.database.fields.models import Field +from baserow.contrib.database.search.handler import SearchHandler +from baserow.contrib.database.table.models import Table +from baserow.core.models import Workspace +from baserow_enterprise.data_scanner.constants import ( + FREQUENCY_INTERVALS, + SCAN_TYPE_LIST_OF_VALUES, + SCAN_TYPE_LIST_TABLE, + SCAN_TYPE_PATTERN, + STALE_SCAN_THRESHOLD_HOURS, +) +from baserow_enterprise.data_scanner.exceptions import ( + DataScanDoesNotExist, + DataScanIsAlreadyRunning, + DataScanResultDoesNotExist, +) +from baserow_enterprise.data_scanner.models import ( + DataScan, + DataScanListItem, + DataScanResult, +) +from baserow_enterprise.data_scanner.tasks import run_data_scan +from baserow_enterprise.features import DATA_SCANNER +from baserow_premium.license.handler import LicenseHandler + + +def convert_pattern_to_regex(pattern: str) -> str: + """ + Converts a custom pattern syntax to a regex string. + + Tokens: + - `A` -> any letter `[A-Za-z]` + - `D` -> any digit `[0-9]` + - `X` -> any character `.` + - `\\c` -> literal character `c` + + :param pattern: The custom pattern string (e.g. `AADDAAAADDDDDDDDDD`). + :return: A regex string equivalent. + """ + + TOKEN_MAP = { + "A": "[A-Za-z]", + "D": "[0-9]", + "X": ".", + } + + parts: list[str] = [] + i = 0 + while i < len(pattern): + char = pattern[i] + if char == "\\" and i + 1 < len(pattern): + # Escaped literal + parts.append(re.escape(pattern[i + 1])) + i += 2 + elif char in TOKEN_MAP: + parts.append(TOKEN_MAP[char]) + i += 1 + else: + parts.append(re.escape(char)) + i += 1 + return "".join(parts) + + +def _check_data_scanner_access(user: AbstractUser) -> None: + """ + Verifies that the given user holds the DATA_SCANNER enterprise feature + and is a staff member. Raises if either check fails. + + :param user: The user to verify. + :raises FeaturesNotAvailableError: When the enterprise license is missing. + :raises PermissionDenied: When the user is not staff. + """ + + LicenseHandler.raise_if_user_doesnt_have_feature_instance_wide(DATA_SCANNER, user) + if not user.is_staff: + raise PermissionDenied() + + +class DataScannerHandler: + @staticmethod + def create_scan( + user: AbstractUser, + name: str, + scan_type: str, + pattern: Optional[str] = None, + frequency: str = "manual", + scan_all_workspaces: bool = True, + workspace_ids: Optional[list[int]] = None, + list_items: Optional[list[str]] = None, + source_table_id: Optional[int] = None, + source_field_id: Optional[int] = None, + ) -> DataScan: + """ + Creates a new data scan configuration. + + :param user: The staff user performing the action. + :param name: Human-readable name for the scan. + :param scan_type: One of `pattern`, `list_of_values`, or `list_table`. + :param pattern: Required when scan_type is `pattern`. + :param frequency: How often the scan runs automatically. + :param scan_all_workspaces: When False, only the given workspace_ids are + scanned. + :param workspace_ids: Workspace IDs to restrict scanning to. + :param list_items: Values to match when scan_type is `list_of_values`. + :param source_table_id: Source table ID when scan_type is `list_table`. + :param source_field_id: Source field ID when scan_type is `list_table`. + :return: The newly created DataScan instance. + """ + + _check_data_scanner_access(user) + + scan = DataScan.objects.create( + name=name, + scan_type=scan_type, + pattern=pattern, + frequency=frequency, + scan_all_workspaces=scan_all_workspaces, + created_by=user, + source_table_id=source_table_id + if scan_type == SCAN_TYPE_LIST_TABLE + else None, + source_field_id=source_field_id + if scan_type == SCAN_TYPE_LIST_TABLE + else None, + ) + + if not scan_all_workspaces and workspace_ids: + workspaces = Workspace.objects.filter(id__in=workspace_ids) + scan.workspaces.set(workspaces) + + if scan_type == SCAN_TYPE_LIST_OF_VALUES and list_items: + DataScanListItem.objects.bulk_create( + [DataScanListItem(scan=scan, value=v) for v in list_items] + ) + + return scan + + @staticmethod + def update_scan(user: AbstractUser, scan_id: int, **kwargs) -> DataScan: + """ + Updates an existing data scan and cleans up stale results when the + configuration changes in a way that invalidates them. + + :param user: The staff user performing the action. + :param scan_id: Primary key of the scan to update. + :param kwargs: Fields to update (name, scan_type, pattern, frequency, + scan_all_workspaces, workspace_ids, list_items, source_table_id, + source_field_id). + :return: The updated DataScan instance. + :raises DataScanDoesNotExist: When the scan is not found. + """ + + _check_data_scanner_access(user) + + try: + scan = DataScan.objects.select_for_update(of=("self",)).get(id=scan_id) + except DataScan.DoesNotExist: + raise DataScanDoesNotExist() + + if scan.is_running: + raise DataScanIsAlreadyRunning() + + simple_fields = [ + "name", + "scan_type", + "pattern", + "frequency", + "scan_all_workspaces", + "source_table_id", + "source_field_id", + ] + for field_name in simple_fields: + if field_name in kwargs: + setattr(scan, field_name, kwargs[field_name]) + scan.save() + + if "workspace_ids" in kwargs: + if scan.scan_all_workspaces: + scan.workspaces.clear() + else: + workspaces = Workspace.objects.filter(id__in=kwargs["workspace_ids"]) + scan.workspaces.set(workspaces) + + if "list_items" in kwargs: + scan.list_items.all().delete() + items = kwargs["list_items"] + if items: + DataScanListItem.objects.bulk_create( + [DataScanListItem(scan=scan, value=v) for v in items] + ) + + DataScannerHandler._cleanup_stale_results(scan, kwargs) + + return scan + + @staticmethod + def _cleanup_stale_results(scan: DataScan, kwargs: dict) -> None: + """ + Removes results that are no longer valid after a scan update. For + example, if the pattern or scan type changed, all existing results are + cleared. If list items changed, only results whose matched value is no + longer in the list are removed. + + :param scan: The scan whose results may need pruning. + :param kwargs: The update kwargs that were applied to the scan. + """ + + if "scan_type" in kwargs: + scan.results.all().delete() + return + + if "pattern" in kwargs and scan.scan_type == SCAN_TYPE_PATTERN: + scan.results.all().delete() + return + + if "list_items" in kwargs and scan.scan_type == SCAN_TYPE_LIST_OF_VALUES: + new_items = set(kwargs["list_items"] or []) + if not new_items: + scan.results.all().delete() + else: + scan.results.exclude(matched_value__in=new_items).delete() + + @staticmethod + def delete_scan(user: AbstractUser, scan_id: int) -> None: + """ + Deletes a data scan and all of its related objects. + + :param user: The staff user performing the action. + :param scan_id: Primary key of the scan to delete. + :raises DataScanDoesNotExist: When the scan is not found. + """ + + _check_data_scanner_access(user) + + try: + scan = DataScan.objects.select_for_update(of=("self",)).get(id=scan_id) + except DataScan.DoesNotExist: + raise DataScanDoesNotExist() + + if scan.is_running: + raise DataScanIsAlreadyRunning() + + scan.delete() + + @staticmethod + def list_scans(user: AbstractUser) -> QuerySet[DataScan]: + """ + Returns all data scans. Requires an enterprise license and staff + access. + + :param user: The staff user performing the action. + :return: A queryset of all DataScan instances. + """ + + _check_data_scanner_access(user) + return ( + DataScan.objects.annotate(results_count=Count("results")) + .prefetch_related( + "workspaces", + "list_items", + ) + .select_related( + "created_by", + "source_table__database__workspace", + ) + ) + + @staticmethod + def get_scan(user: AbstractUser, scan_id: int) -> DataScan: + """ + Returns a single data scan by its primary key. + + :param user: The staff user performing the action. + :param scan_id: Primary key of the scan. + :return: The DataScan instance. + :raises DataScanDoesNotExist: When the scan is not found. + """ + + _check_data_scanner_access(user) + + try: + return DataScan.objects.get(id=scan_id) + except DataScan.DoesNotExist: + raise DataScanDoesNotExist() + + @staticmethod + def delete_result(user: AbstractUser, result_id: int) -> None: + """ + Deletes (resolves) a single data scan result. + + :param user: The staff user performing the action. + :param result_id: Primary key of the result to delete. + :raises DataScanResult.DoesNotExist: When the result is not found. + """ + + _check_data_scanner_access(user) + + try: + result = DataScanResult.objects.get(id=result_id) + except DataScanResult.DoesNotExist: + raise DataScanResultDoesNotExist() + + result.delete() + + @staticmethod + def trigger_scan(user: AbstractUser, scan_id: int) -> DataScan: + """ + Queues an immediate asynchronous run of the given scan. + + :param user: The staff user triggering the scan. + :param scan_id: Primary key of the scan to trigger. + :return: The DataScan instance. + :raises DataScanDoesNotExist: When the scan is not found. + :raises DataScanIsAlreadyRunning: When the scan is already in progress. + """ + + _check_data_scanner_access(user) + + try: + scan = DataScan.objects.get(id=scan_id) + except DataScan.DoesNotExist: + raise DataScanDoesNotExist() + + if scan.is_running: + raise DataScanIsAlreadyRunning() + + run_data_scan.delay(scan_id) + return scan + + @staticmethod + def run_scan(scan_id: int) -> None: + """ + Executes the scan logic synchronously. Typically called from a Celery + task. Iterates over the relevant workspaces and searches for matches + using the workspace search tables. Results that were not re-identified + in this run are removed. + + :param scan_id: Primary key of the scan to execute. + """ + + try: + scan = DataScan.objects.get(id=scan_id) + except DataScan.DoesNotExist: + return + + now = timezone.now() + scan.is_running = True + scan.last_run_started_at = now + scan.last_error = None + scan.save(update_fields=["is_running", "last_run_started_at", "last_error"]) + + new_results_count = 0 + try: + if not LicenseHandler.instance_has_feature(DATA_SCANNER): + scan.last_error = "Enterprise license no longer active" + return + + if scan.scan_all_workspaces: + workspace_ids = list( + Workspace.objects.filter(trashed=False).values_list("id", flat=True) + ) + else: + workspace_ids = list( + scan.workspaces.filter(trashed=False).values_list("id", flat=True) + ) + + pre_computed: dict = {} + + if scan.scan_type == SCAN_TYPE_PATTERN: + regex = convert_pattern_to_regex(scan.pattern) + pre_computed["regex"] = regex + pre_computed["compiled"] = re.compile(regex, re.IGNORECASE) + + elif scan.scan_type == SCAN_TYPE_LIST_OF_VALUES: + pre_computed["values"] = list( + DataScanListItem.objects.filter(scan=scan).values_list( + "value", flat=True + ) + ) + + elif scan.scan_type == SCAN_TYPE_LIST_TABLE: + if not scan.source_table or not scan.source_field: + pre_computed["skip"] = True + else: + source_table = scan.source_table + source_field = scan.source_field + model = source_table.get_model() + field_name = source_field.db_column + values = list( + model.objects.values_list(field_name, flat=True).distinct() + ) + pre_computed["values"] = [str(v) for v in values if v] + pre_computed["exclude_table_id"] = source_table.id + + if not pre_computed.get("skip"): + # Compute trashed field exclusions once for all workspaces. Three + # separate indexed queries (one per trashed column) are combined with + # set union in Python, avoiding an OR that would prevent index usage. + trashed_field_ids = set( + Field.objects_and_trash.filter(trashed=True).values_list( + "id", flat=True + ) + ) + trashed_field_ids |= set( + Field.objects_and_trash.filter(table__trashed=True).values_list( + "id", flat=True + ) + ) + trashed_field_ids |= set( + Field.objects_and_trash.filter( + table__database__trashed=True + ).values_list("id", flat=True) + ) + + for workspace_id in workspace_ids: + if not SearchHandler.workspace_search_table_exists(workspace_id): + continue + + search_model = SearchHandler.get_workspace_search_table_model( + workspace_id + ) + + if scan.scan_type == SCAN_TYPE_PATTERN: + matches = ( + search_model.objects.annotate( + text_value=Cast("value", TextField()) + ) + .filter(text_value__iregex=pre_computed["regex"]) + .values_list("field_id", "row_id", "text_value") + ) + new_results_count += ( + DataScannerHandler._process_pattern_matches( + scan, + matches, + pre_computed["compiled"], + now, + trashed_field_ids, + ) + ) + elif scan.scan_type == SCAN_TYPE_LIST_OF_VALUES: + new_results_count += DataScannerHandler._run_list_scan( + scan, + search_model, + pre_computed["values"], + now, + trashed_field_ids, + ) + elif scan.scan_type == SCAN_TYPE_LIST_TABLE: + new_results_count += DataScannerHandler._run_list_scan( + scan, + search_model, + pre_computed["values"], + now, + trashed_field_ids, + exclude_table_id=pre_computed["exclude_table_id"], + ) + + scan.results.filter(last_identified_on__lt=now).delete() + + except Exception: + scan.last_error = traceback.format_exc() + finally: + scan.is_running = False + scan.last_run_finished_at = timezone.now() + scan.save( + update_fields=[ + "is_running", + "last_run_finished_at", + "last_error", + ] + ) + + if new_results_count > 0 and not scan.last_error: + from baserow_enterprise.data_scanner.notification_types import ( + DataScanNewResultsNotificationType, + ) + + DataScanNewResultsNotificationType.notify_instance_admins( + scan, new_results_count + ) + + @staticmethod + def _run_list_scan( + scan: DataScan, + search_model, + values: list[str], + now: datetime, + trashed_field_ids: set[int], + exclude_table_id: Optional[int] = None, + ) -> int: + """ + Searches the workspace search table for rows matching any of the given + values using PostgreSQL full-text search. Processes values in batches + and bulk-upserts results. + + :param scan: The scan being executed. + :param search_model: The Django model for the workspace search table. + :param values: The list of string values to search for. + :param now: The current timestamp used for result bookkeeping. + :param trashed_field_ids: Set of field IDs to exclude because the + field, table, or database is trashed. + :param exclude_table_id: When set, fields belonging to this table are + excluded from results (used for list_table scans to avoid matching + the source table itself). + :return: The number of newly created results. + """ + + excluded_field_ids: set[int] = set() + if exclude_table_id is not None: + excluded_field_ids = set( + Field.objects.filter(table_id=exclude_table_id).values_list( + "id", flat=True + ) + ) + + all_matches: list[tuple[int, int, str]] = [] + batch_size = 100 + for i in range(0, len(values), batch_size): + batch = values[i : i + batch_size] + + # Build a list of (sanitized_query, original_value) pairs, skipping values + # that produce an empty sanitized string. + sanitized_pairs: list[tuple[str, str]] = [] + for search_value in batch: + sanitized = SearchHandler.escape_postgres_query(search_value) + if sanitized: + sanitized_pairs.append((sanitized, search_value)) + + if not sanitized_pairs: + continue + + # Combine all sanitized values into a single OR tsquery so we + # execute one database query per batch instead of one per value. + # Each individual tsquery is wrapped in parentheses to preserve + # the phrase (<->) operator precedence within each value. + combined_raw = " | ".join(f"({s})" for s, _ in sanitized_pairs) + combined_query = SearchQuery( + combined_raw, + search_type="raw", + config=SearchHandler.search_config(), + ) + matches = ( + search_model.objects.filter(value=combined_query) + .annotate(text_value=Cast("value", TextField())) + .values_list("field_id", "row_id", "text_value") + ) + for field_id, row_id, text_value in matches: + if field_id in excluded_field_ids: + continue + matched_value = DataScannerHandler._find_list_match( + text_value, sanitized_pairs + ) + all_matches.append((field_id, row_id, matched_value)) + + return DataScannerHandler._bulk_upsert_results( + scan, all_matches, now, trashed_field_ids + ) + + @staticmethod + def _find_list_match( + tsvector_text: str, + sanitized_pairs: list[tuple[str, str]], + ) -> str: + """ + Given a tsvector text representation and the list of (sanitized_query, + original_value) pairs used to build the combined OR query, determines which + original value matched. Extracts tokens from the tsvector and checks which + query's terms are all present. + + :param tsvector_text: The text representation of a tsvector value. + :param sanitized_pairs: List of (sanitized_query, original_value). + :return: The original value that matched, or the first value as + fallback. + """ + + tokens = {m.group(1) for m in re.finditer(r"'([^']*)'", tsvector_text)} + for sanitized, original in sanitized_pairs: + # Extract bare terms from the sanitized tsquery, stripping dollar-quoting, + # positional operators, and wildcards. + terms = re.findall(r"\$\$([^$]+)\$\$", sanitized) + if terms and all(term.lower() in tokens for term in terms): + return original + return sanitized_pairs[0][1] + + @staticmethod + def _extract_matching_token(tsvector_text: str, compiled_regex: re.Pattern) -> str: + """ + Extracts the first matching token from a tsvector text representation. + + A tsvector cast to text looks like `'nl23ingb0001234321':2 'test':1,3`. + Each token is a single-quoted string followed by `:` and position info. + We test each token against the compiled pattern regex and return the + first match. The token is already lowercased by PostgreSQL. + + :param tsvector_text: The text representation of a tsvector value. + :param compiled_regex: A compiled regex to match tokens against. + :return: The first matching token, or the raw tsvector_text as fallback. + """ + + for m in re.finditer(r"'([^']*)'", tsvector_text): + token = m.group(1) + if compiled_regex.search(token): + return token + return tsvector_text + + @staticmethod + def _process_pattern_matches( + scan: DataScan, + matches, + compiled_regex: re.Pattern, + now: datetime, + trashed_field_ids: set[int], + ) -> int: + """ + Processes raw pattern matches from the database and bulk-upserts results. + + :param scan: The scan being executed. + :param matches: An iterable of (field_id, row_id, text_value) tuples. + :param compiled_regex: The compiled pattern regex. + :param now: The current timestamp used for result bookkeeping. + :param trashed_field_ids: Set of field IDs to exclude because the + field, table, or database is trashed. + :return: The number of newly created results. + """ + + all_matches: list[tuple[int, int, str]] = [] + for field_id, row_id, text_value in matches: + matched = DataScannerHandler._extract_matching_token( + text_value, compiled_regex + ) + all_matches.append((field_id, row_id, matched)) + + return DataScannerHandler._bulk_upsert_results( + scan, all_matches, now, trashed_field_ids + ) + + @staticmethod + def _bulk_upsert_results( + scan: DataScan, + matches: list[tuple[int, int, str]], + now: datetime, + trashed_field_ids: set[int], + ) -> int: + """ + Bulk-upserts DataScanResult rows for a list of matches. + + :param scan: The scan the results belong to. + :param matches: A list of (field_id, row_id, matched_value) tuples. + :param now: The current timestamp. + :param trashed_field_ids: Set of field IDs to exclude because the + field, table, or database is trashed. + :return: The number of newly created results. + """ + + if not matches: + return 0 + + # Build field_id -> table_id and table_id -> Table mappings in a single query + # using select_related to avoid per-table lookups later. + field_ids = {field_id for field_id, _, _ in matches} + field_to_table: dict[int, int] = {} + table_by_id: dict[int, Table] = {} + for field_obj in Field.objects_and_trash.filter( + id__in=field_ids + ).select_related("table"): + field_to_table[field_obj.id] = field_obj.table_id + table_by_id[field_obj.table_id] = field_obj.table + + # Filter out matches for fields that no longer exist or where the field, + # table, or database is trashed. This is done in Python against a small + # blocklist rather than adding a large `field_id__in` filter to the search + # query, which would be slower on instances with millions of fields. + valid_matches = [ + (field_id, row_id, matched_value) + for field_id, row_id, matched_value in matches + if field_id in field_to_table and field_id not in trashed_field_ids + ] + + # Filter out trashed rows. Query each table once for its (typically + # small) set of trashed row IDs. The intersection with matched row IDs + # is done in Python to avoid a potentially huge `id__in` clause. + trashed_row_ids: set[tuple[int, int]] = set() + for table_id, table in table_by_id.items(): + model = table.get_model() + trashed_ids = set( + model.objects_and_trash.filter(trashed=True).values_list( + "id", flat=True + ) + ) + for row_id in trashed_ids: + trashed_row_ids.add((table_id, row_id)) + + if trashed_row_ids: + valid_matches = [ + (field_id, row_id, matched_value) + for field_id, row_id, matched_value in valid_matches + if (field_to_table[field_id], row_id) not in trashed_row_ids + ] + + if not valid_matches: + return 0 + + count_before = DataScanResult.objects.filter(scan=scan).count() + + batch_size = 500 + for i in range(0, len(valid_matches), batch_size): + batch = valid_matches[i : i + batch_size] + objects = [ + DataScanResult( + scan=scan, + table_id=field_to_table[field_id], + field_id=field_id, + row_id=row_id, + matched_value=str(matched_value), + first_identified_on=now, + last_identified_on=now, + ) + for field_id, row_id, matched_value in batch + ] + DataScanResult.objects.bulk_create( + objects, + update_conflicts=True, + unique_fields=["scan", "table", "row_id", "field"], + update_fields=["matched_value", "last_identified_on"], + ) + + count_after = DataScanResult.objects.filter(scan=scan).count() + return count_after - count_before + + @staticmethod + def check_scans_due() -> None: + """ + Periodic check that resets stale running scans and dispatches any + scheduled scans whose interval has elapsed. Called by the + `check_data_scans_due` Celery beat task. + """ + + now = timezone.now() + + stale_threshold = now - timedelta(hours=STALE_SCAN_THRESHOLD_HOURS) + DataScan.objects.filter( + is_running=True, + last_run_started_at__lt=stale_threshold, + ).update( + is_running=False, + last_error="Scan timed out and was automatically reset", + ) + + if not LicenseHandler.instance_has_feature(DATA_SCANNER): + return + + scans = DataScan.objects.filter(is_running=False).exclude(frequency="manual") + for scan in scans: + interval = FREQUENCY_INTERVALS.get(scan.frequency) + if not interval: + continue + + if scan.last_run_started_at is None or ( + now - scan.last_run_started_at >= interval + ): + run_data_scan.delay(scan.id) diff --git a/enterprise/backend/src/baserow_enterprise/data_scanner/job_types.py b/enterprise/backend/src/baserow_enterprise/data_scanner/job_types.py new file mode 100644 index 0000000000..061ebb1d1f --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/data_scanner/job_types.py @@ -0,0 +1,279 @@ +from collections import OrderedDict +from uuid import uuid4 + +from django.core.paginator import Paginator +from django.db.models import QuerySet +from django.utils.translation import gettext as _ + +import unicodecsv as csv +from loguru import logger +from rest_framework import serializers + +from baserow.contrib.database.api.export.serializers import ( + SUPPORTED_CSV_COLUMN_SEPARATORS, + SUPPORTED_EXPORT_CHARSETS, + DisplayChoiceField, + ExportedFileURLSerializerMixin, +) +from baserow.contrib.database.export.handler import ( + ExportHandler, + _create_storage_dir_if_missing_and_open, +) +from baserow.core.jobs.registries import JobType +from baserow.core.storage import get_default_storage +from baserow.core.utils import ChildProgressBuilder, Progress +from baserow_enterprise.features import DATA_SCANNER +from baserow_premium.license.handler import LicenseHandler + +from .models import DataScanResult, DataScanResultExportJob + +DATA_SCAN_RESULT_CSV_COLUMN_NAMES = OrderedDict( + { + "scan_name": { + "field": "scan_name", + "descr": _("Scan Name"), + }, + "workspace_name": { + "field": "workspace_name", + "descr": _("Workspace Name"), + }, + "database_name": { + "field": "database_name", + "descr": _("Database Name"), + }, + "table_name": { + "field": "table_name", + "descr": _("Table Name"), + }, + "field_name": { + "field": "field_name", + "descr": _("Field Name"), + }, + "row_id": { + "field": "row_id", + "descr": _("Row ID"), + }, + "matched_value": { + "field": "matched_value", + "descr": _("Matched Value"), + }, + "first_identified_on": { + "field": "first_identified_on", + "descr": _("First Identified On"), + }, + "last_identified_on": { + "field": "last_identified_on", + "descr": _("Last Identified On"), + }, + } +) + + +class DataScanResultExportJobType(JobType): + type = "data_scan_result_export" + model_class = DataScanResultExportJob + max_count = 1 + + serializer_mixins = [ExportedFileURLSerializerMixin] + request_serializer_field_names = [ + "csv_column_separator", + "csv_first_row_header", + "export_charset", + "filter_scan_id", + ] + + serializer_field_names = [ + *request_serializer_field_names, + "created_on", + "exported_file_name", + "url", + ] + base_serializer_field_overrides = { + "export_charset": DisplayChoiceField( + choices=SUPPORTED_EXPORT_CHARSETS, + default="utf-8", + help_text="The character set to use when creating the export file.", + ), + "csv_column_separator": DisplayChoiceField( + choices=SUPPORTED_CSV_COLUMN_SEPARATORS, + default=",", + help_text="The value used to separate columns in the resulting csv file.", + ), + "csv_first_row_header": serializers.BooleanField( + default=True, + help_text="Whether or not to generate a header row at the top of the csv file.", + ), + "filter_scan_id": serializers.IntegerField( + min_value=0, + required=False, + help_text="Optional: Filter results by scan ID.", + ), + } + request_serializer_field_overrides = { + **base_serializer_field_overrides, + } + serializer_field_overrides = { + **base_serializer_field_overrides, + "created_on": serializers.DateTimeField( + read_only=True, + help_text="The date and time when the export job was created.", + ), + "exported_file_name": serializers.CharField( + read_only=True, + help_text="The name of the file that was created by the export job.", + ), + "url": serializers.SerializerMethodField( + help_text="The URL to download the exported file.", + ), + } + + def before_delete(self, job: DataScanResultExportJob) -> None: + """ + Deletes the exported CSV file from storage before the job row is + removed. + + :param job: The export job about to be deleted. + """ + + if not job.exported_file_name: + return + + storage = get_default_storage() + storage_location = ExportHandler.export_file_path(job.exported_file_name) + try: + storage.delete(storage_location) + except FileNotFoundError: + logger.error( + "Could not delete file %s for 'data_scan_result_export' job %s", + storage_location, + job.id, + ) + + @staticmethod + def _safe_attr(obj, *attrs, default="") -> str: + """ + Traverses a chain of attributes, returning `default` if any link + raises AttributeError. + """ + + try: + for attr in attrs: + obj = getattr(obj, attr) + return obj + except AttributeError: + return default + + def _get_row_data(self, result: DataScanResult) -> dict: + """ + Extracts a dict of field values from a DataScanResult, with + AttributeError protection for nullable nested relations. + + :param result: A DataScanResult instance. + :return: A dict keyed by CSV column name with string/int/datetime values. + """ + + return { + "scan_name": self._safe_attr(result, "scan", "name"), + "workspace_name": self._safe_attr( + result, "field", "table", "database", "workspace", "name" + ), + "database_name": self._safe_attr( + result, "field", "table", "database", "name" + ), + "table_name": self._safe_attr(result, "field", "table", "name"), + "field_name": self._safe_attr(result, "field", "name"), + "row_id": result.row_id, + "matched_value": result.matched_value, + "first_identified_on": result.first_identified_on, + "last_identified_on": result.last_identified_on, + } + + def write_rows( + self, + job: DataScanResultExportJob, + file, + queryset: QuerySet, + progress, + ) -> None: + """ + Writes all result rows from the queryset to the CSV file. + + :param job: The export job with CSV formatting options. + :param file: A writable binary file handle. + :param queryset: The queryset of DataScanResult rows to export. + :param progress: Progress tracker for reporting export advancement. + """ + + # add BOM to support utf-8 CSVs in MS Excel (for Windows only) + if job.export_charset == "utf-8": + file.write(b"\xef\xbb\xbf") + + field_header_mapping = { + k: v["descr"] for (k, v) in DATA_SCAN_RESULT_CSV_COLUMN_NAMES.items() + } + + writer = csv.writer( + file, + field_header_mapping.values(), + encoding=job.export_charset, + delimiter=job.csv_column_separator, + ) + + if job.csv_first_row_header: + writer.writerow(field_header_mapping.values()) + + fields = [v["field"] for v in DATA_SCAN_RESULT_CSV_COLUMN_NAMES.values()] + paginator = Paginator(queryset.all(), 2000) + export_progress = ChildProgressBuilder.build( + progress.create_child_builder(represents_progress=progress.total), + paginator.num_pages, + ) + + for page in paginator.page_range: + rows = [] + for result in paginator.page(page).object_list: + row_data = self._get_row_data(result) + rows.append([row_data[field] for field in fields]) + writer.writerows(rows) + export_progress.increment() + + def get_filtered_queryset(self, job: DataScanResultExportJob) -> QuerySet: + """ + Returns the queryset of DataScanResult rows to export, applying + any filters configured on the job. + + :param job: The export job whose filters should be applied. + :return: A filtered and ordered queryset of DataScanResult instances. + """ + + queryset = DataScanResult.objects.select_related( + "scan", "table__database", "field__table__database__workspace" + ).order_by("-first_identified_on") + + if job.filter_scan_id is not None: + queryset = queryset.filter(scan_id=job.filter_scan_id) + + return queryset + + def run(self, job: DataScanResultExportJob, progress: Progress) -> None: + """ + Export the filtered data scan results to a CSV file. + + :param job: The job that is currently being executed. + :param progress: The progress object that can be used to update the + progress bar. + """ + + LicenseHandler.raise_if_user_doesnt_have_feature_instance_wide( + DATA_SCANNER, job.user + ) + + queryset = self.get_filtered_queryset(job) + + filename = f"data_scan_results_{uuid4().hex[:8]}.csv" + storage_location = ExportHandler.export_file_path(filename) + with _create_storage_dir_if_missing_and_open(storage_location) as file: + self.write_rows(job, file, queryset, progress) + + job.exported_file_name = filename + job.save() diff --git a/enterprise/backend/src/baserow_enterprise/data_scanner/models.py b/enterprise/backend/src/baserow_enterprise/data_scanner/models.py new file mode 100644 index 0000000000..91d7a4b8cf --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/data_scanner/models.py @@ -0,0 +1,110 @@ +from django.conf import settings +from django.db import models + +from baserow.contrib.database.api.export.serializers import ( + SUPPORTED_CSV_COLUMN_SEPARATORS, + SUPPORTED_EXPORT_CHARSETS, +) +from baserow.contrib.database.fields.models import Field +from baserow.contrib.database.table.models import Table +from baserow.core.jobs.models import Job +from baserow.core.models import Workspace +from baserow_enterprise.data_scanner.constants import ( + SCAN_TYPE_LIST_OF_VALUES, + SCAN_TYPE_LIST_TABLE, + SCAN_TYPE_PATTERN, +) + + +class DataScan(models.Model): + SCAN_TYPE_CHOICES = [ + (SCAN_TYPE_PATTERN, "Pattern"), + (SCAN_TYPE_LIST_OF_VALUES, "List of values"), + (SCAN_TYPE_LIST_TABLE, "List Table"), + ] + + FREQUENCY_CHOICES = [ + ("manual", "Manual"), + ("hourly", "Hourly"), + ("daily", "Daily"), + ("weekly", "Weekly"), + ] + + name = models.CharField(max_length=255) + scan_type = models.CharField(max_length=20, choices=SCAN_TYPE_CHOICES) + pattern = models.TextField(null=True, blank=True) + frequency = models.CharField( + max_length=10, choices=FREQUENCY_CHOICES, default="manual" + ) + scan_all_workspaces = models.BooleanField(default=True) + workspaces = models.ManyToManyField(Workspace, blank=True) + is_running = models.BooleanField(default=False) + last_run_started_at = models.DateTimeField(null=True, blank=True) + last_run_finished_at = models.DateTimeField(null=True, blank=True) + last_error = models.TextField(null=True, blank=True) + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + blank=True, + ) + created_on = models.DateTimeField(auto_now_add=True) + updated_on = models.DateTimeField(auto_now=True) + source_table = models.ForeignKey( + Table, on_delete=models.SET_NULL, null=True, blank=True + ) + source_field = models.ForeignKey( + Field, on_delete=models.SET_NULL, null=True, blank=True + ) + + class Meta: + ordering = ["-created_on"] + + def __str__(self): + return self.name + + +class DataScanListItem(models.Model): + scan = models.ForeignKey( + DataScan, on_delete=models.CASCADE, related_name="list_items" + ) + value = models.TextField() + + def __str__(self): + return self.value + + +class DataScanResult(models.Model): + scan = models.ForeignKey(DataScan, on_delete=models.CASCADE, related_name="results") + table = models.ForeignKey(Table, on_delete=models.CASCADE) + field = models.ForeignKey(Field, on_delete=models.CASCADE) + row_id = models.IntegerField() + matched_value = models.TextField() + first_identified_on = models.DateTimeField(db_index=True) + last_identified_on = models.DateTimeField() + + class Meta: + unique_together = [("scan", "table", "row_id", "field")] + ordering = ["-first_identified_on"] + indexes = [ + models.Index(fields=["scan", "first_identified_on"]), + ] + + def __str__(self): + return f"Result(scan={self.scan_id}, table={self.table_id}, row={self.row_id})" + + +class DataScanResultExportJob(Job): + export_charset = models.CharField( + max_length=32, + choices=SUPPORTED_EXPORT_CHARSETS, + default="utf-8", + ) + csv_column_separator = models.CharField( + max_length=32, + choices=SUPPORTED_CSV_COLUMN_SEPARATORS, + default=",", + ) + csv_first_row_header = models.BooleanField(default=True) + filter_scan_id = models.PositiveIntegerField(null=True) + exported_file_name = models.TextField(null=True) diff --git a/enterprise/backend/src/baserow_enterprise/data_scanner/notification_types.py b/enterprise/backend/src/baserow_enterprise/data_scanner/notification_types.py new file mode 100644 index 0000000000..4d261d3763 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/data_scanner/notification_types.py @@ -0,0 +1,97 @@ +from dataclasses import asdict, dataclass +from typing import List, Optional +from urllib.parse import urlencode, urljoin + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.utils.translation import ngettext + +from baserow.core.notifications.handler import NotificationHandler +from baserow.core.notifications.models import Notification, NotificationRecipient +from baserow.core.notifications.registries import ( + EmailNotificationTypeMixin, + NotificationType, +) + +from .models import DataScan + +User = get_user_model() + + +@dataclass +class DataScanNewResultsData: + scan_id: int + scan_name: str + new_results_count: int + + @classmethod + def from_scan(cls, scan: DataScan, new_results_count: int): + return cls( + scan_id=scan.id, + scan_name=scan.name, + new_results_count=new_results_count, + ) + + +class DataScanNewResultsNotificationType(EmailNotificationTypeMixin, NotificationType): + type = "data_scan_new_results" + has_web_frontend_route = True + + def get_web_frontend_url(self, notification: Notification) -> str: + base_url = settings.BASEROW_EMBEDDED_SHARE_URL + query = urlencode( + { + "scan_id": notification.data.get("scan_id", ""), + "scan_name": notification.data.get("scan_name", ""), + } + ) + return urljoin(base_url, f"/admin/data-scanner/results?{query}") + + @classmethod + def notify_instance_admins( + cls, scan: DataScan, new_results_count: int + ) -> Optional[List[NotificationRecipient]]: + """ + Sends a notification to all instance admins (staff users) informing + them that new data scan results have been found. + + :param scan: The data scan that produced new results. + :param new_results_count: The number of new results found in this run. + :return: The list of created notification recipients, or None. + """ + + admins = User.objects.filter(is_staff=True, is_active=True) + if not admins.exists(): + return None + + return NotificationHandler.create_direct_notification_for_users( + notification_type=cls.type, + recipients=list(admins), + data=asdict(DataScanNewResultsData.from_scan(scan, new_results_count)), + sender=None, + workspace=None, + ) + + @classmethod + def get_notification_title_for_email(cls, notification, context) -> str: + count = notification.data.get("new_results_count", 0) + scan_name = notification.data.get("scan_name", "") + return ngettext( + "%(count)d new result found for %(scan_name)s", + "%(count)d new results found for %(scan_name)s", + count, + ) % {"count": count, "scan_name": scan_name} + + @classmethod + def get_notification_description_for_email( + cls, notification, context + ) -> Optional[str]: + count = notification.data.get("new_results_count", 0) + scan_name = notification.data.get("scan_name", "") + return ngettext( + 'The data scanner "%(scan_name)s" found %(count)d new match ' + "during its latest run. Review the results in the admin panel.", + 'The data scanner "%(scan_name)s" found %(count)d new matches ' + "during its latest run. Review the results in the admin panel.", + count, + ) % {"count": count, "scan_name": scan_name} diff --git a/enterprise/backend/src/baserow_enterprise/data_scanner/tasks.py b/enterprise/backend/src/baserow_enterprise/data_scanner/tasks.py new file mode 100644 index 0000000000..b978916ab6 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/data_scanner/tasks.py @@ -0,0 +1,56 @@ +from datetime import timedelta + +from celery_singleton import Singleton + +from baserow.config.celery import app +from baserow_enterprise.data_scanner.constants import STALE_SCAN_THRESHOLD_HOURS + +SCAN_TIME_LIMIT = STALE_SCAN_THRESHOLD_HOURS * 3600 # 2 hours in seconds +CHECK_TIME_LIMIT = 15 * 60 # 15 minutes in seconds + + +@app.task( + bind=True, + queue="export", + base=Singleton, + unique_on="scan_id", + raise_on_duplicate=False, + lock_expiry=SCAN_TIME_LIMIT, + soft_time_limit=SCAN_TIME_LIMIT, + time_limit=SCAN_TIME_LIMIT, +) +def run_data_scan(self, scan_id: int) -> None: + """ + Celery task that executes a single data scan. + + :param scan_id: Primary key of the DataScan to run. + """ + + from baserow_enterprise.data_scanner.handler import DataScannerHandler + + DataScannerHandler.run_scan(scan_id) + + +@app.task( + bind=True, + queue="export", + base=Singleton, + raise_on_duplicate=False, + lock_expiry=CHECK_TIME_LIMIT, + soft_time_limit=CHECK_TIME_LIMIT, + time_limit=CHECK_TIME_LIMIT, +) +def check_data_scans_due(self) -> None: + """ + Periodic Celery task that checks for scheduled scans whose interval has + elapsed and dispatches them. Also resets stale running scans. + """ + + from baserow_enterprise.data_scanner.handler import DataScannerHandler + + DataScannerHandler.check_scans_due() + + +@app.on_after_finalize.connect +def setup_periodic_data_scanner_tasks(sender, **kwargs) -> None: + sender.add_periodic_task(timedelta(minutes=15), check_data_scans_due.s()) diff --git a/enterprise/backend/src/baserow_enterprise/features.py b/enterprise/backend/src/baserow_enterprise/features.py index 9dc59be75d..fef1e0f013 100644 --- a/enterprise/backend/src/baserow_enterprise/features.py +++ b/enterprise/backend/src/baserow_enterprise/features.py @@ -16,3 +16,4 @@ BUILDER_CUSTOM_CODE = "builder_custom_code" DATE_DEPENDENCY = "date_dependency" +DATA_SCANNER = "data_scanner" diff --git a/enterprise/backend/src/baserow_enterprise/license_types.py b/enterprise/backend/src/baserow_enterprise/license_types.py index f78f9e7d45..8bd750c5bf 100755 --- a/enterprise/backend/src/baserow_enterprise/license_types.py +++ b/enterprise/backend/src/baserow_enterprise/license_types.py @@ -8,6 +8,7 @@ BUILDER_FILE_INPUT, BUILDER_NO_BRANDING, BUILDER_SSO, + DATA_SCANNER, DATA_SYNC, DATE_DEPENDENCY, ENTERPRISE_SETTINGS, @@ -102,6 +103,7 @@ class EnterpriseWithoutSupportLicenseType(AdvancedLicenseType): *COMMON_ADVANCED_FEATURES, ENTERPRISE_SETTINGS, SECURE_FILE_SERVE, + DATA_SCANNER, ] def handle_seat_overflow(self, seats_taken: int, license_object: License): diff --git a/enterprise/backend/src/baserow_enterprise/locale/en/LC_MESSAGES/django.po b/enterprise/backend/src/baserow_enterprise/locale/en/LC_MESSAGES/django.po index 2d9b9d358c..25fb169057 100644 --- a/enterprise/backend/src/baserow_enterprise/locale/en/LC_MESSAGES/django.po +++ b/enterprise/backend/src/baserow_enterprise/locale/en/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-03-16 14:50+0000\n" +"POT-Creation-Date: 2026-03-16 22:06+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -187,6 +187,87 @@ msgstr "" msgid "REDONE" msgstr "" +#: src/baserow_enterprise/data_scanner/actions.py:18 +msgid "Create data scan" +msgstr "" + +#: src/baserow_enterprise/data_scanner/actions.py:19 +#, python-format +msgid "Data scan \"%(scan_name)s\" (%(scan_id)s) created" +msgstr "" + +#: src/baserow_enterprise/data_scanner/actions.py:45 +msgid "Update data scan" +msgstr "" + +#: src/baserow_enterprise/data_scanner/actions.py:46 +#, python-format +msgid "Data scan \"%(scan_name)s\" (%(scan_id)s) updated" +msgstr "" + +#: src/baserow_enterprise/data_scanner/actions.py:72 +msgid "Delete data scan" +msgstr "" + +#: src/baserow_enterprise/data_scanner/actions.py:73 +#, python-format +msgid "Data scan \"%(scan_name)s\" (%(scan_id)s) deleted" +msgstr "" + +#: src/baserow_enterprise/data_scanner/job_types.py:34 +msgid "Scan Name" +msgstr "" + +#: src/baserow_enterprise/data_scanner/job_types.py:38 +msgid "Workspace Name" +msgstr "" + +#: src/baserow_enterprise/data_scanner/job_types.py:42 +msgid "Database Name" +msgstr "" + +#: src/baserow_enterprise/data_scanner/job_types.py:46 +msgid "Table Name" +msgstr "" + +#: src/baserow_enterprise/data_scanner/job_types.py:50 +msgid "Field Name" +msgstr "" + +#: src/baserow_enterprise/data_scanner/job_types.py:54 +msgid "Row ID" +msgstr "" + +#: src/baserow_enterprise/data_scanner/job_types.py:58 +msgid "Matched Value" +msgstr "" + +#: src/baserow_enterprise/data_scanner/job_types.py:62 +msgid "First Identified On" +msgstr "" + +#: src/baserow_enterprise/data_scanner/job_types.py:66 +msgid "Last Identified On" +msgstr "" + +#: src/baserow_enterprise/data_scanner/notification_types.py:80 +#, python-format +msgid "%(count)d new result found for %(scan_name)s" +msgid_plural "%(count)d new results found for %(scan_name)s" +msgstr[0] "" +msgstr[1] "" + +#: src/baserow_enterprise/data_scanner/notification_types.py:92 +#, python-format +msgid "" +"The data scanner \"%(scan_name)s\" found %(count)d new match during its " +"latest run. Review the results in the admin panel." +msgid_plural "" +"The data scanner \"%(scan_name)s\" found %(count)d new matches during its " +"latest run. Review the results in the admin panel." +msgstr[0] "" +msgstr[1] "" + #: src/baserow_enterprise/data_sync/actions.py:21 msgid "Update periodic data sync interval" msgstr "" diff --git a/enterprise/backend/src/baserow_enterprise/migrations/0059_datascanresultexportjob_datascan_datascanlistitem_and_more.py b/enterprise/backend/src/baserow_enterprise/migrations/0059_datascanresultexportjob_datascan_datascanlistitem_and_more.py new file mode 100644 index 0000000000..ae49782d08 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/migrations/0059_datascanresultexportjob_datascan_datascanlistitem_and_more.py @@ -0,0 +1,246 @@ +# Generated by Django 5.2.12 on 2026-03-24 09:07 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("baserow_enterprise", "0058_assistantchat_message_history"), + ("core", "0113_alter_notification_options_and_more"), + ("database", "0206_rowhistory_database_ro_action__6ea699_idx"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="DataScanResultExportJob", + fields=[ + ( + "job_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="core.job", + ), + ), + ( + "export_charset", + models.CharField( + choices=[ + ("utf-8", "utf-8"), + ("iso-8859-6", "iso-8859-6"), + ("windows-1256", "windows-1256"), + ("iso-8859-4", "iso-8859-4"), + ("windows-1257", "windows-1257"), + ("iso-8859-14", "iso-8859-14"), + ("iso-8859-2", "iso-8859-2"), + ("windows-1250", "windows-1250"), + ("gbk", "gbk"), + ("gb18030", "gb18030"), + ("big5", "big5"), + ("koi8-r", "koi8-r"), + ("koi8-u", "koi8-u"), + ("iso-8859-5", "iso-8859-5"), + ("windows-1251", "windows-1251"), + ("x-mac-cyrillic", "mac-cyrillic"), + ("iso-8859-7", "iso-8859-7"), + ("windows-1253", "windows-1253"), + ("iso-8859-8", "iso-8859-8"), + ("windows-1255", "windows-1255"), + ("euc-jp", "euc-jp"), + ("iso-2022-jp", "iso-2022-jp"), + ("shift-jis", "shift-jis"), + ("euc-kr", "euc-kr"), + ("macintosh", "macintosh"), + ("iso-8859-10", "iso-8859-10"), + ("iso-8859-16", "iso-8859-16"), + ("windows-874", "cp874"), + ("windows-1254", "windows-1254"), + ("windows-1258", "windows-1258"), + ("iso-8859-1", "iso-8859-1"), + ("windows-1252", "windows-1252"), + ("iso-8859-3", "iso-8859-3"), + ], + default="utf-8", + max_length=32, + ), + ), + ( + "csv_column_separator", + models.CharField( + choices=[ + (",", ","), + (";", ";"), + ("|", "|"), + ("tab", "\t"), + ("record_separator", "\x1e"), + ("unit_separator", "\x1f"), + ], + default=",", + max_length=32, + ), + ), + ("csv_first_row_header", models.BooleanField(default=True)), + ("filter_scan_id", models.PositiveIntegerField(null=True)), + ("exported_file_name", models.TextField(null=True)), + ], + options={ + "abstract": False, + }, + bases=("core.job",), + ), + migrations.CreateModel( + name="DataScan", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ( + "scan_type", + models.CharField( + choices=[ + ("pattern", "Pattern"), + ("list_of_values", "List of values"), + ("list_table", "List Table"), + ], + max_length=20, + ), + ), + ("pattern", models.TextField(blank=True, null=True)), + ( + "frequency", + models.CharField( + choices=[ + ("manual", "Manual"), + ("hourly", "Hourly"), + ("daily", "Daily"), + ("weekly", "Weekly"), + ], + default="manual", + max_length=10, + ), + ), + ("scan_all_workspaces", models.BooleanField(default=True)), + ("is_running", models.BooleanField(default=False)), + ("last_run_started_at", models.DateTimeField(blank=True, null=True)), + ("last_run_finished_at", models.DateTimeField(blank=True, null=True)), + ("last_error", models.TextField(blank=True, null=True)), + ("created_on", models.DateTimeField(auto_now_add=True)), + ("updated_on", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "source_field", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="database.field", + ), + ), + ( + "source_table", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="database.table", + ), + ), + ("workspaces", models.ManyToManyField(blank=True, to="core.workspace")), + ], + options={ + "ordering": ["-created_on"], + }, + ), + migrations.CreateModel( + name="DataScanListItem", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("value", models.TextField()), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="list_items", + to="baserow_enterprise.datascan", + ), + ), + ], + ), + migrations.CreateModel( + name="DataScanResult", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("row_id", models.IntegerField()), + ("matched_value", models.TextField()), + ("first_identified_on", models.DateTimeField(db_index=True)), + ("last_identified_on", models.DateTimeField()), + ( + "field", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="database.field" + ), + ), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="results", + to="baserow_enterprise.datascan", + ), + ), + ( + "table", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="database.table" + ), + ), + ], + options={ + "ordering": ["-first_identified_on"], + "indexes": [ + models.Index( + fields=["scan", "first_identified_on"], + name="baserow_ent_scan_id_694735_idx", + ) + ], + "unique_together": {("scan", "table", "row_id", "field")}, + }, + ), + ] diff --git a/enterprise/backend/tests/baserow_enterprise_tests/api/admin/data_scanner/test_data_scanner_views.py b/enterprise/backend/tests/baserow_enterprise_tests/api/admin/data_scanner/test_data_scanner_views.py new file mode 100644 index 0000000000..7e9f384236 --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/api/admin/data_scanner/test_data_scanner_views.py @@ -0,0 +1,1703 @@ +from io import BytesIO +from unittest.mock import MagicMock, patch + +from django.db import connection +from django.shortcuts import reverse +from django.test.utils import CaptureQueriesContext, override_settings +from django.utils import timezone + +import pytest +from rest_framework.status import ( + HTTP_200_OK, + HTTP_202_ACCEPTED, + HTTP_204_NO_CONTENT, + HTTP_400_BAD_REQUEST, + HTTP_401_UNAUTHORIZED, + HTTP_402_PAYMENT_REQUIRED, + HTTP_403_FORBIDDEN, + HTTP_404_NOT_FOUND, + HTTP_409_CONFLICT, +) + +from baserow.core.jobs.constants import JOB_FINISHED +from baserow.core.jobs.handler import JobHandler +from baserow.test_utils.helpers import AnyStr +from baserow_enterprise.data_scanner.job_types import DataScanResultExportJobType +from baserow_enterprise.data_scanner.models import DataScan, DataScanResult + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_scans_unauthenticated(api_client): + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:list"), + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_scans_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:list"), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_scans_without_enterprise_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:list"), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_scans(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + scan = DataScan.objects.create( + name="Scan 1", scan_type="pattern", pattern="AA", created_by=user + ) + DataScan.objects.create(name="Scan 2", scan_type="list_of_values", created_by=user) + + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:list"), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + data = response.json() + assert data["count"] == 2 + assert data["results"][0] == { + "id": scan.id, + "name": "Scan 1", + "scan_type": "pattern", + "pattern": "AA", + "frequency": "manual", + "scan_all_workspaces": True, + "workspace_ids": [], + "is_running": False, + "last_run_started_at": None, + "last_run_finished_at": None, + "last_error": None, + "list_items": [], + "results_count": 0, + "source_table_id": None, + "source_field_id": None, + "source_workspace_id": None, + "source_database_id": None, + "created_on": AnyStr(), + "updated_on": AnyStr(), + } + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_scans_search(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + DataScan.objects.create( + name="IBAN Scanner", scan_type="pattern", pattern="AA", created_by=user + ) + DataScan.objects.create( + name="Email Scanner", scan_type="pattern", pattern="99", created_by=user + ) + + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:list"), + {"search": "IBAN"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + assert response.json()["count"] == 1 + assert response.json()["results"][0]["name"] == "IBAN Scanner" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_scans_query_count_is_constant(api_client, enterprise_data_fixture): + """ + The number of queries when listing scans must not grow with the number of + scans (no N+1). Adding more scans should not increase the query count. + """ + + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + workspace = enterprise_data_fixture.create_workspace(user=user) + + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["value"]], + ) + + DataScan.objects.create( + name="Scan 1", scan_type="pattern", pattern="AA", created_by=user + ) + scan2 = DataScan.objects.create( + name="Scan 2", scan_type="list_of_values", created_by=user + ) + scan2.workspaces.add(workspace) + + DataScanResult.objects.create( + scan=scan2, + table=table, + field=fields[0], + row_id=1, + matched_value="test", + first_identified_on=timezone.now(), + last_identified_on=timezone.now(), + ) + + url = reverse("api:enterprise:admin:data_scanner:list") + + with CaptureQueriesContext(connection) as captured_2_scans: + response = api_client.get(url, format="json", HTTP_AUTHORIZATION=f"JWT {token}") + assert response.status_code == HTTP_200_OK + assert response.json()["count"] == 2 + + num_queries_2 = len(captured_2_scans) + + for i in range(3, 8): + DataScan.objects.create( + name=f"Scan {i}", scan_type="pattern", pattern="DD", created_by=user + ) + + with CaptureQueriesContext(connection) as captured_7_scans: + response = api_client.get(url, format="json", HTTP_AUTHORIZATION=f"JWT {token}") + assert response.status_code == HTTP_200_OK + assert response.json()["count"] == 7 + + num_queries_7 = len(captured_7_scans) + + assert num_queries_7 == num_queries_2, ( + f"Query count grew from {num_queries_2} to {num_queries_7} when adding " + f"more scans — likely an N+1 problem." + ) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_unauthenticated(api_client): + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + {"name": "Test", "scan_type": "pattern", "pattern": "AA"}, + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + {"name": "Test", "scan_type": "pattern", "pattern": "AA"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_without_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + {"name": "Test", "scan_type": "pattern", "pattern": "AA"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_pattern(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + { + "name": "IBAN Scan", + "scan_type": "pattern", + "pattern": "AADDAAAADDDDDDDDDD", + "frequency": "daily", + }, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + data = response.json() + assert data["name"] == "IBAN Scan" + assert data["scan_type"] == "pattern" + assert data["pattern"] == "AADDAAAADDDDDDDDDD" + assert data["frequency"] == "daily" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_list_of_values(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + { + "name": "List Scan", + "scan_type": "list_of_values", + "list_items": ["val1", "val2"], + }, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + data = response.json() + assert data["scan_type"] == "list_of_values" + assert data["list_items"] == ["val1", "val2"] + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_list_table(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["test"]], + ) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + { + "name": "Table Scan", + "scan_type": "list_table", + "source_table_id": table.id, + "source_field_id": fields[0].id, + }, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + data = response.json() + assert data["source_table_id"] == table.id + assert data["source_field_id"] == fields[0].id + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_with_specific_workspaces(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + ws = enterprise_data_fixture.create_workspace(user=user) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + { + "name": "WS Scan", + "scan_type": "pattern", + "pattern": "AA", + "scan_all_workspaces": False, + "workspace_ids": [ws.id], + }, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + data = response.json() + assert data["scan_all_workspaces"] is False + assert data["workspace_ids"] == [ws.id] + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_pattern_missing_pattern(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + {"name": "Missing Pattern", "scan_type": "pattern"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_400_BAD_REQUEST + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_list_of_values_missing_items(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + {"name": "Missing Items", "scan_type": "list_of_values"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_400_BAD_REQUEST + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_list_table_missing_source(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + {"name": "Missing Source", "scan_type": "list_table"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_400_BAD_REQUEST + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_list_table_incompatible_source_field( + api_client, enterprise_data_fixture +): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table = enterprise_data_fixture.create_database_table(user=user) + boolean_field = enterprise_data_fixture.create_boolean_field( + table=table, name="Active" + ) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:list"), + { + "name": "Boolean Scan", + "scan_type": "list_table", + "source_table_id": table.id, + "source_field_id": boolean_field.id, + }, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_400_BAD_REQUEST + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_workspace_structure_excludes_incompatible_fields( + api_client, enterprise_data_fixture +): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table = enterprise_data_fixture.create_database_table(user=user) + text_field = enterprise_data_fixture.create_text_field(table=table, name="Name") + enterprise_data_fixture.create_boolean_field(table=table, name="Active") + workspace = table.database.workspace + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:workspace_structure", + kwargs={"workspace_id": workspace.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + data = response.json() + fields = data[0]["tables"][0]["fields"] + field_names = {f["name"] for f in fields} + assert "Name" in field_names + assert "Active" not in field_names + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_get_scan_unauthenticated(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=user + ) + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_get_scan_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=admin + ) + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_get_scan_without_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": 1}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_get_scan(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + scan = DataScan.objects.create( + name="Test Scan", scan_type="pattern", pattern="AA", created_by=user + ) + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + assert response.json() == { + "id": scan.id, + "name": "Test Scan", + "scan_type": "pattern", + "pattern": "AA", + "frequency": "manual", + "scan_all_workspaces": True, + "workspace_ids": [], + "is_running": False, + "last_run_started_at": None, + "last_run_finished_at": None, + "last_error": None, + "list_items": [], + "results_count": 0, + "source_table_id": None, + "source_field_id": None, + "source_workspace_id": None, + "source_database_id": None, + "created_on": AnyStr(), + "updated_on": AnyStr(), + } + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_get_scan_not_found(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": 99999}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_404_NOT_FOUND + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_unauthenticated(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=user + ) + + response = api_client.patch( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + {"name": "Updated"}, + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=admin + ) + + response = api_client.patch( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + {"name": "Updated"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_without_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.patch( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": 1}, + ), + {"name": "Updated"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + scan = DataScan.objects.create( + name="Original", scan_type="pattern", pattern="AA", created_by=user + ) + + response = api_client.patch( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + {"name": "Updated", "frequency": "weekly"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + assert response.json()["name"] == "Updated" + assert response.json()["frequency"] == "weekly" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_not_found(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.patch( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": 99999}, + ), + {"name": "Updated"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_404_NOT_FOUND + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_already_running(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + scan = DataScan.objects.create( + name="Running", + scan_type="pattern", + pattern="AA", + created_by=user, + is_running=True, + ) + + response = api_client.patch( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + {"name": "Updated"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_409_CONFLICT + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan_unauthenticated(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=user + ) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=admin + ) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan_without_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": 1}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + scan = DataScan.objects.create( + name="To Delete", scan_type="pattern", pattern="AA", created_by=user + ) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_204_NO_CONTENT + assert DataScan.objects.filter(id=scan.id).count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan_not_found(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": 99999}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_404_NOT_FOUND + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan_already_running(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + scan = DataScan.objects.create( + name="Running", + scan_type="pattern", + pattern="AA", + created_by=user, + is_running=True, + ) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:detail", + kwargs={"scan_id": scan.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_409_CONFLICT + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_unauthenticated(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=user + ) + + response = api_client.post( + reverse( + "api:enterprise:admin:data_scanner:trigger", + kwargs={"scan_id": scan.id}, + ), + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=admin + ) + + response = api_client.post( + reverse( + "api:enterprise:admin:data_scanner:trigger", + kwargs={"scan_id": scan.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_without_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.post( + reverse( + "api:enterprise:admin:data_scanner:trigger", + kwargs={"scan_id": 1}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + scan = DataScan.objects.create( + name="Trigger Test", scan_type="pattern", pattern="AA", created_by=user + ) + + with pytest.MonkeyPatch.context() as m: + m.setattr( + "baserow_enterprise.data_scanner.tasks.run_data_scan.delay", + lambda scan_id: None, + ) + response = api_client.post( + reverse( + "api:enterprise:admin:data_scanner:trigger", + kwargs={"scan_id": scan.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_202_ACCEPTED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_not_found(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.post( + reverse( + "api:enterprise:admin:data_scanner:trigger", + kwargs={"scan_id": 9999999}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_404_NOT_FOUND + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_already_running(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + scan = DataScan.objects.create( + name="Running", + scan_type="pattern", + pattern="AA", + created_by=user, + is_running=True, + ) + + response = api_client.post( + reverse( + "api:enterprise:admin:data_scanner:trigger", + kwargs={"scan_id": scan.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_409_CONFLICT + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_results_unauthenticated(api_client): + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:results"), + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_results_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:results"), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_results_without_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:results"), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_results(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["test"]], + ) + + scan = DataScan.objects.create( + name="Results Test", scan_type="pattern", pattern="AA", created_by=user + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="test value", + first_identified_on=now, + last_identified_on=now, + ) + + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:results"), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + data = response.json() + assert data["count"] == 1 + result = data["results"][0] + assert result["matched_value"] == "test value" + assert result["scan_name"] == "Results Test" + assert result["table_name"] == table.name + assert result["field_name"] == fields[0].name + assert result["workspace_name"] == table.database.workspace.name + assert result["database_name"] == table.database.name + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_results_filter_by_scan(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["test"]], + ) + + scan1 = DataScan.objects.create( + name="Scan 1", scan_type="pattern", pattern="AA", created_by=user + ) + scan2 = DataScan.objects.create( + name="Scan 2", scan_type="pattern", pattern="99", created_by=user + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan1, + table=table, + field=fields[0], + row_id=1, + matched_value="match1", + first_identified_on=now, + last_identified_on=now, + ) + DataScanResult.objects.create( + scan=scan2, + table=table, + field=fields[0], + row_id=2, + matched_value="match2", + first_identified_on=now, + last_identified_on=now, + ) + + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:results"), + {"scan_id": scan1.id}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + data = response.json() + assert data["count"] == 1 + assert data["results"][0]["scan_name"] == "Scan 1" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_results_search(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["test"]], + ) + + scan = DataScan.objects.create( + name="Search Test", scan_type="pattern", pattern="AA", created_by=user + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="NL12ABCD0123456789", + first_identified_on=now, + last_identified_on=now, + ) + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=2, + matched_value="something-else", + first_identified_on=now, + last_identified_on=now, + ) + + response = api_client.get( + reverse("api:enterprise:admin:data_scanner:results"), + {"search": "NL12"}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + assert response.json()["count"] == 1 + assert response.json()["results"][0]["matched_value"] == "NL12ABCD0123456789" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_results_query_count_is_constant(api_client, enterprise_data_fixture): + """ + The number of queries when listing results must not grow with the number of + results (no N+1). Adding more results should not increase the query count. + """ + + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["test"]], + ) + + scan = DataScan.objects.create( + name="Query Count Test", scan_type="pattern", pattern="AA", created_by=user + ) + now = timezone.now() + for i in range(1, 3): + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=i, + matched_value=f"match{i}", + first_identified_on=now, + last_identified_on=now, + ) + + url = reverse("api:enterprise:admin:data_scanner:results") + + with CaptureQueriesContext(connection) as captured_2_results: + response = api_client.get(url, format="json", HTTP_AUTHORIZATION=f"JWT {token}") + assert response.status_code == HTTP_200_OK + assert response.json()["count"] == 2 + + num_queries_2 = len(captured_2_results) + + for i in range(3, 8): + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=i, + matched_value=f"match{i}", + first_identified_on=now, + last_identified_on=now, + ) + + with CaptureQueriesContext(connection) as captured_7_results: + response = api_client.get(url, format="json", HTTP_AUTHORIZATION=f"JWT {token}") + assert response.status_code == HTTP_200_OK + assert response.json()["count"] == 7 + + num_queries_7 = len(captured_7_results) + + assert num_queries_7 == num_queries_2, ( + f"Query count grew from {num_queries_2} to {num_queries_7} when adding " + f"more results — likely an N+1 problem." + ) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_result_unauthenticated(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=user + ) + now = timezone.now() + result = DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="test", + first_identified_on=now, + last_identified_on=now, + ) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:result_delete", + kwargs={"result_id": result.id}, + ), + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_result_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + table, fields, _ = enterprise_data_fixture.build_table( + user=admin, columns=[("Name", "text")], rows=[["test"]] + ) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=admin + ) + now = timezone.now() + result = DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="test", + first_identified_on=now, + last_identified_on=now, + ) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:result_delete", + kwargs={"result_id": result.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_result_without_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:result_delete", + kwargs={"result_id": 1}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_result(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=user + ) + now = timezone.now() + result = DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="test", + first_identified_on=now, + last_identified_on=now, + ) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:result_delete", + kwargs={"result_id": result.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_204_NO_CONTENT + assert DataScanResult.objects.filter(id=result.id).count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_result_not_found(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.delete( + reverse( + "api:enterprise:admin:data_scanner:result_delete", + kwargs={"result_id": 99999}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_404_NOT_FOUND + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_workspace_structure_unauthenticated(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + ws = enterprise_data_fixture.create_workspace(user=user) + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:workspace_structure", + kwargs={"workspace_id": ws.id}, + ), + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_workspace_structure_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + ws = enterprise_data_fixture.create_workspace(user=admin) + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:workspace_structure", + kwargs={"workspace_id": ws.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_workspace_structure_without_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:workspace_structure", + kwargs={"workspace_id": 1}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_workspace_structure(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text"), ("Notes", "text")], + rows=[["test", "note"]], + ) + workspace = table.database.workspace + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:workspace_structure", + kwargs={"workspace_id": workspace.id}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_200_OK + data = response.json() + assert len(data) == 1 + db = data[0] + assert db["name"] == table.database.name + assert len(db["tables"]) == 1 + tbl = db["tables"][0] + assert tbl["name"] == table.name + assert len(tbl["fields"]) >= 2 + field_names = {f["name"] for f in tbl["fields"]} + assert "Name" in field_names + assert "Notes" in field_names + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_workspace_structure_not_found(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.get( + reverse( + "api:enterprise:admin:data_scanner:workspace_structure", + kwargs={"workspace_id": 99999}, + ), + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_404_NOT_FOUND + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_export_results_unauthenticated(api_client): + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:results_export"), + {}, + format="json", + ) + assert response.status_code == HTTP_401_UNAUTHORIZED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_export_results_non_staff(api_client, enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + _, token = enterprise_data_fixture.create_user_and_token(is_staff=False) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:results_export"), + {}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_403_FORBIDDEN + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_export_results_without_license(api_client, enterprise_data_fixture): + _, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + response = api_client.post( + reverse("api:enterprise:admin:data_scanner:results_export"), + {}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + assert response.status_code == HTTP_402_PAYMENT_REQUIRED + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +@patch("baserow.core.storage.get_default_storage") +def test_export_results_csv(get_storage_mock, api_client, enterprise_data_fixture): + storage_mock = MagicMock() + get_storage_mock.return_value = storage_mock + + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + scan = DataScan.objects.create( + name="Export Test", scan_type="pattern", pattern="AA", created_by=user + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="matched", + first_identified_on=now, + last_identified_on=now, + ) + + stub_file = BytesIO() + storage_mock.open.return_value = stub_file + close = stub_file.close + stub_file.close = lambda: None + + csv_export_job = JobHandler().create_and_start_job( + user, + DataScanResultExportJobType.type, + csv_column_separator=",", + csv_first_row_header=True, + export_charset="utf-8", + sync=True, + ) + csv_export_job.refresh_from_db() + assert csv_export_job.state == JOB_FINISHED + + data = stub_file.getvalue().decode("utf-8") + assert "Export Test" in data + assert "matched" in data + assert "Scan Name" in data + + close() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +@patch("baserow.core.storage.get_default_storage") +def test_export_results_csv_without_header( + get_storage_mock, api_client, enterprise_data_fixture +): + storage_mock = MagicMock() + get_storage_mock.return_value = storage_mock + + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + scan = DataScan.objects.create( + name="NoHeader", scan_type="pattern", pattern="AA", created_by=user + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="val", + first_identified_on=now, + last_identified_on=now, + ) + + stub_file = BytesIO() + storage_mock.open.return_value = stub_file + close = stub_file.close + stub_file.close = lambda: None + + csv_export_job = JobHandler().create_and_start_job( + user, + DataScanResultExportJobType.type, + csv_column_separator=",", + csv_first_row_header=False, + export_charset="utf-8", + sync=True, + ) + csv_export_job.refresh_from_db() + assert csv_export_job.state == JOB_FINISHED + + data = stub_file.getvalue().decode("utf-8") + assert "Scan Name" not in data + assert "NoHeader" in data + + close() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +@patch("baserow.core.storage.get_default_storage") +def test_export_results_csv_filter_by_scan( + get_storage_mock, api_client, enterprise_data_fixture +): + storage_mock = MagicMock() + get_storage_mock.return_value = storage_mock + + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + scan1 = DataScan.objects.create( + name="Scan A", scan_type="pattern", pattern="AA", created_by=user + ) + scan2 = DataScan.objects.create( + name="Scan B", scan_type="pattern", pattern="99", created_by=user + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan1, + table=table, + field=fields[0], + row_id=1, + matched_value="a_match", + first_identified_on=now, + last_identified_on=now, + ) + DataScanResult.objects.create( + scan=scan2, + table=table, + field=fields[0], + row_id=2, + matched_value="b_match", + first_identified_on=now, + last_identified_on=now, + ) + + stub_file = BytesIO() + storage_mock.open.return_value = stub_file + close = stub_file.close + stub_file.close = lambda: None + + csv_export_job = JobHandler().create_and_start_job( + user, + DataScanResultExportJobType.type, + csv_column_separator=",", + csv_first_row_header=True, + export_charset="utf-8", + filter_scan_id=scan1.id, + sync=True, + ) + csv_export_job.refresh_from_db() + assert csv_export_job.state == JOB_FINISHED + + data = stub_file.getvalue().decode("utf-8") + assert "a_match" in data + assert "b_match" not in data + + close() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +@patch("baserow.core.storage.get_default_storage") +def test_export_results_deleting_job_deletes_file( + get_storage_mock, api_client, enterprise_data_fixture +): + storage_mock = MagicMock() + get_storage_mock.return_value = storage_mock + + enterprise_data_fixture.enable_enterprise() + user, token = enterprise_data_fixture.create_user_and_token(is_staff=True) + + stub_file = BytesIO() + storage_mock.open.return_value = stub_file + close = stub_file.close + stub_file.close = lambda: None + + csv_export_job = JobHandler().create_and_start_job( + user, + DataScanResultExportJobType.type, + csv_column_separator=",", + csv_first_row_header=True, + export_charset="utf-8", + sync=True, + ) + csv_export_job.refresh_from_db() + assert csv_export_job.state == JOB_FINISHED + assert csv_export_job.exported_file_name is not None + + close() + + from baserow.contrib.database.export.handler import ExportHandler + + with patch( + "baserow_enterprise.data_scanner.job_types.get_default_storage" + ) as mock_storage: + mock_storage.return_value = storage_mock + DataScanResultExportJobType().before_delete(csv_export_job) + storage_mock.delete.assert_called_once_with( + ExportHandler.export_file_path(csv_export_job.exported_file_name) + ) diff --git a/enterprise/backend/tests/baserow_enterprise_tests/api/audit_log/test_audit_log_admin_views.py b/enterprise/backend/tests/baserow_enterprise_tests/api/audit_log/test_audit_log_admin_views.py index 6a764bfea7..de5fb73142 100755 --- a/enterprise/backend/tests/baserow_enterprise_tests/api/audit_log/test_audit_log_admin_views.py +++ b/enterprise/backend/tests/baserow_enterprise_tests/api/audit_log/test_audit_log_admin_views.py @@ -52,7 +52,7 @@ def test_admins_cannot_access_audit_log_endpoints_without_an_enterprise_license( @pytest.mark.django_db -@pytest.mark.parametrize("url_name", ["users", "workspaces", "action_types", "list"]) +@pytest.mark.parametrize("url_name", ["users", "action_types", "list"]) @override_settings(DEBUG=True) def test_non_admins_cannot_access_audit_log_endpoints( api_client, enterprise_data_fixture, url_name @@ -129,54 +129,6 @@ def test_audit_log_user_filter_returns_users_correctly( } -@pytest.mark.django_db -@override_settings(DEBUG=True) -def test_audit_log_workspace_filter_returns_workspaces_correctly( - api_client, enterprise_data_fixture -): - ( - admin_user, - admin_token, - ) = enterprise_data_fixture.create_enterprise_admin_user_and_token() - workspace_1 = enterprise_data_fixture.create_workspace( - name="workspace 1", user=admin_user - ) - workspace_2 = enterprise_data_fixture.create_workspace( - name="workspace 2", user=admin_user - ) - - # no search query should return all workspaces - response = api_client.get( - reverse("api:enterprise:audit_log:workspaces"), - format="json", - HTTP_AUTHORIZATION=f"JWT {admin_token}", - ) - assert response.status_code == HTTP_200_OK - assert response.json() == { - "count": 2, - "next": None, - "previous": None, - "results": [ - {"id": workspace_1.id, "value": workspace_1.name}, - {"id": workspace_2.id, "value": workspace_2.name}, - ], - } - - # searching by name should return only the correct workspace - response = api_client.get( - reverse("api:enterprise:audit_log:workspaces") + "?search=1", - format="json", - HTTP_AUTHORIZATION=f"JWT {admin_token}", - ) - assert response.status_code == HTTP_200_OK - assert response.json() == { - "count": 1, - "next": None, - "previous": None, - "results": [{"id": workspace_1.id, "value": workspace_1.name}], - } - - @pytest.mark.django_db @override_settings(DEBUG=True) def test_audit_log_action_type_filter_returns_action_types_correctly( diff --git a/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/__init__.py b/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/conftest.py b/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/conftest.py new file mode 100644 index 0000000000..aa69ec6d02 --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/conftest.py @@ -0,0 +1,36 @@ +from django.contrib.postgres.search import SearchVector +from django.db.models import Value +from django.utils import timezone + +import pytest + +from baserow.contrib.database.search.handler import SearchHandler + + +@pytest.fixture +def populate_search_table(): + """ + Returns a helper that creates a workspace search table and inserts + tsvector rows for every non-empty cell in the given rows / field. + """ + + def _populate(table, field, rows): + workspace = table.database.workspace + SearchHandler.create_workspace_search_table_if_not_exists(workspace.id) + search_model = SearchHandler.get_workspace_search_table_model(workspace.id) + + model = table.get_model() + for row in rows: + row_obj = model.objects.get(id=row.id) + cell_value = getattr(row_obj, field.db_column) + if cell_value: + search_model.objects.create( + row_id=row_obj.id, + field_id=field.id, + updated_on=timezone.now(), + value=SearchVector(Value(str(cell_value))), + ) + + return search_model + + return _populate diff --git a/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/test_data_scanner_handler.py b/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/test_data_scanner_handler.py new file mode 100644 index 0000000000..900a4ef81e --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/test_data_scanner_handler.py @@ -0,0 +1,1655 @@ +import re +from datetime import timedelta +from unittest.mock import patch + +from django.core.exceptions import PermissionDenied +from django.test.utils import override_settings +from django.utils import timezone + +import pytest + +from baserow_enterprise.data_scanner.exceptions import ( + DataScanDoesNotExist, + DataScanIsAlreadyRunning, +) +from baserow_enterprise.data_scanner.handler import ( + DataScannerHandler, + convert_pattern_to_regex, +) +from baserow_enterprise.data_scanner.models import ( + DataScan, + DataScanListItem, + DataScanResult, +) +from baserow_premium.license.exceptions import FeaturesNotAvailableError + + +@pytest.mark.data_scanner +def test_convert_pattern_to_regex_alpha_token(): + assert convert_pattern_to_regex("A") == "[A-Za-z]" + assert convert_pattern_to_regex("AA") == "[A-Za-z][A-Za-z]" + + +@pytest.mark.data_scanner +def test_convert_pattern_to_regex_digit_token(): + assert convert_pattern_to_regex("D") == "[0-9]" + assert convert_pattern_to_regex("DD") == "[0-9][0-9]" + + +@pytest.mark.data_scanner +def test_convert_pattern_to_regex_any_char_token(): + assert convert_pattern_to_regex("X") == "." + assert convert_pattern_to_regex("XX") == ".." + + +@pytest.mark.data_scanner +def test_convert_pattern_to_regex_escaped_literals(): + assert convert_pattern_to_regex("\\N\\L") == "NL" + assert convert_pattern_to_regex("\\-") == "\\-" + assert convert_pattern_to_regex("\\.") == "\\." + assert convert_pattern_to_regex("\\D") == "D" + + +@pytest.mark.data_scanner +def test_convert_pattern_to_regex_mixed(): + assert convert_pattern_to_regex("AADD") == "[A-Za-z][A-Za-z][0-9][0-9]" + assert ( + convert_pattern_to_regex("AA\\-DD\\-AA") + == "[A-Za-z][A-Za-z]\\-[0-9][0-9]\\-[A-Za-z][A-Za-z]" + ) + + +@pytest.mark.data_scanner +def test_convert_pattern_to_regex_iban_pattern(): + assert ( + convert_pattern_to_regex("AADDAAAADDDDDDDDDD") + == "[A-Za-z][A-Za-z][0-9][0-9][A-Za-z][A-Za-z][A-Za-z][A-Za-z]" + "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" + ) + + +@pytest.mark.data_scanner +def test_convert_pattern_to_regex_dutch_iban_with_literals(): + assert ( + convert_pattern_to_regex("\\N\\LDDAAAADDDDDDDDDD") + == "NL[0-9][0-9][A-Za-z][A-Za-z][A-Za-z][A-Za-z]" + "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" + ) + + +@pytest.mark.data_scanner +def test_convert_pattern_to_regex_empty(): + assert convert_pattern_to_regex("") == "" + + +@pytest.mark.data_scanner +def test_convert_pattern_to_regex_trailing_backslash(): + # A trailing backslash with nothing after it is treated as a literal backslash + assert convert_pattern_to_regex("A\\") == "[A-Za-z]\\\\" + + +@pytest.mark.data_scanner +def test_extract_matching_token(): + compiled = re.compile(r"[A-Za-z][A-Za-z][0-9][0-9]", re.IGNORECASE) + + token = DataScannerHandler._extract_matching_token( + "'ab12':1 'something':2", compiled + ) + assert token == "ab12" + + +@pytest.mark.data_scanner +def test_extract_matching_token_no_match_returns_raw(): + compiled = re.compile(r"[A-Za-z][A-Za-z][0-9][0-9]", re.IGNORECASE) + + token = DataScannerHandler._extract_matching_token("'hello':1", compiled) + assert token == "'hello':1" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_pattern(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="IBAN Scanner", + scan_type="pattern", + pattern="AADDAAAADDDDDDDDDD", + frequency="daily", + ) + + assert scan.name == "IBAN Scanner" + assert scan.scan_type == "pattern" + assert scan.pattern == "AADDAAAADDDDDDDDDD" + assert scan.frequency == "daily" + assert scan.scan_all_workspaces is True + assert scan.created_by == user + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_list_of_values(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="Blacklist Scan", + scan_type="list_of_values", + list_items=["value1", "value2", "value3"], + ) + + assert scan.scan_type == "list_of_values" + assert scan.list_items.count() == 3 + assert list(scan.list_items.values_list("value", flat=True)) == [ + "value1", + "value2", + "value3", + ] + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_list_table(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["test"]], + ) + field = fields[0] + + scan = DataScannerHandler.create_scan( + user=user, + name="Table Scan", + scan_type="list_table", + source_table_id=table.id, + source_field_id=field.id, + ) + + assert scan.scan_type == "list_table" + assert scan.source_table_id == table.id + assert scan.source_field_id == field.id + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_with_specific_workspaces(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + workspace1 = enterprise_data_fixture.create_workspace(user=user) + workspace2 = enterprise_data_fixture.create_workspace(user=user) + + scan = DataScannerHandler.create_scan( + user=user, + name="Workspace Scan", + scan_type="pattern", + pattern="AADD", + scan_all_workspaces=False, + workspace_ids=[workspace1.id, workspace2.id], + ) + + assert scan.scan_all_workspaces is False + assert set(scan.workspaces.values_list("id", flat=True)) == { + workspace1.id, + workspace2.id, + } + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_all_workspaces(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="All Workspaces Scan", + scan_type="pattern", + pattern="99", + scan_all_workspaces=True, + ) + + assert scan.scan_all_workspaces is True + assert scan.workspaces.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_without_enterprise_license(enterprise_data_fixture): + user = enterprise_data_fixture.create_user(is_staff=True) + + with pytest.raises(FeaturesNotAvailableError): + DataScannerHandler.create_scan( + user=user, + name="Test", + scan_type="pattern", + pattern="AA", + ) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_create_scan_non_staff_user(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=False) + + with pytest.raises(PermissionDenied): + DataScannerHandler.create_scan( + user=user, + name="Test", + scan_type="pattern", + pattern="AA", + ) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="Original", + scan_type="pattern", + pattern="AA", + ) + + updated = DataScannerHandler.update_scan( + user=user, + scan_id=scan.id, + name="Updated", + frequency="weekly", + pattern="99", + ) + + assert updated.name == "Updated" + assert updated.frequency == "weekly" + assert updated.pattern == "99" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_without_license(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, name="Test", scan_type="pattern", pattern="AA" + ) + + from baserow.core.cache import local_cache + from baserow_premium.license.models import License + + License.objects.all().delete() + local_cache.clear() + + with pytest.raises(FeaturesNotAvailableError): + DataScannerHandler.update_scan(user=user, scan_id=scan.id, name="New") + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_non_staff(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + regular = enterprise_data_fixture.create_user(is_staff=False) + + scan = DataScannerHandler.create_scan( + user=admin, name="Test", scan_type="pattern", pattern="AA" + ) + + with pytest.raises(PermissionDenied): + DataScannerHandler.update_scan(user=regular, scan_id=scan.id, name="New") + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_not_found(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + with pytest.raises(DataScanDoesNotExist): + DataScannerHandler.update_scan(user=user, scan_id=99999, name="New") + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_already_running(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="Running Scan", + scan_type="pattern", + pattern="AA", + ) + scan.is_running = True + scan.save() + + with pytest.raises(DataScanIsAlreadyRunning): + DataScannerHandler.update_scan(user=user, scan_id=scan.id, name="New Name") + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_workspaces(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + ws1 = enterprise_data_fixture.create_workspace(user=user) + ws2 = enterprise_data_fixture.create_workspace(user=user) + + scan = DataScannerHandler.create_scan( + user=user, + name="Test", + scan_type="pattern", + pattern="AA", + scan_all_workspaces=False, + workspace_ids=[ws1.id], + ) + assert set(scan.workspaces.values_list("id", flat=True)) == {ws1.id} + + updated = DataScannerHandler.update_scan( + user=user, scan_id=scan.id, workspace_ids=[ws2.id] + ) + assert set(updated.workspaces.values_list("id", flat=True)) == {ws2.id} + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_list_items(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="List Scan", + scan_type="list_of_values", + list_items=["a", "b"], + ) + assert scan.list_items.count() == 2 + + updated = DataScannerHandler.update_scan( + user=user, scan_id=scan.id, list_items=["x", "y", "z"] + ) + assert updated.list_items.count() == 3 + assert set(updated.list_items.values_list("value", flat=True)) == {"x", "y", "z"} + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_table_source(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table1, fields1, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Col1", "text")], rows=[["v"]] + ) + table2, fields2, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Col2", "text")], rows=[["v"]] + ) + + scan = DataScannerHandler.create_scan( + user=user, + name="Table Scan", + scan_type="list_table", + source_table_id=table1.id, + source_field_id=fields1[0].id, + ) + assert scan.source_table_id == table1.id + + DataScannerHandler.update_scan( + user=user, + scan_id=scan.id, + source_table_id=table2.id, + source_field_id=fields2[0].id, + ) + scan.refresh_from_db() + assert scan.source_table_id == table2.id + assert scan.source_field_id == fields2[0].id + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_update_scan_clears_workspaces_when_scan_all(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + ws = enterprise_data_fixture.create_workspace(user=user) + + scan = DataScannerHandler.create_scan( + user=user, + name="Test", + scan_type="pattern", + pattern="AA", + scan_all_workspaces=False, + workspace_ids=[ws.id], + ) + assert scan.workspaces.count() == 1 + + DataScannerHandler.update_scan( + user=user, + scan_id=scan.id, + scan_all_workspaces=True, + workspace_ids=[ws.id], + ) + scan.refresh_from_db() + assert scan.workspaces.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_cleanup_stale_results_on_type_change(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + + scan = DataScannerHandler.create_scan( + user=user, name="Test", scan_type="pattern", pattern="AA" + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="test", + first_identified_on=now, + last_identified_on=now, + ) + assert scan.results.count() == 1 + + DataScannerHandler.update_scan( + user=user, scan_id=scan.id, scan_type="list_of_values", list_items=["x"] + ) + assert scan.results.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_cleanup_stale_results_on_pattern_change(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + + scan = DataScannerHandler.create_scan( + user=user, name="Test", scan_type="pattern", pattern="AA" + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="test", + first_identified_on=now, + last_identified_on=now, + ) + assert scan.results.count() == 1 + + DataScannerHandler.update_scan(user=user, scan_id=scan.id, pattern="DD") + assert scan.results.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_cleanup_stale_results_on_list_items_change(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + + scan = DataScannerHandler.create_scan( + user=user, + name="Test", + scan_type="list_of_values", + list_items=["keep", "remove"], + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="keep", + first_identified_on=now, + last_identified_on=now, + ) + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=2, + matched_value="remove", + first_identified_on=now, + last_identified_on=now, + ) + assert scan.results.count() == 2 + + DataScannerHandler.update_scan(user=user, scan_id=scan.id, list_items=["keep"]) + assert scan.results.count() == 1 + assert scan.results.first().matched_value == "keep" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_cleanup_stale_results_on_empty_list(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + + scan = DataScannerHandler.create_scan( + user=user, name="Test", scan_type="list_of_values", list_items=["val"] + ) + now = timezone.now() + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="val", + first_identified_on=now, + last_identified_on=now, + ) + + DataScannerHandler.update_scan(user=user, scan_id=scan.id, list_items=[]) + assert scan.results.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="To Delete", + scan_type="list_of_values", + list_items=["val1"], + ) + scan_id = scan.id + + DataScannerHandler.delete_scan(user=user, scan_id=scan_id) + + assert DataScan.objects.filter(id=scan_id).count() == 0 + assert DataScanListItem.objects.filter(scan_id=scan_id).count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan_without_license(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, name="Test", scan_type="pattern", pattern="AA" + ) + + from baserow.core.cache import local_cache + from baserow_premium.license.models import License + + License.objects.all().delete() + local_cache.clear() + + with pytest.raises(FeaturesNotAvailableError): + DataScannerHandler.delete_scan(user=user, scan_id=scan.id) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan_non_staff(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + regular = enterprise_data_fixture.create_user(is_staff=False) + + scan = DataScannerHandler.create_scan( + user=admin, name="Test", scan_type="pattern", pattern="AA" + ) + + with pytest.raises(PermissionDenied): + DataScannerHandler.delete_scan(user=regular, scan_id=scan.id) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan_not_found(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + with pytest.raises(DataScanDoesNotExist): + DataScannerHandler.delete_scan(user=user, scan_id=99999) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_delete_scan_already_running(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="Running Scan", + scan_type="pattern", + pattern="AA", + ) + scan.is_running = True + scan.save() + + with pytest.raises(DataScanIsAlreadyRunning): + DataScannerHandler.delete_scan(user=user, scan_id=scan.id) + + # Verify the scan was not deleted. + assert DataScan.objects.filter(id=scan.id).exists() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_scans_without_license(enterprise_data_fixture): + user = enterprise_data_fixture.create_user(is_staff=True) + + with pytest.raises(FeaturesNotAvailableError): + DataScannerHandler.list_scans(user=user) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_scans_non_staff(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=False) + + with pytest.raises(PermissionDenied): + DataScannerHandler.list_scans(user=user) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_get_scan(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, name="Get Me", scan_type="pattern", pattern="AA" + ) + + fetched = DataScannerHandler.get_scan(user, scan.id) + assert fetched.id == scan.id + assert fetched.name == "Get Me" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_get_scan_not_found(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + with pytest.raises(DataScanDoesNotExist): + DataScannerHandler.get_scan(user, 99999) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_without_license(enterprise_data_fixture): + user = enterprise_data_fixture.create_user(is_staff=True) + + with pytest.raises(FeaturesNotAvailableError): + DataScannerHandler.trigger_scan(user=user, scan_id=999) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_non_staff(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + regular = enterprise_data_fixture.create_user(is_staff=False) + + scan = DataScannerHandler.create_scan( + user=admin, name="Test", scan_type="pattern", pattern="AA" + ) + + with pytest.raises(PermissionDenied): + DataScannerHandler.trigger_scan(user=regular, scan_id=scan.id) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_not_found(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + with pytest.raises(DataScanDoesNotExist): + DataScannerHandler.trigger_scan(user=user, scan_id=99999) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_already_running(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="Running Scan", + scan_type="pattern", + pattern="AA", + ) + scan.is_running = True + scan.save() + + with pytest.raises(DataScanIsAlreadyRunning): + DataScannerHandler.trigger_scan(user=user, scan_id=scan.id) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_trigger_scan_dispatches_task(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, name="Test", scan_type="pattern", pattern="AA" + ) + + with patch( + "baserow_enterprise.data_scanner.tasks.run_data_scan.delay" + ) as mock_delay: + DataScannerHandler.trigger_scan(user=user, scan_id=scan.id) + mock_delay.assert_called_once_with(scan.id) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_not_found(enterprise_data_fixture): + DataScannerHandler.run_scan(99999) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_pattern_scan(enterprise_data_fixture, populate_search_table): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Code", "text")], + rows=[["AB12CD345678901234"], ["not matching"], ["XY99ZZ111111111111"]], + ) + field = fields[0] + + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="Pattern Test", + scan_type="pattern", + pattern="AADDAADDDDDDDDDDDD", + ) + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.is_running is False + assert scan.last_run_finished_at is not None + assert scan.last_error is None or scan.last_error == "" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_pattern_scan_dutch_iban_with_escaped_literals( + enterprise_data_fixture, populate_search_table +): + """ + Verifies that a pattern with escaped literal characters (e.g. \\N\\L for + the fixed "NL" prefix) correctly matches values in the search table. + PostgreSQL lowercases tsvector tokens, so the regex must match + case-insensitively. + """ + + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("IBAN", "text")], + rows=[ + ["NL23INGB0007704001"], + ["not an iban"], + ["NL91ABNA0417164300"], + ], + ) + field = fields[0] + + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="Dutch IBAN Scanner", + scan_type="pattern", + pattern="\\N\\LDDAAAADDDDDDDDDD", + ) + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.is_running is False + assert scan.last_error is None or scan.last_error == "" + + results = list( + scan.results.order_by("row_id").values_list("row_id", "matched_value") + ) + assert len(results) == 2 + # tsvector tokens are lowercased by PostgreSQL + assert results[0][0] == rows[0].id + assert "nl23ingb0007704001" in results[0][1].lower() + assert results[1][0] == rows[2].id + assert "nl91abna0417164300" in results[1][1].lower() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_list_of_values_scan(enterprise_data_fixture, populate_search_table): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["secret123"], ["innocent"], ["secret456"]], + ) + field = fields[0] + workspace = table.database.workspace + + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="List of Values Test", + scan_type="list_of_values", + list_items=["secret123"], + scan_all_workspaces=False, + workspace_ids=[workspace.id], + ) + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.is_running is False + assert scan.last_error is None or scan.last_error == "" + assert scan.results.filter(matched_value="secret123").exists() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_without_license_records_error(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="License Test", + scan_type="pattern", + pattern="AA", + ) + + from baserow.core.cache import local_cache + from baserow_premium.license.models import License + + License.objects.all().delete() + local_cache.clear() + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.is_running is False + assert "Enterprise license no longer active" in scan.last_error + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_with_no_search_table(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + workspace = enterprise_data_fixture.create_workspace(user=user) + + scan = DataScannerHandler.create_scan( + user=user, + name="No Search Table", + scan_type="pattern", + pattern="AA", + scan_all_workspaces=False, + workspace_ids=[workspace.id], + ) + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.is_running is False + assert scan.last_error is None or scan.last_error == "" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_removes_stale_results(enterprise_data_fixture): + """Results from a previous run that are not re-identified get deleted.""" + + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + workspace = table.database.workspace + + scan = DataScannerHandler.create_scan( + user=user, + name="Stale Results Test", + scan_type="pattern", + pattern="AA", + scan_all_workspaces=False, + workspace_ids=[workspace.id], + ) + old_time = timezone.now() - timedelta(days=1) + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="old_match", + first_identified_on=old_time, + last_identified_on=old_time, + ) + assert scan.results.count() == 1 + + DataScannerHandler.run_scan(scan.id) + + assert scan.results.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_removes_result_when_cell_cleared( + enterprise_data_fixture, populate_search_table +): + """ + When a cell that previously matched is emptied, the next scan run should + remove the corresponding result because it is no longer re-identified. + """ + + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["secret123"], ["innocent"]], + ) + field = fields[0] + workspace = table.database.workspace + + search_model = populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="Cell Cleared Test", + scan_type="list_of_values", + list_items=["secret123"], + scan_all_workspaces=False, + workspace_ids=[workspace.id], + ) + + # First run: the value is present, so we expect a result. + DataScannerHandler.run_scan(scan.id) + scan.refresh_from_db() + assert scan.last_error is None or scan.last_error == "" + assert scan.results.count() == 1 + assert scan.results.filter(matched_value="secret123").exists() + + # Clear the cell and remove the search table entry to simulate the user + # emptying the cell. + model = table.get_model() + row = model.objects.get(id=rows[0].id) + setattr(row, field.db_column, "") + row.save() + search_model.objects.filter(row_id=rows[0].id, field_id=field.id).delete() + + # Second run: the value is gone, so the stale result should be removed. + DataScannerHandler.run_scan(scan.id) + scan.refresh_from_db() + assert scan.last_error is None or scan.last_error == "" + assert scan.results.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_upsert_result_updates_existing(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + field = fields[0] + + scan = DataScannerHandler.create_scan( + user=user, name="Upsert Test", scan_type="pattern", pattern="AA" + ) + t1 = timezone.now() - timedelta(hours=1) + DataScanResult.objects.create( + scan=scan, + table=table, + field=field, + row_id=1, + matched_value="old", + first_identified_on=t1, + last_identified_on=t1, + ) + + t2 = timezone.now() + DataScannerHandler._bulk_upsert_results(scan, [(field.id, 1, "new")], t2, set()) + + result = DataScanResult.objects.get(scan=scan, row_id=1, field=field) + assert result.matched_value == "new" + assert result.first_identified_on == t1 + assert result.last_identified_on == t2 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_list_table_scan_excludes_source_table( + enterprise_data_fixture, populate_search_table +): + """ + When running a list_table scan, the source table itself must not appear in + the results. + """ + + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + source_table, source_fields, source_rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Keyword", "text")], + rows=[["secret123"]], + ) + source_field = source_fields[0] + + target_table, target_fields, target_rows = enterprise_data_fixture.build_table( + user=user, + database=source_table.database, + columns=[("Notes", "text")], + rows=[["contains secret123 inside"], ["nothing here"]], + ) + target_field = target_fields[0] + + populate_search_table(source_table, source_field, source_rows) + populate_search_table(target_table, target_field, target_rows) + + workspace = source_table.database.workspace + scan = DataScannerHandler.create_scan( + user=user, + name="List Table Exclusion Test", + scan_type="list_table", + source_table_id=source_table.id, + source_field_id=source_field.id, + scan_all_workspaces=False, + workspace_ids=[workspace.id], + ) + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.is_running is False + assert scan.last_error is None or scan.last_error == "" + + results = DataScanResult.objects.filter(scan=scan) + assert not results.filter(table=source_table).exists() + assert results.filter(table=target_table).exists() + target_result = results.get(table=target_table) + assert target_result.field_id == target_field.id + assert target_result.matched_value == "secret123" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_stale_running_scan_reset(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=user, + name="Stale Scan", + scan_type="pattern", + pattern="AA", + frequency="daily", + ) + scan.is_running = True + scan.last_run_started_at = timezone.now() - timedelta(hours=7) + scan.save() + + DataScannerHandler.check_scans_due() + + scan.refresh_from_db() + assert scan.is_running is False + assert scan.last_error == "Scan timed out and was automatically reset" + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_check_scans_due_without_license(enterprise_data_fixture): + """Without a license, stale scans are still reset but no new scans are dispatched.""" + + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScan.objects.create( + name="Scheduled", + scan_type="pattern", + pattern="AA", + frequency="hourly", + created_by=user, + is_running=False, + last_run_started_at=None, + ) + + with patch( + "baserow_enterprise.data_scanner.tasks.run_data_scan.delay" + ) as mock_delay: + DataScannerHandler.check_scans_due() + mock_delay.assert_not_called() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_check_scans_due_dispatches_scheduled_scan(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScan.objects.create( + name="Hourly", + scan_type="pattern", + pattern="AA", + frequency="hourly", + created_by=user, + is_running=False, + last_run_started_at=timezone.now() - timedelta(hours=2), + ) + + with patch( + "baserow_enterprise.data_scanner.tasks.run_data_scan.delay" + ) as mock_delay: + DataScannerHandler.check_scans_due() + mock_delay.assert_called_once_with(scan.id) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_check_scans_due_skips_recently_run_scan(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + DataScan.objects.create( + name="Recently Run", + scan_type="pattern", + pattern="AA", + frequency="daily", + created_by=user, + is_running=False, + last_run_started_at=timezone.now() - timedelta(hours=1), + ) + + with patch( + "baserow_enterprise.data_scanner.tasks.run_data_scan.delay" + ) as mock_delay: + DataScannerHandler.check_scans_due() + mock_delay.assert_not_called() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_check_scans_due_skips_manual_scans(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + DataScan.objects.create( + name="Manual", + scan_type="pattern", + pattern="AA", + frequency="manual", + created_by=user, + is_running=False, + last_run_started_at=None, + ) + + with patch( + "baserow_enterprise.data_scanner.tasks.run_data_scan.delay" + ) as mock_delay: + DataScannerHandler.check_scans_due() + mock_delay.assert_not_called() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_check_scans_due_dispatches_never_run_scan(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScan.objects.create( + name="Never Run", + scan_type="pattern", + pattern="AA", + frequency="weekly", + created_by=user, + is_running=False, + last_run_started_at=None, + ) + + with patch( + "baserow_enterprise.data_scanner.tasks.run_data_scan.delay" + ) as mock_delay: + DataScannerHandler.check_scans_due() + mock_delay.assert_called_once_with(scan.id) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_result_deleted_when_table_deleted(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["test"]], + ) + + scan = DataScannerHandler.create_scan( + user=user, + name="Cascade Test", + scan_type="pattern", + pattern="AA", + ) + + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="test", + first_identified_on=timezone.now(), + last_identified_on=timezone.now(), + ) + + assert DataScanResult.objects.filter(scan=scan).count() == 1 + + table.delete() + + assert DataScanResult.objects.filter(scan=scan).count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_result_deleted_when_field_deleted(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["test"]], + ) + field = fields[0] + + scan = DataScannerHandler.create_scan( + user=user, + name="Field Cascade Test", + scan_type="pattern", + pattern="AA", + ) + + DataScanResult.objects.create( + scan=scan, + table=table, + field=field, + row_id=1, + matched_value="test", + first_identified_on=timezone.now(), + last_identified_on=timezone.now(), + ) + + assert DataScanResult.objects.filter(scan=scan).count() == 1 + + field.delete() + + assert DataScanResult.objects.filter(scan=scan).count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_result_deleted_when_scan_deleted(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + + scan = DataScannerHandler.create_scan( + user=user, name="Test", scan_type="pattern", pattern="AA" + ) + + DataScanResult.objects.create( + scan=scan, + table=table, + field=fields[0], + row_id=1, + matched_value="test", + first_identified_on=timezone.now(), + last_identified_on=timezone.now(), + ) + + scan_id = scan.id + DataScannerHandler.delete_scan(user=user, scan_id=scan_id) + assert DataScanResult.objects.filter(scan_id=scan_id).count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_source_table_set_null_when_table_deleted(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + + scan = DataScannerHandler.create_scan( + user=user, + name="Table Scan", + scan_type="list_table", + source_table_id=table.id, + source_field_id=fields[0].id, + ) + + assert scan.source_table_id == table.id + + table.delete() + + scan.refresh_from_db() + assert scan.source_table is None + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_source_field_set_null_when_field_deleted(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, _ = enterprise_data_fixture.build_table( + user=user, columns=[("Name", "text")], rows=[["test"]] + ) + + scan = DataScannerHandler.create_scan( + user=user, + name="Table Scan", + scan_type="list_table", + source_table_id=table.id, + source_field_id=fields[0].id, + ) + + fields[0].delete() + + scan.refresh_from_db() + assert scan.source_field is None + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_excludes_trashed_field( + enterprise_data_fixture, populate_search_table +): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["secret123"]], + ) + field = fields[0] + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="Trashed Field Test", + scan_type="list_of_values", + list_items=["secret123"], + ) + + field.trashed = True + field.save() + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.last_error is None or scan.last_error == "" + assert scan.results.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_excludes_trashed_table( + enterprise_data_fixture, populate_search_table +): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["secret123"]], + ) + field = fields[0] + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="Trashed Table Test", + scan_type="list_of_values", + list_items=["secret123"], + ) + + table.trashed = True + table.save() + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.last_error is None or scan.last_error == "" + assert scan.results.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_excludes_trashed_database( + enterprise_data_fixture, populate_search_table +): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["secret123"]], + ) + field = fields[0] + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="Trashed Database Test", + scan_type="list_of_values", + list_items=["secret123"], + ) + + database = table.database + database.trashed = True + database.save() + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.last_error is None or scan.last_error == "" + assert scan.results.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_excludes_trashed_row(enterprise_data_fixture, populate_search_table): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["secret123"], ["secret456"]], + ) + field = fields[0] + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="Trashed Row Test", + scan_type="list_of_values", + list_items=["secret123", "secret456"], + ) + + model = table.get_model() + model.objects_and_trash.filter(id=rows[0].id).update(trashed=True) + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.last_error is None or scan.last_error == "" + # Only the second (non-trashed) row should appear. + assert scan.results.count() == 1 + assert scan.results.filter(row_id=rows[1].id).exists() + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_excludes_trashed_workspace( + enterprise_data_fixture, populate_search_table +): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("Name", "text")], + rows=[["secret123"]], + ) + field = fields[0] + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="Trashed Workspace Test", + scan_type="list_of_values", + list_items=["secret123"], + ) + + workspace = table.database.workspace + workspace.trashed = True + workspace.save() + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.last_error is None or scan.last_error == "" + assert scan.results.count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_pattern_scan_excludes_trashed_field( + enterprise_data_fixture, populate_search_table +): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=True) + table, fields, rows = enterprise_data_fixture.build_table( + user=user, + columns=[("IBAN", "text")], + rows=[["NL23INGB0007704001"]], + ) + field = fields[0] + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=user, + name="Pattern Trash Test", + scan_type="pattern", + pattern="\\N\\LDDAAAADDDDDDDDDD", + ) + + field.trashed = True + field.save() + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.last_error is None or scan.last_error == "" + assert scan.results.count() == 0 diff --git a/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/test_data_scanner_notification_types.py b/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/test_data_scanner_notification_types.py new file mode 100644 index 0000000000..1d29734970 --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/data_scanner/test_data_scanner_notification_types.py @@ -0,0 +1,277 @@ +from django.test.utils import override_settings + +import pytest + +from baserow.core.notifications.models import Notification, NotificationRecipient +from baserow_enterprise.data_scanner.handler import DataScannerHandler +from baserow_enterprise.data_scanner.models import DataScan +from baserow_enterprise.data_scanner.notification_types import ( + DataScanNewResultsNotificationType, +) + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_notify_instance_admins_creates_notification(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScan.objects.create( + name="Test Scan", scan_type="pattern", pattern="AA", created_by=admin + ) + + recipients = DataScanNewResultsNotificationType.notify_instance_admins(scan, 5) + + assert recipients is not None + assert len(recipients) == 1 + assert recipients[0].recipient == admin + + notification = recipients[0].notification + assert notification.type == "data_scan_new_results" + assert notification.data["scan_id"] == scan.id + assert notification.data["scan_name"] == "Test Scan" + assert notification.data["new_results_count"] == 5 + assert notification.workspace is None + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_notify_instance_admins_sends_to_all_staff(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin1 = enterprise_data_fixture.create_user(is_staff=True) + admin2 = enterprise_data_fixture.create_user(is_staff=True) + enterprise_data_fixture.create_user(is_staff=False) + + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=admin1 + ) + + recipients = DataScanNewResultsNotificationType.notify_instance_admins(scan, 3) + + assert len(recipients) == 2 + recipient_users = {r.recipient_id for r in recipients} + assert recipient_users == {admin1.id, admin2.id} + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_notify_instance_admins_skips_inactive_staff(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + active_admin = enterprise_data_fixture.create_user(is_staff=True) + inactive_admin = enterprise_data_fixture.create_user(is_staff=True) + inactive_admin.is_active = False + inactive_admin.save() + + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=active_admin + ) + + recipients = DataScanNewResultsNotificationType.notify_instance_admins(scan, 1) + + assert len(recipients) == 1 + assert recipients[0].recipient == active_admin + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_notify_instance_admins_returns_none_when_no_staff(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + user = enterprise_data_fixture.create_user(is_staff=False) + + scan = DataScan.objects.create( + name="Test", scan_type="pattern", pattern="AA", created_by=user + ) + + result = DataScanNewResultsNotificationType.notify_instance_admins(scan, 1) + + assert result is None + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_email_title_singular(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + + notification = Notification( + type="data_scan_new_results", + data={"scan_id": 1, "scan_name": "IBAN Scanner", "new_results_count": 1}, + ) + + title = DataScanNewResultsNotificationType.get_notification_title_for_email( + notification, {} + ) + assert "1 new result found for IBAN Scanner" in title + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_email_title_plural(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + + notification = Notification( + type="data_scan_new_results", + data={"scan_id": 1, "scan_name": "IBAN Scanner", "new_results_count": 10}, + ) + + title = DataScanNewResultsNotificationType.get_notification_title_for_email( + notification, {} + ) + assert "10 new results found for IBAN Scanner" in title + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_email_description(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + + notification = Notification( + type="data_scan_new_results", + data={"scan_id": 1, "scan_name": "IBAN Scanner", "new_results_count": 3}, + ) + + desc = DataScanNewResultsNotificationType.get_notification_description_for_email( + notification, {} + ) + assert "IBAN Scanner" in desc + assert "3 new matches" in desc + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_sends_notification_on_new_results( + enterprise_data_fixture, populate_search_table +): + enterprise_data_fixture.enable_enterprise() + admin1 = enterprise_data_fixture.create_user(is_staff=True) + admin2 = enterprise_data_fixture.create_user(is_staff=True) + enterprise_data_fixture.create_user(is_staff=False) + + table, fields, rows = enterprise_data_fixture.build_table( + user=admin1, + columns=[("Name", "text")], + rows=[["secret123"]], + ) + field = fields[0] + workspace = table.database.workspace + + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=admin1, + name="Secret Scanner", + scan_type="list_of_values", + list_items=["secret123"], + scan_all_workspaces=False, + workspace_ids=[workspace.id], + ) + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.last_error is None or scan.last_error == "" + + notifications = Notification.objects.filter(type="data_scan_new_results") + assert notifications.count() == 1 + + notification = notifications.first() + assert notification.data["scan_id"] == scan.id + assert notification.data["scan_name"] == "Secret Scanner" + assert notification.data["new_results_count"] > 0 + + admin_recipients = NotificationRecipient.objects.filter(notification=notification) + recipient_ids = set(admin_recipients.values_list("recipient_id", flat=True)) + assert recipient_ids == {admin1.id, admin2.id} + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_no_notification_when_no_new_results(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + workspace = enterprise_data_fixture.create_workspace(user=admin) + + scan = DataScannerHandler.create_scan( + user=admin, + name="Empty Scanner", + scan_type="pattern", + pattern="AA", + scan_all_workspaces=False, + workspace_ids=[workspace.id], + ) + + DataScannerHandler.run_scan(scan.id) + + assert Notification.objects.filter(type="data_scan_new_results").count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_no_notification_when_only_existing_results( + enterprise_data_fixture, populate_search_table +): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + + table, fields, rows = enterprise_data_fixture.build_table( + user=admin, + columns=[("Name", "text")], + rows=[["secret123"]], + ) + field = fields[0] + workspace = table.database.workspace + + populate_search_table(table, field, rows) + + scan = DataScannerHandler.create_scan( + user=admin, + name="Test", + scan_type="list_of_values", + list_items=["secret123"], + scan_all_workspaces=False, + workspace_ids=[workspace.id], + ) + + DataScannerHandler.run_scan(scan.id) + assert Notification.objects.filter(type="data_scan_new_results").count() == 1 + + Notification.objects.all().delete() + + DataScannerHandler.run_scan(scan.id) + assert Notification.objects.filter(type="data_scan_new_results").count() == 0 + + +@pytest.mark.data_scanner +@pytest.mark.django_db +@override_settings(DEBUG=True) +def test_run_scan_no_notification_on_error(enterprise_data_fixture): + enterprise_data_fixture.enable_enterprise() + admin = enterprise_data_fixture.create_user(is_staff=True) + + scan = DataScannerHandler.create_scan( + user=admin, + name="License Test", + scan_type="pattern", + pattern="AA", + ) + + from baserow.core.cache import local_cache + from baserow_premium.license.models import License + + License.objects.all().delete() + local_cache.clear() + + DataScannerHandler.run_scan(scan.id) + + scan.refresh_from_db() + assert scan.last_error is not None + assert Notification.objects.filter(type="data_scan_new_results").count() == 0 diff --git a/enterprise/web-frontend/modules/baserow_enterprise/adminTypes.js b/enterprise/web-frontend/modules/baserow_enterprise/adminTypes.js index 1c9c996baa..7cee5e2d53 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/adminTypes.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/adminTypes.js @@ -3,6 +3,7 @@ import EnterpriseFeatures from '@baserow_enterprise/features' import PaidFeaturesModal from '@baserow_premium/components/PaidFeaturesModal' import { AuditLogPaidFeature, + DataScannerPaidFeature, SSOPaidFeature, } from '@baserow_enterprise/paidFeatures' @@ -79,3 +80,37 @@ export class AuditLogType extends EnterpriseAdminType { ] } } + +export class DataScannerType extends EnterpriseAdminType { + static getType() { + return 'data-scanner' + } + + getIconClass() { + return 'iconoir-search' + } + + getName() { + const { $i18n } = this.app + return $i18n.t('adminType.DataScanner') + } + + getRouteName() { + return 'admin-data-scanner' + } + + getOrder() { + return 120 + } + + isDeactivated() { + return !this.app.$hasFeature(EnterpriseFeatures.DATA_SCANNER) + } + + getDeactivatedModal() { + return [ + PaidFeaturesModal, + { 'initial-selected-type': DataScannerPaidFeature.getType() }, + ] + } +} diff --git a/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/all.scss b/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/all.scss index 36ff700ccf..533258413e 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/all.scss +++ b/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/all.scss @@ -24,3 +24,4 @@ @import 'assistant'; @import 'assistant_onboarding'; @import 'date_dependency'; +@import 'data_scanner'; diff --git a/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/data_scanner.scss b/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/data_scanner.scss new file mode 100644 index 0000000000..f1aed2cbef --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/data_scanner.scss @@ -0,0 +1,52 @@ +.data-scanner__filters { + display: grid; + grid-template-columns: 3fr max-content; + gap: 12px; + padding: 16px 42px 25px; + max-width: 600px; +} + +.data-scanner__clear-filters-button { + margin-top: auto; +} + +.data-scanner__tag-items { + display: flex; + flex-wrap: wrap; + list-style: none; + margin: 0; + padding: 0; +} + +.data-scanner__tag-item { + padding: 0 5px; + margin-bottom: 5px; + background-color: $color-primary-100; + display: flex; + max-width: 100%; + + @include fixed-height(22px, 13px); + @include rounded($rounded); + + &:not(:last-child) { + margin-right: 5px; + } +} + +.data-scanner__tag-name { + @extend %ellipsis; + + max-width: 100%; + color: $color-neutral-900; +} + +.data-scanner__tag-remove { + color: $color-neutral-900; + margin-left: 5px; + font-size: 11px; + padding: 0 2px; + + &:hover { + color: $color-neutral-500; + } +} diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanActionsContext.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanActionsContext.vue new file mode 100644 index 0000000000..0a2653cf93 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanActionsContext.vue @@ -0,0 +1,117 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanForm.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanForm.vue new file mode 100644 index 0000000000..3914aa3c0a --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanForm.vue @@ -0,0 +1,514 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanFrequencyField.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanFrequencyField.vue new file mode 100644 index 0000000000..82465f7328 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanFrequencyField.vue @@ -0,0 +1,28 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanLastRunField.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanLastRunField.vue new file mode 100644 index 0000000000..8ea523934f --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanLastRunField.vue @@ -0,0 +1,28 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanResolveField.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanResolveField.vue new file mode 100644 index 0000000000..9f627d01d1 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanResolveField.vue @@ -0,0 +1,56 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanResultsCountField.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanResultsCountField.vue new file mode 100644 index 0000000000..3e044cae7a --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanResultsCountField.vue @@ -0,0 +1,31 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanRowLinkField.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanRowLinkField.vue new file mode 100644 index 0000000000..d0210a7d6b --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanRowLinkField.vue @@ -0,0 +1,46 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanStatusField.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanStatusField.vue new file mode 100644 index 0000000000..f10d55b630 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanStatusField.vue @@ -0,0 +1,33 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanTypeField.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanTypeField.vue new file mode 100644 index 0000000000..1980e582a6 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScanTypeField.vue @@ -0,0 +1,27 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScannerResultsTab.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScannerResultsTab.vue new file mode 100644 index 0000000000..fc8583766c --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScannerResultsTab.vue @@ -0,0 +1,183 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScannerScansTab.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScannerScansTab.vue new file mode 100644 index 0000000000..37a90b7e2e --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DataScannerScansTab.vue @@ -0,0 +1,234 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DeleteDataScanModal.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DeleteDataScanModal.vue new file mode 100644 index 0000000000..f0f44138f7 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/dataScanner/DeleteDataScanModal.vue @@ -0,0 +1,66 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/forms/DataScanExportForm.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/forms/DataScanExportForm.vue new file mode 100644 index 0000000000..c7dd3049a7 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/forms/DataScanExportForm.vue @@ -0,0 +1,28 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/AuditLogExportModal.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/AuditLogExportModal.vue index 05cd407e77..de447b0b06 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/AuditLogExportModal.vue +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/AuditLogExportModal.vue @@ -89,7 +89,7 @@ export default { lastFinishedJobs: [], } }, - async fetch() { + async mounted() { this.loading = true const jobs = await AuditLogAdminService(this.$client).getLastExportJobs( MAX_EXPORT_FILES @@ -105,7 +105,6 @@ export default { this.loading = false } }, - fetchOnServer: false, methods: { loadRunningJob() { const runningJob = this.$store.getters['job/getUnfinishedJobs'].find( diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/DataScanExportModal.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/DataScanExportModal.vue new file mode 100644 index 0000000000..db622d325b --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/DataScanExportModal.vue @@ -0,0 +1,180 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/DataScanModal.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/DataScanModal.vue new file mode 100644 index 0000000000..bf254a36c3 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/admin/modals/DataScanModal.vue @@ -0,0 +1,106 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/notifications/DataScanNewResultsNotification.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/notifications/DataScanNewResultsNotification.vue new file mode 100644 index 0000000000..25fdd9ce20 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/notifications/DataScanNewResultsNotification.vue @@ -0,0 +1,43 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/features.js b/enterprise/web-frontend/modules/baserow_enterprise/features.js index 776e68e59f..03f89b719a 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/features.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/features.js @@ -13,6 +13,7 @@ const EnterpriseFeatures = { ADVANCED_WEBHOOKS: 'ADVANCED_WEBHOOKS', FIELD_LEVEL_PERMISSIONS: 'FIELD_LEVEL_PERMISSIONS', DATE_DEPENDENCY: 'DATE_DEPENDENCY', + DATA_SCANNER: 'DATA_SCANNER', } export default EnterpriseFeatures diff --git a/enterprise/web-frontend/modules/baserow_enterprise/jobTypes.js b/enterprise/web-frontend/modules/baserow_enterprise/jobTypes.js index 3827e8a2f1..4ad18d44da 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/jobTypes.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/jobTypes.js @@ -9,3 +9,13 @@ export class AuditLogExportJobType extends JobType { return 'audit_log_export' } } + +export class DataScanResultExportJobType extends JobType { + static getType() { + return 'data_scan_result_export' + } + + getName() { + return 'data_scan_result_export' + } +} diff --git a/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js b/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js index dcf91db63f..d5f6966f92 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js @@ -104,6 +104,7 @@ export class EnterpriseWithoutSupportLicenseType extends AdvancedLicenseType { return [ ...commonAdvancedFeatures, EnterpriseFeaturesObject.ENTERPRISE_SETTINGS, + EnterpriseFeaturesObject.DATA_SCANNER, ] } diff --git a/enterprise/web-frontend/modules/baserow_enterprise/locales/en.json b/enterprise/web-frontend/modules/baserow_enterprise/locales/en.json index 361e46b3f6..c8c23bc0fe 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/locales/en.json +++ b/enterprise/web-frontend/modules/baserow_enterprise/locales/en.json @@ -61,7 +61,92 @@ }, "adminType": { "AuditLog": "Audit log", - "Authentication": "Authentication" + "Authentication": "Authentication", + "DataScanner": "Data scanner" + }, + "dataScanner": { + "title": "Data scanner", + "scansTab": "Scans", + "resultsTab": "Results", + "createScan": "Create data scanner", + "editScan": "Edit scan", + "nameColumn": "Name", + "typeColumn": "Type", + "frequencyColumn": "Frequency", + "statusColumn": "Status", + "runningSince": "Running since {time}", + "idle": "Idle", + "neverRun": "Never run", + "runNow": "Run now", + "edit": "Edit", + "delete": "Delete", + "nameLabel": "Name", + "namePlaceholder": "Enter scan name", + "scanTypeLabel": "Scan type", + "patternLabel": "Pattern", + "patternPlaceholder": "e.g. \\N\\LDDAAAADDDDDDDDDD", + "patternHelp": "A = any letter, D = digit, X = any character. Use \\ to escape a literal (e.g. \\N for the letter N). Example: \\N\\LDDAAAADDDDDDDDDD for Dutch IBAN (NL00BANK0000000000).", + "listItemsLabel": "List values", + "listItemsPlaceholder": "Enter one value per line", + "listItemsHelp": "Enter one value per line. Each value will be searched for across all scanned workspaces.", + "sourceTableLabel": "Source table", + "selectWorkspace": "Select workspace", + "selectDatabase": "Select database", + "selectTable": "Select table", + "selectField": "Select field", + "noCompatibleFieldsTitle": "No compatible fields", + "noCompatibleFieldsDescription": "This table has no compatible field types. Only text, URL, email, number, autonumber, phone number, and UUID fields can be used as a source.", + "frequencyLabel": "Frequency", + "frequencyManual": "Manual", + "frequencyHourly": "Hourly", + "frequencyDaily": "Daily", + "frequencyWeekly": "Weekly", + "hourlyWarning": "Hourly scans can put significant load on your Baserow instance.", + "workspaceScopeLabel": "Workspaces to scan", + "scanAllWorkspaces": "Scan all workspaces", + "filterByScan": "Filter by scan", + "allScans": "All scans", + "scanNameColumn": "Scan", + "workspaceColumn": "Workspace", + "databaseColumn": "Database", + "tableColumn": "Table", + "fieldColumn": "Field", + "rowIdColumn": "Row ID", + "matchedValueColumn": "Matched value", + "firstIdentifiedColumn": "First identified", + "clearFilters": "Clear filters", + "addWorkspace": "Add a workspace", + "scanTypePattern": "Pattern", + "scanTypeListOfValues": "List of values", + "scanTypeListTable": "Baserow table", + "lastRunColumn": "Last run", + "resultsCountColumn": "Results", + "viewResults": "View results", + "results": "View {count} result|View {count} results", + "noResults": "No results yet", + "resolveResult": "Resolve", + "resultResolved": "Resolved", + "exportToCsv": "Export to CSV", + "exportModalTitle": "Export results to CSV", + "exportFilename": "data_scan_results_{date}.csv", + "exportFailedTitle": "Export failed", + "exportFailedDescription": "The export job failed. Please try again.", + "exportCancelledTitle": "Export cancelled", + "exportCancelledDescription": "The export job was cancelled.", + "emptyTitle": "No data scans yet", + "emptyDescription": "A data scanner allows you to automatically scan the whole instance or specific workspaces to check if patterns (like IBAN) or predefined text (like medical IDs) are being used anywhere.", + "emptyResultsTitle": "No results found", + "emptyResultsDescription": "There are no scan results yet. Results will appear here once a data scan has been run and matches have been found." + }, + "deleteDataScanModal": { + "title": "Delete data scan", + "confirmation": "Are you sure you want to delete this data scan? All scan results will be permanently deleted.", + "delete": "Delete scan" + }, + "dataScanNewResultsNotification": { + "titleSingular": "{count} new result found for {scanName}", + "titlePlural": "{count} new results found for {scanName}", + "description": "Review the results in the admin data scanner." }, "auditLog": { "adminTitle": "Audit log", @@ -327,7 +412,9 @@ "builderCustomCode": "Custom code for applications", "builderCustomCodeContent": "You can add custom CSS/JavaScript code to further customize the look and behaviour of your application. For example, you can integrate external services like analytics or social media.", "dateDependency": "Date dependency", - "dateDependencyContent": "You can define a dependency between two dates and a duration as start/end date and duration. With date dependency, if one value change, other values will be adjusted accordingly." + "dateDependencyContent": "You can define a dependency between two dates and a duration as start/end date and duration. With date dependency, if one value change, other values will be adjusted accordingly.", + "dataScanner": "Data scanner", + "dataScannerContent": "Scan all data in your Baserow instance for sensitive patterns like IBAN numbers or medical IDs, or match against known value lists. Configure automatic scanning schedules and view detailed results." }, "assistantSidebarItem": { "title": "Kuma AI" diff --git a/enterprise/web-frontend/modules/baserow_enterprise/notificationTypes.js b/enterprise/web-frontend/modules/baserow_enterprise/notificationTypes.js index b9f4311e98..7e674671c2 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/notificationTypes.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/notificationTypes.js @@ -1,5 +1,6 @@ import { NotificationType } from '@baserow/modules/core/notificationTypes' +import DataScanNewResultsNotification from '@baserow_enterprise/components/notifications/DataScanNewResultsNotification' import PeriodicDataSyncDeactivatedNotification from '@baserow_enterprise/components/notifications/PeriodicDataSyncDeactivatedNotification' import TwoWaySyncUpdateFailedNotification from '@baserow_enterprise/components/notifications/TwoWaySyncUpdateFailedNotification' import TwoWaySyncDeactivatedNotification from '@baserow_enterprise/components/notifications/TwoWaySyncDeactivatedNotification' @@ -81,3 +82,27 @@ export class TwoWaySyncDeactivatedNotificationType extends NotificationType { ) } } + +export class DataScanNewResultsNotificationType extends NotificationType { + static getType() { + return 'data_scan_new_results' + } + + getIconComponent() { + return null + } + + getContentComponent() { + return DataScanNewResultsNotification + } + + getRoute(notificationData) { + return { + name: 'admin-data-scanner-results', + query: { + scan_id: notificationData.scan_id, + scan_name: notificationData.scan_name, + }, + } + } +} diff --git a/enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner.vue b/enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner.vue new file mode 100644 index 0000000000..334825a409 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner.vue @@ -0,0 +1,53 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner/results.vue b/enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner/results.vue new file mode 100644 index 0000000000..f9ed186624 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner/results.vue @@ -0,0 +1,22 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner/scans.vue b/enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner/scans.vue new file mode 100644 index 0000000000..a804288e8d --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/pages/admin/dataScanner/scans.vue @@ -0,0 +1,20 @@ + + + diff --git a/enterprise/web-frontend/modules/baserow_enterprise/paidFeatures.js b/enterprise/web-frontend/modules/baserow_enterprise/paidFeatures.js index 58c94d99ad..4fbc4b4013 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/paidFeatures.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/paidFeatures.js @@ -286,6 +286,32 @@ export class BuilderCustomCodePaidFeature extends PaidFeature { } } +export class DataScannerPaidFeature extends PaidFeature { + static getType() { + return 'data_scanner' + } + + getPlan() { + return 'Enterprise' + } + + getIconClass() { + return 'iconoir-search' + } + + getName() { + return this.app.$i18n.t('enterpriseFeatures.dataScanner') + } + + getImage() { + return '/img/features/data_scanner.png' + } + + getContent() { + return this.app.$i18n.t('enterpriseFeatures.dataScannerContent') + } +} + export class DateDependencyPaidFeature extends PaidFeature { static getType() { return 'date_dependency' diff --git a/enterprise/web-frontend/modules/baserow_enterprise/plugin.js b/enterprise/web-frontend/modules/baserow_enterprise/plugin.js index a4bd79f25d..d738a012ef 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/plugin.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/plugin.js @@ -1,10 +1,17 @@ -import { AuditLogExportJobType } from '@baserow_enterprise/jobTypes' +import { + AuditLogExportJobType, + DataScanResultExportJobType, +} from '@baserow_enterprise/jobTypes' import { registerRealtimeEvents } from '@baserow_enterprise/realtime' import { RolePermissionManagerType, WriteFieldValuesPermissionManagerType, } from '@baserow_enterprise/permissionManagerTypes' -import { AuditLogType, AuthProvidersType } from '@baserow_enterprise/adminTypes' +import { + AuditLogType, + AuthProvidersType, + DataScannerType, +} from '@baserow_enterprise/adminTypes' import authProviderAdminStore from '@baserow_enterprise/store/authProviderAdmin' import assistantStore from '@baserow_enterprise/store/assistant' import { PasswordAuthProviderType as CorePasswordAuthProviderType } from '@baserow/modules/core/authProviderTypes' @@ -56,6 +63,7 @@ import { } from '@baserow_enterprise/dataSyncTypes' import { PeriodicIntervalFieldsConfigureDataSyncType } from '@baserow_enterprise/configureDataSyncTypes' import { + DataScanNewResultsNotificationType, PeriodicDataSyncDeactivatedNotificationType, TwoWayDataSyncUpdateFiledNotificationType, TwoWaySyncDeactivatedNotificationType, @@ -68,6 +76,7 @@ import { BuilderCustomCodePaidFeature, BuilderFileInputElementPaidFeature, CoBrandingPaidFeature, + DataScannerPaidFeature, DataSyncPaidFeature, DateDependencyPaidFeature, FieldLevelPermissionsPaidFeature, @@ -124,6 +133,7 @@ export default defineNuxtPlugin({ ) $registry.register('admin', new AuditLogType(context)) + $registry.register('admin', new DataScannerType(context)) $registry.register('plugin', new EnterprisePlugin(context)) $registry.register( @@ -137,6 +147,7 @@ export default defineNuxtPlugin({ ) $registry.register('job', new AuditLogExportJobType(context)) + $registry.register('job', new DataScanResultExportJobType(context)) $registry.register('license', new AdvancedLicenseType(context)) $registry.register( @@ -190,6 +201,10 @@ export default defineNuxtPlugin({ 'notification', new TwoWaySyncDeactivatedNotificationType(context) ) + $registry.register( + 'notification', + new DataScanNewResultsNotificationType(context) + ) $registry.register( 'configureDataSync', @@ -219,6 +234,7 @@ export default defineNuxtPlugin({ new BuilderFileInputElementPaidFeature(context) ) + $registry.register('paidFeature', new DataScannerPaidFeature(context)) $registry.register('paidFeature', new DateDependencyPaidFeature(context)) $registry.register( 'timelineFieldRules', diff --git a/enterprise/web-frontend/modules/baserow_enterprise/routes.js b/enterprise/web-frontend/modules/baserow_enterprise/routes.js index 85b527efa8..72bd380c3c 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/routes.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/routes.js @@ -12,6 +12,24 @@ export const rootChildRoutes = [ path: '/admin/audit-log', file: path.resolve(__dirname, 'pages/auditLog.vue'), }, + { + name: 'admin-data-scanner', + path: '/admin/data-scanner', + redirect: '/admin/data-scanner/scans', + file: path.resolve(__dirname, 'pages/admin/dataScanner.vue'), + children: [ + { + name: 'admin-data-scanner-scans', + path: 'scans', + file: path.resolve(__dirname, 'pages/admin/dataScanner/scans.vue'), + }, + { + name: 'admin-data-scanner-results', + path: 'results', + file: path.resolve(__dirname, 'pages/admin/dataScanner/results.vue'), + }, + ], + }, { name: 'workspace-audit-log', path: '/workspace/:workspaceId/audit-log', diff --git a/enterprise/web-frontend/modules/baserow_enterprise/services/adminWorkspaces.js b/enterprise/web-frontend/modules/baserow_enterprise/services/adminWorkspaces.js new file mode 100644 index 0000000000..309004dd95 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/services/adminWorkspaces.js @@ -0,0 +1 @@ +export const ADMIN_WORKSPACE_OPTIONS_URL = '/admin/workspaces/options/' diff --git a/enterprise/web-frontend/modules/baserow_enterprise/services/auditLog.js b/enterprise/web-frontend/modules/baserow_enterprise/services/auditLog.js index bca3c6b006..4024b664d2 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/services/auditLog.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/services/auditLog.js @@ -1,5 +1,6 @@ import baseService from '@baserow/modules/core/crudTable/baseService' import jobService from '@baserow/modules/core/services/job' +import { ADMIN_WORKSPACE_OPTIONS_URL } from '@baserow_enterprise/services/adminWorkspaces' export default (client) => { return Object.assign(baseService(client, `/audit-log/`), { @@ -13,10 +14,12 @@ export default (client) => { return userPaginatedService.fetch(usersUrl, page, search, [], filters) }, fetchWorkspaces(page, search) { - const workspacesUrl = `/audit-log/workspaces/` - const workspacePaginatedService = baseService(client, workspacesUrl) + const workspacePaginatedService = baseService( + client, + ADMIN_WORKSPACE_OPTIONS_URL + ) return workspacePaginatedService.fetch( - workspacesUrl, + ADMIN_WORKSPACE_OPTIONS_URL, page, search, [], diff --git a/enterprise/web-frontend/modules/baserow_enterprise/services/dataScanner.js b/enterprise/web-frontend/modules/baserow_enterprise/services/dataScanner.js new file mode 100644 index 0000000000..1394122552 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/services/dataScanner.js @@ -0,0 +1,48 @@ +import baseService from '@baserow/modules/core/crudTable/baseService' +import jobService from '@baserow/modules/core/services/job' + +export const DataScannerScansService = (client) => { + const url = '/admin/data-scanner/scans/' + return Object.assign(baseService(client, url), { + create(data) { + return client.post(url, data) + }, + get(scanId) { + return client.get(`${url}${scanId}/`) + }, + update(scanId, data) { + return client.patch(`${url}${scanId}/`, data) + }, + delete(scanId) { + return client.delete(`${url}${scanId}/`) + }, + trigger(scanId) { + return client.post(`${url}${scanId}/trigger/`) + }, + fetchWorkspaceStructure(workspaceId) { + return client.get( + `/admin/data-scanner/workspace-structure/${workspaceId}/` + ) + }, + }) +} + +export const DataScannerResultsService = (client) => { + return Object.assign(baseService(client, '/admin/data-scanner/results/'), { + startExportCsvJob(data) { + return client.post('/admin/data-scanner/results/export/', data) + }, + async getLastExportJobs(maxCount = 4) { + const { data } = await jobService(client).fetchAll({ + states: ['!failed'], + }) + const jobs = data.jobs || [] + return jobs + .filter((job) => job.type === 'data_scan_result_export') + .slice(0, maxCount) + }, + deleteResult(resultId) { + return client.delete(`/admin/data-scanner/results/${resultId}/`) + }, + }) +} diff --git a/premium/backend/src/baserow_premium/locale/en/LC_MESSAGES/django.po b/premium/backend/src/baserow_premium/locale/en/LC_MESSAGES/django.po index 0a79fbb0e6..0c9632227a 100644 --- a/premium/backend/src/baserow_premium/locale/en/LC_MESSAGES/django.po +++ b/premium/backend/src/baserow_premium/locale/en/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-03-16 14:50+0000\n" +"POT-Creation-Date: 2026-03-16 21:52+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" diff --git a/web-frontend/modules/core/assets/scss/components/all.scss b/web-frontend/modules/core/assets/scss/components/all.scss index 51543bd040..c0abfeb16a 100644 --- a/web-frontend/modules/core/assets/scss/components/all.scss +++ b/web-frontend/modules/core/assets/scss/components/all.scss @@ -206,3 +206,4 @@ @import 'field_constraints'; @import 'workspace_search'; @import 'formula_input_context'; +@import 'tabs_body'; diff --git a/web-frontend/modules/core/assets/scss/components/tabs_body.scss b/web-frontend/modules/core/assets/scss/components/tabs_body.scss new file mode 100644 index 0000000000..64f5b93fe4 --- /dev/null +++ b/web-frontend/modules/core/assets/scss/components/tabs_body.scss @@ -0,0 +1,15 @@ +.tabs-body { + height: 100%; + display: flex; + flex-direction: column; +} + +.tabs-body__tabs { + flex: 0 0; +} + +.tabs-body__body { + height: 100%; + position: relative; + background-color: white; +} diff --git a/web-frontend/modules/core/components/Tabs.vue b/web-frontend/modules/core/components/Tabs.vue index 6cf70258e7..2bdb1c4025 100644 --- a/web-frontend/modules/core/components/Tabs.vue +++ b/web-frontend/modules/core/components/Tabs.vue @@ -153,7 +153,7 @@ export default { selectedIndex: { handler(i) { if (!this.route && i !== undefined) { - this.internalSelectedIndex = i + this.selectTab(i) } }, immediate: true, diff --git a/web-frontend/modules/core/components/crudTable/CrudTable.vue b/web-frontend/modules/core/components/crudTable/CrudTable.vue index 84536f0984..ccd0fd0a18 100644 --- a/web-frontend/modules/core/components/crudTable/CrudTable.vue +++ b/web-frontend/modules/core/components/crudTable/CrudTable.vue @@ -1,104 +1,124 @@ @@ -211,7 +231,7 @@ export default { emits: ['row-context', 'rows-update'], data() { return { - loading: false, + loading: true, page: 1, totalPages: 0, searchQuery: false, @@ -219,6 +239,11 @@ export default { columnSorts: this.defaultColumnSorts, } }, + computed: { + hasEmptySlot() { + return !!this.$slots.empty + }, + }, watch: { rows() { this.$emit('rows-update', this.rows) diff --git a/web-frontend/modules/core/crudTable/baseService.js b/web-frontend/modules/core/crudTable/baseService.js index c7108c8c08..ceb78d5526 100644 --- a/web-frontend/modules/core/crudTable/baseService.js +++ b/web-frontend/modules/core/crudTable/baseService.js @@ -41,5 +41,15 @@ export default (client, baseUrl, isPaginated = true) => { return client.get(baseUrl, { params }) }, + /** + * Fetch items by a list of IDs using the `ids` query parameter supported + * by the APIListingView. + * @param baseUrl The base url to use for the request. + * @param ids An array of numeric IDs to filter by. + * @returns {*} + */ + fetchByIds(baseUrl, ids) { + return client.get(baseUrl, { params: { ids: ids.join(',') } }) + }, } } diff --git a/web-frontend/modules/core/static/img/features/data_scanner.png b/web-frontend/modules/core/static/img/features/data_scanner.png new file mode 100644 index 0000000000000000000000000000000000000000..73b30ed02453ec52e567297cb45e52670da6ab34 GIT binary patch literal 125099 zcmeFZcRZW@_dl*$9aMFprB++iuC4Z{TBR+5m~HJ5TVjv4+G=amtleRhh!sf?LaDu} zl^_%~L(L$@_j13-XWZ`l{rB(j`}`r2E4g0Rb)DBa=XK8WJkKlQp`HdkEekCb6&1af z=3RX%s*~eXRMgPZCxIh{GI#rdf5*J^HB_kzJFm?FA7pGzwCr?ssrZ5Sr>Ty;a-=$O z_!Hod1^5F_{Yd?bsi>5xwC<`H`W{;xJr!tZJJ7VEc3O)w z{O0!P88O0!%JL>MH)sC+4bQnyPec`e6z)->%yA zo;}ZbriOy^uyaifsDXW0oZ0v4G~ndsp8k59E0l`*bXp&EUS3{oTAJXKCr>V(Kku$* z=DvEG^YE)bz0~H>x+iTI(hRd7J8@o_>aSloj#0~SoXxm?exsg zZK=W$a`^9erwBZQ;uU%IzrX0XAJNfeig+@~yOJg|s;6Gec26T+N+74^1eD3vgW#eF@RL-XUDIkCAk%4-$85&4cyPUYt$tOXZDX;6)S z{o0-H(dTk=KU@-u(nqU3Lz;90Rezd9biA)T2^v$rR-Tc&nekul?Cu zHGctoh2MIe$Iu`1nfu^zeWR{e@a|3NB|@9XM!yqAZ%i6X$%M5QKe>^$973sRiee+^ zB=9Fxw<$ajysfRP+n@OQnEuy?j}s9Wgg)G1y3Gb%pgGBIM&z#wNB1*Ds8L4*-*XhfC!s7Jja5z$>vK}3jGo0r^}sibpjI5T>cJvM!?)KT8MV z6DEPTOhSo?La6~sgiBtsc7fkJL`;5v$RuhzTKX17DEL7?00q;+%%Da$<&jPB$qwIJ@aA1-$e^a!?;oH-BW)SgYtO68NfqG`j1`tL!~gZ z-ze*ulk*-@A;IgQ3fKQllu`yy9~j%u)ejVA2B{TpcAVRSYs){V8=I_n0keB9@-^@9 z3a)*LF`}|d#Ej7Z$+G)B=tOnzks8k>*^>)0UQ*mQZ#KV4fZH74j`tRTC%r*2Z5#N- zA6@S)+GrVDv5?+<-bZH6!j@HrO;^n)sZFpseFs~87IBS{tlB0+CD0TpVqHVJ+$4(D zE0dsWXqdHxNsH$P_XZR?lDnLbvJWJ03}(49T|s*6>K`)apgZyE1eUX{F$}4kYgsKD zY|olAiBlMwQ8a#k`Mi5zl`vgEf#=dFSuu8lRjRq?243z_?z1qOY(0{x=Ole~wgnaq zIGn{_Zo?OE98ie4VjjJhOUR`gE*xCnxSk?TAFNH?(8nyTkoUJHxm>X}^fvq5AMG~N zwuL!MlY4p|dDd?zOm7n0R_G>h`#W;mE4fzYwZJK; zuOz-xtQ#{+b)Fli3Ob~UO~Wv*y7VqWp-6J_9WV6DGP+UwDHES8)Rg@7Yza+Nm%)Y; zCcw=u44mfwJU>{(YqTffId$dg3cC9 zE?z4xr?n|FkV`jJDv?K*1=|m?;t1S|U@vO6fQ|7dSq104G|(`|Hhxm8Nx~3=iQx~K zB&MfZs7v=Ko7i!L@5}}f;J1!J{zHq@4vdk{r zoGKmiDWGVqx)RIusi&uBDG=>C`YVwzlf96@X2G3-f~SdECkk4i&3PW|TeK$f#W^=s zw_UgeefHBT zo_a@h%)C>kl=%;t2i|$Hp=OzBQRlJC9?LcDi-p!bApG-}Jd8Yf)!ohFOIt3KqO>bu696 z&medIll2!#M!LAqlJblFNGz#786658 z1?_nDj8QthY0QIU8l7vL%AZyL3jJ|>_crhdhboCfpx z-pk(WtPmSEQB7Uv`OOaTu#*lX{SvuEeR@hHtTjQPbreSAs}8W5#LW~`cBl$FYsgHk z71=g*rmr^FGnCWjR0nKUcr*fvlp{pjmi4^Y&vC|V7VWa_nY=M`js7>f)pAd9c(O5_ z@RjRfsk4fMbVJYi~6L;NkFvY+i&*zr@@gHB?ruCyiZOuDg%T&itAdRErW?d7$i za?X66q$tlBHcWY<%kY>AyQQdC0XNj#*c z#`6S>Tc-1|?K--mRui^CM3ba3R{6enn&e&x+gpK~7BY!FIHBFEih)@t{j9WH7L6d+ zeeara55#IG@VEMHEHn?w-ZQ%w2eM`Ce)F{~&Q5YJ=QNx3VgxUHkH=VWQiSZiG@3_V zGo3I-3n?kQ&){YZtLtpKD|SS-p8s2lL-)XJk&^@~Gl{|xJirur=3*gZ7_MenC=l7I z3Qr+?nfwL*kO1Wemu98iL+ zkJBqGkgZl|kQKBr=obI0BegtHH(7iN$34Kaut!#cLP{2AQ|8M>j4g*(UNHP{l@~TG zGii*>6l5!G-Zlb3y}U@38>K;nbA$73mNTfi`h%cFS95g5)^)_Wyb0Krk;=+8v(-Y! zz{8ShAdguQ#{BM4m0=Zk7%a3H+m=_M3ab(+_L$_6UNXwl=eGF$>x`S=>k@4uohb^q zo0Mh~#0#XyD2t>lG2DHCzZZ9jMEO>iR=~q5UU=Z_Rv!BGL1w_z&Vs++@te*!wt+a( z9_5El0u?;8bp4@JTg(C9Nl^&M2$Y)ANtSdjWSuMo!G8r4A;V)UU~oz3U8n3!H=Ry9 zf3}?a$vnu7z!V3%)|eRU$zhsudP+JCx`0& z9Bq2XMjreiL1V$)@U_QUyy_e5BDS+fck$i~?{i|pei4q7e6Wx?mfCb@hbPz1-YR%I z@+{lVbC+(Z6vy1>rx;qAI6^)_YJwB5&=Uf4`9b_`;tvkY#{2?8&AoJ`iR$tiRAKVM zxBc6F=Y|*T{H7DaJZp2azs!_Dm&b%yC9;rwhR6L=3v{0V`PW^YVVfUcOA7ZsnSDGc zQPSHYVS0L(vj?Zhj(wwI@Iq}(P45Yd+$d5tF%!L%4}!15YM*rd;@xvtE^%17NZzvx ziFxwmM*}lvV7ZVtOXayWV~oPO(GT5UagBlxiSP3?LUSiCZ6}+GC<% zO)rj^As|>>wV+8ut(#(@=|(t6%xQ=Q=z&|i`aO15W?op2eXzvlAnC@vcRj4mVZ9Ft zF5rakX?NCa`+jxkSLkQAOaI^o*2nm2*+kv<*TXk>zwbU_>FZmL?Q!V0 z+$8Txp<|Y97^$G*J*EMSLM-nKQ7I5Nq2fTe!s4~XCR3YX`iIyG2T+hy*QT4a2?;tjA}I3FHigfDu2uC?V1# z01_8@!3N~IDJvuM9p#4HmP*u57Rz3jS?n+2KMnVn@ti8Qe13gt7eSBLW8-a&8fN?9c93 z>n6)uys1_i0-a|Tv1|{&Ga$VBiX-H`r1O9lkV0ocNhz-On{ES*pwury?5A-1Xgcjq zr2~1gLh<8o*ZtNIu<4lKqB4iEA1H};QBlwZmjQlLLQ;^DiLbSF``-GRJ2@z3$xuMd>d2cVubA{zF$q9bGX45;BC+79b zMPU%XQ*Dk9I+cn7@d=w04_R~p`DZ0D8Y@wpq8Rj&qLwb}zi%PS0l$rriZ<9tA;4x6 zjG?jGi~{qLcdipqz3suThIjQtP>3>4!Ag_(xMankoS~t>hV>Nuat0VV^BOeHJ-^Y} zd3hojbFzGy((eyxb&PSY!7K)Zbf)>a-mgrNbZ)kVh=Bx7Maee~W2*-r`Ou-V}te#M7;UZn53e2M>Sc%u{@TAjKr6;8s zeLPncoYF>u$mv%zqG+UR_IFZQC!6xo&z?VDe`ym`p=Xw2D={3|lo`Aaa{BRgzVRhx z74BxUy$WKv#@1i6S_H&17L)w6jdPK05pg*Vp=CPMDEa{ar|_n%v)RvdV8-GQ9)zqQ z3fgm{-?$Q!s$gFjxZQy1-7J;D=Oe<$3fOtzMuqP$BkmR(fsHNtwcJ#t+h)6aJCN>< zLn1e9meOcEmDR*kwPz!L>j8*RsuXiJUFuJi?8MBI8#ZI+7FPAe<1w7DxT;r1d0;v* zXD)l`idSd1O__wv3#8=@yrX4}#xP2iQg%Y7QVEs{MHjFh3QvAsCI<&>F7|J{YHcPq zYGYiDD_6!APcFr9Syix>lg9#*h9aS?uN=k~o~JdDFvO`S$kyve7g;ayhCidNpnYSh zfOQrSr6cWYlSbrT!^^jRzWr2cs#LEbseG1>!D~`{Sa+nTW`AsSD`YKlKLio9_$0-$ z>B5bB3mXxzHlZmb)KiK*6BGjA`L!))e&k!!M@DXL&!|nqi)s>*@&4WWx~3mpiCP2x z=x1nX)$&+~gQmf^>IfRKbUB|G)Q;PbYSFRdW3i?9*xtGj_~X+23gL9QSq`JLnf*Ni zxt3;UxxdkF;Q*hik9N6A-YfH4W7^9Qe7_XUruXYU9_74%yYXvo$SvrA5=)n}7{D(q ztQUfwgF%`1)Sj*KA{We&k$Z~yU}Poh7a={hyt$BFK4Yu?;R@+q&i+m^*)gQ~#GZw0 z$e8MxUSjlSfaj~1PMb|kGyO#~dy0kT6{S({F}0BtJJ7q$8m_4M<%T^LQ#?n#3F}kr zk`&Dx41j1r7LO|f$m~}EjOs-=)B6+jO)l4zocEGP=j&FdTLF;q14S)*1uur*Z_aWa z)Z4j|P09O+$V#d-%H6~^@7MCEhqw)MWDRjV{xLutw$*$3Zphs0*q!r#wh7Km+9^1d z>&nmU!qtxaP{BYQvz7$o!cVB~(vNw8;huQ@Mb%;44a7>el;E|)kZ|04B;Uzn5Y@~*!ETVt=UE! z^?*YG1>&f1Z$hSze%I}JQ!eLP(;%)xw9YFq!#pZ@Q!7sDw%+#|xW+6I$*T0IzBs(F zwKLJ|x6P-qQ_IzJ{JZi5oAq)C0ctEM=DK@RtDj#2dyM`3L*-u!9`}=Fbu$82A656@ z8?bqxw?SJIO@;p7UGJOD+Dh}2Ue13Iy)=c|-y8W5YKL~}cY?jkh$Nk?*t;z9=)REt zsJ&y)XKvRL(6K@@%-CL;i=Uv(eQmv7(J-U@LF$VAm)0t|2zzO51T}Ks&quPGVj&VY* zxR)ClPB{(;?}dU8D))N&YKsI^_)bO(b!roPz{Ah$E36QRGm$|mLB#6bQU&GQ-tm`X zW6SRF-r%1RuPhkTK#s#4RwiGqxkf=OoZ$YEJUt^CMTc%GKdTy%9Np9j)oYYl=Tj$~cw^4lH{<=tqUaH!T8J)Db9>%?Gp5YjDFn!2!h|{M-mg z_8wS$Z^vgHMpoaEvrjVu;dF5PFn7{?TVor7c{TRe{7#zQ1c*= zV2VFztjKp}-kXfxxpQ&AZbk-t6e7z>p{6PG)MmXlO=?kPpa^(56KsSoE({k#SXiU0-YAMmIaL{9X z2};4tSI%UHG>qOyZ^j>#FF+Bei)(&bU3a7Rw^Ga7a2@#)y-$W#4#c>g+ToCsrkl_3 z-|;)s?^KJ~;3*w_{iCZOP;y=FYCh-*DBmz~f6vO@S_EwShK=%yBlx;5i*&r)QWcnk z)iJa}4Y#{$whfMMF~tR~Y--EcCS;C2te&5D zwvZgZ6s>fV*}^*v4x}h%PoGW=+fyo77u~A+t?&*ePW@*vqrqJzOkdUOvr~;uY<8DC ze{=avzv!pN=2r)l_DDGihi>Pkdxhx>pVVGuxew_@y$6Bg+c)3$2R<;fgrARk_s%=P0?hWnh|&sgTZALS| zu-qLLwcW53{G*)nzbd9O)zsUcsXJX`8X6mA^8hGhdiZyTYazn40v~PxtuKNQq?XcL z-hHAqrMK^YJm6WJfc9P=kty@2DX472yGlCtK4cG`4c=SWwN3zS8gHlEZt-GSa!8Dy zh>uO970zk%s7&HOu3t|K>}kL}meyfLzPykH|SOg|}T7~YHBqq z(%KI)XoPkFpXourl>#q3q9eL2X3DM0r${3`5@|K>Sj?EjRMbTtm#~D-`IB*t0dYU@ zK1^qCePy&!u`~vB>+XgOA-Eu=a}A$(FlwlwOA1o3UhyH}he4U|wjl55K#1779G?ZL z=Fbgn<|cl9x`W#Gh~Sd7GVK((<#G@L)_fKcG9w4h&KO-pc~G07T$ZYTp`7~T?K6T$ zP&xtzIq#Nv4&AE3`QjBpf&G3bF)X%Z^JNA!JOmR70`=$nRK3|+9#HC;-b{Z=93I{t zQ|t+vN!TD!upfep?CeV)x78U@T9#}w-y5VyRl8}v#Fp3^O}HuD|8tR=in_vT+?QA| z5>k-N{wXrSj`55XIQ$vST|bXff)Bh z`j$?^FZX(dxR1-|{g^ z2GXG`gDLHuM0^n%_b~zfOhS$4&r!&~$p$jWLt_v&5%}n`m}D(<+`3HtNq&?N)UxvS ze0y)k7M9fSG3fG6lJ%{Sm{?t;$FJeHxesRT5T}t+Q6RV>^zG)d(pSX!XwU&Tps~#0 z@nfz$^SsYS=@30^b2U0t2!a))D{?6fZK4+FvRUx6a<;XA=DgcmID-`HLgmzyg0k}>(o`nr}54+Mi^HJ8wG)MZkjujEVbto#Z<1aGh{HQ3Rin| z(OenhzaeXHUOc|y)wXy=&lVSXX1`wd2xBEQw0NNZs|pLh&`rdUOyNA!3q}>5>Umj8 zV@}ALa}s=9S_K_HOKsb1DVuD

%X~ah}tyl}wQW#)Z8l1H15X!K#v#y46*iiF{BU z%Ax1}#fukh`sXEA`ySM^zP|9qb~WD7lx7eQs!I^C8c4U>_MD#D4;j6|^gX?Koh%s? zQqvewm1%FmYTSx9aE%=0k%Ki-BA+-ww1CRPO+L{Ukz#<`TS=4NU(197@K7=b11dMh1^Fbpf4z{iFQ25^O`rAM$ zE#a<)hMzR6Qt;>R4Ujclr_O_d>{)O6_(UNqJ|W3K=b_>-xVM~DlZ*h%1+K|PUGIb^ z02#qM*LK0g;p!z;0i7h_Bsvk7X?jt_^rYJj4d5&>1xDluc3+NANWs%$VnMm^pUZf5{+6Plt@wNfb<|V7bkqr?99f z0zJnxWl^NGFbIiR-ar<+9qiZe@0}j+JtlViuZ^PbR?PCDQJ~|b?tO5`RfDsaj^D(b zau>JT3&lbNwSzO)=0>#c}Vn9S4&yyMHmBay2-uPP*T-WmKW>avN7Y9gvaA zOP7M9Eb+o#OFA1S0g!(V>#24o(+|C6TGoiz9rY$9iUr_{IVUEb$PQSjRqREjPD#?c z^XR3`2g@&51^cl|D~|o1_%gb`)*G@INI*FYAZOa+xOEcSgfRPk;rX9c;vxf>FJCS| zFuTP9s4KRw)Cpx)=I_##QN2onDYZ^Mke~cT2ywY4LAkcRG-@F*-bwDn@LI^iVv@z} z+ooC!M$0`%v9JXD%x#QWQ`7_yUni+NA4@8+<}J@W%$klPErTgC3Csge*LD^H+AI;F z6acDlO~c;t`h}%!9$+quX_Q4zXmAAaYeZdRc!VJRdI8jeX=c3rbejTMw|3+zM;cL= zm*I6Av*5;yr1@LQxnH}h~==#M{ z1n-y3!-dmjn;RojZTj>3S#Xk`aLeKBxD!rPl0Pw`!XJ=EiOdeZUVAQZR(Q@o9|8!X(BRH)Lh#@OB+Tw*s%qL6(x&Ua{iP0pU-MSoR<+Cv0!T3aq zJ=vb$vg!lj6{)SSu?HzAQTAG5Kq^LkT~tu;hq&D6mvlFv44<{^b?}}en1$ypvyhkd z6pdqpF#ust&AVExUyDkwvA3o}y4FnE8Kdk#UZ>8nds}}@5Cp4MLP>(m`EK|;-PE^J zH3vc3_oA-dz(PqqUYk~vjjVqX>(9YQpy z=|ov#LJ&m}kr4F7!Zw2~NhmrhD*mu2B5WZ$wUgiFP1XLtqVE6pku&r3Wlb%u9{=k( zyVoB>t}?L8b8>NoZ}1L@D7UrEI2`-&N$m_?PvRTu3JRJQVmxZQfZuW-_}pU}vvbnq zDtPFF*DIF;F0QL+*D{as0=BGh4@b3zcbl;5Yx?^77GIqvzky%RCKu!%ma-C$9Y5vh z++Moc$DpS0cH~RNY|Rtq%9AIV;!m@&E-#z!LTl+x>|ZSIf_Zv+qW2M@FaOFcV^GSt zqR_UT^)+HIC)={nty?pVylq0NO#I;Kv^^VGX$cjH7aJGuG{N%2RtoH6W@6HL5Lc3z z4CIkrAl$~R2g+#PXO*4Yo85+5b$yfNQ0)I?fv%S=rNSg3HxhItl>aY>P@%>-)AwS; z!}1ypwckC?>(|Y%zrEKyFX=oTDNTG2AfoE?KM+E2V?+X=ayt6@rOrNndp>uhm&`N1 zO(-8#HU0U;Ot*Q1rWX9l2phP!UbQygQcWJ= zwr$;tZb>@0Eb|4)r%cz1%sefZQ|z z2*$L%z2lw5lkFe8JKlBakkj^4%p>Ng$o@2Ej0Qj&U2P8kc^i0$us%M3w<_y1EPXV; zHO-m&*bM%R7w+h>{w2)QX?g&+qc5)w_t%0s)Yr~Cz|XvvV!i7Ae!N@*fMV)BU+VTZ zfi`gSUV6aK(y&u$w~xO2e-a{U=MK4u>6MGDf4Sx!elebQ>9qVz=Z8!8{s>Mv>B9U{ z4N6T?l$gU$F~_iQZ)cv^UYqwcI_lQ#)k3ild$n}(-NSR6QPj(Ny6vbYXyv&Aqn2l@ zzH z_9lP;&OTp*0^=&XGRXvPxw|S>iF>=ybaqpwGB9`i*T1@xyt+2VHUFWV|IKLtH@b>b z4vUYE|KfKbA#?mt)L{k}G<9_orlxGp0P$sLXsE$3BjK;2xtS^|7`SUY8er5P#XR}< zu{(f@CB>yI()|bNGXs+}qeM;h?-P$4b}9=XwCFjNiu?!Fusl?agVR+1R?#Dejfn${ z>pQA)&q#v(^)(e)ju@ouU=$k0P1mC1RQ$*8=D0T%}nj`Jw&@9fKJ^>VqWKKcrh5wk) z)ly|Ar~I3|yff1V<3oSpufoi4)LwC~_@(d8d88KpTbZ_105Z0cxq9cZztwKeL7-z| zIz{JO*uT#_a@?s?fTHcXGu}D%_v&p9Q1z3Xg=tP5!L$2sSBT*_^!4?iscZkG%SXNQ zj0B9Tu;TKy(|;w#Kjum~_&l%-F&Mx1`k#K}e-GI#30T$t3GaW(^8bYQzt!YWiT?kl zdFOr6QB%$DZye^?nC?|HL&aL! z@wbEl>Vp19ANaBs2|7T;w zk-iG>txM-+CeL*3?@?@PYcYrUT>N;lgcUQQPb{z?ktkT+< zisW>I1a~IYB|J*wVqENcpN9l2P*sNZ@15PHi!zhP(r8>bf+X(dgAg}4f@L%TuC3s= zZz@Y4g?#775N}6w*6XN)#+*$!i)(lhy0H4p6k`Bt5dCB04IEiWX0HZn6imL zzTCYW*?qex-`NUmf+44)zkhGO2N3vx3V31qbQGJMMO$0*XLTP-ko17*T zuJpkcSk+iQeL9(CK3uAga$>Zo`G5`=oMKBA5*5`-6t>{eh-ObXNk6|JMX&)%y0P(T zf`i^Qt!!_E4j;Z>qx-`T9H^HQa$_yoMu0^DtrO z;`-(qa%6AIe??)O%XDcaRGoFmD~M1z-|>&;g`bR#3|>dklRF8=dl996Uhj6v+r~DO z&GgmTFASQK_voiWm)ro{boJY*%+}8I6|r$tuqg;ykeyB96cv5+GI;Hagv*e!W#Gis zLkngS0sq23R@~o0t`c6$nkpC%P%LX6P}DN~=luYr_+8c3ZeO8(XB1`7N@FgdhX2#2 zPb}x4=V*GVb$>tmWD zkhSK9=4j9KLSu|d2G9`JQQ{+d6fUmN7`jfk6&q)8Qx)FH-dEe%7%%YfV`!OEzq!E7 z#pwj&vG`%+jQ{a+5&_c&a7TI6C2Uw_#?|v4eW|?s1DedlF6B0M7dUZg7$@S`Z_0}> zrCZ6Uk78T#To4-6N$O-9^@VP-OMz|8QgwvmTS5GXbOAwtx4pCnsAI`x5hiZ(82M}a z;2TA184>OWN39?KIu<4`;(qVA4E&)$zBL;BH36C;2gA|T2hNx|x6lbY@S&Xhy}5?m zY9bSW7K^v2uYzBLI3V&esde@BQ?0^m)05FP@fBXoTQu(Bv^4WlGXZK+Ko8dxi?r+4 zZ|3FgyIyGXpp676vqMn1q{M7`E|W zx@!*suEk+&7pv2DsgPym{V|sHPq0LkTiM6ONNG#YW9BX{X)gH7lcs9>zb9`m`)BaV zK^NU07#En6@_)*M9MI7Kq~>R|rudSH?|K-`aa8(r3CZP~K1A0s!CxS;s8^JL+bmr0 zE49^mcbWEVyd++=S2|R{EGJ7&y2~FI&Bw>*+KwqNN3bk40A$LNol7?k8=w$S6Ji(j zoxiJx0ZHX9mwO|1xiWp>=PzGuXQC`vG#jwc7~BhpNQFs0C~q-+qZRJf9LwFteN$$r z1KHi0161K!Vz}n&Ma>4v9M#VBImy7qkYr@bv+`%ApAyp@>l-^NNf&^s);sviHiuNe z+v+^k*T46+w?ju$GsfywbnJ4?0r*#!)h%_2b)Z1He<+IAEf^5->)}S zLVzljo1*favCR*YZ$F7m3keQVsyJ)`*&DI}If#mS6jB$OlsHHixeS*st^AZiOjp=v z#Ds=gPZm@pD;|`T3>V;+Y;xWJtz*T74>GaY0T^otpE&-3p`qbDWM79_v3p5XzRFY0 ziN|xq1DZTA^5tCzWru`NpyPY>SL$*$;pNr;;K|L#-_f`mMHm&iLh?Y!%JqP0pSfXV zs$qAnz!%4{DjJqrNWXGn+K)VgHiUl3Q@=cuysE65xv#@LdwPcHzKP*O?ABNuZaE}) zMMOkAwZ^5l+CaFIge|xLalN;CN#E}=C%i%!1qh0JmTUjPxrCffGtV2+$_M~ho!Jyg zW36wY&pg`@2k}7HLK{Gn82JEUG9eK|l*$hHV7cSP!4Qq_YI13y$B+;Jtfu2Y9p6rwvYbYMum1-y8YK-8MD(02lxH!o)>JwmD2kMAa=&epVHEiOTO| z*QM3wFAsDE1?03zcuDgiX}?v8t9LvL%A8?&&K%n?3d##%RTK3C=;yKw_WKBQS?R;) z`P;@60&MYRR|{pAmN&n7{s&d&J@kr`5%1o`=Vk=nO_7EO5(6pE13xD?w`jF^ZvDP+ zhTl-r6z#H8D`lA>k*u_30&jcflUA-;NqREZJ>gj~ zlOdPwnXQdWP5ZqDaH(2C2QA}`DK%V0vs;h}StB!Tr#CNNh68iBkw|c>0mIj5?VM{4 zt1Jhz6}aC|L$6fPhvO7x}V&uUutuP&s3r=++51l9N&VL^ZorX>H;$_5aeY&e9R@CXnuws zbN%}DT_a3lB9^@Pe)j7vU{%^MON@`dDF^)yc#q=4u*NO9T4T4+;v#(c!X;MThPRBD z=ng)AzMkzqQTOvsArQljHtF3dAc?mn2`pe3y6DWBIz^?R@b8TcT6ZjpVx#QsRi<)u z&2Gxew<|CJElDjcElwr6R?PigXUMo{d#gVxq`5k^IQXL6t@{VFwfw%x|60lZBmh0j z@d6rm-jHUOh{G^0rtc845^CK)`h;T896%7F;!4&2ULU={oWIQ#{W{tHc*?N$6HwBaP+KA*=KG89?M5DJ z0J6iW{^oLGclSf`6?S&v!^&YO)qW$ljttY0IAlyTaDS^S4bwmV5Qm=SKhb?s@qxyQ7D`!)e&Jk|Emi$f)SpeiAg zV$EOe7Mufhz2a>BF|OmaFdzpK>#e4ME?)7KuU#vszj3tMP60cZl<#^8-MMtAVPSoO z56c`LRc!qn0BkTMr!TbP%dt<&| zVZpl2kKrBfUrhPrp=s&0p8Eq8>{Jho=oUh`Ffl=eQP{Dspk?M3$vj4bx~2#(hy=WpXGH^ z{#jAb*N2v$uix;Yguat%d++y(fg>@Jl~gX>yh)M+0EK$}!oWL{d?M2uqB1gDS;QvO z*RI1Czfg#R0K(ukh%}sw)yL?uMZ9?rv^L=)NMVA@YrW1X#?nw%_wF3k|E zm9rNMGxLmTy1&u6vuA&-2`!`0J_7|{-i5)UL8o@Mo#9*j00X117K65ryU$^OdLz!m zV?@Z!@gFQP$^(>Y^<)DOPgEYDN}(kG;p%{B@gne-6uDP%)nj@7LI4YAv2Z4_Lv&~O zZKAA`MWOjsKEtmuWRp_=rz9(q*Q{Ll%a@hbFx7%* znK(E^fJ%MRx93W6T5-88-;TfPeRPuMR7>O_IcBUnK>PX04Rd81u8+&>4M}R#D@~be z>C@!hjX~li%F6wNok5FL{hFEHa1-c#Cd>A;J7&0@9kXrQEeEY$PCn9Q7Sp@3Fjo=G zNZ|9sxRKuQjo|)PZ?&0y#eSt_Zz;%&SE5OR;GRSR@$(ow-b_VtsY3TqKV0Hy;3((7`?d#k4BrPb|wS0v`%(_u< zS#kbH9$#ND$7JJ;L;B8Hmi4e;bk3V_N1OrjSL9TooEs=kPdL7xdlF zm4mwQ5`)*0SqFdf`h*#Gk!C~x0!dqQu*r<4rBH?&bMMWjJP2B&<7*l0r z=4~w{q8b2WOwSn!7|Cm!!q?bfBiDiQ{QKG2ulJQS1lIAAZao872D;+`8#vRG2g_{R zYhA;uPiyu8#+LW;D$u|~dn_&K`Jb)Uo#k*7pBuv1KT38oFdD(f*q!`NfbvvtF`cUM zbpKgHsnfI*AE-3TK9Rgq9L0)xI<4)u);zyh(_t5ek0{Yk0S;K1*$|HgcV?Jb)H28!*AssfVg zV3Cqh-iIF6temWQzrE|1c=v@B{LO_P$nSrQ?_b#k$P@+UNhG5E;T=R<6d zlk3?2-l-{oij(M8Ljmu53<|9_-w3)XC@9gl(I?!E>Gj%Eu?*UvAN~jmxT=e^??_4z z7-%*{L@o5qj0x5klo6U5APHopD+QTNX!r3;UQL&~fOR_o*K`j3|@KS0Uf0Coa_- zh=Or}pqEXZ_m41DZ@M%b`=eY+^RUuC9DIfO$EOW%OuzLti^vq;o=LLOu=AZVL%Y1U z&m_%IXBE^*6C;K0?yMt=BtQDgS!~2D`}0x)YtYq!iz~U)^$bcKVu5R;H8cWhHQSGx zzOl~o4+bWsH+vDf;`92ZhzE1<{`=rpZW()~RCXjj%08KKn}iX~7@lK?Ef3!isTO<4 z9<(?oST9Yahb@h8a|Hg57ztWdy9gKJ5>H_gGQA$e9#|J10}gk?5%9sMa4Ur`p>9qM z;AhL91v9(Sr5|Jj6JkR=Z#AC|#j1{WG=GUh}`=Efb;zcBvm16>a28R?6 z#Myck$gka30+LD3mu(8-ujpC1q7`?_+EX#N`+j#I3j=p?arDg3Oh52nu&fCQw%?HqtfvYXXPpuvNa)9~t&Dc>m zx4-2DG~qn6ieb{1xl1?&jnNUVFcBEh)=Wcd?rvf;r=peY`~;@XU6Z*5zjw27@ zxWbhsq-#3fvr7Sk9lV56m}pIKu4-G<@-ESwLx7%UVPRrgukwyA!tGS}^^ocac~i(qMIBeWPq@fspcGf#ou$^RQ8-f8VWn;kYGKh*Nv`z;l^lC1pHDt#9C8{Vkf zqs^9gx7U~*H-@xJx)OEOZ@is1@s+e0-H9xPESx*UW=cfOB&R5brnC09`1xDK2^0tG zCWWovA?wJ}Ho2~hAzwFz0P}3?ntsp8?nom7(B0*CK!4MDp>x8N-Z0SbM%)Efd-hoVoC)JmP$_BM?HRG1-RvBl_7oZJ4L8 zrItdKdFL~uF{}B*DLM8e8a5-3DY=)%Ktm|dXw-%6zIpriR?#A>Ta_MrNL_xf;@Z+6 zf1>3ee?GfHrJn!*|P`A1LZaE)ISx3+L;)W7% zC6QYvsRiiS{juQt8=yQg!m2VhCT;lX3~gPAA~j8PQILkh;;aD%%fc8tLgT|Ru;QVW-*$RKFR=Z|0Y(G>hFUuqR0Kl ze8dBQ@bTE5^h0H*fqWi@%?dM93AY?;>H&8tIcOFGgE)a8mP-%hZ%L22Bc!L+ z7E6l(HrJAO_>Vk`KJhjMVQ{7D#fzmxqaN5KVm?0B?koy$AQk5uFX8+Kq6JyFM5?z6 z%zd8-I1)CW&J{ETEt+o3tr3bZ>jZS z$t&6_W?YAHtF<2ar9G1}BcHyBup2DTd;W)|aOKwo_{jjE=d4T&5kXWWg@-4>#ysGc z1(vKL(bi3R5rgBRat1awFRFg;8URSPbY;G{c*&lrtHy-7&^FF_gYt( z*PL_NMlStJW>!N5K({Rz!WYrXEyV!4veGzI zk)*cu%mzxaUgnm;S*BV5EXb(2xlR5PGVSWWijA<<>wtXed5HFnWddff+??-araK(- zpK1dNlt!`^+4~o&CclSd)4q_A`tadgoW|YI_O3eoQ~bf{i!!BBmy*i|mxedo zzh_z1ZR-MJld65BT8Y!dm9b&G!t3{AiycO~@$Qr@tl_kQ(xR~S3*&0*AA29Q;U4#- z9PE^BPbRkzk%PYb9syR2JGw6nN^AQ`dI-a+s4?$uT2xJGNSoUjTOLsS5L_-D!M-f9 z7paHX4khBu)0!}(T1cW5y|-mKd;4E88~^0v_W2oT`yYOTk6BN$IjU}aaM8S(w!qP? zK+Dg!RQ!n-mEQg?Vxq|}+WcblPc_kPeKqw!;U7G3`6!CHsNS6UNgTYhg>VI5-Kt+@ zesZ)`)uJtSpsgU2^?;oYYuOQ>y)2g2yHHPn1SWu0_tXHZigllal`87ADK6EuBu9Tx zsb%=1i)~BQW%k3WzIz_Iv`=N&3@U90d6&&&v_&oTw4*&hsq2_L(4*QEy=*w#yd2>+ zJRJv_d?!b`kjCZO0V*>US}xNgya2RDQ-#_7Ujt39ftZ{IdThUy_4&U)OB@Q2mv1Qu z1%Kz)tLbno--p>@iuxQa6PdSbA%#nG%=N?Kt}3wtF#u`#c`5p+{mR|vPXU8KD^D59 z;0)tmK>@%u1E~EYdAHYzPZ zxjzH>ls?sdk$pdl41nm#v^E0v&}ELK(C~y=M9sRxr!8EN zYRUxj+VX92JT_KVi{;dC08{TTf?@%IuzakbG52ky@Ms^&AcK8wvTk@e$VOWep)ud| z6jjvGrIn{!2H5II_JR2yM}&o_R*dL-hs3xaiVW+%*2c|B@f((Djk<3@*h>p?NlPx> zg9}!&%M8pIk>xy>LTFtKFIZJ9?QdoW^$EZ>lZwXIFdwgEy#!a03U5IKhXlt{md7}r zn1^|MC99?Ys15)afkF0H%N1;^WsSLkhDQ?RFtF~tx6b$w5Yp}W;9c0|pw87Vrd_V5 zgX4*?!TkMERC4P?)1mwfJX~nKOX@u_66@C#wDnc9W&3plHUE~!Dd>bAALIrzbDV5F zH_{)Z%^1w}+QIc=g#+=iiM&KneyRb6V7_?0VNa~eDQsO*9#6@OjE+bO32i0u*S+%T zm=i0J@k90>=kLcKuUdavM0pO?QJa<9UenRY*DcmCVq1+3yDl*relxaJfVc!q!de~j z)1MnazTiZ;O6@mh_}A^n;>>@3LtJ65;BSZMVt#!2q9x%p{wj9S!Y#v&GL)H6vyG>f z92jDL=>098uiLHRUAC#5sFH3=L|l8_3ON`ZYd^W;DvCD3BU}=qB(4nU?ur8TcKXDv zRMli*@Fk8PlH&Z@d~MWk0FBR6ucv!6TQ}Ly3Sz)+!xHlp{g-Wj<4y958Yt>&oMOb4 zj?=ZjG&kT#y41{8>fGamI6x-dXX)|pH3nqN|YEWPl z97wmC4wz*QJ**{`gp}AH!K;jt3zf@R9itp|Q7;DpK^@Am!4*+mJ-m!RaCKX!@h5)7{-47i{ zZ(Z9X&}n2SNRZYT!auCa?nr+OA70pIyq@eUE!HjRx8tsLcHaOX9Y|9)X&-KMSDBh5 zlgz<-00Fz-PFHh@bs+DhBH@!#Yr7!b7dflW4Ty-b3?29Ns0m$rAcWk=$atc*YlzY* z^lT&)o&4)Aa1sR~Mpuv|=}#QpN$p&oez}siHTe%(E9vO+=Ea^2>}>W3#WcQ)ijTi%Np>Jx4Qhixk^RgU@L0rmQ~XdjDx z;iCnN9Q=rzP(gBCcKOm zXTMZ?{wfz4(u4%_wqr<_oEW{C;^8TmIA!^QH?bQV4A-vRwP`_?I`2}^oVMPL-(qpy z8xDN)7hSXeQK=i(DtL4c@EAJ>m?Zk2K6i&+!F0?m3|h~!x>9T!ijiwoZro&Az7j=$g=&RFj!SFmefLQ> zfIjVL@$URwvXxI-mh!Ch*zC>`owt9N)%!DtvZcZQ=+Z`XP6j$2E!R;#y1m~$rFmDo z|7TzDu!^AUEZDRu6VuoY>>YhnWD3)mWp|=Uw+sf|z6$C~k=C=eQt4{x{<%n4xV51T zGRuZtZ-?$hM-ETP-XrAeKWiPPcO@>YjgAVo0KhfAH>(9Y+a-f`?u&h4G3YFT4AHN4 z`A8uhZMCEzsYLD(0ltuFDKMsO8_4tE*~U2Z4-5>u78+u3x`2Ief4oDt+)-k_;hr~P zP_vs*cl7?q^s|ZClm)MV0JLIczAtC!=iN}na#PG%*`w*n@2>%R!XG+x7vssz;q7Rf zKCTKv8vE$znTb$H`3wV|Io z)6khew{5V|W1{LnfB*ij6=6yJXh{${m1D59KAnUEy~k!kw-y+;7f^yQiF?}0Z!h2e zsb~RiA#Y)tWxXg~jxJk0{3K0WJkO6w!hE0Aq~Ed&hSG0ogFq1NHA{narSgB-lj)It z3SQ1FB+8**SqjI1un6RQLsHlgne#~5pm?%XATBB)Vg6)iXT9cC#5y0YLdup^a|IZs zRJe^%?pO1k4j-eeuo)rHjez@FU+Xi**JDScbTX%TLOY`urs8butG;~64MAnaR)I6! z>V~!|U*nH$KL-UM)qdcx;&7k0V*Qj~akZP&0H!q-0u4;<_FK2Bv+&zv_A>=uWf+mu zFRS^~cp7=n&l(5DO~7&rB!l6X%H=3WhxfJEo7LN)#|!npTZMEvfV#fbVigGeGcL@< zQGq$m9L(icfTr}HfJ_=a3cDoUR1a-0$;>et%qlzqA5=OfYMWJ3UKpG)J`e79e8aI6$3%8 zYv=%v_B%T3XTC>$70cGWR!oK|;_T|kn4^F2^kX=Y>8kN}6QOaEcl82K(a zsqXMmk}%?iYn?-~oL|Esk#xjGK*N-mU5_VF4v)0hPR4F@1sAs%=ojF1 zYPb8gBaeE;WFB}nq7YZ4AK)E1^9@MGY)h}rG;<3yZVX0?TC{zAk)K^~B}>~#+lE)W z)>_X@Dfq^`-(lp@l37>&MX=d|V54H8nqL*Ze4Xlm)p1xXf06ND)Xm zf@9;8eX!50l-%kHOJUH0Y`D6cAfDOCxA$;d?%*>{YcS|L_nm|B_mRsnI=+p*3v9-; zIL&9zKDy$Yx=9C}JJ|hIOWmAn{UXzHIgw%gIl5%lOF?WjN}y_`CT zH4@qs#J~M(eRgShR(tsCfj1}fN(~fWgVoH8^sT&EdGGFBV##8=m&p)k?B~xyjkO!gaurso zdzrYwvAkUZ>ExWIdY!rP{-g>=K_~iK!+TI67nqBQk+Fi{LY-VkE<6lu4gDJ}oAd>s zwxnw+{U-Y7fLi(smCYz9qnV8yS4UYCwdz`O*qIRX+ZdF7B{hzdV>auH*<&#|+8jKx zSlZcT>y{>$-Sw)7BGtSp<^8_fA~kAMZT?wi?VAQ^d-c659d|#EAnM!>yQIrds?W?k zeLZ5aK5grh?M@d~Ti4~Ds5;mpoHA3lD^@&~#kR1p_KfrP3VzED?L#PVjZx_p#!?c} z}h5*AHj%Y~5a=S)Ci~b|;4Spw-bESG&8wo5I1dB;T!5yZ%JV zH?!w(^5`HYSRs5FzBkvTPI2>r5slo}u93@+b~w>&1klOkYJI~xUrh{D#$`9&XCv1?#Qp^~K6Bz|-Z^%e|930z@UY0+OTr8DT2#&*=^}yNX3K@4Y zo!@F9R>N)p>gGD}&SQJ=Xj2CiV?`_v=n0&BNKP3$yh!RQd86~GB2mZ8n$kr&cw|uQ zJ)D29SaB3$YPVQEG^H+E+~GSVcDM$uc=o6xNvz#>OUJL(vMZirF;)njE>)93zsfeY zTP?Vg1b;8Ba@9_+l0C;HlwY$k!kxZrHIXiNLa(d~U}Qzn+>|OFJ+yF87-* zY|tz9I%I9_1P|udT}Pfn__93^q;@_1zk~J(e`SSqV{pBQh=`uagblBt;Lb|CKIL@k z@TZ%KjqVcBTOYluibtyZ9F|=Xc?pab}u1YHK$vW6ttU@!!6k z)pCUC&!12~+7wfTJULuD+ST+r!oo%^CbnEzXtoa<7!(TmNU*$i>XTMm~U&p7RK9#lvR12)~&lwLnd+Nj8Qa0>E!*^%E7 z^HBt4q?^Lto)&LK^4ba6DM64MjSm9T%HxbZyGHldylRPKC6BDST&qHooz0fojXl5D z9!%>%{m&-a*w{eSn288_wRWSe{@3J*$WgT1k=z)X)|v4`uM0BXYq>m5ktO$qhP~>rC!?*M64$#Z^s={`P_W1r2QsR0}?b_8tF_B@08;gzse-{V%sNNSCVOiZ4@=0O_ zjE;ka#&kpxfb~{iU}$rROIO%3LUgHli!Fmn&CHKa_rVx&L&UXeyXrwM6Jew9X3LK) z)>561ehlUm29Qfe2_M^)2(HAGD9Rt!>e~hz_v>A%IJ)0pmfd?I?b7rMj4qPXeu2+; z%31@1Z_;U(WME;6O`3w%c3>o3lI^qUX~xmJ)3QfLx)wG~KkXBP>7(y42Hcn|U^>|A z@T~1l=0>-*Ay?}hlY@h&TEaJIjU8l~2fl||xc>U_l5}t5U8G!`{Ub|NT7nBEebLC? zvFw81n5l)uv};$374KSo5n?{X!Xg|!GO~8_zSdoQ3c5W`(7ci|4@qf3(c~o?KZ#7rp4h6Qhwt2a8F5(IB7f*Dmt*j!;FVs~VP8fi}F*@JC zx)5SEeeK-oq-wjLxY_Dk6r*x>Dc50mrvU-XcHdp)*)`kSK{*CSqo?Jr3HM?q+KMp} zRfB|OetsD%zr^w0^L@eh=iMo# zeMWbk=2kBo011PCt*{nZ|C4c(^(?6^1`Sx+44$_6ewSDN|0D!6ZR$4!b0?CoQ(JGDgngt?%V$UKE0vs9zM+g8qEZ! zDMcb}QTh3A27f&`>|Aa(v9Y$kFm=vmG%xQZ2ORE(7XW2ZKq@L1%~)&>Zpy^b7?qZk zn1Ul0XbMY89-b~69v*gx8nDqD@&6ZJ?tj;Q|7i6#b^K*7jx&fOXj@S_27D5FVDk+{ zmVUPD;2)sm_}QtVi)bJ6r2ywHhF9~oxrDY+zQUZo2o|BnRHO2T0lLT$Dc_|4egm{H zx{H=sAhG$g)a%Vk(iqOE4y_Ea0Af?ydQF=kUY6Qpw%fQUQf^P!B_B~dQ$Tiiqq#QQ zopd!8MgJgC7OxZ-$C*$57O^kMV1x)~i-V%lj4bx{JW`Nnw<_ON?8xadQ>pBsrhkQ% zi=fgjnc=^mb;cJoHr^tv&flY>Zf`2vk6i-$M*_AuL`Q*36MAJ3nl|Clkr{r%(&qf+ z(59#N`Y*ua$~JJ?{Vh}eRB1QB0nR57r?}I$gP_CfdP8(o5v(FSOC*G5?|ysZ`lfVu zqC^v5+00-dEB3oh9yiVS*i!a(}h?%LGyNEI!790g4n#9T<9 zoUI}dkUur3^&rH%$7Ln|maQQ5JLH1Kk8E3ka2+w%B^J8;sHKs;(3{p?J4CuAHn}|* zn^0!He*RD1U$sZ?IfH~cX=v`@#nW=O@LTk1E5n$vGk5O{;YCn2sP>!7q89_-qyT^TBt()ua0_pRgwH1~n}HVJ$5GS^%41d`@S zc#@93=O~7Tf4!WpRItskeIlDbIfWIz5f)vn%I#w3 z;Be_=RJFENMF^nuI)}5P89(Fo0sd*lhc1_akC8D29Y{MH`~>+9bTdxj2eYxefQu%A zUbPPuqDGWe!r877(pQ&abc|?Bl2!zHc`Hb@hwtfg!C-J;!K18&GV5AXy7}4zJSt-W z6v>_EJxkodzFtmV-Y$5Y=H&{?tpTdq3pkqv*Vm~!69^s<`Hlg#t=vH`!Q^_4{aA%w zsbx^T8o5wFv?aW~7_6Ru4)V+vGra*M_y|YigkYPP6e@8K>5UHCqy;b(8E{=%T9@53@xevDwEf?WB) zfZeFiB$#fTPb(F>ovm{)DxYE3asM>iqDcx*AA@)bKYuv2Q=Fh!pvA~4@)3v))*CP7 zn><~!|5~?eyy3h_;;thw8=*P*s_fwWXnuBesWNzPi~El}1%xA!6G3$970t~`bO!?S1+M&KsKuGRA-^?7Uz+`XN_KOEL79_$?wv5S z9#OqZLaYf;_>4M89Q>E8_s_7V?2l1z*q<^Ywm^~j`4?4m${`vWY-x@6VH3F_MTv>+ z+F%GYQ4M0QWR}I97mDvkL@(DjC~D6@_z!W$N1$ABT8MZA>ouN62G1uGn!=I35j)RI zcgZj*A6?J*KNvk;R$h}+FL1+MlZ)hFi;ZPM;`$lk%~6^|Zmk%c);ZBp#m9k8Yap_i ztG#xm3L%fFt^*F|9dNJeMe?BXtKs)t!V%A<%C!qsmxcfi7ZIWVvK=Sx-V~G_SRgqS zSGX0~%1Uq+I{2Nz(78<&0Ejn|=BF8M6Nn9#h>8vlC{nGnPA*Hq0HB2#mqNuSnjQ_! z;Gh+5m_dV~wQ1QExGq~`CZk_5Nrg*f`Vc!c{>;>P933d ziByl9nfi5cD~*O4W6Qpy($_ywMxCVSy;NgPdRJ=!8Qqbf~Bm^Feyl;grbH z!QLCi!A~^fNEvTrZf?<3i~7E}9ogFu^N{?p(C;o^Au-HtH59Ewlx!A*rFl$0e=`*6 zLf3xfNdrv#Z_QL_6iRDz-o{J3|E!FGc2LRuzFa=9cF;g0nA{ zij5w8);ZzIUJo;@_2rcHcCR#udB@)gHFNk#RabL_!vG4FJ&6*XP-XHnOCYA z_~fRO*GvM=B<;%8vj&Ngov|}!>5%OLHeBEo)*8RU9nEE6X{n?;B0|^VGKNFt$EKt# zH%yEdb<)%*MRHg$3&sv}Es702veOBE@sg~yY=%eRK{2m2b5t+2Uo(GykW<^?-A(Y_ zFT<0fxoDTyr`soLQ+?SUnt#x_6j4kIh)QZ!^vb) zrS8Hla-)bBvefwy1Eead zpar<_4$qZle<8+gU_rSPmfJd&fCKvv_ligf077*k}khP_7m!83_C6o4>8;pwZJ^QJ1# zpI-pG(Z~V=3(jqKJ=G7ttIwtH`^)PT8bKH3is;afiRV)#H0=Q%{zbDeV16vE6+>)A zf`yI=1*BRbq0?>>62rv{)lzz?&BB@=wmlp}prNZAOlQ5A!)cBf6fRzix?XLvWV^>o z%15KSFm`$8WgbOzbd-gKT__y41&o+!&Odsn^vx2?{E3F%ZgC;6cgG^$O$I^#5Z6?j zAq&;w&JB?UpG8vfx1$DxjSRf|%obK&5_N2^C8}@q;~+F9GrFXKmpCr~9Wo)8ab_un z(@eF)3}a(aLd3F;L_GdQOiquM*>{^TaWLr#A=_rlyKPy+=RH^l99(4oEBABkJQ+gT zu^H>@>odzpS?Sxvt3e$PO3I`~mUEi@QAp%@vpFEhT|=H^9MRsBY;=biv9nNDLEo$a z@}3p#$9`UtuDOeY*3vB)$i0pKmS`f8kJHXLDGWR_3YXWOZHQKgKRA%4(^cBs4Je!$ ztau&mTtW4TDcJfwmLrX z;xl9T)WFsZao-565Y8Gik8^yZ@Vz-d=hx*wr$z)oNkCNl(DGM8+@I;gKb2Pc(+Rbt zoR#w`D&2uAmtm6?Ih|g*7l>-Md?n|vzQz)ozNU7yx%_#eDgc`l4D|)K|Mn}p-B_)s z5{IIi>HFf%u=(#ax00IgFW6ovmU{4t33#0G&wv+_*DvG$=7m%*!|=2wxP4(OlOg@? zJ=q@6+?MF0Q@=|KzzHXwzvHrVBl5RV0C@Vxq~nY@`@R3)vEgq&x$r(dG00Cx_qVF7 zf9InfDd07YdI*X9Urz^&pJ44x@Bj8^{kvJ-1RTE#+>BJie|JuSU!nrQG ziUn-d7E_ot|J#fD=hO7$1LC#{Y3biahyS;9|9dmi zcUS{J;5*i`_p0ltkk)y*xxzAqz(A(lr0M4hiN>pPQ?jr`+IjK|<1h6K;Aj^EyrTHM z-{24aQcUizPL5=&Y?)-0mX<;%zez9JJjh&|WMyN^&&X)N6xkazvy&+NmRj*H>*g6g zC5L@Q1wv-cLt?(`cB1a9;|3kQWQ>7?#yLVTx$4|7M|t3*|aJ;gC391_Zixek-xhw@Lz}I+=!xo()l(SLu+*=z@2vMK|-@j zC|#O^m?NOTI4cJQis1y3YkRm~P5D-0ij09RF!7jJ0nk>A(?sUZRh)Ga?d8|y<^^?%+WvYPIF7sZFE0SLCBpB~p_PjI zQ?%nNmMyPeH|r}j-q73{p;*Ty2^cs(dv>lKKnrx=ya{DwN=2_twQTY5EjbVt78Vo+ zKPU#>30%lufiDY2BHC1memr5xk`@}6PK_rcB^HsxOT(o#2E84wk;HD}G7!*cQMUvA ztBXEPQ$4?^>WvlQ!{ys7f<@7n-2QUr{+&YWDfI65sVad9PO!rn#>`yq@m?qxLX6Y~ zgQst5CRl`XSnN^n0pxc9&5BP8>#x6km`Kq^6fgBNnm&JCa*fq~XWSr%7FT*k_~9qAu1uM0rsW$XZCXzVWO``;Y4;^G!j4 zFy*l_>~WA7>S0+y=2tPG0eGx;M}ogbwqIuY)6Ley16}tBNo^2Meczi5ST_99MWBQP z?iw&A6*de!Kof!4s)99g7Sy*O;#_AG+FQGR^&sY=x4Dn_b4 zsO7#xp373VpDmENd7#mw`QnqK;4X_DJH}JXEyinyy(m1*0qx0oTOCA@G;0<%DD?!N z!)HPTC4HJlJ!aoOXeJgOW*J|aBC<=o<~yCD-!kG+3f)4+#HOZlyhEi(?F~XBxGfKMeNOTmDQl< z-7FqfLZg<0gECI+|c5I+0e^#e% z{rjr^-cW_j(c$~;-I8pyWhYX%9syjGm4;q!PO|3uz&C85@e^b8JH*q=nJ^!)t`F&&J@ziM* zgnPF0$OX6{B*vsAGHp|6I4D4Xi}CgGBy*wSjS+QD2m)h+)ssuxk}EQ!4X`f}4WI{Q5KeQBsmpu5;NNgg1cdFk?0{t!6g%W-J0{!>Fpks2@>o;!{$6Z%@Ob8_w zUgthBFg*b~(R|Um2!CL^oS0+3(9hkV+;{(h}> zHyk6+dL)TL%dc{&5C3doB0MEEYs5HEQ}1mP2mR=q-vWBokAR-xKA#G~2lZ-Q)w%dA zBrWJI2<%MD+*u(!rUZjk!P?P9kYGCYwT0qj`o{wUxP4zE+Ke3JHbaQbUJlljgee3Z zw7ZNibT=hSlzP`xC5ZMm^+t}YeG2wgmtKqKH+&$622Ch2k0EQ?Th+0N>hh^Gy&I@m zEl8Lz+AL>z6x*%Uxs$204x~N1iqqIj+J(M6x_9p0+e3dDk|O!C7(D`LTeL9|%a-GR z-UIz#!j=24)B6@Da@Lvw(lO^^!HbL+CKL1YjU(PcXFPrgxALYpkJT&+I3f%hAsE-- zdS&7*+XG<{Fc?9#R1+-6D!Al8B_8-}pY^~aF0?v-g*i)rWycgeOGKMt-qzt$&t)JTE6a6 z%Fs}}f^kr_Rllw~t1gRD&9hw7K1zwFC+@jlN{nuTn;7zW6F>U|51^c#7_#X{S8xonDo;_OL{@m|Ai?d>w|h;sn8^*}=> z%of1;hyI|Oxa9uh_2eHJ)3fR|aVYF`3BxaI)ymx$+}}}$!k1F@`?J6DRAo9`!$B=x z_9m>lZ!Sj>5MC;eC!BpUQaG$|n?{ zyMTNFiWKqYIL9ITx*v1HaM-2>EoBI;C`H`h=SFO3ZhBiwIF zTP7VZTzhyYz?J<}V4+@!huweyyUt4Hl7nR4a;8Jz0D{ z=7Se@a#i76sd@T~7Yzg8<=GTR4PCHe)AZ^iV#}vcx;~d2G&irEmV3=GM9MWE0sTP7 zD`L_(-W{y5xm-A8%(D+J2Y z3_E4;720FOnFdH5TQWZWKcF-uEHaUn@`IvF$??Lq4KN+kdU?;f*143HV>e2Z!qUG zUzC35ov)^0$#On()~jeh$R0y1@won90>htW&0NLFtxCWo7?|5;=VVU^(6VqPdH<+u=x(c;m^J{1U*(1w`Dpv`e|`Ly^dW zUD6hZG>mcm+xCP2s;VqmxR6Elna$#J=Yw5mwEJ4)&?IRvJk@@?Cqt3n$K7)%;L-Q* zHy?+OalnS%;g#Q7n%J&a`_y=re^*Q8xzN3VFGtxQ)>IYOg9~1YC9SnvyJ5K4rXQJG zgkNHen4XTwn0$%*^0Lp9VH=5PeEs7SdBZbwphf+^Acw~Ta29teisW0(PucWpQ!WVJ z=P;0mTp~~5a{&7jYVMbr0@?-1@F~V>WqaF_sL4{-355i(wzlV|Xhu=4!ImpP=l5>R zPbjPuZp#nyLzfAU4B|wUBZ${sMSR?Lva_>2nIp57mxh1#P=4hNKjMFM`*ug> zvbftyjZO^oBG?=#hD4d@T`M{YIQlj&g98wztF9nMb5>3eqZOB0GEK%HfU_45Gg#J= zM}y&H;Y%qr=R`)+@Ib+DI+5gC=U&{Q*wr<4!qzMEM1JkzOr?L~tTuH0*E67wcaZX) z`_P#rMYyN>BOK(pwEJ;YzsHWBOA6DsctiB&?>QC*=QbBls1fpdtN`DcWl`gdn8x_a zqbZryJT(OP%H?j4!5|tG6JPw8#aw2(vOEYG41pPOy8+=WoI^(Pw1(zAbtEMp_ z>~Sbc>MYnkr~8Y0MM}<6i({Q#ySIIvkXehao?aj@xD<+4|IvA|{rS_%nRZsTW?#aR zzsbAq!W#v7c^#q-_yu_b1-|sqf$UjhVvH?B%5AyiOGfL&)PcYC;?XinmdB^vcQ?DyO4@wh7$MfUBY?b(o!u7Jl@3;nGeQ7%GE z20D*mZ;W9u&^cqR7||9~ivx}!DxCdMGeCdCE`BowR8~g0ZJN}B(dq)F6fzxf7H27o z?<~y2R)v-<2>>rZQGC!j!P9#!liGFn$4kHqNZDPNQRihv5_?V^ku*O)4{#uTTSvWi z6boyltQ>3hwC$N-+2U@a*MZW;W~o*RTikQ1EhL18?t2@QSiJ=hn)$r8V&WXC!~d6J^v4UZT0v^%8_kb3GCd1^}GX!O;!kBJ{+Z=e+i5GW# z`1mA=4v@sc^c&sx$ZwpQQCG+O{8FYpUPul!DJQt#qI?Xixj6hf0K$@O3wCljT3t3A z6blUM&^9UatUt!?812|R@r9nVq zvfPni+TPt=y0zt)vB$$AVzs_`;1hzMso!^ua$A)YOg~hk&BbOCq&?e zMR|E8v9U8n>_}TJxA>9}N18)nEOJ(9UsgFHQK|53zJa{e;kOzoq-mHjbH!zMda8Yl zqm9kl;98NZ9~Ygef$rb1z5fJ!R*Pt*i%zV3(DAjo@F4J7pY&!3ok_J~{?6O!enevr z^>m&d4GABtPwXTxM{+;b5WmXV<5FP+^Fny^XBKxEtVe(wTX+nCEUKd^5-VbPgCW z^-!iT)*t@;WT76*;Qq?2-p0G&m>K18AGe&qEWMR`_s#_xvp3Vzpc~$cpINT{rjh`# zo6YR{Q>eFnYD$nR`{FKGn(?Tps5zb@LF2o5z)c%xYECdZojT;UJp1>ZSUMdwhc0#U z_DeIU4c`Nw-v1;<{0EZ3a*pc!DHE=t-(O2`^7MWvFe&l#xHoj^H^~){sh+&~_=MPv zR3+EnJBy~*0oNvM#aX>u{*UMRk8fG%fqBoVNgH#n-~S%}|2^KvQ>KDL=l`)ND;bZ$ z;PRN5ID>_rjMzA_hAUU_#)Yq^nlGXd`Q@R^?Tz|Ys%Zl@iC&cwd^Er&oMWcy?Cf+! zm-Pd^?Wn7l%SZ=6xq0|dX~b3k9-S!)I0ZH?@z`) zm45JwdQ+~V8ZVdhC58~g8^_OU7v>j^E-Mq3^}8@+mBRiftjzM1s^&fWQy?b)$>wrW zKi!qeUk(_R1Y2M?e{P66o2Fg_%oB+v-7%>ZAb_h#hO4HUD+XP#*%7ZhF7Ho)#@DCs z5o)2qkZLVI0tA??m&Nd=-Fb`2XSjM})tdH8oX*e;$N+TmC?EfYJNiI3>e&f5-Ww~N zpQx(G)tRY%57zcqANg`ID#;!0Y9z^c#a`r)7T^9v1*Jp|9F2E$bVwa~XFfLcOWH@! z+5}x>V=_4VuSAgU2J8UrhN? zO+M#i&xziFOvkkc$<+e*JQ7pA+Z*lkkOQdqkZ5>ush+lh#&^@0WEdiz@bmLe=zg6tY=O7*yKL1)7)S=qyE9V?T3l57NRZCL zmMpjj<7rz706e{Zbc@CJ)++sZK-D0)DcR#3Jr#KpAWB>$r7)~7k|3lcm*k!?EIc%E zWl%`IuPG5H;Y_!$z0@Sgmq!`2Ab zKv`S!yUcYL@%Q*KnB_>c3X#FKdhXT->ncgng+*-wlZP!9U$DEA*86aGc!$cf3-DRI zI(oz#@4HldSJd~Qa&kyZ!uTz16qQXNX`nN$OxoMBl3f|6hcDO$Y&>;CKOA32ZIUo zvT7rn^O{ZPZ*>;FA^`^K7r5+GeO&W_M&ekG+tivF78bTD?**Xk+AjNL>=ND`g-*~5 zoaTUr&}<=u`|ww|>|Cn2^Uq7{k_()k1ho^{IXMKNGvVjr!eBb5g@6Z1Z=sUhgGZ07 z@!!l?4%kVT_v9N^7pQ>brbi!BSIPluPS6xCuI}KwYfFB0wWJpbOV^Yc%kR~(zOiv& zdkA%%X#DOJ2B;b9(NOZzMnPFrW3hd2CUx<&G9i7+7(gG2Qjg~bSKmn;W2kios*9r7W`_`RL93iqZ3=6b`?G;uq8|OGB;n2fXPlBDUt@dw#!u2+?uT zP<)0M+{c?t+QO_Gyi z?LG%t(uyeyCF%>hR*1*zmB4!;?Sp#jtlLb z?3}b^nkNTmfEG}Bv<#I-I6k)!>D7;R9A$A@n zBr4U;w7aMReq#$;Q%JB|o~Cy7@?1Y(e?Iz^OAOCrAZn)=S&}fm4jc zysegJl@e`7Np%G|sdBx^y2@IiU!=yei0qHZulB^yr5l+6u8o&vixw0efv{4USyowj z$J;t$rq{(&wa>TOxuRJ`UE&R~8YCq?7bBZlo@^I%(V2Et8@J#3D z+bw2XoFdrfOm1Gm3LRq)iR2gkKDLr;dC_5#=dX#dAGu5f@T|MsN&;f`Ddoq0W5@*w zpMBMHUHHRo$8QlQX((cjqXQ1XTd4~GM{|3r)D90G;?COzFk>@DsYC=4$#dFGKhl%a z+Q9*J|EDJao3Uh%^8$b8kQq5_37K%(i?S4#(%?%my^x6WYuZ*mnJ|D@RV8gr8dz9>#YYPu%$bfbBZl&|Ji8VaFdgxA~k#TsqS(A?^cCH-xK;m$# z;B2|C4AYpcogF2!3FhkuJ0RY48W?L%6j3LW-=lk~iKqWb+^#=y`Nmte8|rs3f9g}5 z=W(XU;`?khWUhkw9OrGaK)OSz%M)ybmn8bh z$z-()98#{g@{G;eV0jBK#B)M9T@1Cv-4?W7=JE+(bUB)9^s3fdoxqguq0CLY;*woF z)|%jtX>psh_tXNm5Ue-E+zeU7W@|}&0EfvR!_?EdyNi6l|Dq06l5b{_j2v(==?^$R zMRAlSqQ--U*RFCY?!tFW7Cc@7eR&423?x@e%LOl}jb~H{yQvr@otI0?EXVx=@MGPA z@2Ggz!RC5Ct2HP%x~VmM16;tH$jUHwalb=DPPs=ir!A2WG*~is9Fg!)DxrKul2}V* zcX3Xd(KQ7IcHXT~4gV|-O~y1TdTd=^;2EUC&oy;sOK#!bC$F|R#^-&~8yS|9_yAX2 zPPlQIwXeAEXr*%#Z-T&X37%(mvo>bH@ARf0wBcP9y7MR!&<6yCs0L~wnfi_f0DQG) z`72!WxHpa5L*uowJ+_ysoQhKqgDow>&mSpoCHC5N*?%(Ljk??%-gjg~-DgmN1}CA~ zpu;8E8Sf0`mxDPAXdAYdh96N(Yp2FyIUQUaYhzpzts)~QdT_Se<)bwwKJ;5hH$FK2 zH3r!6zR>&X^rHP`cJ#Ze%cTlR>{Bk%;Kx~JS=GXW#iFcDD&((Qh>+(k8q{x)E1KnL zv7cjN*cDQYNOvBM==di*0yoIqH++#fyYj2GaW#8ZJ0^UA6$a$Ji7Itf)eG`4S6A0_ z;~E-4K}Q*SdQiycg{rrE1lfh{?3p}bG?2jmcN>oB80 zNL6$SZY(X94UAaa%Ij99=@6+!%l6GVO4D2+nX`BNg9h^04$!N<&m)&GyN zw}6VPO}d2xfnXs(aF+xK?ykYzf&~p0+}$BqZ~_5>ySuvw8iGsX8l1+VarqDLJ2Us* z@1MyxYxT-%$m!;sN9w7nU3*t2{JDV|=@(~cWqUSS4KbG-=~s@jZZ=)-;*t_)2T5w~ zDRzl~^4i1M?6{$3eNvGeOcq}A9GG8VCsrxadS_Vab+4W0WeUlvKJ8Y{q&4nm3Cz~# z@{>Co;q~t)PuDnO*MMZid61t1V$>xvyP8emAZHr?G?J(@VNw2ingSK%t)tfmY@l=c z8$;LKx2uizb8A1F6kc(%Pld!C@{SY(M}$R;8Ouz?3L-A)P8DkfQ5LyhXAgO%k}&nw zdJ1g)&X@eFTgQ75YIrN<$xtuo>�*@gvF=c{h9A z@cdT1;+eWhN^+F{ygo-{BECLPu2Cs%I7dg{rs|~lN29)oSGr1jZmCI4WPxtTr&FIt z(|bLCl8tw?@3wUm6RHW6Up^M!FhXs_2Jbma?KF(g_Sk}(wcD3G?|zD@#YQ3f>L)9< zdF=QWFb0P{aGoc<*U#`hZ=}ZaAl~!!n9|(@!mu8f9q5k(HQVM-CoT&<09+jOH`;Uj z3{Yn#0rvf0>dbE3aOW_ZYF2xDlGHH`R&y2rBDhz3f;{80zi0x$qp1Fg3b4C$G)1?* z4FL!Z&ZIf;djS?s6O`lzcKh`tkSiFq!D$3jjx;Nk8L<=j=#six_K-M!OP_B}`^VvLE)Q9B~YkH|R(g}O= zc5x7#Ze?zG?3AeA;iPAIR940p0o|A22?q`zjJD=7n>XOvm(C@GE7{ga3u{?x{f+CZ zneUMfqI^A$3NEuO#=YiU)L=_?xd~R-%NT9G+VZ|AVU;oJOyhT^b@D8_j7I*1HLzmnK9E4I4GE?8@qppZs?2@{%G`hX%{7AG{=eQ*o>Ki-<$eTz! z5}MscWP2Z6UH{fQ?09I;_vkF^bOS{#Rjq31?pU_%>0J~RsNaYObAf7F5+SA^|1NPo zNAN!!MZ5@MFedyvgW}c+dvjcf){fHVnXM?agjFsg0G!M%WfKSsMgTA2%H^$&Krvbm zZ)k3bAQ(EH@p!FYpYou0mwiBxF1SdWQqt{)8rOmdUq=FOd!iw^67F2{)O9D58n#$R zxxtKJrn)+tHqf@1UwM+Hqy*C!dDh90x4o!ZQ=WRMPA<|n2boXzDib;8dO0Yyd2eqv z8*_&EXxsI#`QatmIcnZM*jn-O;&S1wA4(}O64}nb6w`|S;?ovAaogZ=L}7ciP2RDk zB(4NH9;S}e#R4xbPS|uccV-%WxNH#@FA#N&1_S|Lfl8w$&G>$oc-sp4ICjh*T0XWk z&#BqvhY1lAg3I)$oc66s%m}b(;p$!yBY+paTr?F|^Xk;*h4CJ=G&Ap~xz|$pEGxpm ztH(d+6ai)B_#%b^(KjlajeS^Ck7_|x0Ql4ZZ~~!cQN%6X5OLi0p=1t$^PQ|isTI$o z9y@;n#Y;(FE(?$QLPG_ioQ}Nd=?h=NWQB?fN?DxSm5oIu7~d|sYt>M#1F+{!6QDkd zww4ews+FAnH^ka0jM?Zua4)>TcJa5mr@tqCCS4#9AQB6jx$w_r#{t?9&IZh$&ba48 zY?dOp5_^Z@mn+fvakR)+Q|t*L3gu?#j^0Hna!Umk81e;7PY3of8C?Di_{ahZg|3Of zr!lJrg??!&Pi~%4!EINbl-@$o^DfP|DOD}YHqx>=yt2zs;D)?)ak`tdSB_`ql^*Fu z@c(Ly_|gpq-cGqNKOF%}2G}lYb7n7#0akN&m85~c$X#K0AF%yPt4OW0CS|qU@S%OM2ok+fW)aPv`in8Ff zaa)KHshjIJ1OfDRe5+lKyvLmJMvI$8jJlMyS!ZAGI%r@PXuf+MNDTn`*VE5i-Iwbe z6%fy}59M-9#DsE8iXgXFIFSwW^B&-y#V|Ya2$C7oH<4`wE9ciz;7nj~-B+=uu$sh{ z>73-Yod)3=t@Bz$a37%k%|!F}ryWZf9DGp>bNdg;&uF;w_f8|Ja|T23&g-vNoLISe zL$l*pZYq<$eOuE=o(`hpTQ6wSXsAY`@@3Ft`(yJ#y zqX=q;is?azrwr0E;*JQMGgF=o1=u<{>RWO1j)e|2g9bG+xGkaiL9H%K!A0YWMGf>R zk`Qya00d6UQ@A4Lw(CbmKJ7t*)|kp3#Av*+6rj^!M8kfk#R%N`i*i5OfPDFs&p}sqyY1S9iCZ3)*FRGa)3u z$3@L8XoyrSIDF~?@tV z_0pxIWI~u%ApgDPnCVA2 z)ue>OQTaHHuW7t$c-`y*`W|VPciY+B*~din-Z*A0c;cQI&eFw?IlaA_f*WNciA_q~3+9u8cR8C76siW5pmomm-{DPc9)8`UEuV;Aj}9IcX*c#eR-k8;`jd2bE1>;tX_4Wepq%EjesR{(jga^Cl17*k+>FiwqC zDeL9{98Y#QXRY|YjVYJ0n+!k3eC?JkbD-KWS~SxG=xD_Ur2K4mGLPp(GMJpcEwPQ3 zM#1;Yquh^9nqsT@(d~d&I-f*tMC|hS>~u3v=S=zTs=Sv0UsjEy0%!OoWb96PQap%& zOG(YrBXYNnbK$W)L7@Jvj$NZ+ufrp08_V56p|($3eaK7c3v@4;EBe?ZhP8{18B~-IcV_~dxrAVUx6ZO(1U*y$24)WLSP{Z zdD`3QkjX-Go@SYkkQs2%i9Bg-&cTG!%>=>>{Moj4weJo%#L5{V=Sg+I3vt>F1^E$} zU_YSVcg$V}M5)^)Xf{?B%|TFL4VfP8W?`bUtkwT9HtlVrv5>pTp#LCNXj}v^wg*Rs zhoS3-UtUpv#}v`%Ktc+*zjA%&)Kaop^vf8aRQgnVe{NZ~)PVdJghO|$g!Mqny%hC0 z!hZDGQP<8}4-laWQhMMl)SbKomd*Jy3Yyeo4d;|)x}s#W=~Dc)M)Wr72}O7nKtj-J zhd@JjcNZ`Jm%Z??D*XL{A5}-6*5M z_HC*`xx%0OW4g#yC*fZeC+y8mq2?##9r|%})GKx&mH8s?INt(Ffu5T6D*-FvBrY6* zGcgZ(=f0|=H6nFa`P-LuKq2XHN`psTUY|l6Mql|f&}xve;b6hljfr@3?JP{MAsmO{ zIIOWID_P-j6nZTt?<))VmM9tIs{REQ4Rp(OZF)T(_q#{ZleA;jkQctCm#sp|wonF+pBgZ=h=k^AB>>pSiUO#NW-OrWbs4Rrj^lIC`EK38{ z7`WY9W2io|#BmVG9mxI?3<^`rv$ua0^tuqA1n1tMmqK^p6rEy=n5Kt$pRC6rT(PCa zeuxkG7e)mo?33=GGd}r;i-jYP%ci_6PZ_nS9k?_`m9Z-Jz26O!av3XGaiilwXuL23 zCr9cki_x7qE{D^oevRVT(^70h^nz&nZeG^S1I0N4I~nsxKE!qpq8h`0Mcq=Vdm&=>~KyF`tc$H=50LFVuKL* znMkenFAcZj+15vgnngyoc(OIwtP#6;>n1jb+!bG1gch8HF3yzo^V3b?O9iaQA=xFn zVDY?mkNIvcEv}*Y8W8|;11E?`3JU`8Xw0i{c+YJ*1@iXM|9;T_bJ^eT_=RC^h}`_HN(ufw0{@RE`Lz0EPFuIX z3;h4IZ2zCnx%mOuAR2mx+P??mpRx^qozndNKObsk2{x{Z(WZUw=Q&z@?mcE}0|Vg* z`Q0kfeE8qq?JF(}*8d&cK7Hcfz8h0s^+&MdvRSMIG$zz_b*-_nsZW~`a#*~5uLCmuw({?R`o9rDAZZ@iNmjwza*|muv8Mg;W6~2< zq`6r(A%?lXFJj=dlVMop!-t;Fx=jw0fezT%YMf>?e zWBJ`Kz)>)YwU8gFU7A(ZCECSu5zpkyQZIjLotMG%6sFdj9tl?53|3}b-Dq^T;WFyg z+WOu)%SuaE)mf(31LUE_lh3K#?!Tr1fg%70ns*#6ulxB^&=-I4q@L|Rw2u3Gn#NY( zZ(4`43U_J`W~bGbXwNtujCKYXXuDo?dp{t7F6ku)lu( z?5etDa!*Vn6kIYE^10QW^b#;1)=2L0p?Rs+DE$W#7qDy_U17aRI(0jImP+Qz7qUbp zyEScqL!$<8vML;u2}i>9j0{KBFe5aF0zZbMl7ZL#lT^KTdLzg9Ir-V=SAeI#Z6N%(gy zfp)}WUX{E6T|qxSUQJ!gqr9Nz!mW8PSqmt26xo4QG014{=hs%Th>VQXWz5OSB9Hhm zv$(jpYwz4pBvYe3T*8t_OG{T)UCdirFbONzA=$&B-;zj@)6k5X!ER&Dd#VPA@!9tM1cXc>VHRw~CWEEk%Si3GiVlzpuE#ounhmF5Geht&$* zuJ{@N#RCmsMInK(kcNNC_yI{g5jP& z)wv3PB11WMf0me-XbR}hT?iL-fmmfv?N?&0cgC_HZ63U)R&Vb1^!JL*YPwwx3vfO^1ITT~tFsYvaDaalT zjsB<9ZoI0l!1PJ`{(coX-iNQw6?%>(N|+Vcn_w`%0k&$I9<_{hx;iyJ~b+0I}8}9e=VwTje+UZMkbGzJ3-U$Y|*CR88^! zoaX=6k3gxO)D9$4oy>7l=FD2d^Kj9f4}xuaL(5cwXw>JND!mrhont9bhy&ra)#%Vr zQ}gV1VT_>W5_Hv}S5mo3*jCfuQ>gA2EpD+UXi#7f5Li;R&Oery7pke2Na^c<4CY(3 z?Cm9wfsLnkl*thz^nF!SV&&8MX%_Rw&dHXSCi7`mO)3H|99 z_m`A-Iza(Z>HY7~82K#Ast4dIpz80G?3WclYpV)j2O4gd%tvAuAYj8jsono3R`u@1 zA#q)wt-@>{NtIy&kU3(F4MM>2n zFt|O&K;02Qh@)L~D%dh_ns@QNt3KSehEGSzK!`vl~Tb5YA7+YOgrb+H?+5p12Q`c@D|9k z1Ox;<-X|!8T#TToTQ>`5=R^^y_~c{)Pfa!NX8kM;eU2cDd407q-Bu->?)slUQ;Y`8 zwTs*4<+qf_^3m;$Kj#Cia8&=?FOyd;fImUH9ktishWU@E<=>hsAnbawRNcixLl}bP z+IecKs(?jS!=ES{}dF`e3EWZueoLqYbXMhUEZq^_@eEc`fDb0hR?5geBr8*mrv6eZY zg{6;`7Q^Ln)|<6UptOI@)S()|?hTtLVw35S58V&;anL{tI>{urFW@bS!RyGy5dOhL zAHPEaj;n%7Q$Tx^MBVzO+9yEwHfwNTIg<{Z_i(+gXyGSNx%_d8!O%7VL#Qm_Nfwc= zwR{0cWi{^~sw|{!V|ya+0Q2QV)G}G`WUl>;%U#9)V6JAAZbM;V;i%Qt@$+bH@;m{b z!$za4=^RnX_webD0D1ON3Ycq2vW8y^P#e=sJ!kV#4&;LJxadw>A2@&MbQiG))DQp+EyeOXu0hEsgG$D~^wJ?(Ai zd$;m9wfy)=SMy;JATC<9r05DuK#ukbV7sK_P~`H11UZ))1P~YM4{8`e=@5&D9YJT5 zW`|x(fJ7jdarx8)uj^q$@uA^KLwX-@z&H)M-Lz|NiqMPANrGI8>pf5Mwp(G3A5D+= zpXI!pXTHl5Aq!Lx*$E6>1`5?e3lYDz8FqbvHW)rvg;Rd7nmj9vu`6|?<`EK@EKGKu z9&_w_lbj$w8$cUgPR}0^f1~PJB=W_%K6TU#@&DqWJS1Or)!;_=) z2rJTla8iGMk+GpmxuS8SSUlWa=c*Q=qjuqu+lQeu-2YPxz}}wr7z>9!fP1r8*o!aA zy`Rjh_a_l%ph#^ud2_L^z~Jh8gmJo(;2-o#WWLoY*?MX79C>~hY-FZ>Kl}RdZF#;b z4<~abPz0e27@+d(=8==5;JjqN4_6sw|Edk6;J#N`Y6PmUHW}m$<#EwzyXJqZxXiDj z*P~2POLg=62jR%fjE2QbvBBv&P;NT7ELn0iS1Wj0rQ7JH`~~F&KFlkTA0@FGmjU&s zkSgBkF2VU)8*_&<;)n=V*f&v>uIV}|y!5I)K~&gN$Qz?fXcv1nBLge~uTncQGlem6 zoYtwdXJw8J$YEZS{rTlTB+@C`TS13!Ux_|tXk;Y+sumHt_{Huy%qu-L9>*6^8wfFFRQby91kqSGu)u1wcP~Y+DEOoo-@+ z?&C}`^LJ8AOp0mGE>sL8<+#NVWq%L_Z>#e_(Fb#`hbNLR4?YtcI*@n`9H3z?0s5oz zHc)4VhM(1t`F?SeRKl-rZj>5!3iUP%%z5b#Gk72FhGgnoI z?GZ{C7?=T$p3VPwhyS{HI}k=cGa9$X+h~$VWyJ`G;FbO6p{w>@?ZJv~MBdJ8du1Y?8^&iB3+n*LYq2)fPuTuB2RMGfX94 zZk7R3=GMY$+9AFA89novYdmeM+s8lcH8KXccb$1bBn=#Jzt`6{0UPcBbO#F67;Uif z`3QDoY1xXB)mGs9O$qZ_N(dHfwY++{#dQTNbQSuiEv)k14_1dZ(;o9_`8G}>D*3f| zNEjIzRXlVl{^2p-#vsEk7tIEz#5_ZKdUX=ZcFxRPMhS3|;QjTBzkmHd{uRDKKbFf_ zvSa$wH~(uIV|akO86?_Hl9c|>%kuwRt?)EFm}ZB;e|p@13^17^Fyx6mBW%&J|IO8D zz>UaKk1eA9=STb3lL~K<3Bxctrkava{x?_iVgNUykl94|znKrfxcCn}P1Y9oD8I~q zbF~u%a3g8Lj$HrEXeR|cgKr$NPj~!zd1;@{Ch3etq5f<{`kBjohdFw0_UkygILtkXnZ?_6;2!x}xZJ4I{YzvIU zp%^R)2@-94sPNab)6;BpSL$?Q63qYnZJpM#WW%sT{L6+$>o(fY_PL)fU5tXGS6mMa zbThhJJvflZICSCvw$Jv5FCmsD1M{phUlWp>GZ>tsM5-zCa6ulUl7OM<>n5G^!d# zxvt-3=ac!c9x%Q9Aiz&pB9E4=(SY* zGXw(8lgPaV;@ja>|8KyfxWKI>B(M^w$<%oqqG|Ao9iQhGlny=szq=R@|p2MM8RMpU~7!#jgcytA$XY zq}QgPrnmhMt7tLw>CY{NlDWw}4TN36lhH0nl_o*>!`vLmkg35EO`wg>P{`xEX3RT{J9JGIaRiiEvV6k;aiEicpI9Pw1Njursz)j1h zRn_mKBc}B92!)}F*MAzc+#um#Bp~T7Ie2^IOApkL;G!v_?LvjQiu5be;ovryF*{FA z5;upEY8<$4zRD%B>f$i!&@V49Pt;q}xu1X2BS>fU)r#U9s1Hx1kDsgAmiF@c;+}^**^?YQkZaXU}me`+Lu=7mS?p! z=oNOIjMPfIcdp2_gET+q>DDk^Zy0tw(Ed>>{uj1MzebPo-V}Z;8<{YP&lSu-G6w{e z?<2y6)Nz|(Y%JwAH#T^- zRIiOzKtbjCw}t;`HpP~$HGP#A zfGUsNwgW&pBpgUy2d})8nj$&f(4Dw3t!o1iJIrLv16|v$dZ374a>jjNFHsG)RzE#m zZ-6skUBLS$V%K#f6^15wsnzWy8woe;eig6!^DqTE&~g6siX4Q+k8~meLycI?iXr#7_kc{Q5S9E z!a-5QiEh}gWb|q!>T^T|9UUD-Ef!#cr1MMFxmNd*YtOarylT%&k`rZr^W#3lLzK;U(uZZ}~ z#9D}WU~?rRizl+Y2^|&a$zPER`(D{cU@Vh(REFH$9`Sz60>K&_0_6u7fx;&LMPv)1 zTy4K`ZewUEL^F7!BaZ7vt@chsy)Dn7u`vFQ0+ISyt{bx2CAA*;qN4Hq9r{{jg}u-t z6=2^J^9B=SmzwJiV5b$=eZHNnKUSkCACG-;1boE3w1lab8%e+oCr@ zI7;`eCnRSWg+x!isZeCj^T^k#^?sFSYaL=FM*h_3o+>t#i>T3JDTpWD96&vtAAQzs56zDC+2x7U^fZu|V$zsm`{USV56D?=02m1E|;5 zqXi;_4mM*>@2=wft8G`_uj`sZk%qW-?K^G{wfhKP5vD4mwLl?U$o!~hn}a3y$$87R z+YwwkIy$Ls@_$+Oe|iG{KKtMgAzyVDh@aHD1PEUetHONPbO`{P4*eTo%B|}oBY3|T zbE>pph{)hDX#GUw{Pr#65k=n&Gm%Om)#?-kwEO+Q>ewfi#NwUZdU7*J^>Q;;`a|SS z@V(34l=93YgFHA6HP1BKhShi3DT&K9`I^vhEE`}o4JKGiAbOk)YZ(7pSvT1_@7OTG z9zvIR4T}R;W{Hg`^+uH}KxsPDM>pvT7!Q+EIPb_rT1uc8m81F?C7!&OEdkZD`yWryP!$Vx}X(8IQE68j0) zZeI85=GEXSaIhAd>bu1p5L!TiKStjdKkwy2QDXv`&>gb`&whg~rX#%sfI>20$%7kEJI^g5?VG3W_LtNdVOBc5rwr7233Zppo2i-iq z9LF9blzk>v+in6fqdt!M4NNdzqotk*5e$buM@pkE6?SXQhhQ5n6>f#xhR!28fkOsc z010yNEjz*MKy&rQUG;CiV&Hj9FU2AME?_e_6AMTyb0R>E~S?f$?x#(>@tNSTClwuK}=Khr0 zoiLcDl3tD`;wior$TuKs%XK**_{EoTGX323tKO|@BQgpq#~>L%UFVdgc!MTg5OuJz zTaZy16799>f9N3NSZA$P^@?Y~ZgOjV{SDBLdg3x3Hv*&u_%!?JHU|rJ>Qy$uNNfaI z%Ta~fnlg|TMZt%!!5#F*7fX84G%{_QKG!C#Cx@c9QOkaMd&DyeJs(|~N@@m(Lt=8! z8C(w+834?C>Qa|B42Od&DF<{%zqWRpYY^aZI%|X`o6gUPQRZ@x+^2nbHX733uWZEe z)+|Kq`oatBSP>p98suh{h6@rr1A;x_tvAnLAeSqT-*sCkahy!J~N&QNoNt^g=u0@KEPXj0qOpbLp=ZS`Je)j6SdRzlNH{J~6VEx@v!H(q+ z4-kGmIQyJ^_Gk5nuNE7QP^cn~>*_&2pNYJ9(_xt*IEwcMcM2qSh^zln4b4WoU9|*N zN7T{0a9x;V@tsC}t!kmtNFYxA^jR9P2D}j?h@f$SmWa-vFJ_2MQ_Lk|p@u5CQshM+ zYnrciz4-APPfXp#bBFbWtfh>3L z78gE&)}8{t97!As?AM!futrbb7lI1cmLnUsqm68>Sn+MuQeqB^RJT3n>$Q*y36}1= zfj}f*>8ju28{?8Ba@EAR|MD&uiC}kWJIPlFe-eHN?l6Dv>neaY0lwAT!AO4O2$W zv*2~_tFH8X;tjo;s1CDl-CuB#sg$Bgg&0B3*;rrj=+9r5zm*!QGMZ-GKkAjm;_Ak` z2>P<DuYb6U{r(Sr&>-!{ z>Z@czb-7n!7E(&Jme_LWs^vQ5M@`qGLH$kMn@5EtV&4jxX7`hGp(9bg+^~6N7Gg9prZQOwMzuu3#+M1negq!KA ztVB;+t;k>do*`?vZ$J4e1~u*(5*{p|Q#doQYC_P>7adGJ`Z-~(9r}IixXp=19T^^B z;7v3?nW7N1P{504e2EAak>PPBmu5maa;MlL${0ezSLfN&##&AXh0J`T&hVN&`21-s zF@x&32J-_8EoMSkgz0i;j+KX?yFjnKn(Xq_R`u{gsi`BEk$sn@5GKHIC58yRnI ziRn7K-kHx;$8TK*Nik!ss@z#DvS7yu!mV3uj~q;WY7AI_U8FQxT5QABBeJe%2C8Z|F6egGBWS_lsOE~Q3j zK4&&lzVz8qpEM=YsIJa*R4++zofP$;Q9+=0BpFF5@Mq=EZU`!Fl`}*=DcgYb+ZpQb z*Hkp)%jVdFN@RArB^yd5UE3Z)l%(G5!hl{E1;)J4SA~a%xjg&z7Zb&5C}~7AW+U(- zL!DpHMJV(};H}p-HJOo`0ab{D^_OwsXx5!A7_hqlJAcY^R8{~lHdueKd<8}}ux@^^ z`1g;3f)EU%@p2=7?yaL*TOjYWPW;tF{VEWMKT?kK-Gu#T)|#P?LjpV8ka166VF z0FR)m;`01lg>jc@Yo4zY4TwS;*kLxlr)I4Ke&D5UO~x&|(9ZAP!H6rD;V>?DlddoC zS?xMQzSy5=rB84*5pw0aZ-m|7)kHg~G*z3U8S$61{jeipyD-HUxWR>bE$3V-ti)+? zl&r*fS7QL9e2GXl(b;m;o^Jpqd~EP%kedfcN4bxF?^!=}esb)X-VRze#|c`5C~uY4 zSueQYbK8AoDKAlJqWEY!Qqe{1IEC&&EuWl^A-`8*pKDtUf;XD!U>|-C=PTy*|AIaK z+VOu)W@)ifR%@r0^`r!L5CsK$t29yrDHh8~y&5U&$tSev!eX^arcxXQGC1Z9+5kVz zOkp3-!T}(D*lgO?A0HTDvp7f+XE>3SE>g+M^=g_(?yyRy*RT3@aF&sY;|H)@C#A#R z)3)leF6&SNb&jki-Eu9C+l*64%Xu{QA%fFo+S;OgQ7V?*Nvxh1{MhLf9Q2#YJ+@mz zbEglUw)o`8hWI3Q^wI5DgfqxM=#3ueBt}|g4dCmM2qNOB2%L7SpX&VD_q?}e(_ID+ z&LR(DOKi_FTILi@7atmxD)m0dQq7lIfMpyqf`7YTZ_b#U|5z~|%dFUhWMx=J z&GJh(a$gWfYI-Y9kv%>E7dEgz&VyHv?r!Nd$|IW4N>rQD=|5JhkoXi9$f6lq%60Me zvGG&Cu^sGPW`fd+PdjJDU+K{&n+I4rB1_*F&NZdde3{Y;10Ck z0l8Z8)kfZoSo2NG@792YrUGDyx{+&Q{Mh{KEV&Bp1( zoHvS6UE7P=e*2#0Zy2xh2m+ zY{27QIzTX9tswyK1B87~Ar>R)FI{5TZf=Di%;Sg#Xy^N%#ke(~HhrvaS`0e1KLZWL zMwmCaIwKKnRNC)OE!UwhMyt2ey=A^Up3#eP|1uf;fWA_`Cy~FvLK_2Aiw#>@euiEL05P)5)sbJXz*ye6gRkH?5DGyC@GWIU$1g~K(q z#6cP~tlQFUGB*)l{n!ap=Fq!QCH77nkNH#fOq0$OWK@BJ|6}<~9MDioMx8^LCMQk> zI6qvXCszA>GVZ~Tte$jN($yeUP@c!H0E zDua%z>D0^Bjau8NhIqR*)r(c%7i(0|iX$(a!tq_aSSR4uZ?IW1IK=~*8-MM#FjLZx z9R38i#eWhhedc|8k+SlrKUdR+o4mu|S& zVLOfZ`Wj_p@^}`#~J}+_2!tT4XALZ#f9c90cb27T{8YGMUpTd-td)ZxoKx z=fA8tZbWMl@cN)TJ{1Fx5t}=gf~ZCv5;;!$>Nx%Kkq>0-0{kH;Mc)G0gsegw^v)2i z$BcSWMh)D-5R7|q9}|#KHCW+6U#xK!NUW7=R?SEBqO&36(&UZ?aZLF!4W08`mo-tb zVVF(kCj~d?(lT3!96DmroL1QS`oY@NYRMU zbJ_MevN+i%(=gl4$898?*Et_~&VHikgXdh4vMo??`Z=)kDa8_Tk*AQBd_lB6OCA`l zHa|)!tI@BUwvbYb6h-V`S#LLPc$U)MB3mG4&~%dGXTapJ3N%ig!8_fW+ICqCSDRUE z`@xj8vnCank%qL&=Odhfk^pH$eh!_Tl~pM>m3tAD>ny-y2v=dBFB?y~hS?l`sEr{# zllAJ6`TCLNCI(WhNugG{>}3?&>CxC307NUw$(-h9rPpN#YX%Ln)EG-NQkm^G_b<-K zma9bhFmG!7wiGPE#d!mx{weGmQau6B1M4V~9bdsNs-n^W-u*8Us@EWdEE{NtWuA0S z@Pr;U20wr6##mMlRRTB3A|R+~bEM+-lf+X~>bBeI$`9#N;8Lh6laQMRg)clmc_}Oj zB3hVHkm)B0c|7g=H9i2PK6w9Zy*~#{h-|duIC*GtAl<_m@)o1k?k%EXtKB&3ZZ3}- z*-hi?RXc`{^;SXWw|1GOI#m*p#Ow`8VNwXM#FC`f4liBf3(`I0DyRirxDm_P<*5T- zcwa}BYBwlT`*U;@1Um@k$CU!HB&!7a#ji(iK_^$D!36Im&*IKJNu}H%kDGn%O1C^117mM357wbs{4XxpUa#dwa z(pLb|&qx~bxvxu{Vyg@|d}9=QFRbLLdM3x9a)B;oZ~N&aT+l%*81?pj`=d|9mSWr6 zEZSCbMg2wvTBo*_9Ix3UYnt*)OnSim;!5vC8ioL$<#>UBE5mLc;<*fOA;{hphbH5J-=7!n9FCcX?I-n?z1FcYpqLxkw$rx0z0AJO)W zko&U6jeI!o=|51ZQcso4-!^!w`hgYh?ME}cF`$LC zPK8m*5RadRxbX)0rpiZ>X_nTO0p;&3eMHi{L-6=Anc5BIw05@2{?`6_C45PqueM)_ zixYE7a2aG#Zk96=>mu7-wLP3q9xGPmm-%FzN7Z z%A~mGPP6QEWmYk^M+G3F=(J3pRD-fD@k4mC3 zP=;UJg}|jEc4|H^k`a9LDwLa2D&5P;Uw3<-mcx@RL1U7um|>b)FLga?GN{r2_%Ndw zsatRMuz9z2^YR;^qmH5g^y|rI#U||Of2Akw!~yt*+bLUjVcG(UU$lk+B2>1l5EQ;R zyf>zRXqMD8HgyK9N528#L1gKf%FwFaV%qWH-G}UYqF)!_*ikE{e`l~TXjbkmGOJ7N zF*aGzNS_IT(N7ZJpd=ck`JW8+8;99mA$nHkZe8xy+2Lquh5Ih1W-nL0@^$Z>sVXar+0g%HFsyT%F-)dZ?$k9 zZm*mZF|cIu6LOYjr0cG?1L&-cBkaL^ojT0EW!nD7y!Cpc5jQcu~*yXqV1n$M9= zrS`;BZY@#Yj@oallFS}Fc7)k>k)yfXoB+NX=;r2UU=&bK7|?coz#DoOW#gzI4k#4g zWv~}kz-ETI@VN#LZtHtd+IE~(wb{s!+6fQ~-XUCzQ2VS^v3yz$d7Y10_1QtS2Mdkl zuI%o}Sm^p}G^1A^m5yx4y8;9Sj~zL^ekUanc_YEl*)wE}4cExEawEKeXC*SMOSf+; zb1wtAVKtF~%#zST*p0F2XBc2?Vilxb67!0nN@^vEogIsHdC1+o(N~d1^}nCDWJ(UL zuf6bJ0OEcfO26BYXd#k9%sJet$=tBz8r;f?K(+hM(cpeV>;S+dnLxQqn3zWbE(h>9 zoB}>YDXrZLd9MhvyozRya7AwwoJ@^D5C8PG(gWHtaof~Ro9dl=nbAWjwea5m-Ni1o zK3>Hd*Rq>*a^ZVIs&TZP8c+=Rip55lEoBG+TkG_tcneG_xXXUsXXf^b;&k%I(NqqD z=24ep%0nmHv+jk49?4$3KGZ%NZ1JH}X zOG3MfFxXL!#hDy>Wm~i3>cji4mwJX-29(2%pJ^%26xStCb4!B@A2(-^CORHJx&ICg z**s(<9%8|!HO8EJRg!r2CR%_F0gdQ80_ncZVPpUp6a6|FIF;2R2|_(mjjH2h>y}FYlfl6U$TY*w$vUzjm9~1L9l7~Jecs*0t=O9@b@frr?P~Q5 z=yb)(Tie!$?E9p}s#k~l59#~>(t0<6hZ2d;CU@j2ga2G^v4el|mhw!1u|PyTt`C)% zD;|I=2Q4c)m_{#rboPoG_#=LN@;l#y*bW|Jyn#00Q+)jym+-~ZW@A97LPWCTa%Jl4 z3+J~TVF6dVuO1I2y)PcrL^#rVF%|j0_xq7E+-}sn;=l|cc^AEK@Ks377I8r1Xf`W2 zlYZ|YCcZB8@Z)Vo$3uGO7-4rMAH316<7fwMyTxoU#EKzu2qwUOQQH?tUY#Z^pRId5 zKQjGSjfy{nI))sU$i_HBoz9ovuKUIk$c3@0^-^g@Q>;12LyLKx_i_cfD-y0za0n7T zG{ZicY?bm*t!PFcUVaR>ndK{8f4savkY1QH{T!I)b{5(GIlHsiFJxDb_Kxfbjb8TT z^aE1~%FQ#>@MXhZloTyUL@a^+&D0pU>lLVD5|IvixoTcmLDA}3y1H%>o zKiQH(>zg<&ts(~(O~|EJ_UIFr(RUv#WqP5a)}zgCJ3Y&v zyN-u0h?($3- z6YNHW2PDtcU-No1@gADBC!Qw!u+U0$8zb4X#eX)n?iFJn*gXok4>Yg+D9?L%VXjA7 zG73#ksNU>rlNiq8VWz%mCj}xrLss{b-zJO;K^3S`fs zF(O=luuJ&==%6;tsOnp3HVHvq5!e{!(r3Y()kI@?@xqEhn=^hSZGNC;R4}yn;&iSv zdV>jCFaG_77;EPd!DR;o+=(iIpzn1O8YQm*;@^ph(SpC6`7qUgupYhg zjau!y$g&w$#0PM(dIB#rd0^DTcTo$}WI?buT(vQjB-$5+Z}9T}WACkk>e{w7Tp)PR z5Zobw1Shz=dvLel?(Ul4ZV3bn?(Xic6L$-46PG(!Yp;FIsk`^u=f2&CQ9tIeu-<*ragcCi`7Ak9SE7}sQSYVO`TTBx(VB16@FQ5; zTzaBxmEVv6Rut`UO}SQo(|F?0YM*ggqwHfEc+OcW0*@7yH#B4p1VpU^FCQHXY`Rtb z>JV%S5rsAFfP0_51B~N!Vy~P$0-iV782MgDSrq#wQw-nX4>+tA_~>BR11AF6&nCmh zQsCdr!6E_tf{!w3H6V$V0s9P57PIl6bSe!rp~;MDJoqjlCo^a%J`yuj2}fYG!OGBJ z^dnEuaT0gyeuE=4HhB$kX-e9?zaxm859$W4@H2a*XVdeN-6g}alNZG9>u)d zGC>lPI@w}qQ;wTnjvL%utkW>=EmO093TPyMHdm1tLE)0MXKI!s9bE|8I0$60S|y$x zrn1?@tZi#~GKWhQeD29<+F|lF!G_K)3r1l=eYkdM>T@;cuyJVdfh%;fR`&W31%mS` zYG;B(^{?*;sgbkiK(tkS+Wz@Ofg~NzYx*r1;saVf2f4d5+lt40)qNVE0BAR7FP8;- z(zQLYb_D93DWM+%1e0Exf8}ozb@~X92$j~tc`TRd*CyiGp|Tx=s9bN3 zABPn`w`y^ua9Gsxx)Lis9(_Z={yGqHMYsSTIZq{iw^U*1fjMSBF#VBfJJ9_pQ%+!? zQC>BukJU=!eWk5ckers)Gv2cO1l(i80UHi}w|l!n#6&lIC>nVZ9FH=5QyWE|G9ae3VaC21(>g zctl&0Awh#Y*N(FeW>xT_+CY=Gf0`tZ_SIJynm;=^O`y?ONz-PmTJCzO+VNMv2>5%+j5fjf22EUPs&f-Pl-e=3F4PkLavPbVAhsLOhdrGGO z&p_d0u+&>~VoBfeK0jbAl${V_H$8-ZJp&r8a8bAEd7l|#Zi0~+(#hYEQv~Q0`$EZ8 z{z|<-q3`oI6isHfXkIBH9s5*C@|LjkHL)srEH$1AE{s>V6M|9WilO7g5;eO?J*&TA zna)?rW@(WpA)uX58yXGIYpXwFCd}O5itF)~wC_!4L7nM@c-NHkXUL;MNW$Z1hFa3< zR0qHD;3ZQ50hsW-xArCL>F8}nmLW^=7dy1fac8&Z+bmGn3wtX(6_kkFGkW4GE$*0v zY?>4)AG)82eGUa~uNH~HymoSLAW;lrsnPc+33nNTmjD;kxCR58vSHK|e~1TR z(oq!!!-<$|??mkMlOG?>k&iik$z;&-cq|!CQjoNzIdp*AbXqufw(C(g2Sku1B9+T35du_9nB9bDb7S+y9~N>q-{K)J?=5pi4g+T*Bm zB>`rwsRxYVB5N%X;^Hso>_7j3#7MfJ*&Q)3yjzI&N=wg-q+R*zhr7-(RI*%?djhD4 zy3sb!*k1L32~kbzf8*T8xtp1p3NLj#x7^biLb`i}q}Q_w7Of>L_`|`dfdm)=e#S|4 z?XKbIeOzElVZa20_kpI9W;274`}{DUM4DQBHv+{fiiljhE?5d^iUk2vyd`Py-o!}r z(r?OrHnT}_TEzogYu7#Bxu7d8OzvXlsjsua8pLr_$%o}v2U^4!$@yVlzo;P^E0l@8 zi|plH^Nym_KLDpz@NUHnzYif;Ox`_PZ;Yr@AJJeLCm)-`F{8BD4$FD+5b)C#HGxmS8f!9l- z3PVKjq2B~?%qBcx6y|o#_E9M9LyUp~o+L$YAbjn41iQ^EAs81#`_#Z96GRH_A3VIp zM~v$MNe&$~n?Geg2;p!SyKUN*h!XFP)Zq@39BPPS5nv#FiY^=2h+4`4h}^O2{aly7 zcPqWSjLGdg$%s}w>U+xp9XKZfqLZCBzf%(EZ9GIy@$6~plnFhzUM8OCN2f2iKF*d1 zFCeJ}Cga2|W`u|I3FXi}d1dTiFXlt3eEhK#&>!XR!7JUJM60RMwH6s?I-;sBmQ5Q^ zueyL^sLqj4j3R6yTLVvczyJ$GVqGnnQSy+T-%}0@_hJvjRC9 z=9hp-xq988Iu`p}(k+|!UcbbTy_fCUj=g(xcuTcIP>5M|7AGbc@q!G~f)^K!Q+?~w zFkz?6G4#0zTS7{e#&TknA4%QnXg?PSi0+KZa2ZgInyP+E}kSpqH zyOfy0MDMq@rspuCQ|p=Fek&b~J#C5&b4|W#PNW$q6r`3eCe*D{)P4i6lYHY%N>6jF zjdFnV>ACG5J8;k8=_&Dpg3|7X*l(*hJg|j+v^!*aBu?arJII!8L#9l%gcB97pf*v! zH$kVttrahl$<58sj>qR5)AGm(RpfNS0~k!C$&(_FkD6%5M5`<~S%%-8m7`A>czHVl zH}W8Mzo8`o3c!o7-(G|8&a1Q8`4PVrOAzJ!7wILUwcr_AaGHj+tRQm-GD9He0qNU} zljF&GR}-&`qic7?rx0|X;rpAo4Qo@bdZa^I`Vb`y>4p}^>^L`)d0aQ(c17gwozI|D#*9TO%ujT zV(ED$i?tT{TCVj8Y}fe!UlzSWjNDIH!aG9RZ~Ow2I$qVVQa*21@p%^t+3Xg~Mjqad zAg6zNHDOq)mT8$49Y5;9?|Qox&L(6_VrW+=j{z0z0$OW5H=7gfQ$ zKWw1&ww(C#1YOWs@~Mj44CAQP!MrpL&O9uLugM1a;iaWIY+xreyl=Nr$wwExlkzUF zu}A~pgsMswtKAr{@~ZPDyye!!2{gQpQ+Hc8!evLy4#3YAH)n}0Oi>F zuPw-y3108huhJdk*UBda`dwlJ7GI^Xw=-Wi-4X&~fYYP*>{z)N&Z ze2av_+^1G#?6jGHB4Aq@8~`|bL|hpEesT%i`6*=Gx-!O10WW5TiF0^4K0c#;yl+b5 z9Fy%dxn;(QTgTtUL9bYJ@p#^GZ$n`LZCKOg0_+(FYnyJl_liz8|~O)u9k}a6NehmEGR=vYSSPMcz9+g_{6cCECH7%y%t; zRe6xA?g!gY6veVdpfv?UqtfMsavI(+!TltGQGthq4p4i9dG|oNSNYevuH8DD2yk}C zqPTUmTJ_?``{%xhh(uJT)tFnkuWVdJ+o@5=0DMIZ9V!CZdn5w1*s%tJi39j=n>9TR zC{MMen)RY&Z$0fhVfQaO-HU1COU;#-exDSiMs9>xTTKZD zRTr7T6k8R$2xH)|pGsk#^qQu32%dj`x+*kN(-5BOXb}(=2GAKyCHrh(!p>mBjt3a~ zgtof{E{bo%4tS$AK|9UVijCFixa(i1kDfxNXwEr~<1$WYpcvkEc-<5z=8atV6=jF1 z3SkH@!=upcnLA|}G5qQP7fK~vo%iC?Q)=wqz?G_Zz$jXe+2MQtbH1e;&po($p%ohm7;O_h*=%bx>a*7{KW^9oI}PU?9?mYWgSTA1^N>rMMJWx2N=h1ylS z@VVg|2bBS_W$rlPzpp4 zozK$mV}Lx+Li_gdicbz!I`tR@HOWfT1xM;onR+MfE2#y(n$PC?lRYeco1Q<;F4Q%& z)4{atR*-&`{53=IQ2fUdqHnp|z_>bVgA(x`IB+=mEQ)OTS{WmKS;1C#PO5JS^p$m%kEE5a}7gkZ8E0*6Lekq={WC1hR#VW_U3ps)c=I0_1QIKZ&z#0s>$m9 zN;r&kqs#So_gBT#Y-d3DC!{998+WrB8Vfo-Qi}qz3dkG;1Vp6xkvk-)6A9Jr9A36S zu|pRrU+R3&M>(){Knw!+C}D<3aqW)`h2rg(R<%Dh1=}e8-o8ltvJ5&4(qTQIdnO!W zw-V~F5ZR7Q<_kA@f84UTg^GXnh*Q&@PT_HNlq5#%M$*GWGHj2I3{#~^sTSh`hgum4 zvmWY=cbzs6@@)V+zS-?UGL8b7S!~Q%y$GXew1FRmD17?CL6I;`;);nPG?lr)raK=< zk>)Rk)$!LdJyLgm9o+9P;!%wxmZ4y*C)Zmo)$5R*Auj@R>?vO_p-13i6krAU4W5A$JaI3j$V`}*kO0T?7WfV1QTj{pXF7GB3QMNJ5|LFHZj4!dd$ zz2SuSVc*BmTKa{w>Q86Nb>Y|5G7a1arRWC$?N$1z;M@MXaZpA`y>gMUvOseG0NyWj z@H*Im2S&%`Lx0e3SF4T+jTNgzxU3nveXG^QNM6amSOBexS7wp5TZresX|)?W1J@jj zF$y>YD5YE}liD`g9GaJ;(ddmQTi8dp}+uz0|Ltx&9{ zk6CbmcrMJjYGb)jt=^)8A%@ug6yr?p^ThpGsOy3g8wbmB!E$J1uQ}2u4g#K}*^Uso z&2#6HBjPh7`M@^PrzeH z0X@gjb*s1I((sy2itaW*vA)`7br4>0Pbe^$tLqDTU`tR~Vt?KvT*`xxfI#i(Ctx1t z(`{s;=~Eqx#q0v5n4Bq50$}}VM%Uc-dBG?ZWEhJ>>KHu)Q|3kG`c!oLG5Ym>!|wec z95E6@L%fdblbHv~hr_pVXF{!b6kd1DrTV{q;aaOj5-`9U)p*{Te^f24-We0;IAp0c z1de^;k`QfjTQ`;78If%h^sP6NLhZ;g8}GjrKq$_-mgrUDLCu(9UO%VsjHQ<$uh<>T zT15khE!wXiP#~@LtJU#FE6aB59|jIldHk>k3Y9ICnp1K`(W&+dR8y68_AiRhaGq|a z1U?;PG&px4!NMpiAG!gX%q*ei?Ng`Ya>i+ z*@6@W1tklyBg;sJt?PqlyS{uJMZVH5)1jSuiz6yV%s9k&+!cWAwvKl7sR4>UQ`hHF z$k=?8_q4ah0BGqvKcmd}&0-n1BUoa3Zu`EDCow>xOy~Mw&P)ET`KHlQNdQWa6!DH@ zqbX*x>3WDljIz{<=G~e&M6VEi+vRtl_FZDw`(WG~4ji)wazi3Fo$GfCKj&sK>iDBY zonQ&W!0VEWMI_j@N!u2u{~8Oml^5@Ncs%QSce}HQPUTL_T*mkO{T+SFdE(n~+p&q26b?Y#7UMoYq5Q?;xN~eJ$ql)0K)eB)q4&zh?J^ z#|6a17<-D1{#MBLiCs_iVQC;pAAP&7{4}~)mcsp=*o$5MBt|vWb$8OXkly z$SHvceNnu4`5bQU;=Z!YiSlF7JgHrW^3(cEe&3=k8|=3$>p;coh9hy)Lt9*4ryY4C z!VZ29Zbm1quK=3;t^3~Yf{L8SqlH(0*}wuE@}1IJ3xoX@J2*Y_M(?Y*#7#^|+N}KPfo$ z>+~c4PanmLrFa?zf|+G1_}rUyxM!w~<>#mCOgt%1dO)t(}pS%hBZ;1fUKnT z{rym4>Dy5EILc?^PW_eT+_-Qe7TH!doy94q=gc>`AbeM}O$Rp?VYNEzIET0$Acz01 zGJ~*UlJ2HnS@08x>7?@wODc63^jENHPP3Gsd6FS9-$)2idgHz|B@fp7NAQs)|^8X_B z4N*{&U$n1zKZ-FePZ^E=so1iCq2{S9;~iXqE-T+a3RAl@FBg;$5o=%(0sGo3rW9PS zz0hKLxnh9cT{J$U!D_8WN2|#^1-D*w-}P?tdBt7%&gVRR=hazUHm6;+a#wIrP*8sc zMz7@WPx{8X>W`I$VnJ3mt_*!&=>g zoYQMi6|!{q@JI>Pb9>ss_nHaI&8SUM?lL)!VJTw6p*}|^hn0#Vy2cei0~h=Aez^vD zIyBmd2@GqTJVf4!KSkc1Mphu+5k`}TxW{fWq*EmKcMb`SNo`YY&lqkfs5gE|lr?{( zA|-u)=v50$zwl9biA6k!3QA(CPMk%{3O`UY?#rlFsj%QOxz}IrdFy65nl zVPo~S1Kk7699`q3EJS()1@K&P{g57J*Kuo8%g%PQCC<@m-I(QTJL?Yr6sQ0M9A6sUTFI*-HjOG{)*J6#^mdn5tHcL& zHiV?dU5(C1ma|R=&T_Hf{)99Dh<~wIQ!Ai0D|Shwx902pCYmXPksEk~Q(P^UPsEU7 zWPdEAUD-Zowh30H95tnmz;{HhS`^7N!|lxldUJZK>L&-C0SC3!iG29nkru>o&lT^c zGVaBDi7B>+osvz#Mc_R6S>H36(Vz3z26Lwp5j$G0o4H4Uey8m~-pVbzwFTb|kG>@o zakw%~N{BBD(Wb}lO-uJR5iFKr9}*}OMW0hNBlc*~S;|weStpM-ftUW0mB{vnh{HPM z1l1eEmBNnZW`z4{F5tXpm$)F$nb%>p*>u8w^U)KeePlFxIFMRZUAal|GAsC}YG6wU z&COgbwLvmwZ8BcgaBMx~4w8f5SY7k; z)gl4fyYEO2llCCzmBZj6vAniIzl_ArNPpIh<0#$#RgJVhum)q#Od)z+WFji5Z;l3w zLWV`*w^jo2`gk#St%)$Z4Px`~^!#)p^0$;AxXEMWsK09Pmm(LeJjo4PKKprD9X2b>Bs4=w1 z_TPOYHgl-&!qJ)m`BvI{0yt%9f#?{#MF~0Iam=Kl`RkgO@xfPxSP558e1y7s@G@(1!V(@w;1e)#gwZAuxPXF$#JthWwfGK%m1H6d z1y3Dr@$q;jz~5h5IHEe5KUr^yja<&YeC2{G%*o6wZSk+5X#59%1S3SYL7j=3kLVBB z#gaWh6I!;<3nV1_>zw)LY4gAT-~R;~$Ny8@4_hKSpx*pL&Ud!T<%wel{h!sje-4=a z^TpD_3*RG`^{~Gr!oTIEG2f3XfM27_Vj1_X>)U^-MgQW(pSTk{d$vtkoa;Z92#5^g zgw4HOI}8olZSpf;Vf$XeI!&BLyB4HQ{AX zvet$=mHc(RSmNP5$Mlb^@4w&Af4ue^7I~qjpa}#9H;8R5bbF;v;P(arBFc_qT zgyb>$wzCgMx3k5uFa9$<{a2TDKpt%jiPF^7T~=3DhrUMAx!&`S`*{Jcmg)Pkd6~P{ z@^9e=pQT>VL9?SJ2>J&$FEyqA@kag25BZJBqxT7HQ;^?ACH=Lp|1G&?iIg95lu)u5 z5S9PZ9{=WrH2{R!rTC!o*NDi!e^*`}0~dgUI_So8nK)|rOe5gSAU)*m9gn9{I zsPAb%)gu1ycK@}6{=eRC-#V^@$tqa@QXniw9P=8RE_Hcl5R(Mp)nm%ZnFsxLBdVwW z`<3>`SN*2Z>$jon0f)6A02#WyA(94>L@7r%5c@t17;?=yhHt?9U%jI_4@*AHkfBV3WVL$c|d?{`Ntm@+D!fTE=R)uH+Y+iYK7-*eqaf@62L_TNOe|62dP z9?+kLU!inr3U}*p*sl4$rNMdkUh=JKS)G*qdAEe}hOlH3Lfll|nE013IRu<UozHT$?^Zt>q_Rn^r zzj=b%{rG{%0%%y~$peI9?-PVGcXin4c#<8j&^o-X2+gW?G+vok#HKyW3P4XvOjHhc6F7wAAZ=3J1-U$PK7_@F~iv%E!sEs@u2M5%-k_nDpj-ZCi z_~5qhTrPfxi$3k49)LF*6e>HsPrra_w!3lxKvzzM=7aa-az6gVar_OFmI?-OOKy7T zVs~t4#p`|>K(M^Q!lL3Jv|r8STc=W4;9ehnzYyv3UvKKa48(tYatROmld!MQjYO#U zsV%iOr?jBphEjCcTh}VUBYRo47X(rSs8j|zr=p8xooZ(rJzJCSk(CRG99*L!T1?9_ z5>Y8-)8i2^o?^Q~(2plosZw}w?YOO*DIP9lgGror8qp8x)Gzn&4(`uM= z-!0TxBmlp3Ze)8%)qHP6D3;A8T^Nc$c);WM$RM&wJf^eY_!Zl`{}%B6T{x_Fd%64~ zwTkBN91gu(t^wGZL5SPaQ?xc&+2K)9Y(APCEA~4`XqIrO6~Js<1e`h%WpakA`(7n1Nn6 zIW;x40y|6Yg5PPm!Gy@Wx&8aMer*S1*AT5hmwi_EW8KQo&{Q>yP4gSTOj%899IIAJ zyguvx@)NimDczZfD1D#Q_2~3_EpJaM>$*&+m)`wQ(&o`)=chh{nkXyuCk!vU(oMCD zjQ_m;NSM(3(IDw8W{XqXXv9V8m#HrE*K#({5d#SWQ@8=VOevRzFd@7R&_JbFR!y^G zGu>j}Lm>}iej^$xYDS9|D_cYYe@5f;G`R$lHIfc z60gVAF%zl4SUL~&8P%lGP`V77B-6W`9CQf6#a)yC9FF_LUlvb*9wsKH5qB{FWG#V8 zD#1rvuC<(}qGUca)LJrf;6+3EB$vLl|ESWj6*!bQB4=MD4;b7tl0%aUzlUzAehdGl zm6!E|{t|Knvy+b#3tkSCMG7U%?#sI$IJ;hz&j(FrosZ>QQsb# z0JJVD*x-Vjy4g#F$4{Nv?$enk)GEcXn^bf2%@55=#bGR@oJAJi+@~FORsT6)@EcRC zfsdO`S~G|sNR)ty6x>WivZ|gM*W3ur~_db2{4) zFQbNVP$E+*_!r0wm_of|7|0I!>kzGleXT7N}KG5a3Fo z{DIR`9(&;y*a@G|7{Z0R#70q?<}A;wccL&o+*BH{Ffgf`FGFv`8EyTpg92I>^H~MV zihd~iYycp|=q|#`kMHidh>Xa74G_uIS}>x_mPIEE3_S`o^AeBg!wQHAy{26|Gv!_y z7wU*1!zXT*4W(_FKh-}*2J-xEf*wv8Z4F=~`~6L3f#xddd_lgp;?E=uZ9uYsgLt|n z2;T>UJ;@q#;MdDZK5Dke`SbO`OtTWS2=P1W+;)i=#jmGu+vg7T$qmSEkh-HGR((^fX04Z z6E|o_Ct}REQG|Tff^vdd)zJLR$bNJg?oSc8_YPS+q@Ft;k;B}! z#KVa+>Bx@-phXH)%OZ)QlCGGVw&2TuR0-o>p#Co&=r7QdGQI(!4(blU0-jVV#mNZN zUmr{pqa8#Y$}{;L{TaSshDw}P`q$w%calJKaAiyXW;h#5*XG%YOty_jJDvPIZd_BC zkSlL1Fw#{kL#B+xU2r>pb;PH4p5B3v;8tJC{63><#U;QlLNj;;Iv zh!BQ2v^8sI1Y}wx?A2ngjwTc+f*TzbQM5JLqGVXqW*{Bm!4@eg+#w{>O5Q15+(M(m zpeFGoBiJ(o?`bwU2Ozj0P_XlX_vZy2D9l6u*u2}VBq2EH{u4vFluFyotS7>X)Yi?z zBNferavJ|N#`jYlYOVTc^y!C(2WjdgI_*zR#T15y^k&H-Y4yf#&ZN={$II*f+{W;E z@D)SncvZDhM;>~5kbVsC6{t`IR~4!}*CaW#iYN6zD4_bD?107`Bn5yVyIl%*tZvg`3<{tsO{s+9?(mx=!|9a2&t40%NgGAYo z78egiz;t`KVZ;m&CaLYnrWHf+*^~(SZrI1HI}b#_2on_mQ?|phXw%K$YD_9jju^t#{1rTom`Z9{aGU5$G;DFUdu&eWm}b2n!&tMGB1LC-fFyB9*D zX#Z=}D26Nodyv!KxC#%Bm0)3!bHg8XY->0uoa_Jep1v&Od>C|OdeG)58!)}-^wZ}6x&WW?d4;(B#pXpu$`WOg*z$GRMzZ8AEsGH_5CqEf zI!H!@OopUPj)>D;01BvzO<6OAoc464(3`?cLQ+-%wM#HKMyI8*>5`&pcA!GS1ypF~ zFx%0E`ksQK7d}e`N1o=DZs4pVogOO6_F!TRP}@fo^d>V|P4xkp^^YD^TDSxb%&6wc z((cOK{#j^Jf!-VO(ELH?ocM3go7NIg#|_fUYo%Rjh(xeJsF&A8bC3siU|K*<{%ljF z=f_5%0|+ZIfH+myr_QPfM@sU7Wem*f3_{FR_Y}&84{87s3|v0_EE)jkutHoV^aw{v zht7&YN~uu!xVs&68g?2egd-iQkZ6Ltua`zn^|mwM+|k39ijqL|oXH^)|>&{I9i z(PFYgfPhxlD-CN><;nhtZQ^u^2fB-l(~8xra3E1}gGoj) zX+I0w7m@^fZQQIYCyCq#yg)}0AE!ZUz9)n+R(B4uk*8djgE438(MvWs55L zhu{Z(2Y?L)`EhbxGRACB(Pd>eSq`sGs`B$ckdI@6JPl8&0zkO-{XqarNa@=B*|%L) zLF9w0+hY+agKDuBZ5lJ`kGi^~voAuWoPo_W8ad_F*2{uxk3moh#RWSeT)AF;{7#p9 zjM|Aa@MiEPZch&!9~FO9L&IW~yC5tEC!nx&D!bN?WEX4HP>D7ReGm`a4D7FSIng1H z7RV_AvexNB<*pS`4L*__lWbA2G66U3{WWnkfUQX}^xykxNTZn@KAgg}+KLY!5HxLA z!LR3C;v-M-z$n49=NZ8Ocfv-D$&~Ud1-S(o5aHn&bX3)%Jnh-C_-`zLF*s<8n;`(Q zh{Jd^nH#oH9rDeSBUd!y^XC$CBk~)LW7NmyHcizO9+Py-)mFtIfk1sFIaaWN1rYP5 zi`BIq@{H`3g!e&3hS0@ELn?zXnB{1*=rpzuXs_^Dis{qwg6xkLB1D%sy*C2SE(6dK z0XOsQERS*z7hg20B&F}E;78lj!uP@&UP5H zikxm22?KsLb5I5}G*o{1Z9fkj7!hOj$ld84UN94){EnAOj!WtPyTj1w_=835P1JAK zX|%=Jl107qHuZJNb(vr-TeMv?!1LgnJer$IiH0HfmZxWP1V4-%^IJLPe1CucoX@%6 zM28z=mc=pvYB}@nBJSokY*a=3H>)(YI7$=)Re(z?A;`g52RX(^kX>ejwP^*HIaHzd z2&0{$3{zblJnv`;*xES*A@^GVYZOl(SSqPBy5Ry#ya#bRMnAV-}@}HVh>K z?AQq#Jz?fQsfqyAO++k(5X7(3+5wVp3M_Vw)vFblE=o)*ANz%2iwN|OJi zXruaRiR;OWk#~p8ETdYL;?V(16kfb@OO~=1A`(nlvuYymEz^OvFy4qDgWW+y9P*!9YH6t0t zl6uB@`*ut8%e+^Ul@HCcR=X&Pj++P*3np@B3BY@eausWz*8!N_6m={1*=Z-ED;QXR zXM_gfxAi(_fWQ`;lShiH!Ry|c!}}pnlh$TM9;h?fejD?1&UEPiFgD-p1cyQ}-WFhe z+(?4#yfc;|*D2WLxTX0kytDm47y=Sg4~3l<_TTs~@ed;8G(l($wlH4>zO0*QEl4Qa zfdm7de{*FdW{|-g@2$G={>{((Ywf+$^o7u>EKo!LcQ$W-`oH<*z`(cv-;evBc3}T^ z@BZHr;r|xO|6C(~7+(JWeW8#r%v)5F{b{L;E4&7j@hk1OsQ?UiGPBK!QstMgj}6=| z$KQv)b7;!`;3O0yBO&3JjQ;@ecavg)no^G&T?H3V%t`=wCj-Ftp8Gi$@iXvU-71)4 zc4O%gF#s1GaAvrDqkRp`qg8r8iq1DM2?EgF4}ie_wR8vrh*9l)=*et0et=c$yU||& z$1|4OP4H4fO%U;IKc#a9%!@Z~bUV=e{AIn9?;QYr{|p>z27vlX^9+FJr5Z{qi^rgp z74HfL1Deab6;1%vu3@c7+vIX$x1B>Kg4m}?$wI^q%+&zwf0Y;7JorF2af>u7JKG4T z0#;gptQ*b;cQjguW4Bm~qwQc&h`8Uq#(KzXD}c!{pBX%MNF+cmdQGEItyJw!2AuJx zk188SfX%8X1H=+b?PA?fY<;wK_|54b4+IGeG*mU3D=*Ad7u_G9qSJ?b4XbW6;?KgI z`t1;)vk-YSX9LJUX?#g6sZbRqtBT;bLFN4cKzIVs`<5%gWcQ>IDs6yxFt9y>F9lTl zS#Yqu1%~?jAMKxCMfFb?Q~}uw*Y+m5P)$}b%QiIDm>qd_$1m+gj~v}q(4|_%?pE8~ zdRIuUSQPK4!vHye&fD<0R7?F?xjqVr{SVZB_}0zZzob)g2~jE~_0}>7-;QP^P(&s)$HK}#ow|d3cpN^A$1okbiym|{;R7mK_ zB43!S1-`isC(%?so5I%{_vr_2Ev3>aoI=NK&k7ZKpmP0JIbKcoUaEF=B~ufdgy|Cp zlSt|no{{k%6**Tp3(nfVb6ji{lsmHZt__cf7 z5+wDP4}KzHiQ?J4puv*5l@DEfu3~)jZn5AyR04Jqmpbd+eNtT?@Yk&o{GHk~_A_j^ z!wT62fwET5M{wT20ygzOOe)*ku*F@-*NWeGX;3eiWR2Dnak0j>KbfLEUE8pF{9$t{ zj_dcGBVelo?gY#VYjGbv0Pc0VUU;N0KA(3iv-Og^#)?x~o!9{k+lB}Rag_tcyRqR|FDF4k)-RxcT4kSCZQU)TJt;|@j^B*-*wxn{gH%Rb2;u3)|pe~-02`6SG>ejr!Dfp^#6)v1oGLZ=|4LG)(e-B0NI$tN-u*57yj*9~hBV!~ zW1jl0cT%D9C3KpR>Jg$_H~!`{vztO*>C(Ji{;U{z@9oche;9WDG#;h(ZP_Od1!(TN z&`@R1C6C0AH8Ue5vD{j&$fn%V8s<1@01WS-C)@ZV2{>}h6C_jpKZ%Q#$wXoMhDGvH z136woG3NrSXc;||$*4m|Jx~yL*Zux_vP7L;ljY|5kV$S_b}eBY_Mf+h-yKan4&r<6 zWTg@#aL7a(val*JDqge9UOik5#2Qky;#e;^w}Jtemou=%uk3W3<+MteauNOMI=*HA zuXZP%_Pm!qOREiGJyYm(hWC|w*A8%VxIIr}{H*Qa#dl}wF<)&~gyel9^!w%#gRXv8 zee79R%FbM8(g;Qc%M7nG>gHV zNgB0s-?vms@&{BVj_HLWskxig%#)GWiKT8si6!=fquRSCNQj8`ecG)CxnIT-wW(A~ zT{|oqPB`rF{laPVR&T#y4oLv&tI@_-o4I@Ms2zLpvffH_5NB*{g|MjT;N*MiWlvjH zpH!0yi~e4KpcW6i4*)JV+~)4fcx+Z>@I_Sf^#piy+(7AT+N6H?=^We6zroaUruj;f zRhb1I@Ru6J@}C+MwBieG4GwS?^0Z~Ii-XJsK!_!9s6>utEi`H`GB@(%4+nCj z;RkJwnYQAa$A$=^0Pd0il-?TuXs&L71)9KuA z6a>giG%1XRiCP{?z`MyVe(x&tX_CI*nT~0mVj^ zBvPT3M4~6D5CA;Z+B2vAPFfCXqUE zL=GKK`a*E;@q8DnT*|m_znC0ZmJ8NeYph?Lw3S8i;eKp!Z}~XpxKC2nx-fEWW^KX` zdC+Z;lFj?%nsd1~FP9}N(r_zx>z=Mg=l^k5L!z@)cx9^saE-qtic+w%=UqjQ?7HXgpbWfks3x;;Xt|{i zs&OfNoHqdCys7I`FVX>o#QH$XKluujw`}7ch!*<;Q>pUqDSM4 zX9r8Lj%nLRFwi})*yFybf!a*A@cJXi5c-d;d5WA`^njjAttxXv_}4EBnSUQ)&dL3< zZgUB;P^F0G_M(sV>5gc^`<0kC<6y2zD>yN2$!4|Xy4Df0o^gTS^A>aDeG#iTW$j#O zxa^Pg@!K8N0qU~LnUzV*9J4sl=Y2MY3d1vY;76az!B|$SC(`&Gk6r15eF4quDeqN@!{joo1_K>+KGgmu!G}*XoShc42)N@BQ#IkOs|gwk@h2<6WCa?JjS! zSg5#Mm$e<64`93@%kaLcIjDHn+NE3;PX*M1`o`cAOXPJ73h7v)GH?O|tyrCvN0bui z;KCTWdjudWEYzK>;bAv%UpqE8_7A5xZ0$h?aM_t)}E|vQ6 z+(tKHYjUw7h@Ofu%= zCL%V6m2^~N_WHR^^5!5R1AB#{<*aY zoQzp>?8Bc(tNF1E>?tG$Cn8>#)L4(wWt|QEWFo-Z1Ph4jGZCWui@VAoeReOzPLn1pRW1xImD@{Sf>-96nHe#H;kvcNETe+*=4*$bmguBQyR>_8+gQ0jOzRw-4I zWi=O#;KQOzjQo)fQrLv6!H{hI(lBRIta+{TY5N*?p`omT*;F3n^seG$VY-0HDB_nZ zA=gfr4@)DRI%TKip8@wl42I4fVl>X+lg$bPo6E>5LaMsW=liOa;@S)#H#A-Ms4G&_ z%zZjJ9D06S%Fq67xg`Ktf?>!!>Ma7`R2^fp?Vo^!g#FM6n|3wCQuFO==9v*^pjqDi zJH+>h58RG7O-ze#O_3|KTz!S_?{=WQO zrES7;P;MMHWNEh-&8ihL=yhE)Injh#AME6D3@G;QJR?*9f`8IU^b~qcW7R9ExTrzE z*py^gw3g?=mP>QF=_M6Cz?jiOl$64qZv<=8mE^d#S5XKp+*SzE|^%|6X(nMH?&c8+vITSdh)aIe<&8XoGa$DmTprF=#yqtQM z)LH9!TdBcHMV{Lk7B=JKayYlnNiwekbXMd?iI}5hRa#J&AJOQ6Rh~QOZj7__i%d1t zie76*k9=NqKD=qhZ*b3zOl*-JGMJ6jn|HFrm&(0&eqKo(tt`ODwYU!poqIp#>@D%? z3!h}wuZZnLO>1e}QVx7SW{H5SWtL?99}$4n8HG$Lv$U;tl&_OutK`@wzKqOMgY&rg z;?)W}p~(?vmFdLi7Ty`4-DDEJaI=JFoo*X`dn78K zvFrRFEBjfNWPN`4d6^euc>%E&Ypt?J6yhPxHzh)e%k4aedU%2Nd8GBwvsj@WaX8WB zNiZz8z1nFsJwk@jInzn;oP#T!5JZLC8aCp@sNXqOsq!iyeeKt-Thqy?GG$q3xOmrmD6vjL zT->_u+X&-y;|!s7I{ATZ%(Ov-(DjN(U9|;B)cet?FcRgq^RbkohH#7=$l41&>VepY zuqJ-bV}ChXK(_8qcG9To89Zhc+^&)f831gkwvb!r9TH1AM4=aJoPr`mQPL#!>-%gci{d0qq@~tz0;8^{oI2l@PhU) z$NFWbu!zA1-B->jPM52^$cTra|A)Qz3~Q?Gx_geQ{ zbB;O2n4LGcPqR3`2=*B*YwhprP8HScP-?-p3k8nK5k5Uvn)f?*fp1n!vTE>oTtc1Y zcs?il)H}D@QT$sqip9z%RbF{U)XAMv4m{^27#Fa20ji zdo1f$*q@p(@6>3Mr4stco;1b})z2!J|DfCV@2H{7gQb^BD_Z=0Pk*GG5{x zG6w_PAiujrmfalFNXGt~Z0fqqHRE?5z-Z_j59P>F7VWP)o@q+X%}G}EG#uzt5uttsN0UrTHN6PI9t?xr!up#|*1cwm-jBi|#Mn9%7)Qys|cot@TgC7|`}eitTvNLU$6 ztK;?J6)m?)MEd(!^AVfY+Bi%W{O+pHx)O@JUV*DLIu|VIyQPdnlF!uOa_fIQwhQB9 z&@VS{J4-kL)hWPyy$$rkztZwXU~cG4hi^i5U#qXW=xo*S7YSBI?f+uXO^_^*72(1e z$or^p;{vu3x#ZnkO;u7aSPA>rsK4un0aGOlv&9w3#RYo=zZku!E)1omvFBu`&#r>s z*G76=HEhu$Bmy5M3SeI=F|+jGaLkqY$gY$}&{NA1;ggsO6RZ zHc^S^hCLIXanYxeNYf$gS$MdLr{`SVHd&iF6Ybn3RO-E+;FV@ZNVd#c5rZz>8`-26 zUR$4Xe3X79bs6ro-d<(nmbZc5j-*`8weBr_Q#fEa?*HqjiN{LYSu;j5?x5C~G72+A zC%(knkEs_={~mVlLf+nDtSxd(qdtW0!cb41>5Mv=gzQy~6JbZ&O#wa2a2wVKAaXCh zazT!{G5pm_h@n$TN_OaPS%tlbcPz{{n9kCMRoIb#u@ZG-S7G4Twr1KUPY=VQ(7iTU z6cIH(wZw=iep;f&yx-C4P z&wngsd(K5BG$Rd&k}k$2NBDCVufUguAq($X@;qX`Cc2Zf(U6-ozTkY}JaW`V%n z2Swk8H;bkUe&ExuH?qXvjRmdj4l#l^$^n1gbG*Hfrk4|;w~Og_er%TzE>3LV`xZ%q zrs$@F2N=#v;Z7LI1bG~)?u(_$<5|+91syr0j;IIUxDpcPVFywd+^+nQYWf5eCe+wH z3Cx5G>7AIGS4J&vbA<~yjP}AGchL`xOGG6wp%uQTNinh&>cEETZW2B6?>RiCexe5% z>Gualfm-uJ5z-fYDq+Bq`l)bvC+rIP#^lG`)+z@lR93_(%SfD>Sq}kISKj?7!^Ppo zL~K7pg}YQ6X(doE4G(xBCo2g}!no@Uhf&E46|DHZx$Zt=sPS!DwXSnl|rl4d^>*VcdHE*tS_ zNpH`0H??-ZJ)O_tam6T%>aP|di%akhTgOj~dbAbyeE>NIaa9^z>{>DWjA4Hu?Y)$( ziZ`VyC(GSTAJ4zSXn0|Soa%zgmi-7slgZAgawE1{kEq9Gn!Ra8;hMtiSe*+S&I`TX zV;1k4(vc^7;FDZ`8+!g+kDE|fnxLHC?6->HeL7H+40_*9t~2&aZ#g)sO8*`kuv-{Z zsm@?W=<&HbA|Zo3S?e~sVCRPA8F;B*+Cx`3qxWQesgM`5$=OLF9Y>9eJ8&{OZfa_p z_`Hmj$$m1;hf4vlhjG?jVX)-o(R7@qyg z2Fimnn&+uEnSVylKO^tGUL4%NI}*9N)$vD0sY0!nv<{HXM7t71Kfz#KYGI+;K3RdM zBy#88l33d=(?B~(`anS07{8m#MH2dIqDwFMwPgJ&ZlS$ZG{jYD463=~xK`q1*g{3B z7<9xKxz(k_a+akT^!Cn~naQ3AuDn#b>+HL+zybR0z2#4`JFb5ORy%`(anb2bul5s7 zCLV>@;juf>N@}@bH(iKF z=uCH-7u~Dg-j80a?wT3z`=5GQD{yv?|mo(nAk%ULIq=f8VV^L-1X3 z$Mm0t2Fpd=$uCrs+!Q&R?2>^}%{gJEmrUr?$|vI^-d6|+RKE@jgU|Dvu=*xy_o0%> zIo^Y8mW<-8*MOYf3BdiXEa7$vH}kmttM_DYvuo=BldTf(7mC5V5t+IeeM- zEqj3Wg==U?tMcP_F(sZ!=ka`;R)UkY?jx?kCpS(xWO{x1@niMuw&TkK6E82$x|*)> z{l}{^hZs;Mo~oms8FY-17i{uFIb4FGak~hsWKh_UO|;@kFq^W9AkQ|qc+8u+Zuz3 zb|u6%$BEB3MPoDs&1uvwNNxmyAtk!=>7>B}sXIsZ`XH8E+_55~3o0~0MGj*hx9M}Q zH#sxjvcJK%hZ<5Hm;+>j73{>u2A3{OrQ7ikEvBP+SG%A#rHJrtYa6bx{1a-*NN?{C z`s+#zC#k>0_n$jySZyDB%6{dRilZg2E@ zHoPq8nM!VEWpgkQ$&h&W6LCQZ7te04zsk9c-ag96sjNiNFbLP@aeveoYl!od zM&b>VUm1vrIw3j^fm~YhWZ>T7HQlmzgKbj|zij`YI|RpFDve}Ci^K|>d=)y?MA?V7 zwfd0saIXDjRMo(Fx+pJk^BtzXIhiUA+L!KM){POy30iEbJ5p(r4kXDleA+5Lo6EVZ zPaP9MR%`g46j7_;EYAdJ4ySm(G;%UhNIZM^80+c8Rku}Ncqb6)=Mssu?&}xXJ|ODt z6=s7Rv!}^$7Mk{ODeDp|FxK#Qyz>t7(EnPY$?!{^Jk#R{bEUNN0gq$7M!=zWIsT6A z=8^bH?TBH?R%#MOOVKm@@`#)HTFJAVY|B6Z?S7fJ5$-VuWNi0<_QmAK^QF&+BA*_H zN3>Sii>F^L*P$u_G8O!tD(E<6zAj%tvZU%Wo3ULgoIP$Nhs-$HU7_p|Mar3Cqzw5{wM?!0u57ZYKJ23uwxdD$K*tYQ;8=aJ3Y@3#BJC4%F zdHcs~8T+`qZ9`g)6x3=X*t5u!B0Up47G1?P)fZLsEGVe)`gS&~%MYlrySzdz$vi^+ zji}oM_yi8&Fkj`}fqry!o>}a&Q{tsda6zm67s5^{UE`KE&D^!tfm=R5eyG;P(a|+j zn113p?&UH#)^;13mb<44O%=~(ZEtXwpDKCjO%LK5Jk0FY>NnPp;1|JY-^w29rTJ>X|hEZ2WwL4166Lyu^5182J0L zLgX{z;aOk0Tlu@(S0_l!>|O>s##Pa-AhNjSFY^3&EqK|7jyg4TY2QVM`I9v9g6{d6 z!Qa(RhAVY>@n}xhw#jclEYQn!uxA;PRu*dqqh|{a1Bc;1Sm!-XGG zJ4*L-^SPJVRnZFed0ztiJexC(1pJB$A<(-dd&4yI#d6yI-8#n@!K8so zJ$f`956olz7INr_v8+32i@}am0WHP_X0Q+03dEteD?O6(d3kuc>Hf&vgEwwD_W=%6 zNA=ZrLf<=HX{5^)O9t$1a3%)yt)-5&@{y)$$Yk8z-BaaY4lgBt6@4LyKdmQB<9Ky4 z%U%VVh0mJT><;=RO8T13ua}(|1ppg_Xn>oSbZgKLUZ{*LrQUn^$?FLSWda|=+oAw z&@H_LkFH)f$SUp92gG%w5x5mfV#`{b(ckEYq!Wr&KhUk_=Rh@%R$B!JG38w-6?Q zs`H9SHSv2hz;d?+?j6DNHlf#yaic(|_|8CXSB#Z7>T43Kt2<#ceNr^X&kFS<4vaiij56B!y<>RuE)n|UD$&YKWadUU(#HaFmD#QI- z`1vDW9Yxr9E^Chpem!a2{RYnOp~6|Zdo0=>?rC{U1YaZESh2yxEe3(&olIGIi^j%V z{F4Nv)~}vRu1NeYuDbfILNatcD@rO6fQQO$F-;p1eCs9=q#JsTQn)IQK?4$WyIq`k zKDLF4I3re?*dLtI@x_i2^LhX9q=@E;j&7XKCc8ke z?`Js*NO04-fxV+R-o?s|On-?X<@pG^J1-cPOF7YPN8cXuF8v2CKrW4cvBCGx5|NYD zlu-#xCL9wHuXySZ8h+J%F~F&Iias$$q~tJRWNkX?O67FhtvU^c42M%2p$P`b3AJBy zKc)(Mex_WXTL^$5dY4?%D=Hs=&;cwwSSuBmM@4#EsKD-1Qa#B@ug0Za)35GtL)>BlE9*K)R_f+QulxV=ouBAa^We#tXY1q zl~ZhUC#+STW88}dQ{rV9&N3|C?C_{ut$%HI7e%N}f#jiHF#B;kH9Tmgm@86Ce<-2^ zyvd3sb|w$Bs?`>~TIRr+fKVtz`}knbW7AjasiUKd|0}AR^T32p+tmaAQjEX&X%^)w zx{REQ2z9hd*!(7jeN#x$_q^QoREDaxs=-ZBL5#` z!9oUJqZ=b)97f{8WFl=e`nTlI4Da_uNp59!6um(Cf-%bIOHTR%Ikt2*1NAu)MX02b z4X^QteYNu+EEJb8+uAqoo4!Tsv$roep*%;v7OmI%(@~vUU`YuL3Y|YhkGvS7=ypeoyglh$* z0bFWqa}3}LV}2RPtQtv;>c|m&)`~99UwCUyqve_|rtMCbrIGF<6Y)153#HpX>67dk zdRz!5sx6i%ws$DqAAR7zZQ&+4B09QI|FeG_XpGSnFWKDHt2?h(pQ4~> z;>>_k?J|>{zH`?Uw>eyg%NqCWc&M0i)UIP$RUTRFl;!G&LLu=6Nv-W}Lj0|xEZ^7O zr~qAPla}Nt^2%tp#aN$Rn=t~t+$%b4Z}6c%=LclBOKfz{@g*jHxKknt>w|2GL`*cL z=f%?y3{1n*YSIGD<{k8}r28$2I$YV&u68trVd@)@@L4G3<;`BdXXWnnf8-kb!7WnK!wCWw3mov1|}pq!iKcP zm^`sr+3j#0miF@p)(Tf~p=WPD{4dP6?V3^p1>Y`dXUxvv7MhOhz8)Kv-mk;XfR{_+ zSpKaY$#Z>9AMnlAW;^QS$|WsB$x_~qpJm~cB)4f^Pf%Tf7KQzh!qx^s z1|(71NV^eYBm{o6Vn$QDm6+NTITg5!ax=oeokID$Ms^%;K^a$y!$rYnm<8Qndjd~q z&f9hA^vxwr>oP*oH|!zzkhxsA;1kQX%SCBp?Q9#BcFyO{pLbv6FFBR*N7l?m+6-(>Zt4guZY_xpv^g{JL#g4I%27w0KM;)6!s3Cv&C~sCZSp z`Mk#=1;nNLg=~;|w0zx`5 z(cbFG=J67HV$0ZKs~VPA_kpsPiWWml zRTH9PyyR+&7dHeeM6_&-+uf4TYn0gzCq}!}HGKYS=VHcP^IJuXr>+V=_8dR3z*=ile&fk!pvcixPa~Vohud$NH zx*ZmU{Pl?c6}f|C6lnG8T;tvG6Js6|5pRciq9oR2US++0Sa6}_Z$$cHZ5&1aofB53 z-hxNS`>^f=+}hi44!UmHuNokG3{+2cJ3oxf93w|7=OD;5G8;L(%>bX^$rq92`lia-pT zY9d_pz69QQ0SBi3wO3(`Ie|BatLPF>T4=;;)eHHeeC`txe{7OJioHL7$S7PZ-Myn4 zclTcWy@c4gMB(_tl6r~0!Gs&*?=i>Z%|9wbBU`eQVwddfVJ+c)b70A!vaAf7I>0!M z^9c(|b;LSvO1hxymBW^m4F|B?V8>&|vI!4dy9DWftK5^_e;fdTilnp<7W%Sc^yi~a z52a!ge5J~o=01pf{mwM+iG~d5y!q)%c;_&GRJ%L;fTzUMM_^UjWQ8?IX7BRbwAl3z z3@-LrabS!6NjmxKS^wX^9@fbuQykl&yKi93Mr!!_)BJ(H`3?|e$H*%>Q`k1PztTz1 zOeyx&ntuYm^W$<~T{pkQ+1{sap;uDvw=$lUvK?pQENgIMqvG{#XVO*Lgm?vtUuv$vbM2N=2}pN!V8AoIeph9tcws)9~d@ETv+A% zYZ3pfvH#23|8>+w1(q&_#-yMK=ZX_!ivxLV{H~}+kCtPAkC@6>tezkbL6{15(*Cr;L$+pt{M`Zw_o zh?yopma@d)cIwX%`X3G&eD03S30`uj;u(o^6H&6cilKkJL`yUTO@c&)9 z|MMySzfax&cuI~pEX)1P3*Y|Q{fJ`5{n96PS{_hvoe~-0`T400M`x%OG9`D@X z%FC^>9CwP(ce(s?T$q%;EpuW;dQD-C|BqOprEn}c=d57eI_~@b_e1~j>kXYC(MkSg z!yanI|MUL+w-}&Aa~!umw@dx=_x}CA{6)uci-cbP{_mgDk_VS%`?Xy2-{a9aE)chV zAU7MF|NV1=7eU;jFIYbAv;4!N|MfSPA7??xFLjdSyZP^*6Fdb%{+oNGV-)^BAFs9a zIOPBTxO6g;O@6bAQB1J_JZn$-`q1S(!zbXWHrvVztg;e6e!SQg$LZff(UT#~cMSGI z;M+=}YZh!GA|l=RssAcX_@6ggrkS?(>&e<7J9PWQo(2}+O~0=yA#+DD@@9-M_w%K} zd^QJWYgRzKOXy4*aJtCLK9~9d$OjtMS4e9^*I2RMN1Lx|L;v{^`s>?1MpN+Gp5SF? zSBPX3zqN_S#8V0MT6WaA&EIjwN$J-FWKr7|V*;sYmN~Qwp2q~8i0IDgm~D8pco)g4 z7}=)pul^@!^gmDj@2~Yb1-_u$t(QMXwT(v@H(7^&ox83I25y?D5RFTe zfb&rU{#efNpKl%5Nvg8PxjN;vcu&eWuqch=f?2b7r3fVeluqRsvOR%qi_H`8n1urB zLfTDf7b8IazccH8SoP)cSyj3;`R+hb->sr$wD%~w7{YN0zNJ)Mq@U9OV`ef|k&|El zsejNIOwO+xrLX(U6u9hhG2oO)s6Dmkodx zCGi&VLZYVht`;DM+}d5AjR&Nn9Ct5H@BLM(*=}*iXds&M^|bj`1Fo|*#RaRkH1E2-NU1pyx!(0I_sy?=txvw$ab>DWbsl2vL(`R96o8?Vb~4i=<|jGrgBQ z5kAmZ;PcYrT| zU_=I|j;~HF%APdCz_4Ahe7+TyHC5e)FF~>SZy51h*MmdVGm2~9{r%wnL)-=AY;gB< zPl~9H?6iDo>MwmEDeTjkd5zEb9@q^r3{JHMK1ejg)s}@DhzPsEmXGJs{s<@9@V#s~ zyxWy5=xW2s%DN4Yb`@G$CY6~e7aI&gSh~?py_xAZC4HVN`0eKmmYC>{O7*g9-0Mse zXEnzT^Fi*ZP|wvlxFa0Ks*mCB9GloT{a>0k&mfPgSIimfgrS{Yivw_YrPEPRZg1yO zcNMfa8Aude>egYT89>O4QSq4&b)8XimX>(?uUF@6Lv5}SK2Q^FLAP9vVidn^zSvK@ zHmjQj#B15JX zcX}^vYu&yLZQHm}0R~FJvlSu=0K-*Nua`G0QUO0Y`-hPTn4V!L)-mm3d`#F;2HYI1 zl30(BP(92JV`UD^Y{Z1I?8145sS$?_ZU%}=#fI>wa%(d&IzSH6XdEQ2D31B`sY0tv zPBb4<_{P59t;+q=|4TeNQV$lmaZ-3Kyl?H`!j-hI6j*K+3%4BYLIZtcuc;e0{n=#< zNp!Tdv;D$M(g9kEbrFf2D9xyK-r;H?TSu#(j$k3b`Z{Jr28H*M06Wo_+y==23=1Iz z&tZ4rP(s49JdKtlahQEl{~5Ivo!>c1ESW zo@o^i9=QH)yp@RVkL>slulv7t+Hc%dAkQ)aptXD4Fr#rF;Cv}tTfco?plAI3hpNr* zd*Qw-ok`ZBes`eeDLG#{(fsSr_?s2~NK!xf!m_qb{jw$4ZhA!Q7 zjgDSbsGy*t;;2>6%+2bI<#-gi__hvoI`@eE>7rch}E!wYI@e1`^5ehSyD7}BC^YUGU zFQO=`_q|X0`|XkltqL|h#r~1k-)O8Ef`k`e~1*U4R%tiC-kw1*g;rQ>D+&c_LWRkGyPM}In;L9ODc zEthU_G|(o+WMpw^ETNi#e=|_wIb4dNWf={fJ$=wLpiuXWP14@y~Q4KB{ zy!%^}PXb$p^NYh}g$R*j+dN+)J>B|8t|U0Rj#U%Z+PM(fa=YbP#Xj+;6qiQo zg$zn8yKG|~-c>U8uZj{+;CNzn#A|}oVzMTE`oJWKW)P?({J(f;92$}-UyxUw#uz>( zcV-lmxC5zg;7&{TehO>&k`&h;ZP}uk`Vkk9!v4^*GdeHb=Q+7)S^ll{nKo7j=2}G^ z{OQ_sNmBbOitudL*)`Xh)(83x*7<6wJB99qQDt&x_300}6^Vdx#Cqet^IOV2Atm6) zDG)I$oAl5(9XJrl`fPZ~E)BxszbeSc1PM6wD=`Js7B{Rm%Ltnojn`M@C{4X8x7#*O z%KU~}(=@Y2VF( z4fI1+Cq9iiIZcsyBYaV}fU`!-z*DO`+}u&__^njxrP+PAfI2UR!`&dldZ4DnmzEB8 zR;gXN5J!}Xv-8JP5pS&m-9qjVocA*@A6xBg`_B{J|C&7STsVE|k)mP->bbEZz-fU9 zHj1+4yvo$~&|h5@3BCBzx@qXj2WQUPU`Y8?8dtaH_%VlxMtFjaIqLdt&vB0wRsA~o zns@Kt$Nbh}LuBFoYLu6@^wqs>IQccxBxV(sck|$Cw!Z1d=9BZ`DS%T8(9*Z0b^m?2n)R78mq;XWRa4VU>V|_fhkXo9 zXR5wRSmIjXC(^%fp(B@IwWC!`OUIv@)&E#2Onw1H2mCAc=DB}ONGDOe&dOaU6Ahf2 z4-cr!SFjXX6_$dWpt*YNsf_+Sd{Eq+b2UV(zrj7f_P$_6bK0D1Z~v zI<6g6Y|HunITLPOAj{6|T*x?KEd9?%`)dVd-h(SpeX0R{^RI9GkA24w1BZO!$9k;mX zfyf@}clsLre?7lR)Gg=99nr<3G361@+f(cf*b(v>uR(9lG~o(?a~I;0o5oR1yN&0sC14Sm4xN<`_z@HZbuXEv{Pdv22|T?SK-cN!{c6E11nQYH?RT2wAKIN*v453!IaN^T zM9ovhR)WY)RxZzs_^|@3M6=JF%s~~d`#|(~(9PO-fE?TA1dQ{<8K(o_*I>TClaXWA zK^pF&AQz&Ol;}+BAsBHb`(cH z^f52pPYvi3TrYJ8$!EU_ivPAvlx6|mdbts|1cOHAQI~g!=zt?yYKCK3&1rM%QVQW;?btpcO!`l%I3@bK%hKqYcgI69Y&&e@wyR7W!mJc%xZob4S zm6&Mkr?Y}2!Q8j`w^qivJh9#=tP80NR8ZHhSu1&Rw4JfI8|*gUt&9sGH*b%-;bL49 zdXB8(RDF;2`dkF0uFz-G)&;$r=UFth3G5o8r7NcdfoQMppgLt727Dkhap9QR_QK@$ z6sL{DJh+wkDXqsW73`dK2umUTs`;YSt?+O9)omdhJ#BT;R4tDe*Ht~6TPLzOX7f8j z1b8gkEk*RSZ-jT^#HE2D8^4}9f--G|net_X$3TgacWe1| z+xLmXK(qk`^=#sorn7Kh=Me&N>H%ziRm{9^>&II^W`Dk`&6?@o!}5hrE3V*5BE@Hb zj4$!~qHE-7mPrm%Obe(mVh$K#+VI>{3)Tr=9#bn%_#OE@{}iFdd&5}2-krd%c%aa+ zlV+AYF6lb+7LH_;PL~Q~;GW$8Ml2R6sDS9aO}_d=p7S4MiW(;n(JO9Af`*wjo$1qD)<}@#b(~FX_7Q zMI}wL-4OEZ7x_D^?s5VL=9wn=^rl}>MoB($0f*T$>gq``;Ssm1aF+|Iyq8oSZwOD6 zLmY=ol>>Gy-eWzFUh>-xdYT0ntG+0kd=cMH%LZiJT7}Rm^N8P?B_&*7yJ{(vxCHB0 zv<~T-q#J8K?CQpYCe_;RHJsA!T8_ou#=CNl^?Qrs`1q@gHfff8yyVQo3T0y+Z2=wn z7nPjIJ_M;c5l_$m(H;EgcuH=4CT3MSGT|dOFh@_^BIhIX_Kbjj;Bk zbT5JN)Q23$g+;b-Fc+`Wy-VU~72xf1s6G8AB5tHYTnm^-@XnuM%BiVIT9=(Pq25>- zfjA-rp#sT-PWlxdhbqBefxENTHkd=@cf(bfoHG@abAk1*U%ze@C&vwsZ%uiW-_$@m9f6SPBn2@R5XA+pcLPjzrNP7^w8BVqR4D4 zZ^re&L$@PS_Q0rKZ>;1%!;pRsq@pT4=}s!!y2goqyXzhCx&rV!b6?=A&3JP{@?&0t zgNhhr*$e!Z0 z0XS&l##l?+ACbn${$xs6wrq&c+LzpGzH4psGleBc_4_BRA{2x&-bGw}hrUE5bWQGS z_sf`GtYShTRx4DG{!ZbYk<(}K2n#o^P>^aMAC2Wj&{ym#0|`j9#Gc=`R}_51nGRWu z$G$lvi6wfJgFk+kfYJ*c>HgtUIocotO&8+yc=VIEOaUerdRAG7@HlDs@|7#ok!2=| zH6v2?=@Q~-RB`9wPNun0JS1*k#$x zd7<}KTM>*xuvi@i6lcR1U!-pZ=&>q9W-fP2Q_))EYCeo5ehh-(mf>*_-}re|Q{zhO zg8)Mn5faScy$Q*fBy@d5un9>GOgQRv9Wbqo;;dLfA+o(2Xn3AP-!P zkGM=xG(K2r2p2(cSPZy1^SJMxC!aNOmJ z%&_RF=#gRrmN3!P1hG~sX208@GK!h&O1`rrZj1BK)W1^`p!3^a4gC&lG z)U_Qr@zNaU3*QURsV;P z1>|-H2YT!ZFl_xrHZRrHZ+yk&46jum?$>SF`{SCBnmSHr&YtZTO_ej+9c}*!y4}$O zVqrFY(s(i*=Fuv%d~Q-M)YOp3Eh6oCCnbE+5WPa_ z_^*(}Vtd_8nUlIDD$DgLjjECBAN}db$dK4Us@$5Q^)%6XCsJg1yiunAlwxyvhxRH|dvuZ*pouM>8KY#T^ zCh2bWCS~B*>*|`_LPEt$;=80-491+*_-N%a3CYwu&5Vw`_+Z7rax*1cjUdR>yZ6yw z=d9h>d&V&&Gg`y+{9N?MvZ2pgHV9d2dRVvV=A8sp&7c$5ob7a%=0A!O8*bCk!uPc8 zs`*-M7IU4Vu%`kgCRuAhs*+HGJk;O{^Mz1!u-|b)7k!rKo4TZ{zsh4Yl4=day0M31 z9HVa{QXqUzT$xq29w`mj1Bz|i;>_)-unfTYYq60%__Xq{S@0^(Z{1_`$LdLpDWRwQ z#WedO?g)~I1mgO;x}sesEEy6Tlm4!S-EEU&O6KvO9LIgy*Jul8#`-eqFq@%ULN%37 zf`gNCfYr$8i7jbhA`fl&o}*1BwYTtW*XEbGZho0t)qBSfN)HI1hufMz;&9&kGf78g zWe86*(%@0Je}9GH#_Fol4e>2SUGK_uK3&)ciG#tHt|Jy6zXX1m zBIdhMavRnDluTsn)q)Ge0_Tu8Fa~i;>b!8u8I-NXL0mvC3@F>B~tfYg~GQw zw`;AFx0t;K+zmp%*pE1;Ju9eqUPw2lI^g>Y*dbrOs8?Z`CNVK}13@X2P&2VSEBUg} z9r(b>1OJzY!tG7;4Zu*1eW*}7A<=lLXfaj=?onf>QafMYuq?(tOb!Ur{h|v(a65zT zm7caP9>3t@ zr;S41KmviIItxHvvQd!PecgE}vPI)YF$%$fAj6(?zNPuv3bo{o6eFFQYLfbPYJ=dt zz=uoXH*(zeQhS>fxWb1$bD$0+m=xVS+%_Wmht&03fZO>rAZ0(xZf9+|W!q$|1@FJt zvA-J#=4VP)?)PGW=EIjT>H(2#(v#6V*5*j1-CfX%}WX_(}n=Ce;sQp~o%ffIY zQMrAlhdZNNlE-elNom9{X=drYevTTl^YZ05Wi7AYV-$dtwqg``AQ;2B9u7F7bUa}h z&S{E84F{uk2S$>Xzb6kTH2Bif9xJRQCV1AwrZ_g}h#i+t`yLuTj(xF} zlCGbXfmY6J;o?&oDu_qmeI51c=76%JH7_GzR%~^6nr`3g_czk`*G6$-y$P(>N7<7n zQ15^xe^zeunHV3dLuA-=c%hQd4`U0|fk4emD2>lsnys*3Qb8W*Dbux_$^a1+q> zbxXjU7O!QJcJ{|gphlgau)ma75r0&DxQ)Pg@r>!spGo2W4qMeO)ajp=rk$rJqKfAa zXK=Z#0JW~_totFkG46@$DAHfAOuxtPEMsCb_!ViTz3t7gvGPQ}1{y+bq+eR2m9y5b zkN*MshFj<`49qakV@P;Ym?iwf;hY-DeA`T(v?VTawi6@y`zPHb)b9h;K1T9ZT_7*!Q`KG>lVZ z5fA9E)?`K<@Indw5Bfxt<~!3kvvURw7Z;d#tS#oTm+^dyi>oJDP!|?TJgtkH_PZ(_#h}(1ilu?ex(zJ z6nboiUEbHwn=e0fLKUtzce@9>Qop6+d#VOBVRXa>Kju%syh?+4?Xx|u?3sqm4VIJu z=)EP9&3Wy6x+*$`q27)o<=Srociwd-RmhN6a2U;-+L@G=?*kX*zHag|#$CTC2V6;y zLO`(tHx6zJ^QhKLa@X6}f9|ZjA)`$OA)hpv$Z!i z`O3CtqW`<0cWGUaeK)0el+A!;dpP}v3d%8+_;%Y;n=&DdlK!VwaEqT|@&XImXtyO8 znfp3Vu{%88RcNkgFds6m{o2y*v-@zV=1eAK?}Yz0kM5@}#|>deJqxFST%K|tf5ELc(oE(8$c`~i zD3UC_U_&SEULuc4etf2&s{z+JC&X$!W&mUYB;bw|sX=*=PfGn10o@tu`F8o>CU@)& zA718jz8GyL2Y;?X%`_o-G214uEFCYE#ra(C3I*`K@!8#-$C>y2vfvL~A+8{5HuW#o zK0mb#H^n3Xm!&4rSpaN|3##cUQrO9Wu`|Kku_m#Izr6q;)mli#^^i#HD66!e^eBX? zIJnsS-Iu*IF&Hjvm@YZabz#rv=} zy+X2@>&s_}-mRxeLjKj%uMIVc-t=!P=0V|oS02aALVGW6+aCHYP?~-b^eM& ze=X8Dg}=@pBz;PecA3Q{sC1X=8a%EsSbm0ZQXWfGPAX^ffK!%(x%O$)c13WZBYvAy zXye_5@$Og20C(U986EJS;^Z{S4thQy3Y%LJht>UxRmq%x6 zkRE2Eb*$eZ*NUd@mo?E}?1RTu}pu2SDl00EN_n6jg^0xz*>VDT{{&AOb@l5cE> z@4kB2MlL4ixVciHRbx(freCc%Y5ARiUQzVOt2?8KKjC_5mebAZuwbdr-b%&B78btv zw=chu87_M|z`Ge8W(mWt^4E$D<9q96IFm{ke=Yl%Zqkq>VMR2H=}ii$JQ&wK-ZbfI z&5Rnz#h(iO2JF!{9?rEc4cS4)^VL&yc<-B!jV!>lxiAY#@Y-+Ln+!tj?_P4uiW9JA z-%!1!5{}4PP#D^su!yI$LEE(ixi~&5p5NIj8Dglnrn-%ndFeJg1k;9C_9XF9dpw`~ z{UQV_c-Kh$o{w=MvHGy4Yo#qu9O$JZHuEpcxp*)XUYKi-HzXG&AXtO#v%A)23$aSK zdXV-r=?J8J*o8J5934PYtU0Kikw zu&=@_?7F-*r|vHtZ4z*xp6JmBqNus=zg}x6<@CTC}`-C@AcjM=VFyX#1-j zsDv?Yph+t*C|2U&#AqV4AbRe4uObXFS0Lgs@iePh2{UivGHXr!%iyDTTmS4R?16!J z58)%=Y}i^&I1^pC*MGjJErj@wi>1MgD`c1KSTL_f(U&|nmjrq5ANI-L96!&#~0Vk*Z?lfj!4KL>27@hB&z*37_p0#=61FNjfsq$M}^<5DwRV-26O8U*cB}yD2(d;_!Sat8Mana+g z=R5i3Yw*&OlEN`C-JZlF7l!b8k$vAw6qRP4>8PU`Po@)3A%@o`pwF*fdh+Dz#ij{g zBE9;92e}-F`MK7ER|kC69*l2n8IP(G*%O9+xy!cSQ_#|i>)q`}VVcFy|H#bDypC3O zwp_uwu6SClq`IL;>|#$!lkcDV{qRYtG`W~~PtHR(BeBz;Q6%nBJ?Q4rIEI#Mfq(I&y{ui)Itj0A5qE4Xg(V|@wK&_kA>|g|vbtoNP zOc@r*=&LVji*Z=#zxyx|>P7JP6QVp{kJB!2|B6qtGI4ea5ewMp3ygc_p<5DO;?84I z){lr76|Y^n;g!)*Vos5G)@Q}x_o;b?%$oruLe#kqf5dRTV7Eu#W8K>KBnr5ht)Po~ zl2})l2G1iSVXRk5;cJt(uB2Xn`}=`5KN(={Q-Tm(lL1^#h2;3y0!v!inf!~h{gEG2 zEWqT6LWJ?dn|ailG}O_#PvFlNSu2I!726g;a5 z4Txf|mB!pd*BJ-wdgyM_M;Os}{W50O6u1^gn2$Ue0ud_|`QfXNXe5p%?n(ZTq{Atj z%;x94f#1_En4%Y4n%PxL6Z(O43zogKb&#I3{lf3s(@OFcdj5@Dx;6+5ut4g}AvahV zm)hR%fCafu@urn68NU(RCkEo`RPd6!$#zcAMRO`rK*-6JMX^t&+wH;=UM-=%Vb7iy zc|KlpQcHwggFE}>1@)DNr8~!SYxivqPxpv~6P~i3GWzNUo&X4-i#&vhoDT?jGja5` z(9b!$3=D3Qx7CBbD-@-=9{46RkQl-A@)b(p3-%; zXvd|c-o@LPB|*D!RG~HQeE*heQhB`5lF;NK{cP+ey1S-eq~p*z~QS7+N*qVJ?5B5KofH@+ZqGbv+A63G%A`0(UD`;_Nq^%!Nn zfC}qCe!);SKE)}pRA^6?4>fgofUSLi9;$P5(hWN=SxlTld^Y$ES4n^^&^yn!;dXa+ zymXHa$%`V41C6l{nHBaz5uV(Wy;E3$oa0e0a&lb zHL*?TL(n6V_Mq}XfY?=-;OI=jsKI>cqSw6B(nvx47w+)y&}kwv-woOYa63FE#F5pS z_$8Ha?`{=zT`Y{sePF<$ZaZ6u({6$Mr0vr&20Eq}{Bja$vhUVE#zFdkJ;U!|M6>L} z>Nb`4=CmTzJ4@g~hu3z_+U)6ygRS-GxV^(=n_hdH{-ituHjr%8Hnk@%Ms~IxD>57C z^m37KL^bHSj~ z9!8bkN+4Pd6+yj~QNT3|Y;4_mmSO>`f#)g-=-@k z4E@>wA^6^U5$L+j@h~Xjyw9rlj=WWSGr?aW`;2ey-uvypoC9yQak`J0m;8D}9!(o$ zyUXnfcG!H+3l_O(D&pG9wWLnwFb2+P2NkQu!cOOw4V8AIc{~<=#L9jL{Lw}!nl?<6 z7j;&Hf`+a1&r$DpCsJ;3B>%bkEdl$XO|NwR@>FsmVoH>DtFrM$RUZvGy2HIqifWn zKhkRVB)%%OBit<-XTY-qG)~UzENjZ+SWEJiq%?2P=kTkEdeEOzxjXd!;6h9zPe-5H!E(VJ(%P`H_aEbVP>V&ik|U?*{wjRbBfQ zm&%sHs@sy2i7^s4W_|=G+t}G$M-w9s>nVf-P(y;tfXf-#RixhbTv=Mcs3H{c?lZ~- zF5-^9xQ5*iJy6c(()$Ft#7A>!M4_kTUIcK(yqGU(A12T@aUQwL#VK{j*gVi z;jj#cWbw`Zv?lD`nneTX1?8Num?uV{KFn}grq%P{SXekvAJTqFbOaewWs;y+Z8H2L ze7^H>Qg;KV5ej7U>d&N30>7khRPTnpG*-sA599ySy(;a0jb`$Wgq-|mg&_^dYG|`< zj1&i|`oLv7*F917Q_r>6bMal|W{ALGXNPT(_awr>a$TB$DUMJ5Gpu$A+QLfhrdSO( z=^jtUM{ALN!y}QLjjsmAywC2kEVEmEa> zIh;r;SCUEp1Wc|qwH;v`%~o-#yVdRe3>q!xQfV~7MUCvVfqYssF9mhe))IJVhN!vI zq-HEm<#>$-BQ~{maq!CM7zkfY;f$I@T65Tkk%s}XvZEl}8{PmO&%|4Nfr`5WU3H7N zI+)d$h70Re2+vL~UJAx{oevOaCU(0r&@P_&1@>LT$6i`y#Z>U3E}VTV9GkkauWmmF zMePl3e;3GhdZ~Z=_C&%jIfIl7aE>H}IdU#+*f!f{Z?>*`Z*zedo@7aFf`kMLpaVkl zf50zEZl+Y9V~hM6L7J>uoP5@q#WY7b_s;k#2GreqzJ&{?G&VqB?m@q^GMc@N{PxXR z%S+^NNAPZI)M@Mh*^K1RABH@}_=pP|c%_V;yD;;E=6g@4L590D1k~n-fB+qVe?-ZU zOId2Qj(HP46b11IIOGVzOtoz2^>Y5%Wb%wo1%qSD-7~M3CZp9JN1itMjWQMbSyyph zTE*pj7@J&yya$oH63$5iu_vGg_Z3|J(KB?r7o0>u$9W-lV#1gc`RKHF9|DUjrRaC@ z5aK($2C|UJM`OZ=bmg#)lBev;;20Hv$2zuvLu9dz`qHRKEDGvO{suO%A!2K?N5DL| zdCqmHYc$JLh0{Nz<8aj{ZwU{No}jn7UTRP%Qe4rVi(x}ya&LVtoSS`G-O~K$-tL2r z@4h|95w)=)L8Q6Pp5gncQ{5@m%4`~{{`;|4pG%PI=-7EDS@9u`@J=U0F#DhlJMdnN0|A|nGO5SMGO7ki+2`#y$6&WOf8u5xpB}>imh^B4Bjox_kJfAqt2Tbm zp^sYVlq?g~#|%*L*flm}A`@%|i-{|yJD(2kkM+^KSZJ4hh=IL&W~u7#+4ZQbm?-(?L^Dm690k-71^OWe7(4LU5Ur=h;nZ99I+YkxdE^jsU>jR6GoMU z=!F$803kIc`htumOdVU2C}d9@VG~GqCE!|ib-XMzTRC#DDMWv1JVieW(gMSQ|= zXQlI9P-(x<7?8)v)O{RD#-0ng_a$LIuQqHbmNnE8-|{;C5GBPdF;Uc|kzeATTM=B3 znu&S>4-8yaDw1Z`r8>hSWNvrs`$Hc0;^m^lbnb3Z#;Xv&_vMXs zt%H4*1iOlm^TzY9L32G)6;sJJ0u|L?Ibw9S%eNsAw((S+g;nzc7Mb+A_UKyZ$HK$7 zZO2LR@1NSOe>K8Lt(EJd>_bmCf{OHw#EUb^P4VOxSt&Bs%ujVp zX5RrF1`VA;l59WUQ1k^Snjyp6xit}2+l+%U1iYpKtwTXpg(@@>WOz1s51PRpH=w{% z&!Re(%D(InFTGCl9X5%O{Z7$KdhWSx_tUu^-FbNAs!KAoW$EUN-xsd~P(fH=<%d0y z-+T+jYHRBL-hn(^db3O7V^ugX-mwtf{i8 zzsQzpv3v-SK2f52(ul1rT&ztFsLE&*{7prD(c?&Pvruy%;DUjoy%Swk#SN=wmJ$+D zXub)noRRBWm!b`E5JcF>BAr($d~{yu^GrdBb_;z!>}rA|n;gjYy|p$8u($r}a8`c) zhDsqGSf%$0>0s16qi7?ba2KP@#HG!w`=zK{Fc&WQLjys=F=UjjYe6^tk4wg$xkR4R zxZX8CG}5|TKC`Y%kmqnZFU=h?Eq96aeTF35%SNI+mG1ZyF!A=V6++Wx$7xeI3L%$R z&rPXQ8B6cQboGvffEqs1(9&i!9oMMq=!7qNs~M0TT9-r4Q@-xC%$Q#LWC@!kNwu*| zKS;tF!~$jANdI7SFu3IzSFg7LI>I?i#L%|c6wig8>(nyjZQHhzum^?tw4lUsS1EBb z0{1}W`mPM6V_bj1s2iSg+VEdMx74ZO`&W@>MLv-}M~hC?7S{Wd!t4?=egZrB?{sTF z7mt^8>_UvB15RKEn8SA;wi4r0U%?d!&-4xyXDb;I-0LTbpu)GQD6ASt`^~1F?3S9? zL4UY>pl}tgB1s1PHvY2PaNSj9@Qr__S>Cg2j+f%u!moaQjV#U$^DW?#gWrGDa_RfJ zT#|D4&9B5uEP|Y7CFl{>Ks^{?)f1q|5`xmgN6f5e-=ZJKft|qe9-kEYekVs%%<;QK z#1H#g1j}6*<10b)*C8){A<$PC2QP6TA%Mdm7_R&!0p;*Lwng3jHm{Q!az1aW4g7L9 z`htKqZuL5?`rV5)qNcT6Pxi~8^>$Q{ClixUL&k4$B-D7(^gyaJaM^b5wxTb#W< zs<;MNYAD@=3iC$pY2`9%&RGL($RF}IEhN_($3 zq#cv%8GeYQIu8G&rIPOV@cmbv#*!YR0%ydl)AI-}E4Gp4)$I&3Q>1tImBlcYRHt!m z0fH7(c3iVRTLTl83stA<6?xYcuOsuJbyh3ja-}1p)Q=rW(sD`o4YWTM*iIZ;CgX}a z%B67{t*6I2vdS|z1>6Ct)$l(1_HJ_4Q{2Iiu{aGKWuU=nN*9AUDk6Rf-8vH)MU)8pnHn(szKiC07K%l>ql}^{q)hHCh4Jy zrkCnPxjehGPk{w@1}kJufE{R>1j@pwBO*hCAJ9h!9v08|9Ogd-bH`)N4W6cll0&<_ z(2lfKq2!7?*P1vz`-artOxr;GCau=fH0w_K?yp^CXc)**T$5ge0cpjHZT^~nMFnsO z0{_R@()q6(Lt$9bcip0C6p{;i3_Q&et8$GN-vn(7`Dp>BxEW`prq@hgHb)I$LEF&rsx+2~rQ`$uPwTNJ$N4$_SSp$W4l7RQ%IE zXY0+?TAdoh>I=htemKy31J9bzhYb?yrzIb)Y%h+7FD}`FN~;QoglwDmj(c7U3lOiv z_U!PQanKFsAsA^=_m5~ZX@@j6hFUY z;Gg1Kc;q5K+rf$VEOp&9ZXJq&cxs`BalJxVF3a#<)vpP{L=k#=K)*Y_tg5JsHZvg# zxF(&S3oXq7y$MJ)@scAoj8+#%v-N-3xG7{57NIjCjzo+U!Wr$WUl^A5%G{0S={!Kp z9L8a5akBz3b8qysvBUn;2DsmScb}7;2Zwk0Y86G?tS4$;#q(_Vs&EdbPfWx%>RzR4 z%P`1#{DEBPPVfF+ejPk{r{=+P#q_ML``F&aaiQe~#I$?87=GmB;rp4S9Cq3Q4vfMY zaPo2d=EO^D6oAN#_M<#-I6x`0kGnBmXn0+OS2`&GrOL2dixM#AKF$qDtYuhL@b3WI zk$@yb0-qB7&j+4X;XPg(aLG@;cS=Xto$nvsWem;HeSDf*>^vj67?=^(|3vj(@wNwC zWjilV&v_5Z0)M!Y;_6Cb37wVK3NS^dwO)L4zaQm$iaoeqJ}sppc%Htttx)DceP25v z)LFD1M&)PP45qhwAWtfFb`sOAaaBr!mEhpn<7cK5c3-60RjR%itf8Oi9uRXQv<+(e zdGnPl&Cn$BChFmEFS{8%o&)8+XfwMJ< z+3>pS>2YwLz9kJa=us{zRKhTZQi!JdD+)uTvu*&|<^=K|yzee+SS7D0)EY{^RWB!C zIH#L8Jbg9`W6MDYBk)j0s&xJ)|7OmcZOA|V-i&zSNmVw*PxcY@#8 zs{v%yJFvF~6`pzgE$*@YiscEmf$h0L`r%f&1$HW+SnSp~eCTriqreri88Oy@?C|nC z%ckFQs|N?Z&Qlrwf#7I7?Ex0ytB5m79@m9S4=U$Q5wX-6+A=?^PLchH&D0u$S17g6 ziq`iNUmD&Z{&W3cK&Pr~#kCO#d-@fekU`O>LiMtol%Mzhh;1X?72loLRedU@{d(c) zBjnC*?)0semR(=KRmyWB!jQK@=wa5!c^sk7mXpR^7N{>94SLw8<3Uw-A63Oe!%of> zA457u!mx#GqMyi|+V#kqwD&Q)Qi)2w_yV{yeR`G5K>9Y}G6&E8!7`rv7a$isCgjs* z-8@{jh2kD-e3WV4tP)%ss-TNIU5agYvOKf!P(eknH6Obq3N6mk3^B>_*v0DJ?kxI@ zL+!9`sUi0w=Xo&O~#T>Z`+Y$opg4I7J%g$H|j|F?!je`UCdsph-+-EJgSw4Ff z8A`t0&j{KDX8DA`y|#3pLx*uYM*`uWEHoEv>ual!a;RE1RAqLmc{QI4#q+pkNdLCf z#JLN9c%SGI8a}%mcscIH$*62~SSp2mQARC;O3!Wl9$AZrD&R^ge2uSed731`8*H}# zPi-KfJ^pYU%qB~ng|Abrr5(k2k<+Co9G+3#-3x}FG>OcGOuF!YEdbP(l?f@NT=gt? zreU$xWz-`#_6iX(4!c4b%Q?h<@(&OslM;U0he$peBNL-zUTk9*IIe+Q&*_Ae+6d(d1|9r%A%nVuu<16DNWq_G zgG#lJn^QTlLqPSb6jxlSgGZoj(ON&7qQP}QZ&WU@RhSv%Db{n)+k+n~V(b~Nt0gLe zZ83Tq)9Bh5l5#gK^W?Fy?KIPF=KvZLHSSJ8ewRHv?^R@|XQpvyXd%QHo*uNGS%cE?z#mc{j%TSTK(z{7(UzB9n&c zH$L9p7Ct7ovlmmWSwTdjZyG46s*vG?v7yQG;8Tk(*p*sc3fp zPACA8wdw58-`&E$iNm4lIbAf8`Pzg)Oq6uGo)A|(NPqO@v-zW7g@&CH1h06T|E>P^ zmZJL=(daZ<$iO@Mvj8c$4$j4@RUsu5-))-9)@us_MLHY%k}aGHF0Z#2Wm<##{b@xHKq; z?m8XHde*d2nuM-4PcsHvg~5+I!$t0-Gc>hpOsm@;X6jPk-sY!7OiXjZg)HXqth(C&JlKgCn_fM zmXmYt5!7T?#!Zruudq0J#m}bd;t@6BT<&mmPW9IUIXmIIh(ES5)0ZygREgX`cx#f( z9ym+pfVvMSL-WugAs?Dt4sNx26Y9spf70w_THV*SnHeldqMJC73hlq6^s5X*d@V_9 zR_Gp3CCHe0^oNh0KAU+y(g5=}AMQ}2>QfJt(xY2cU1%IXB-A$m+E;}~s-lV{4<~MOh&y5j|rU$bcDb!Oe(s8138JCz4!VawE+<)!v#cnhaSX<0(ZmYzD+a<(unEa*e-Yj- zE93F5w7S587!Y6H{!a97f&-SrdhXBerBcuAaLzLI#WD^YQ0XXRRDWa&O?X4b1 zI`CB6`h|UwLH!uUo#l&DXl`_v$d#9}_U3V|_4E&B$oWd6!)T(P;2|y^-cBRNYEe>X zC&+7Gf+lzD(${pw6%i4nePsOocMJuaOg$uq{L=&d$y;^!%im5ex*6Y(bycnWEw{G4?0#DM z>Mvcw(*FbuQi>kdc3esj?d1VVMlW{VVqj9L9_^= z*8q99LK-hqFL-<;Yj-3#+XHi~nC|y;3lixe^$w+Idskw!L{2aMz#xC1} zkAN1pH<0nMTKuF`*_@E09slrj!s)`Ed|`vO6w>*O0sk>D%EccQPhp1Nhw%u{=AFYX0PP~ z>BuV_ahGx<>Y0W)bFlT9(8JIyJ-3geo1_3cfS8WJ(yCBry9?w$J6S25aZ4-D%wvxX zrvi+!Zj!CUX#MZMFP}bGW&8rp;Pwfi4e=dRz9i z*<-C3!{qf~bQ(1vC+3Ec?)XWHb-+F1#;ZPON%8u5E!9v(-e#gAyDl1SsC?;BN~Jab zDFycF>@x15X}EoucEim|=tdPo;JAQLT*Qr~JpSceHFKOwG5x}GVyGhWZS)5Q`?K_f z-pviGtf0gL0dRX(f{bLA=`#`fTOhSv{GgxbVImWQ?ni_-5fB$5HZuGc84`m z@NZ^}UAp{kq4tplD-^hQw)>d?* z{lH9!NYmN=BFWH4Gw9^9-2()3yBF&8PJ-UX3O_<@pnn1$%0HR)YJ-(e%DcGyZ zuMCJ!Dw{U$L@T@gXf>o7Y~CdaK??YmwJJX`Ji#CA6vToB*OkR8tK#A=F1A&=(#T$v za|o?8Pxy9H#uqv42Vjq0^Q=3nMP?p}zLoe%@f+|fF)_%_Qyw#{vkZFx%GN^KULsUE z4l9pOUPa3$TO7|#+AH=XnR59CGyU&<9MXzbQrmc@7=Zl0UGQ$s8CGe^cmL!7s-Jl% zvzs&AMD>@RvCTrwqzxiHbsFt*RH~cpXng>l6dvgN2a_bnC7VjWK6# zj;A#pJMVm>e}Vw8ix71+uh;~iJ=CayL*kV8&9_ds?ut&VUndp-#U%j)gcGyUDsjRh zjbR!1$G~NRt@E}N^u^yYiEeq_gtg)o<)>xKRKOWJovrI(q5K5PwnM8*m)6VfQ**kz z_hh~NP!DvqF81hneS79emqTWa0p^nP92G=Ov>^tx!LXTo4>At~)A-4*g477yBXdolEXac0 zCqwn9EdmkIcPBkny9Bevfg@t4#A1nt6HV6*LT}Gle0J(d(jQuMHqXlZ+<)Uv53A`Yzb76z~Bs_ z_~4$>e}wr5Xrx^#w}PAq8f~`;d5%>bYHP?sIn+;bf3IX?eU`)&1vD#6{B?qAfNNX8 zS9R96&B@St-SOtCz}x8{`$DFL<;H-B-JJX5ODAThh>GL$$xT1V0^*ku%Q_f{jMf6M zl>6uUIOLkKo)1c}qYXPK+Hlvpr=7{!mb!pB z8?)CTtMsa^k|yVDYO+`qqs+x8%C5s>FH0_tQ~F$279-4uROBx9e$rlKb;Zj5h`n}e zb1Yh+q6e?Jjw;O*y9B_MjEcK_@K>iDUo}_ia)4;ufvx&vtcjr9ldS<#HGX+t!_`Il zT!NX6kst|R6FFA!u0ISQl_=U&L94}}!qkwUvl@eC$TPw+%9(nkfVzNGQJcipwCmfWNPK_<} z@cClH8gro};I0i4SS${WC92TY%y^?pO|<5Dq;$~oC%lWE&AQ7!dXEh?=(|Mw)rB2s zA#lF(`GQm$eu7|p6#1gdMLvn^AO-RXEUNP@%B2`|z3SxJXkM9XnRE=3$d?mr9_19n)Yv>(QgP|?AP%ZUS#O%|AwrV zfS)oj)}9hwLo7}L${|0Eqq_~NjlV`e_peN_dLJHoKb|S1z3z|iW4Ndw(!pB9w69Qa zJFi%%oB?#HoJF0W!F{j1!VkeHjk%o|zxD4o+@t+3$0yahOzeEFYKk0?9Al&I_dUE7 z+(upw)$0PNmvJ}^FR!d)heP)FtoOr}6frzrr-CHT;EshGZ|fyvH|*hhWY(c%6lE$% z8e7@8{Qs8*J-$%|Z0bok?qyowF+Gg(9SnU&pNah}8vIvTE#?>ijq}7sI|c{eJ1?x} zfoFz;HZs`~MNtLnI>nwyH~LnMmWuU)msfe9{mSiix@TX*g?J@zj@Ouitd)NzRkYb~ z>O@W$2pX&TPzcn=Gi3QMkxrgQPi#Kt*Hsa@uBMx$tc-@?GV(u5WGe!<_K_}2)QjD@ zVOe@*iV^xdA~B7%owg3RvaVKSXo!f6M-|Eaf?AuK;x|b8+$_uN{IZg{UFB{f%0-T7 zf3G6Urzl`cC7;FQSUrLBL!!?U<9w( z{Ue|rUW^_dxh4t^%uc{Ygx##bjxg`r(&tZb!-!waMXf#LV#NuXL%RV>P9?{#lcx{m zU^tBdgA_pY?@0PdDR%@oaoHiitp0MhRfnzdJ&lctlW|GArG0Y1lH~Fc_7uOk?!9QK z%RnhVL-LUGL3W_YuJSakqz+;yf`cI9{iNh0Z-r~sc^s*5#_Vb$GnC+g(laDs2 z(ILZRR^PExZDGHA09)jtFidzuc0A<}uWXu!_a;CBqQ#V_%~2jpykFjRejmj>3k6Cb zN!ms}gRUk$zncwVS{J!{ST-~D5``zt!6+BVA%m2D>xM5MVv^qF-C1ZDmr)ujT}n01 zL(Vr9iL_rKv@6rz^OWcSrf>S8H?q4UIX7w8OqEam_m!B08t*vRk-LV+_4Yp2V%XKtO=Jwp zTZQVdqWvjQYoPEEFkk4}-|2S5Bhj4)RafV)<~ZnMnr>A2l*eDbbO>GVXd8V$?!aZT zHFv28T@~bgwqc?v+k3{Kls|O(=vqtpc~(U@PSk4JZsJ;Hf14=8=7#|WWrPk7W5XJ={In{ zu&?rm^67*pYa|-Z5%sEQ&4q}`Sbd=AzRd-)Y2rIhKf1W_ke`s;+d$P}BvNKGlPzZG zvl9k{X*{toB2^mlJ%TH&nOK*$U3 z5@u0oof1Q$7W(rai1#fWu+x=_%n5F*d+(zP`1sl%rs(pHfMC5jyRO%2D@_($TM(@i zp2(x-z0dHseW$OVb-`~&85;w#;YWj9Di2YXf4=$QkcR=%8NCPkq{Rlvv#ElZA?zxs*hyY!*8<6X~Zq92^v59wbC!1 z>HxCiiJU>dsb8+tQ}I25|8hY`eGWiVo3ngYHz4$4qRH_Dq?=g8kXSVwSSDCp?<@VSf%{fZqn5fRgdR_gG<&WbIPOA#A9iFGq9r_S9|i;D&vEodb~JmeIwBqmX$MF9uwQcFPS!+=`=KM}n1Cob z^h@y3Dh=BTF%Tpak8jGOF3EksrkuV-Ln$;f4GL(4Gfccr=kX}68KUo^gWJRg>Z|5m zXFh&r`Xo7LtG=V7wMORTD``$q6wd54qNMPE*Y+5Uq`ZPNWE>3)^5}DZ#;wNcUmm zX*}a+Z835Y*asa|4UO#Sp|E!olLs7pqZxz6#j+&+5X08V1o4U0@7DfCShoPVqjmsqwRD44pvQfyt&nu7v>Rrl`ICu)$3jofc5B<>xyL&MW~;k zBmtNbD7fcqh-P6zZ?zb`eKxj*jy8|@W~4VkO^cHVHxR=(vy*x4aQz+J8Sd^x#1jy@ z#T0-&J(BTr3R}b}q{f?HgbWZ0qcj&Pk(S9`#~D5`rbaQhp?cU)=l*KXd2oTVKSycI zPgx=%Bz#TLaPrmdg4M%zQQgI=S-M_n-gC#7ue2-r0IVj(diY#TkY@JIEPbnj$7n|Q zN<+7F0$1jW7bnP6W9I}^@UhSZlN|p}?R8`BTFa-)zP?h9j*K-$<1Yspqa-Dy;Rl~d zCnpO(dZpVT??xPb)P{{_qQbvLXpHqbJu*idOQey5!ybl{tzq7Yt&cw$b>(dr`F}ZK zZM394L8Rfz+6i6rop>!<(;MC3>dw=y09d1EVCLZPp^pqay{jStZMtfMl(^^^01eQL z2uBoq*&e#U?ERduCjdT8FW|?`g~Z--yz8=7{g8$F!oefbuJwhNSO=`+?EB=ZImM*t zsnuA2x6reeXXh=G6VSm931L163PuUncc^*l<)?RA{|L~so2z)i-XY<5&jilRCVvQ^ z7>h)TWzWqKkVQ_E)+MavnN({9H92HSX|A5nVr5k+hk+lfUs`^mXP~MR({We7+#>B8 zBp+@v1Z7KYFAa}x=_X&E3w8i3xnO~;>s~L+EYuEi?jx+lO^rpYUNTpMoG{~S;qiNL z7)z`iQi~MAM(kC(W2OCuNc!i)egD0y%*zm!f#Jac13W03Q>R1>1$=7 zc!AkP(T|=!`4BYdsCybU`pcqLs!idT5TZFUByrCmg>LjP4c^K2R;U#;y@m35`U|An z%1w^g$bpe!E26=%WMVu3OS-k zyJ5!GJ7TssfdG}L*F^J=|IhHmZzJ6?AfhgWR`=V=E$lA4mOI1ikll7>SI}doc-KeV z9nZtHwybXv+P3b?E6bFM&|~aSG7v5J2Sp|E#EV$#g!IbEhgwta9~f^ElEE>7`LnUoewPMyH^M>f%2vVWUo_H0U0b<#h8 zBfc1vqZ;gXc^KpC)eaC$4j~4W-P-fE3vNtSfQ$9-Q)FFZCeW^3@5yeD_v)WE=cr&} zF@orp%KCjilQ3MxZrsITZ5SLz6`1sBI~awo6}jEnNv7O|2xHA@$q7)Bz7S?L0*fI1 zDvI0f;;)td=jO~CXED5P`^l?S>Xrm56bejE%eq->&tMY({dk_7jQUb%tar?buH(}I z-@=5|+SLAUBhMG48+Y+mo=20{veXPHXKQEB$ww$M?6+6HHUpQy;P|l6+%nZ^!9FkS zP*C8=9Lf#{jM|_Xw)*5`6#fsp`N6SX1SZOj;MOC{h|d8y;k{_>|v+P@IA~EG(|mfz6&9IRZ`C4gzZj4Wms6t@Uul`2I%sty-0>V z_U;~uMMu@ub49viev6KUmH>u~9yxH>G5Ok`qzsP?kLs77o6-r+Mux3u8DKXC#2%vu zUI;7cazTBU%@c0^Xd}gRR-PlxUGzGEUioNZbFW|V(Xq43(GqObT0iX{s_EX?xq_UW zZzcyHq^PZg)*dvy7Q+9^o&NiA@fpFg3OmT;Gf3p*N6j|<;n?GLYV7s2ARm#JJW>&j zDpPs!93~5m04I&3l7Fc^D1rX`E2&|8>GYfwhMadx?@TsOil8V=kh)k1xzL#3@nb;m zZgm<>a<~B6O8_*A-MB06LbLvMu6Z6d)yn7xx1khO!V{V^k}pW_!M=BJ+7#Z0L$CD zFW;E{xlSfPK)T$6(_&-!8L&d!zS5_6H6f(4tHh(ob`yt4MBo+3R586#V(E^LIBm6q z$y0T(8B6oO_-msf{aZJ{6yB+ud}N3sW~(DtM#G}~p*r42M5Lt#;Bx3B!ZCl!r;dLA zf+D-QA&0=n8czl}stSrrWYSi6m@u@J`&GUH`{qftzur-|%D2fK9s1(l3u#lg<+*CG zTiz!AUt)r{%_wiX30LhVIT-#SE*MhW%*OiRFt=PHwmYhj$oS`%_9h-MEF>`m{5hZz zg^d-?_J&snpjf8_6^I)6WWITxE`+#K3TpGpvR@+wo{5v>{`@x%+9>h>wNg@2gO9;8 zz$%Dn75RBw_cD&?8@MvzdZW|EYrkvZxF6}c4$nKV_dBRRt4e{0G6tzYXGl_%S3@j@szC^VhF(wYKxNe<4cW zlrb=V%gV~4qb=8OY809*8~vG-ME@6xe|vDYPppn#S>EHp;` zF=mKAZTD-Wm*Qm13~8J;Z!AXg!6q-z-!cIl)nuhKZUD>SkuMuhbBD)z%rVR9WLT?b zO@F-1goaJ|LzED2G#R^oNXSUIDl%8V`7HR&o8$Pcfp50?Mv*^R4M!%X-QZzp*dsj> zU)6KHUEWb?e2O6#wlCN8JbXYvU>KK^6w?6UekNKe2%+0W){d<}H{nR-+H+UIlV+5I z*3IYVGj2h5c+^Z%|Ip6aL zb9Y=4qvpsb0e@g%baV>$!|}8-J5xClN!YA>e9Nx=z!rY04(fEN`S1_0ra1@zN&&t0 zs&NWfKF8F)1%-w3K(RajFOg@N)^`F{gC#zehUHbJZ3e|3OG`?vpHf){xv%GJ>wGQ? z8#C%2a|aYSMbEopDcr8XSmpcR6V7xfHrlzF| z5m+a0G>*h@C!(`d{{5!^_Yg}xiJs1Zh-Es}833~Y+N^7Xo)8<76=#R4@zh&f=}Oe{ zS_Qhu`nlS{1ntu1#27T4)awB2K&*n%%Qmyw9RCOjdGxt!KcAZ0L?igZWg`8q)_EH5 z`HQl*?7qS{L|*-aqa%rB7RSxc@v^6#6Pdo~CgSAWfV0f#gaeBtr6pw(OaE-mthxb; z33H$$oRrW@wX*p^eI)|SN3DJes6gcD)>Nvco4$NTDS$XFI{xCcDLtuj%+SN5{G_Yl zkN~Wz4mbGqi@d5bM+P_Cy!@5__xt-M4h?1E7Xq3ZfS5nAY|?3#HHp#*OL=BDJT;|S zXZtzMbF*7ajI}u~4uih`1TZDc>%clXGREI*bc{}@&~IAyi}aT;U!BNkLs?lU|K+Iv z3oTu_MIn8Kk&q~=Ov=10i3KRYbH)CawTa&E;{)z^R3r|5k>W2)q90s#zh$A08O){F zXrdacl7utdj^7grnY0u}lg2-g6RSBXk@D0Ucn1ask{)Ocl{38F5ld&Hdi7Sw*PVk* zL9$IsMmymvK7-wrz7*dnf2N6%zu4ayFwZ{*JUT3qI*17X>XXzf4fu4+7rp^6x&z-u zVo#w9ATh4BP)LXK6@71a0<1B(YTmK45BE0}7o!_$O)`@McDXrf9%vc!lXtZJm4E>N z$mR9#9)7%0`4Se(!I%?{IR@dtWNFC2N^PTqUg??6Iy~?T#%o5Jh4*T%ec{o+P^qu(Xu44rNm?VEEz8WXJbMwF?e|mUvkDa+1@)P0e_$ zif>Loqu@__o`Nj4QmxjxCGTEejMr_#K}C~W{$MdT zQd@sA>n0%l&nfjkk7|ev39J|_L7=}}YhjVR$E4rsUqBEVko(Pf2-yoTwi`VPx30t> zl;G6XqpMTvx04+2X?2CH2mFPb9S)sCbPDbG;ds z`S+Rme;9GdDxI_X+VC3O5=7RYABLs^k`(8?ghHE!Say8+U_GQYnn2$q;f8!0?V9m) zWEbF9^lKf}V-*Or zEUa4fvd_fI3|bIuRtgj6Dn72bcD~M9xqnGEC+}8v3~UQaamrUZtSkVgkCA>ttzKO5 zV=&$cyFvLGpNx6hLyC&Kbw*nP@{4hbDRc>R7bdOB53g}(==*!l&PH@kLPzEPdkp<= zzj6KA%?Csa`(@K?+Jy6S#*z?a%r+Ipl|(+uz}RSK54(z~=&`f!4F6NiRP zpUSJFd<3z<{tB+1knAUW%+1Ryp8zJl1kJL=pak?gzq*o zz0YV@-Xuw-N-p6ylETR6&q5KXMajL;N7IInJL zy1Ld%cTqI0TH+M>6GkeCiHR}lB)$&;=EixNVaaa*-c8e8x3pYCU{EQoF|!7NnDW!G zYa<=~?~!^NjsnT^Ej(Pc`D51E8Fh|)q zfK9Jfc^#b;j0oGB5~9?BZ^RAPpeJ+WM-;&T8wz+>W!RAhGRq(?)8u?KmZ@Co1_P$F&Q88D}R{t}*P%i|yWHxfna zc6|bm+Q7EZ#6nJi5%l!S81`x>owUr#^OAthYq!$uC)!Vwp`M2d)Dm+^!f2N$tQ{z zNL^YYttb{zlU4%LIj}43zWl094o56PJzv%Ye>;`a@l7IA7Qt=$iNDZRdzp4s;k28B zc~9s-t^0v1)V6^0up$%W+h{v4j$<(>{&%KDC<+FUtTkUezRzu$#nPSNUgGh^um+ro z>Y&s(8OovpzE{Q4c@*3$(vpFEKyo}@{<@~4tE;Q-_B)kH)YsxBfZ;7?YMLuul=vR8 zOpzqW4{Lg~t@{FZg#u`7@ZhV&wGk8BEy$M!zZRcdrhoEG9H7L2sol09bjA6{<)0;FFOEf^!eE(QNijFY9t z{%cu=T)ay6zXc5-q=@#=i6T~Re)7xY`#Wb-;$$Nvqp7wi{HIa=YlaC4RR~5h=;^o- zbo)DJ3sQ?BvSMeuCjXr^4QevbT@&he;d)Hwa|H$qtUCQwh% zcCHed{5vb-|My`3``!9~5B9%3=l}1){ z4tPqJsJ@o`Ml|lFIqtNtL-lvUxxXm2QHdT<220!>)hMY@v@j*4D;6M;o1H(MAD_*M={@=EqOELE#Lz~KdY4aFa` z#z{H_03Z?oiY(NqG?2SD^l2er(qit*yn$@60rZLtas^@fla+=!#C=CMG%R?cOfumzN2Vk)?Q~dTKz0mZ7A?#-|s>)JR_b{v4Z1_{YJk$ljGA8uE4h zuE2moXpJ$Ty}k)_R;|{xx;HprCMrB=*}jd$b3X|BtCIS^9(M_%ow%EL_z6E%U%%5V z{Yk*6F{JduXY&C*nsV!Tdu9T?SoK$LwAYP(`{KvW4#9FO7wc>#ZonR39oaNIZ8n&m ze1qOL#-hKa=q*7&aX9vOvNhHB&v(dPi~vl~ucN!J;K0jva$u!3pB_fi_c0b*7f9Kruz31M3^WV&wKjzHm z^ZuOI>+yWOo=-26i*Puqwu6wovV!~S4+O`^Kp-bh^y(@f?npzHA!@g_TIl0eq$HMl zPw7Q-z*ViPmj9sDygX5r&fypO)AAGEi;Ys2rB6IU2AMj|AqIK~&jYxsgVu@m3eF)J9Tsvv^Y<$3Awa3mq*>X$p>awZJsSu7yX7FKP<@8V?h@x9UJ zYA+%4rUMC-&c(@wTbp3l`=z?3}Rd#IVwrq9krT504*^~yx=8G-~fLv=Nu>@WLjo) zme;39ue$a68qFPVQE7#OGQhD;_PuM^0|BE80%QQwoRXylKMJRB=pV8VAb8x@hIHOs zP@=*c^HQ`X!r$`(Fg@K14c5FpT?w-kGt7~kUrT9ep&SV&c_5xo&Ie z*qJ%ZMlrhTCP%@L%j%GRl=;Ky=Yb1~neWXi>})yj4Xn{&JXvy2E*g5SyK;I#@BP}} zpPm9lgOR07QAx>R#$sUoYzhIm@yNGA7zQK2$?#R5q$c|hj0(Udd$nl{!-uV#e!D1L zO8Hz{a&ys!a_zBZi&VZACvjStfrGJVv}AlrF42!|DAdu>5umzH%)w`@#7;SP#8~ji zJopXM`}#f?sq({`dlOeU;Xmn;hHgI_z!0x-c+epCkfs_0X_ExYW=fWpw~BLmsblqW8x14+ofWtpV8UE#g;XM ztAR(A4vyr@bl;WTGq3!=4USWtx={d(-vP)&$ z!?}9>$fBelykR_)AJ87npB~1|<)}=@<0uQAKc0GEtC9RcbEyt_Q~S9+=}KZLfQS^l z$7s?rRT}nVnUMB(2~S1wW94SP7qBAZ2=Nlqz7CBq@C)?4CSAj)q}V)cJuyHXat?4D zojy6iUSo+xDHW_w)$}53V}k-NO(3FHwxC7)1SxWz&5|1UD>S^{2z^w+cm-rjetCLGr;J8Z z;GC~yNyG%oWV9gU#p@caMZP8h(o*_+l5t0l8u*SO9RSu22utStVW4UYkv?b(%$hY( zr0enn@`V&u$xzL9W-vH4zn^zHs8ECP6);i*rmR4iqE-+{>cT-*m@2Wd02zU{Tz*Kd z#7I&!`%26uXzP5A)RlU5;&85Akw+1a%rY8x?fUifC(%w?&$gu_FJH$3M1NyqU>g8d zz2(^rOu{^jDCFr>Y0NO?7vk2MJNu;^;CnoHvny)z5@ZNotx}Ai264kOX)3vob!@Q8fovWttQM*GBQs5nh$=HaR1}UAR3cOdr(*Z zN%{fC%W=-(bWvTZNS3!R*#c53s|+TnoygZfHZ|II%P!;eJ#juKrwuusbFH%@I}_v? z{D;?(ZxZhMWW&mH`A0tzXpO$JaErY-hS(b*pF$PJ8_1unJeTZ5#J`jfBub_9aWr8@ zQ~r^iy4n3t_`g@5U91etsC(KeBY^lY>Pk$DMpU9BP$G<6pA>g^bfI0)HI~3(==Z!A z0#^DRFI_pon&6wpUMZls8A)q}4_X|ZT7s`qQn+eN9uAAwkj%-@@Lkwi!t3=3LGP&e zhcQ0mB66!_87VKVo33#7o9w3tL@;}snDv=l9aj@**utQM1IVN|Q}YADUsU2-FE zwZRqgSr3nJbFM87wqQo@v&Xx|$AxPtHa{>~D94twqI1*Dtad=|40WJD3qc+QhoKV^$5 zE=k7rUaqj}h=G|l1jta8zN^BJzO6}+gi6^69i9T7N02UPaZ9bd5|VpDBlV25cOv$W zXn0F2Z1RdQL!Sf$>2%xleGk?=;$I@m4Nn=?2m#fLKF4_*?S56S8f{z{aDE_ zJ^WyyEy^d%@kXC=I#8T!AWc~}SzN5`ZPzvXw3R4gdSj>70JoB zc-)~A?eDzgzm<86yyIzwo9c)>ji30`6L|o6U$98E}Ocn5F=^#ykHFX z-KIF&2=Xe6L9WczfEAPHZ(BvI!D#*7#~&%F2^B;{M&drCrgw;XiXGAzh9n@+S4FqU zfYXq|#D&j!DHrVR5F~fE+Q)Oj`{AeY;O)J+ojz-hAwmvQ7$>9l!F{=Qd9v@^-DB!C zatBWI!+jQpnWMw~xRP7h(5c+3>)R>|tgm&S&qB$HINDXfe$iR0ScI+|!&)Tc6iqIb zq#ox;og(oeJ2kJ8PG8aPLkdV)u$g@spxY0guY(`zO)aD*^z3$=PpO*spr+~oSC8h+>;$2;Vz43_J?L11Iwpw;=FprTZ7Z#|;^#%PQcV@7_k3pMvj zi#;zvxk6&M#3QE$1Q4;a1%KcET zz!<*)L_v_Wi599s{)HjA@#r-$<a+vX(-UCGd|7Ao=*a>uzf|oK#p}$ z*t6bU!SyuOjgH%p!yFef)JJBIXGy zhN~9q!>5=ZOBmR%$`JCR2|Wjt`)^6X(uY&y0WV^YdIr&2K z?=eOj=}vWx{R64MjorkIwhOIm;o>bVEmwP^qWrfqfXSr)C6N8&lkJQa<%E0sy1Hl4 z{IQyYoZm;>&x1%^ai3F2e4Pq$m6A)+L=CMvZJId__f8e}l^fUq0+fuI@T~|_Z5jSI z`tI&bvHU6|@uAsbn*z%R7RKXJ`#%!poQ{*ysYjMD?*x=H>@ACBCZq^}^)C!FQ_t1* zt}-JVcUOi`Vn$gM6CXc|XQj{@+`A`*R%oVIz`xl|^B93|+|EF?3YbrKTa=6y(q#w_ zNw&A@jZmQlyvPpk=6&5_4|$B)+P~OFt;*IJ83Bl`t~O=-DXeC@&oB(M`b)4-jn}?Ch3Z-#&AJl*rhyi-1ZuiSbN; zbUT1okuEN`cUbJ7c@2$H46tuQrSPH3o2JMF3~Hyc%$&Ig5QFrVdeX-=+wG#-Yg4@m z-;!K-avvPdAFv-r9pIJYLU+nf*=hSsy9kfmJf{(wgsQ)9Mso@>F0pJlLh=;PE0p&$0INfOkT>l3+UDoJd7fGDeF zsxa(R6cc=RFx`zO6&?R2Os7dPRIF!{JMLuk7svSKFH$&w~>g*?l0+(0F$Nn%+etzc+;1u2LLs ze`P{Mn8B;&GgoYE>_}l1uUNo>8i)v7X9dj?$F|?;?Riw?$7*->dcN9%O${P7(C4*2 z?MPQ@NA$zi=;+1uo_7Ruqdjh_9l(pW%&}!cCwiFxVH6e1yEAig$O96uS0ZzM`y;oN zdJa=<5-{V@QIYpFBGQt?e4~&V81rXN#KB;i99&$^nTt;>Z`61qXD@?nU3yXyq5UFJ zD@xZNS_3&@!N+ZbJTJyKdw`V=tkzq>-cwQT>a6`rqq^|Dcfndh49lqI`B98gtb%43w4?Y=C<9U4aBifHsa6Ta0)z2sko*adNml z`I1}mD#6bN^frhi%rus#e`5%v(QrPrd`$b$B@9NEsp&R_;fqettvi!3<%?RQCFR3}p>RG$l*9Z54)ozXX z35Jw?l=d6*70g}={=3L?-w*YtHXDf6G%vAjNZpaQNw64Y Date: Tue, 24 Mar 2026 11:49:51 +0100 Subject: [PATCH 3/4] chore: Cloudron embeddings server instructions (#4938) --- deploy/cloudron/CloudronManifest.json | 2 +- deploy/cloudron/embeddings/CloudronManifest.json | 15 +++++++++++++++ deploy/cloudron/embeddings/DESCRIPTION.md | 9 +++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 deploy/cloudron/embeddings/CloudronManifest.json create mode 100644 deploy/cloudron/embeddings/DESCRIPTION.md diff --git a/deploy/cloudron/CloudronManifest.json b/deploy/cloudron/CloudronManifest.json index f382d9181e..9ede2593d2 100644 --- a/deploy/cloudron/CloudronManifest.json +++ b/deploy/cloudron/CloudronManifest.json @@ -1,7 +1,7 @@ { "id": "io.baserow.cloudronapp", "title": "Baserow", - "author": "Bram Wiepjes", + "author": "Baserow", "description": "file://DESCRIPTION.md", "tagline": "Collaborate on any form of data", "website": "https://baserow.io", diff --git a/deploy/cloudron/embeddings/CloudronManifest.json b/deploy/cloudron/embeddings/CloudronManifest.json new file mode 100644 index 0000000000..79eb49ee78 --- /dev/null +++ b/deploy/cloudron/embeddings/CloudronManifest.json @@ -0,0 +1,15 @@ +{ + "id": "io.baserow.embeddings", + "title": "Baserow Embeddings", + "author": "Baserow", + "description": "file://DESCRIPTION.md", + "tagline": "Embeddings server for the AI assistant docs lookup.", + "website": "https://baserow.io", + "contactEmail": "bram@baserow.io", + "tags": ["no-code", "nocode", "database", "data", "collaborate", "airtable"], + "version": "2.1.4", + "healthCheckPath": "/api/_health/", + "httpPort": 80, + "memoryLimit": 1000000, + "manifestVersion": 2 +} diff --git a/deploy/cloudron/embeddings/DESCRIPTION.md b/deploy/cloudron/embeddings/DESCRIPTION.md new file mode 100644 index 0000000000..f2dcb75439 --- /dev/null +++ b/deploy/cloudron/embeddings/DESCRIPTION.md @@ -0,0 +1,9 @@ +``` +cloudron install -l embeddings.{YOUR_DOMAIN} --image baserow/embeddings:1.0.0 +``` + +In the Baserow app in Cloudron do: + +``` +cloudron env set BASEROW_EMBEDDINGS_API_URL=https://embeddings.{YOUR_DOMAIN} +``` From 4a107a32e62668ee90269eefc4d864276f3281ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Pardou?= <571533+jrmi@users.noreply.github.com> Date: Tue, 24 Mar 2026 13:56:41 +0100 Subject: [PATCH 4/4] feat: allow to use django SMTP configuration to send emails (#5001) * Allow to use django SMTP configuration to send emails --- .../skills/add-django-config-env-var/SKILL.md | 63 + .../agents/openai.yaml | 4 + .agents/skills/create-update-service/SKILL.md | 145 +++ .../create-update-service/agents/openai.yaml | 4 + .../skills/write-frontend-unit-test/SKILL.md | 175 +++ AGENTS.md | 36 + backend/src/baserow/config/settings/base.py | 3 + .../contrib/automation/nodes/registries.py | 2 +- .../workflow_actions/workflow_action_types.py | 2 +- .../contrib/integrations/core/models.py | 9 + .../integrations/core/service_types.py | 119 +- ...emailservice_use_instance_smtp_settings.py | 19 + backend/src/baserow/core/services/handler.py | 5 +- .../src/baserow/core/services/registries.py | 3 + .../core/test_smtp_email_service_type.py | 182 +++ ...tion_can_be_used_to_send_emails_with_.json | 9 + docker-compose.no-caddy.yml | 1 + docker-compose.yml | 2 + .../components/AutomationHeader.vue | 4 +- .../core/assets/scss/components/checkbox.scss | 14 +- .../modules/core/components/Checkbox.vue | 36 +- .../services/CoreSMTPEmailServiceForm.vue | 26 + .../modules/integrations/core/serviceTypes.js | 21 +- .../modules/integrations/locales/en.json | 2 + .../unit/core/components/checkbox.spec.js | 2 +- .../exportTableModal.spec.js.snap | 28 +- .../smtpEmailService.spec.js.snap | 1038 +++++++++++++++++ .../core/smtpEmailService.spec.js | 234 ++++ .../unit/integrations/core/smtpForm.spec.js | 72 ++ 29 files changed, 2208 insertions(+), 52 deletions(-) create mode 100644 .agents/skills/add-django-config-env-var/SKILL.md create mode 100644 .agents/skills/add-django-config-env-var/agents/openai.yaml create mode 100644 .agents/skills/create-update-service/SKILL.md create mode 100644 .agents/skills/create-update-service/agents/openai.yaml create mode 100644 .agents/skills/write-frontend-unit-test/SKILL.md create mode 100644 AGENTS.md create mode 100644 backend/src/baserow/contrib/integrations/migrations/0027_coresmtpemailservice_use_instance_smtp_settings.py create mode 100644 changelog/entries/unreleased/breaking_change/4999_instance_smtp_configuration_can_be_used_to_send_emails_with_.json create mode 100644 web-frontend/test/unit/integrations/core/__snapshots__/smtpEmailService.spec.js.snap create mode 100644 web-frontend/test/unit/integrations/core/smtpEmailService.spec.js create mode 100644 web-frontend/test/unit/integrations/core/smtpForm.spec.js diff --git a/.agents/skills/add-django-config-env-var/SKILL.md b/.agents/skills/add-django-config-env-var/SKILL.md new file mode 100644 index 0000000000..e3277edd08 --- /dev/null +++ b/.agents/skills/add-django-config-env-var/SKILL.md @@ -0,0 +1,63 @@ +--- +name: add-django-config-env-var +description: Add a new environment variable for a Django setting in Baserow and propagate it to the few repo files that usually need it. Use this when a request says a config env var must be added in several places or references `INTEGRATION_LOCAL_BASEROW_PAGE_SIZE_LIMIT` as the pattern to follow. +--- + +# Add Django Config Env Var + +Use `INTEGRATION_LOCAL_BASEROW_PAGE_SIZE_LIMIT` as the template. The env var name should be prefixed with `BASEROW_` but the internal var isn't. + +Keep the change simple and explicit. Do not add abstractions for this. + +## Files To Check + +When adding a new setting, usually check these files: + +- `backend/src/baserow/config/settings/base.py` +- `docker-compose.yml` +- `docker-compose.no-caddy.yml` +- `web-frontend/env-remap.mjs` +- Backend or frontend code that uses the setting +- A focused test if behavior changes + +## Workflow + +1. Add the Django setting in `backend/src/baserow/config/settings/base.py` near the closest related setting. + +Example: + +```python +MY_SETTING = int(os.getenv("BASEROW_MY_SETTING", 123)) +``` + +2. If the variable should be configurable in Docker, add it everywhere the similar example appears in: + +- `docker-compose.yml` +- `docker-compose.no-caddy.yml` + +3. If the frontend needs it at runtime, add it to `web-frontend/env-remap.mjs`. + +4. Update consumers to use the setting: + +- Backend: `settings.MY_SETTING` +- Tests: `override_settings(MY_SETTING=...)` + +5. Add or update a targeted test if the setting changes behavior. + +6. Add the related documentation + +## Quick Checklist + +1. Add it in `base.py` +2. Mirror the matching Docker entries +3. Add the Nuxt remap if frontend code needs it +4. Use `settings.` in code +5. Add a focused test if needed +6. Add the documentation + +## Guardrails + +- Do not add a raw `os.getenv(...)` in application code when the value belongs in Django settings. +- Do not update only one Docker location if the example appears in several places. +- Do not expose a backend-only setting to Nuxt unless the frontend actually needs it. +- Prefer copying the closest existing setting instead of inventing a new pattern. diff --git a/.agents/skills/add-django-config-env-var/agents/openai.yaml b/.agents/skills/add-django-config-env-var/agents/openai.yaml new file mode 100644 index 0000000000..5f44641896 --- /dev/null +++ b/.agents/skills/add-django-config-env-var/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Add Django Config Env Var" + short_description: "Add and propagate a Django env variable" + default_prompt: "Use $add-django-config-env-var to add a new environment variable that changes Django configuration across Baserow." diff --git a/.agents/skills/create-update-service/SKILL.md b/.agents/skills/create-update-service/SKILL.md new file mode 100644 index 0000000000..18e8de5e42 --- /dev/null +++ b/.agents/skills/create-update-service/SKILL.md @@ -0,0 +1,145 @@ +--- +name: create-update-service +description: Allow to create or update Baserow Integrations and Services +--- + +# Create Or Update Baserow Services And Integrations + +Use this skill when a task involves creating or updating a Baserow integration type or service type in the `contrib/integrations` stack. + +This repo already has the core patterns. Prefer copying an existing implementation close to the target behavior instead of inventing a new structure. + +Integrations and services are shared by the Application builder and the Automation tool. Each of them should be compatible with both tools. + +## First Step + +Before editing, identify which of these applies: + +1. New integration type only +2. New service type attached to an existing integration +3. Update to an existing integration or service +4. Full feature spanning backend, frontend, translations, and tests + +Then inspect the closest existing example with `rg` before changing files. + +Useful starting points: + +- Backend registrations: `backend/src/baserow/contrib/integrations/apps.py` +- Frontend registrations: `web-frontend/modules/integrations/plugin.js` +- Core backend service examples: `backend/src/baserow/contrib/integrations/core/service_types.py` +- Core frontend service examples: `web-frontend/modules/integrations/core/serviceTypes.js` +- Backend integration example: `backend/src/baserow/contrib/integrations/core/integration_types.py` +- Frontend integration example: `web-frontend/modules/integrations/core/integrationTypes.js` + +## Backend Checklist + +For a new or updated service type, check these areas: + +1. Model fields exist and support the intended configuration. +2. The `ServiceType` subclass exposes the right `type`, `model_class`, `dispatch_types`, `allowed_fields`, and serializer configuration. +3. Related nested objects are handled in `after_create`, update helpers, or custom methods when needed. +4. Context/schema methods are implemented if the service emits data for downstream nodes. +5. The service is registered in `backend/src/baserow/contrib/integrations/apps.py`. +6. A migration is added if models changed. + +For a new or updated integration type, check these areas: + +1. The `IntegrationType` subclass defines `type`, `model_class`, serializer field names, allowed fields, and sensitive fields when relevant. +2. Any integration-specific context data or permissions behavior is preserved. +3. The integration is registered in `backend/src/baserow/contrib/integrations/apps.py`. +4. A migration is added if models changed. + +Common backend files to inspect: + +- `backend/src/baserow/contrib/integrations/*/models.py` +- `backend/src/baserow/contrib/integrations/*/service_types.py` +- `backend/src/baserow/contrib/integrations/*/integration_types.py` +- `backend/src/baserow/contrib/integrations/api/**` +- `backend/src/baserow/contrib/integrations/migrations/**` + +## Frontend Checklist + +If the feature is user-configurable, update the frontend in parallel with the backend: + +1. Add or update the service or integration type class. +2. Register it in `web-frontend/modules/integrations/plugin.js`. +3. Add or update the form component used to configure it. +4. Add translations in `web-frontend/modules/integrations/locales/en.json`. +5. Add any supporting mixins, helpers, or assets only if the existing pattern requires them. + +Common frontend files to inspect: + +- `web-frontend/modules/integrations/*/serviceTypes.js` +- `web-frontend/modules/integrations/*/integrationTypes.js` +- `web-frontend/modules/integrations/*/components/services/**` +- `web-frontend/modules/integrations/*/components/integrations/**` +- `web-frontend/modules/integrations/locales/en.json` + +## How To Implement + +### Creating a new service type + +1. Start from the closest existing service type with similar dispatch behavior: + `ACTION`, `DATA`, or trigger behavior. +2. Add or update the backend model if the service needs persisted fields. +3. Implement or extend the backend `ServiceType` subclass. +4. Register the service in `backend/src/baserow/contrib/integrations/apps.py`. +5. Implement the frontend service type class and form component. +6. Register the service in `web-frontend/modules/integrations/plugin.js`. +7. Add translations and tests. + +### Creating a new integration type + +1. Start from the closest existing integration type with similar auth or configuration needs. +2. Add or update the backend model if required. +3. Implement or extend the backend `IntegrationType` subclass. +4. Register the integration in `backend/src/baserow/contrib/integrations/apps.py`. +5. Implement the frontend integration type class and form component. +6. Register the integration in `web-frontend/modules/integrations/plugin.js`. +7. Add translations and tests. + +### Updating an existing type + +1. Find all backend and frontend registrations for the type string. +2. Check whether API serializers, nested relations, or schema generation need updates. +3. Keep existing `type` identifiers stable unless the user explicitly wants a breaking change. +4. Check whether old records need a migration or a data backfill. +5. Update tests for both create and update flows when behavior changes. + +## Testing Expectations + +Run the narrowest relevant tests first or create one if none exists. + +Backend examples: + +- Integration and Service tests in `backend/tests/baserow/api/integrations/**` + +Frontend examples: + +- Unit tests near `web-frontend/test/unit/integrations/**` + +Minimum validation before finishing: + +1. The type is registered on both backend and frontend when applicable. +2. The create and update flows serialize the intended fields. +3. Required translations exist. +4. Migrations are present for model changes. +5. The most relevant targeted tests pass, or the failure is reported explicitly. + +## Search Patterns + +Use these searches to move quickly: + +- `rg -n "class .*ServiceType" backend/src/baserow/contrib/integrations` +- `rg -n "class .*IntegrationType" backend/src/baserow/contrib/integrations` +- `rg -n "register\\(" backend/src/baserow/contrib/integrations/apps.py web-frontend/modules/integrations/plugin.js` +- `rg -n "getType\\(\\)" web-frontend/modules/integrations` +- `rg -n "\"serviceType\\.|integrationType\\.\"" web-frontend/modules/integrations/locales/en.json` + +## Guardrails + +- Do not add a backend type without checking the matching frontend registration path. +- Do not rename a persisted `type` string casually. +- Do not forget migrations when model fields change. +- Do not add broad abstractions unless at least two existing implementations already need them. +- Prefer matching the nearest existing module layout over introducing a new folder structure. diff --git a/.agents/skills/create-update-service/agents/openai.yaml b/.agents/skills/create-update-service/agents/openai.yaml new file mode 100644 index 0000000000..d6471be610 --- /dev/null +++ b/.agents/skills/create-update-service/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: 'Integrations and Services' + short_description: 'Create or update Baserow Integrations and Services' + default_prompt: 'Use $create-update-service to create or update Baserow Integrations and Services.' diff --git a/.agents/skills/write-frontend-unit-test/SKILL.md b/.agents/skills/write-frontend-unit-test/SKILL.md new file mode 100644 index 0000000000..85981e2f77 --- /dev/null +++ b/.agents/skills/write-frontend-unit-test/SKILL.md @@ -0,0 +1,175 @@ +--- +name: write-frontend-unit-test +description: Write or update Baserow frontend unit tests for core, premium, or enterprise code using the repo's existing Vitest, Nuxt, Vue Test Utils, TestApp, and snapshot patterns. +--- + +# Write Baserow Frontend Unit Tests + +Use this skill when a task is to add, fix, or extend a frontend unit test in `web-frontend`, `premium/web-frontend`, or `enterprise/web-frontend`. + +Do not invent a generic Vue testing style. This repo already has established patterns. Start by finding the closest existing spec and copy its setup shape. + +## First Step + +Before editing, identify the test target: + +1. Pure utility or parser function +2. Vuex store logic +3. Vue component mounted with the shared app context +4. Nuxt/Vue 3 component mounted directly with `mountSuspended` +5. Premium or enterprise variant of one of the above + +Then inspect the nearest existing spec in the same module area. + +Useful searches: + +- `rg --files web-frontend/test premium/web-frontend/test enterprise/web-frontend/test | rg '\.spec\.'` +- `rg -n "new TestApp\\(|new PremiumTestApp\\(|mountSuspended\\(" web-frontend/test premium/web-frontend/test enterprise/web-frontend/test` +- `rg -n "toMatchSnapshot\\(|vi\\.fn\\(|vi\\.spyOn\\(" web-frontend/test premium/web-frontend/test enterprise/web-frontend/test` + +## Tooling Used In This Repo + +Current frontend unit tests use: + +- `vitest` for `describe`, `test`, `expect`, `vi` +- `@vue/test-utils` +- `@nuxt/test-utils/runtime` with `mountSuspended` +- Repo helpers such as `TestApp`, `PremiumTestApp`, `MockServer`, and fixtures under `web-frontend/test` +- Snapshot assertions for rendered HTML when the component output matters + +Important local files: + +- `web-frontend/vitest.setup.ts` +- `web-frontend/test/helpers/testApp.js` +- `premium/web-frontend/test/helpers/premiumTestApp.js` + +`vitest.setup.ts` already mocks i18n, UUID generation, and `WebSocket`. Reuse that environment instead of re-mocking those globally in each spec. + +## Choose The Right Pattern + +### Pure utility tests + +For functions in `modules/*/utils/**`, keep the test simple: + +1. Import the function directly. +2. Use plain inputs and deterministic assertions. +3. Prefer `toStrictEqual`, `toBe`, or explicit formatted objects over snapshots. + +Good examples: + +- `web-frontend/test/unit/core/utils/date.spec.js` +- `web-frontend/test/unit/core/utils/string.spec.js` + +### Store tests + +For Vuex store behavior, prefer `TestApp` unless the existing spec clearly uses a temporary local store: + +1. Create `testApp = new TestApp()` in `beforeEach`. +2. Read `store = testApp.store`. +3. Seed state through store actions or the mock server. +4. Always `await testApp.afterEach()` in `afterEach`. + +Good examples: + +- `web-frontend/test/unit/core/store/auth.spec.js` +- `web-frontend/test/unit/builder/store/dataSource.spec.js` + +If the code lives in premium and needs premium-only auth/license behavior, use `PremiumTestApp`. + +### Shared app component tests + +For many components, especially older patterns or components coupled to store, router, registry, or client behavior: + +1. Create `testApp = new TestApp()` or `new PremiumTestApp()`. +2. Mount with `testApp.mount(Component, { props, propsData, slots, listeners, global })`. +3. Prefer the existing helper in the file, for example `mountComponent(...)`. +4. Clean up with `await testApp.afterEach()`. + +`TestApp.mount` supports both `props` and legacy `propsData`, and converts `listeners` into Vue 3 event props. Match the nearby spec instead of rewriting all setup. + +Good examples: + +- `web-frontend/test/unit/core/components/dropdown.spec.js` +- `premium/web-frontend/test/unit/premium/view/calendar/calendarView.spec.js` + +### Direct `mountSuspended` component tests + +For newer Nuxt/Vue 3 component tests that do not need the full helper wrapper: + +1. Use `const testApp = useNuxtApp()` in `beforeEach` if the component expects injected app/store context. +2. Mount with `mountSuspended(Component, { props, slots, global: { provide, stubs, mocks } })`. +3. Provide injected dependencies explicitly. + +Good examples: + +- `web-frontend/test/unit/builder/components/elements/components/HeadingElement.spec.js` + +## Assertions + +Prefer the narrowest assertion that proves behavior: + +- Use `toStrictEqual` or `toEqual` for transformed data and store state. +- Use `toBe` for scalar values. +- Use `vi.fn()` and `vi.spyOn()` for event handlers and method calls. +- Use snapshots for rendered markup where the repo already uses them. + +Do not default to snapshots for pure logic. + +When asserting reactive store objects, this repo sometimes normalizes them with: + +```js +JSON.parse(JSON.stringify(value)) +``` + +Use that only when the nearby test does it for Vue reactivity serialization issues. + +Don't assert internals, always assert visible result in the DOM. For instance don't +use + +```js +expect(wrapper.vm.values.use_instance_smtp_settings).toBe(false) # BAD +``` + +Don't directly use vm properties. + +## Mocking And Fixtures + +Prefer repo helpers over bespoke mocks: + +1. Use `testApp.mockServer` when the behavior depends on store-backed API calls. +2. Use fixtures under `web-frontend/test/fixtures` and premium or enterprise fixture folders when suitable. +3. Use `testApp.dontFailOnErrorResponses()` only when the test intentionally exercises failing responses. + +Do not build a large custom mock environment if `TestApp` already provides the needed app, client, registry, router, and store wiring. + +## File Placement + +Follow the existing test tree: + +- Core: `web-frontend/test/unit/...` +- Premium: `premium/web-frontend/test/unit/...` +- Enterprise: `enterprise/web-frontend/test/unit/...` + +Keep the spec near the feature area rather than creating a new generic test folder. + +## Validation + +Run the narrowest relevant test command first. + +Examples: + +- `just f yarn test:core --run test/unit/core/components/dropdown.spec.js` +- `just f yarn test:core --run test/unit/core/store/auth.spec.js` +- `just f yarn test:premium --run ../premium/web-frontend/test/unit/premium/view/calendar/calendarView.spec.js` +- `just f yarn test:enterprise --run ../enterprise/web-frontend/test/unit/enterprise/plugins.spec.js` + +If a snapshot changes intentionally, review the diff instead of blindly accepting it. + +## Guardrails + +- Do not introduce Jest APIs. Use Vitest APIs already present in the repo. +- Do not add a standalone mount helper when `TestApp` or `PremiumTestApp` already fits. +- Do not over-mock store, router, or client dependencies if the real test helpers can provide them. +- Do not mix unrelated styles in one file. Match the nearest local spec. +- Do not leave out `afterEach` cleanup when using `TestApp` or `PremiumTestApp`. +- Do not create broad integration-style tests when a focused unit test is enough. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..0dbeaf9a2c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,36 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +Baserow is a monorepo. Core Django code lives in `backend/src`, shared backend tests in `backend/tests`, and the main Nuxt app in `web-frontend/` (`modules/`, `server/`, `test/`, `stories/`). Paid extensions mirror that layout in `premium/backend`, `premium/web-frontend`, `enterprise/backend`, and `enterprise/web-frontend`. End-to-end coverage lives in `e2e-tests/`. Product and contributor docs are in `docs/`, while deployment recipes are under `deploy/`. + +## Build, Test, and Development Commands + +Use `just` from the repo root; it wraps the backend and frontend workflows consistently for local and Docker setups. + +- `just init` installs dependencies and creates `.env.local`. +- `just dev up` starts the local stack; `just dc-dev up -d` runs the Docker dev environment. +- `just b test -n=auto` runs backend pytest suites in parallel. +- `just f test` runs frontend Vitest suites. +- `just lint` runs both backend and frontend linters; `just fix` applies auto-fixes. +- `just b migrate` runs Django migrations. + +For direct package-manager use, backend commands run through `uv` and frontend commands through `yarn`. + +## Coding Style & Naming Conventions + +Python targets Python 3.14, uses 4-space indentation, and is formatted and linted with Ruff (`ruff check`, `ruff format`) with an 88-character line length. Follow existing Django app/module naming and keep new tests in `test_*.py` or `*_test.py` files. Frontend code uses ESLint, Stylelint, and Prettier; SCSS should follow BEM-style naming already used in `web-frontend/modules`. + +## Testing Guidelines + +Backend tests use `pytest` with `pytest-django`; frontend tests use `vitest`; browser flows live in `e2e-tests/`. Add unit tests for backend changes and targeted frontend tests for component or store behavior. + +Examples: `just b test backend/tests/path/`, `just b test-coverage`, `just f test -- --coverage`, `just f yarn test:core path/to/test`. + +## Commit & Pull Request Guidelines + +Recent history favors short, imperative subjects, often with Conventional Commit prefixes such as `fix:`, `feat:`, and `chore(deps):`. Branch from `develop`, keep PRs focused, and link the related issue or discussion. Include a clear summary, note schema or env changes, attach screenshots for UI work, add a changelog entry when required, and make sure the relevant lint and test commands pass before opening the PR. + +## Security & Configuration Tips + +Do not commit secrets or local overrides. Use `.env.local` for development, keep production settings in the documented deploy configs, and report vulnerabilities privately via the contact path in `CONTRIBUTING.md` rather than opening a public issue. diff --git a/backend/src/baserow/config/settings/base.py b/backend/src/baserow/config/settings/base.py index 6796069e99..cf55b38195 100644 --- a/backend/src/baserow/config/settings/base.py +++ b/backend/src/baserow/config/settings/base.py @@ -816,6 +816,9 @@ def __setitem__(self, key, value): INTEGRATION_LOCAL_BASEROW_PAGE_SIZE_LIMIT = int( os.getenv("BASEROW_INTEGRATION_LOCAL_BASEROW_PAGE_SIZE_LIMIT", 200) ) +INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS = str_to_bool( + os.getenv("BASEROW_INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS", "true") +) AUTOMATION_HISTORY_PAGE_SIZE_LIMIT = int( os.getenv("BASEROW_AUTOMATION_HISTORY_PAGE_SIZE_LIMIT", 100) diff --git a/backend/src/baserow/contrib/automation/nodes/registries.py b/backend/src/baserow/contrib/automation/nodes/registries.py index 0800daa126..75152efe6b 100644 --- a/backend/src/baserow/contrib/automation/nodes/registries.py +++ b/backend/src/baserow/contrib/automation/nodes/registries.py @@ -264,7 +264,7 @@ def prepare_values( # If we received any service values, prepare them. service_values = values.pop("service", None) or {} prepared_service_values = service_type.prepare_values( - service_values, user, service + service_values, user, service if instance else None ) # Update the service instance with any prepared service values. diff --git a/backend/src/baserow/contrib/builder/workflow_actions/workflow_action_types.py b/backend/src/baserow/contrib/builder/workflow_actions/workflow_action_types.py index 6320f77dd5..ba0770e8b4 100644 --- a/backend/src/baserow/contrib/builder/workflow_actions/workflow_action_types.py +++ b/backend/src/baserow/contrib/builder/workflow_actions/workflow_action_types.py @@ -381,7 +381,7 @@ def prepare_values( # If we received any service values, prepare them. service_values = values.pop("service", None) or {} prepared_service_values = service_type.prepare_values( - service_values, user, service + service_values, user, service if instance else None ) # Update the service instance with any prepared service values. diff --git a/backend/src/baserow/contrib/integrations/core/models.py b/backend/src/baserow/contrib/integrations/core/models.py index ff5568fab5..1f8f0bb204 100644 --- a/backend/src/baserow/contrib/integrations/core/models.py +++ b/backend/src/baserow/contrib/integrations/core/models.py @@ -132,6 +132,11 @@ class CoreSMTPEmailService(Service): A service for sending emails via SMTP. """ + use_instance_smtp_settings = models.BooleanField( + default=False, + db_default=False, + help_text="Whether to use the instance-level Django SMTP configuration.", + ) from_email = FormulaField( help_text="The sender's email address.", ) @@ -166,6 +171,10 @@ class CoreSMTPEmailService(Service): help_text="The email body content.", ) + @property + def instance_smtp_settings_enabled(self) -> bool: + return self.get_type()._instance_smtp_is_available() + class CoreRouterService(Service): default_edge_label = models.CharField( diff --git a/backend/src/baserow/contrib/integrations/core/service_types.py b/backend/src/baserow/contrib/integrations/core/service_types.py index 1bc3ed452e..566f9251fd 100644 --- a/backend/src/baserow/contrib/integrations/core/service_types.py +++ b/backend/src/baserow/contrib/integrations/core/service_types.py @@ -621,6 +621,7 @@ class CoreSMTPEmailServiceType(CoreServiceType): allowed_fields = [ "integration_id", + "use_instance_smtp_settings", "from_email", "from_name", "to_emails", @@ -633,6 +634,8 @@ class CoreSMTPEmailServiceType(CoreServiceType): serializer_field_names = [ "integration_id", + "use_instance_smtp_settings", + "instance_smtp_settings_enabled", "from_email", "from_name", "to_emails", @@ -644,6 +647,7 @@ class CoreSMTPEmailServiceType(CoreServiceType): ] class SerializedDict(ServiceDict): + use_instance_smtp_settings: bool from_email: str from_name: str to_emails: str @@ -652,7 +656,6 @@ class SerializedDict(ServiceDict): subject: str body_type: str body: str - body: str simple_formula_fields = [ "from_email", @@ -669,11 +672,21 @@ def serializer_field_overrides(self): from baserow.core.formula.serializers import FormulaSerializerField return { + "use_instance_smtp_settings": serializers.BooleanField( + required=False, + default=self._instance_smtp_is_available(), + help_text=CoreSMTPEmailService._meta.get_field( + "use_instance_smtp_settings" + ).help_text, + ), "integration_id": serializers.IntegerField( required=False, allow_null=True, help_text="The id of the SMTP integration.", ), + "instance_smtp_settings_enabled": serializers.ReadOnlyField( + help_text="Whether the instance SMTP configuration can be used and should be the default option in the UI.", + ), "from_email": FormulaSerializerField( help_text=CoreSMTPEmailService._meta.get_field("from_email").help_text, ), @@ -706,6 +719,41 @@ def serializer_field_overrides(self): ), } + def _instance_smtp_is_available(self) -> bool: + return bool( + settings.INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS + and getattr(settings, "CELERY_EMAIL_BACKEND", None) + == "django.core.mail.backends.smtp.EmailBackend" + and getattr(settings, "EMAIL_HOST", "") + ) + + def _should_use_instance_smtp(self, service: CoreSMTPEmailService) -> bool: + return bool( + service.use_instance_smtp_settings and self._instance_smtp_is_available() + ) + + def requires_integration(self, service: CoreSMTPEmailService) -> bool: + return not self._should_use_instance_smtp(service) + + def prepare_values(self, values, user: AbstractUser, instance=None): + values = super().prepare_values(values, user, instance) + + use_instance_smtp_settings = ( + values.get( + "use_instance_smtp_settings", + instance.use_instance_smtp_settings if instance else True, + ) + if self._instance_smtp_is_available() + else False + ) + + if use_instance_smtp_settings: + values["integration"] = None + + values["use_instance_smtp_settings"] = use_instance_smtp_settings + + return values + def get_schema_name(self, service: CoreSMTPEmailService) -> str: return f"SMTPEmail{service.id}Schema" @@ -734,15 +782,13 @@ def generate_schema( } def formulas_to_resolve( - self, service: CoreHTTPRequestService + self, service: CoreSMTPEmailService ) -> list[FormulaToResolve]: """ Returns the formula to resolve for this service. """ ensurers = { - "from_email": ensure_email, - "from_name": ensure_string, "to_emails": lambda v: [ensure_email(e) for e in ensure_array(v)], "cc_emails": lambda v: [ensure_email(e) for e in ensure_array(v)], "bcc_emails": lambda v: [ensure_email(e) for e in ensure_array(v)], @@ -750,6 +796,13 @@ def formulas_to_resolve( "body": ensure_string, } + if not self._should_use_instance_smtp(service): + ensurers = { + "from_email": ensure_email, + "from_name": ensure_string, + **ensurers, + } + formulas = [] for key, ensurer in ensurers.items(): @@ -772,31 +825,48 @@ def dispatch_data( "At least one recipient email is required" ) - smtp_integration = service.integration.specific - to_emails = resolved_values["to_emails"] cc_emails = resolved_values["cc_emails"] bcc_emails = resolved_values["bcc_emails"] - from_email = ( - f"{resolved_values['from_name']} <{resolved_values['from_email']}>" - if resolved_values["from_name"] - else resolved_values["from_email"] - ) + using_instance_smtp = self._should_use_instance_smtp(service) + + if using_instance_smtp: + from_email = settings.DEFAULT_FROM_EMAIL + connection = get_connection( + backend=settings.CELERY_EMAIL_BACKEND, + ) + smtp_host = settings.EMAIL_HOST + smtp_port = settings.EMAIL_PORT + else: + if not service.integration_id: + # This situation can happen if we have changed the + # configuration variable in the meantime. + raise ServiceImproperlyConfiguredDispatchException( + "Integration for this service is missing" + ) + + smtp_integration = service.integration.specific + from_email = ( + f"{resolved_values['from_name']} <{resolved_values['from_email']}>" + if resolved_values["from_name"] + else resolved_values["from_email"] + ) + connection = get_connection( + backend=settings.CELERY_EMAIL_BACKEND, + host=smtp_integration.host, + port=smtp_integration.port, + username=smtp_integration.username, + password=smtp_integration.password, + use_tls=smtp_integration.use_tls, + ) + smtp_host = smtp_integration.host + smtp_port = smtp_integration.port subject = resolved_values["subject"] body_content = resolved_values["body"] - connection = get_connection( - backend="django.core.mail.backends.smtp.EmailBackend", - host=smtp_integration.host, - port=smtp_integration.port, - username=smtp_integration.username, - password=smtp_integration.password, - use_tls=smtp_integration.use_tls, - ) - email = EmailMultiAlternatives( subject, body_content, @@ -822,12 +892,11 @@ def dispatch_data( ) from e except socket.gaierror as e: raise ServiceImproperlyConfiguredDispatchException( - f"The host {smtp_integration.host}:{smtp_integration.port} could not " - "be reached" + f"The host {smtp_host}:{smtp_port} could not be reached" ) from e except ConnectionRefusedError as e: raise ServiceImproperlyConfiguredDispatchException( - f"Connection refused by {smtp_integration.host}:{smtp_integration.port}" + f"Connection refused by {smtp_host}:{smtp_port}" ) from e except SMTPAuthenticationError as e: raise ServiceImproperlyConfiguredDispatchException( @@ -848,9 +917,13 @@ def dispatch_transform( def export_prepared_values(self, instance: Service) -> dict[str, Any]: values = super().export_prepared_values(instance) + + values["integration_id"] = None + if values.get("integration"): del values["integration"] values["integration_id"] = instance.integration_id + return values diff --git a/backend/src/baserow/contrib/integrations/migrations/0027_coresmtpemailservice_use_instance_smtp_settings.py b/backend/src/baserow/contrib/integrations/migrations/0027_coresmtpemailservice_use_instance_smtp_settings.py new file mode 100644 index 0000000000..8311e13df6 --- /dev/null +++ b/backend/src/baserow/contrib/integrations/migrations/0027_coresmtpemailservice_use_instance_smtp_settings.py @@ -0,0 +1,19 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("integrations", "0026_backfill_coreperiodicservice_next_run_at"), + ] + + operations = [ + migrations.AddField( + model_name="coresmtpemailservice", + name="use_instance_smtp_settings", + field=models.BooleanField( + default=False, + db_default=False, + help_text="Whether to use the instance-level Django SMTP configuration.", + ), + ), + ] diff --git a/backend/src/baserow/core/services/handler.py b/backend/src/baserow/core/services/handler.py index 0abad082ac..47fff8a85b 100644 --- a/backend/src/baserow/core/services/handler.py +++ b/backend/src/baserow/core/services/handler.py @@ -220,9 +220,8 @@ def dispatch_service( :return: The result of dispatching the service. """ - if ( - service.integration_id is None - and service.get_type().integration_type is not None + if service.integration_id is None and service.get_type().requires_integration( + service ): raise ServiceImproperlyConfiguredDispatchException( "No integration selected" diff --git a/backend/src/baserow/core/services/registries.py b/backend/src/baserow/core/services/registries.py index ca689f9eae..419e8e4c61 100644 --- a/backend/src/baserow/core/services/registries.py +++ b/backend/src/baserow/core/services/registries.py @@ -248,6 +248,9 @@ def get_context_data_schema(self, service: ServiceSubClass): return None + def requires_integration(self, service: ServiceSubClass) -> bool: + return self.integration_type is not None + def formulas_to_resolve(self, service: ServiceSubClass) -> list[FormulaToResolve]: return [] diff --git a/backend/tests/baserow/contrib/integrations/core/test_smtp_email_service_type.py b/backend/tests/baserow/contrib/integrations/core/test_smtp_email_service_type.py index ceb7658372..9d521ae144 100644 --- a/backend/tests/baserow/contrib/integrations/core/test_smtp_email_service_type.py +++ b/backend/tests/baserow/contrib/integrations/core/test_smtp_email_service_type.py @@ -4,6 +4,8 @@ from contextlib import contextmanager from unittest.mock import MagicMock, patch +from django.test import override_settings + import pytest from baserow.contrib.integrations.core.service_types import CoreSMTPEmailServiceType @@ -55,6 +57,9 @@ def mock_django_email( @pytest.mark.django_db +@override_settings( + CELERY_EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend", +) def test_send_smtp_email_basic(data_fixture): smtp_integration = data_fixture.create_smtp_integration( host="smtp.example.com", @@ -100,6 +105,46 @@ def test_send_smtp_email_basic(data_fixture): assert result.data == {"success": True} +@pytest.mark.django_db +@override_settings( + INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS=True, + CELERY_EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend", + EMAIL_HOST="instance.smtp.example.com", + EMAIL_PORT=2525, + DEFAULT_FROM_EMAIL="instance@example.com", +) +def test_send_smtp_email_uses_instance_smtp_settings(data_fixture): + service = data_fixture.create_core_smtp_email_service( + integration=None, + use_instance_smtp_settings=True, + from_email="''", + from_name="''", + to_emails="'recipient@example.com'", + subject="'Test Subject'", + body="'Hello, this is a test email!'", + body_type="plain", + ) + + service_type = service.get_type() + dispatch_context = FakeDispatchContext() + + with mock_django_email() as (mock_email, mock_connection): + result = service_type.dispatch(service, dispatch_context) + mock_connection.assert_called_once_with( + backend="django.core.mail.backends.smtp.EmailBackend", + ) + mock_email.assert_called_once_with( + "Test Subject", + "Hello, this is a test email!", + "instance@example.com", + ["recipient@example.com"], + bcc=[], + cc=[], + connection=mock_connection.return_value, + ) + assert result.data == {"success": True} + + @pytest.mark.django_db def test_send_smtp_email_multiple_to_cc_and_bcc(data_fixture): smtp_integration = data_fixture.create_smtp_integration( @@ -338,6 +383,26 @@ def test_send_smtp_email_no_recipients_error(data_fixture): assert str(exc_info.value) == "At least one recipient email is required" +@pytest.mark.django_db +def test_send_smtp_email_missing_integration_error(data_fixture): + service = data_fixture.create_core_smtp_email_service( + integration=None, + use_instance_smtp_settings=False, + from_email="'sender@example.com'", + to_emails="'recipient@example.com'", + subject="'Test Subject'", + body="'Test body'", + ) + + service_type = service.get_type() + dispatch_context = FakeDispatchContext() + + with pytest.raises(ServiceImproperlyConfiguredDispatchException) as exc_info: + service_type.dispatch(service, dispatch_context) + + assert str(exc_info.value) == "Integration for this service is missing" + + @pytest.mark.django_db def test_smtp_email_service_generate_schema(data_fixture): smtp_integration = data_fixture.create_smtp_integration() @@ -366,6 +431,18 @@ def test_smtp_email_service_generate_schema(data_fixture): } +@pytest.mark.django_db +@override_settings( + INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS=True, + CELERY_EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend", + EMAIL_HOST="instance.smtp.example.com", +) +def test_smtp_email_service_exposes_instance_smtp_enabled_flag(data_fixture): + service = data_fixture.create_core_smtp_email_service() + + assert service.instance_smtp_settings_enabled is True + + @pytest.mark.django_db def test_serialized_export_import(data_fixture): smtp_integration = data_fixture.create_smtp_integration( @@ -378,6 +455,7 @@ def test_serialized_export_import(data_fixture): service = data_fixture.create_core_smtp_email_service( integration=smtp_integration, + use_instance_smtp_settings=False, from_email="'sender@example.com'", from_name="'Test Sender'", to_emails="'recipient@example.com'", @@ -395,6 +473,7 @@ def test_serialized_export_import(data_fixture): expected_serialized = { "id": AnyInt(), "integration_id": smtp_integration.id, + "use_instance_smtp_settings": False, "sample_data": None, "type": "smtp_email", "from_email": { @@ -437,6 +516,37 @@ def test_serialized_export_import(data_fixture): assert new_service.subject["formula"] == "'Test Subject'" assert new_service.body_type == "html" assert new_service.body["formula"] == "'Test body'" + assert new_service.use_instance_smtp_settings is False + + +@pytest.mark.django_db +@override_settings( + INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS=True, + CELERY_EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend", + EMAIL_HOST="instance.smtp.example.com", +) +def test_serialized_export_import_with_instance_smtp(data_fixture): + service = data_fixture.create_core_smtp_email_service( + integration=None, + use_instance_smtp_settings=True, + from_email="''", + from_name="''", + to_emails="'recipient@example.com'", + subject="'Test Subject'", + body="'Test body'", + body_type="html", + ) + + service_type = service.get_type() + serialized = json.loads(json.dumps(service_type.export_serialized(service))) + + assert serialized["integration_id"] is None + assert serialized["use_instance_smtp_settings"] is True + + new_service = service_type.import_serialized(None, serialized, {}, lambda x, d: x) + + assert new_service.integration_id is None + assert new_service.use_instance_smtp_settings is True @pytest.mark.django_db @@ -446,6 +556,7 @@ def test_smtp_email_service_create_update(data_fixture): service = ServiceHandler().create_service( CoreSMTPEmailServiceType(), integration_id=smtp_integration.id, + use_instance_smtp_settings=False, from_email="'sender@example.com'", from_name="'Test Sender'", to_emails="'recipient@example.com'", @@ -481,6 +592,7 @@ def test_smtp_email_service_create_update(data_fixture): } assert service.body_type == "plain" assert service.integration_id == smtp_integration.id + assert service.use_instance_smtp_settings is False service_type = service.get_type() ServiceHandler().update_service( @@ -497,3 +609,73 @@ def test_smtp_email_service_create_update(data_fixture): assert service.subject["formula"] == "'Updated Subject'" assert service.body["formula"] == "'Test body'" assert service.body_type == "html" + + +@pytest.mark.django_db +@override_settings( + INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS=True, + CELERY_EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend", + EMAIL_HOST="instance.smtp.example.com", +) +def test_smtp_email_service_create_update_with_instance_smtp(data_fixture): + service_type = CoreSMTPEmailServiceType() + + prepared_values = service_type.prepare_values( + { + "use_instance_smtp_settings": True, + "integration_id": None, + "to_emails": "'recipient@example.com'", + "subject": "'Test Subject'", + "body": "'Test body'", + "from_email": "''", + "from_name": "''", + }, + data_fixture.create_user(), + ) + + service = ServiceHandler().create_service(service_type, **prepared_values) + + assert service.integration_id is None + assert service.use_instance_smtp_settings is True + + smtp_integration = data_fixture.create_smtp_integration() + prepared_updates = service_type.prepare_values( + { + "use_instance_smtp_settings": False, + "integration_id": smtp_integration.id, + "from_email": "'sender@example.com'", + }, + data_fixture.create_user(), + service, + ) + + ServiceHandler().update_service(service_type, service, **prepared_updates) + service.refresh_from_db() + + assert service.integration_id == smtp_integration.id + assert service.use_instance_smtp_settings is False + assert service.from_email["formula"] == "'sender@example.com'" + + +@pytest.mark.django_db +@override_settings( + INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS=False, + CELERY_EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend", + EMAIL_HOST="instance.smtp.example.com", +) +def test_smtp_email_service_prepare_values_disables_instance_smtp_when_unavailable( + data_fixture, +): + service_type = CoreSMTPEmailServiceType() + service = data_fixture.create_core_smtp_email_service( + integration=None, + use_instance_smtp_settings=True, + ) + + prepared_values = service_type.prepare_values( + {}, + data_fixture.create_user(), + service, + ) + + assert prepared_values["use_instance_smtp_settings"] is False diff --git a/changelog/entries/unreleased/breaking_change/4999_instance_smtp_configuration_can_be_used_to_send_emails_with_.json b/changelog/entries/unreleased/breaking_change/4999_instance_smtp_configuration_can_be_used_to_send_emails_with_.json new file mode 100644 index 0000000000..0ead27adac --- /dev/null +++ b/changelog/entries/unreleased/breaking_change/4999_instance_smtp_configuration_can_be_used_to_send_emails_with_.json @@ -0,0 +1,9 @@ +{ + "type": "breaking_change", + "message": "Instance SMTP configuration is used by default to send e-mails with the `Send email` action. Set `BASEROW_INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS=false` env var to disable this behaviour.", + "issue_origin": "github", + "issue_number": 4999, + "domain": "integration", + "bullet_points": [], + "created_at": "2026-03-23" +} diff --git a/docker-compose.no-caddy.yml b/docker-compose.no-caddy.yml index d1d7958d0d..354f369775 100644 --- a/docker-compose.no-caddy.yml +++ b/docker-compose.no-caddy.yml @@ -240,6 +240,7 @@ services: BASEROW_UNIQUE_ROW_VALUES_SIZE_LIMIT: BASEROW_ROW_PAGE_SIZE_LIMIT: BASEROW_INTEGRATION_LOCAL_BASEROW_PAGE_SIZE_LIMIT: + BASEROW_INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS: BASEROW_INTEGRATIONS_PERIODIC_MINUTE_MIN: BASEROW_BUILDER_DOMAINS: BASEROW_FRONTEND_SAME_SITE_COOKIE: diff --git a/docker-compose.yml b/docker-compose.yml index 12daf3ef17..a08d0a1b0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,7 @@ x-backend-variables: BASEROW_AMOUNT_OF_WORKERS: BASEROW_ROW_PAGE_SIZE_LIMIT: BASEROW_INTEGRATION_LOCAL_BASEROW_PAGE_SIZE_LIMIT: + BASEROW_INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS: BATCH_ROWS_SIZE_LIMIT: INITIAL_TABLE_DATA_LIMIT: BASEROW_FILE_UPLOAD_SIZE_LIMIT_MB: @@ -316,6 +317,7 @@ services: BASEROW_UNIQUE_ROW_VALUES_SIZE_LIMIT: BASEROW_ROW_PAGE_SIZE_LIMIT: BASEROW_INTEGRATION_LOCAL_BASEROW_PAGE_SIZE_LIMIT: + BASEROW_INTEGRATION_ALLOW_SMTP_SERVICE_TO_USE_INSTANCE_SETTINGS: BASEROW_BUILDER_DOMAINS: BASEROW_FRONTEND_SAME_SITE_COOKIE: SENTRY_DSN: diff --git a/web-frontend/modules/automation/components/AutomationHeader.vue b/web-frontend/modules/automation/components/AutomationHeader.vue index 31ed3cd679..6bc45b5770 100644 --- a/web-frontend/modules/automation/components/AutomationHeader.vue +++ b/web-frontend/modules/automation/components/AutomationHeader.vue @@ -1,7 +1,6 @@