Merge branch '3581-attachments-expire-before-being-imported' into 'develop'

Attachments are not available during import

Closes baserow#3581

See merge request baserow/baserow!3518

Author: dimmur-baserow

backend/src/baserow/contrib/database/airtable/exceptions.py

@@ -25,3 +25,11 @@ class AirtableSkipFilter(Exception):
     """
     Raised when an Airtable filter is not compatible and must be skipped.
     """
+
+
+class FileDownloadFailed(Exception):
+    """Raised when a file download fails."""
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(message)

backend/src/baserow/contrib/database/airtable/handler.py

@@ -3,6 +3,7 @@
 from collections import defaultdict
 from contextlib import contextmanager
 from datetime import datetime, timezone
+from http import HTTPStatus
 from io import BytesIO, IOBase
 from typing import Dict, List, Optional, Tuple, Union
 
@@ -49,10 +50,13 @@
     AirtableImportNotRespectingConfig,
     AirtableShareIsNotABase,
     AirtableSkipCellValue,
+    FileDownloadFailed,
 )
 from .import_report import (
+    ERROR_TYPE_OTHER,
     ERROR_TYPE_UNSUPPORTED_FEATURE,
     SCOPE_AUTOMATIONS,
+    SCOPE_CELL,
     SCOPE_FIELD,
     SCOPE_INTERFACES,
     SCOPE_VIEW,
@@ -78,6 +82,56 @@
 }
 
 
+def download_airtable_file(
+    name: str,
+    download_file: DownloadFile,
+    init_data: dict,
+    request_id: str,
+    cookies: dict,
+    headers: dict = None,
+) -> Response:
+    """
+    Downloads a file from Airtable using either direct URL fetch or
+    attachment endpoint.
+
+    :param name: The name of the file to download.
+    :param download_file: The DownloadFile object containing download
+        information
+    :param init_data: The init_data returned by the initially
+        requested shared base
+    :param request_id: The request_id returned by the initially
+        requested shared base
+    :param cookies: The cookies dict returned by the initially
+        requested shared base
+    :param headers: Optional headers to use for the request
+    :return: The response object from the download request
+    :raises FileDownloadFailed: When the file could not be downloaded.
+    """
+
+    if download_file.type == AIRTABLE_DOWNLOAD_FILE_TYPE_FETCH:
+        response = requests.get(download_file.url, headers=headers)  # nosec B113
+    elif download_file.type == AIRTABLE_DOWNLOAD_FILE_TYPE_ATTACHMENT_ENDPOINT:
+        response = AirtableHandler.fetch_attachment(
+            row_id=download_file.row_id,
+            column_id=download_file.column_id,
+            attachment_id=download_file.attachment_id,
+            init_data=init_data,
+            request_id=request_id,
+            cookies=cookies,
+            headers=headers,
+        )
+    else:
+        raise FileDownloadFailed(
+            f"Unknown download file type: {download_file.type}",
+        )
+    if response.status_code not in [HTTPStatus.OK, HTTPStatus.PARTIAL_CONTENT]:
+        raise FileDownloadFailed(
+            f"File {name} could not be downloaded (HTTP {response.status_code}).",
+        )
+
+    return response
+
+
 class AirtableFileImport:
     """
     A file-like object (we only need open and close methods) that facilitates on-demand
@@ -98,23 +152,24 @@ def add_files(self, files_to_download):
 
     @contextmanager
     def open(self, name):
-        download_file = self.files_to_download.get(name)
-        if download_file is None:
+        if name is None:
             raise ValueError(f"No file with name {name} found.")
 
-        if download_file.type == AIRTABLE_DOWNLOAD_FILE_TYPE_FETCH:
-            response = requests.get(
-                download_file.url, headers=BASE_HEADERS
-            )  # nosec B113
-        elif download_file.type == AIRTABLE_DOWNLOAD_FILE_TYPE_ATTACHMENT_ENDPOINT:
-            response = AirtableHandler.fetch_attachment(
-                row_id=download_file.row_id,
-                column_id=download_file.column_id,
-                attachment_id=download_file.attachment_id,
-                init_data=self.init_data,
-                request_id=self.request_id,
-                cookies=self.cookies,
-            )
+        # Files for which check failed are excluded from the
+        # files_to_download dict
+        # Those missing files are already included in the import report
+        if name not in self.files_to_download:
+            raise KeyError(f"File '{name}' not found in files_to_download")
+
+        response = download_airtable_file(
+            name=name,
+            download_file=self.files_to_download[name],
+            init_data=self.init_data,
+            request_id=self.request_id,
+            cookies=self.cookies,
+            headers=BASE_HEADERS,
+        )
+
         stream = BytesIO(response.content)
         try:
             yield stream
@@ -177,18 +232,24 @@ def fetch_publicly_shared_base(
         return request_id, init_data, cookies
 
     @staticmethod
-    def make_airtable_request(init_data: dict, request_id: str, **kwargs) -> Response:
+    def make_airtable_request(
+        init_data: dict, request_id: str, headers=None, **kwargs
+    ) -> Response:
         """
         Helper method to make a valid request to to Airtable with the correct headers
         and params.
 
         :param init_data: The init_data returned by the initially requested shared base.
         :param request_id: The request_id returned by the initially requested shared
             base.
+        :param headers: The headers to be passed into the `requests` request.
         :param kwargs: THe kwargs that must be passed into the `requests.get` method.
         :return: The requests Response object related to the request.
         """
 
+        if headers is None:
+            headers = BASE_HEADERS
+
         application_id = list(init_data["rawApplications"].keys())[0]
         client_code_version = init_data["codeVersion"]
         page_load_id = init_data["pageLoadId"]
@@ -208,7 +269,7 @@ def make_airtable_request(init_data: dict, request_id: str, **kwargs) -> Respons
                 "X-Requested-With": "XMLHttpRequest",
                 "x-time-zone": "Europe/Amsterdam",
                 "x-user-locale": "en",
-                **BASE_HEADERS,
+                **headers,
             },
             timeout=3 * 60,  # it can take quite a while for Airtable to respond.
             **kwargs,
@@ -315,6 +376,7 @@ def fetch_attachment(
         request_id: str,
         cookies: dict,
         stream=True,
+        headers=None,
     ) -> Response:
         """
         :param row_id: The Airtable row id of the attachment that must be fetched.
@@ -331,6 +393,7 @@ def fetch_attachment(
         :param stream: Indicates whether the request should be streamed. This could be
             useful if we want to show a progress bar. It will directly be passed into
             the `requests` request.
+        :param headers: The headers to be passed into the `requests` request.
         :return: The `requests` response containing the result.
         """
 
@@ -348,6 +411,7 @@ def fetch_attachment(
             params={"stringifiedObjectParams": json.dumps(stringified_object_params)},
             cookies=cookies,
             allow_redirects=True,
+            headers=headers,
         )
         return response
 
@@ -523,14 +587,18 @@ def to_baserow_row_export(
         return exported_row
 
     @staticmethod
-    def download_files_as_zip(
+    def prepare_downloadable_files(
         files_to_download: Dict[str, DownloadFile],
         init_data: dict,
         request_id: str,
         cookies: dict,
         config: AirtableImportConfig,
         progress_builder: Optional[ChildProgressBuilder] = None,
         files_buffer: Union[None, IOBase] = None,
+        import_report: AirtableImportReport = None,
+        field_mapping_per_table: dict = None,
+        exported_tables: list = None,
+        row_id_mapping: Dict[str, Dict[str, int]] = None,
     ) -> BytesIO:
         """
         This method was used to download the files, but now it only collects
@@ -573,6 +641,49 @@ def download_files_as_zip(
             cookies=cookies,
             headers=BASE_HEADERS,
         )
+
+        failed_files = []
+        for file_name, download_file in files_to_download.items():
+            headers = BASE_HEADERS.copy()
+            headers["Range"] = "bytes=0-5"
+
+            try:
+                download_airtable_file(
+                    file_name, download_file, init_data, request_id, cookies, headers
+                )
+            except FileDownloadFailed:
+                field_name = ""
+                table_name = ""
+                baserow_row_id = download_file.row_id
+
+                for table_id, field_mapping in field_mapping_per_table.items():
+                    if download_file.column_id in field_mapping:
+                        field_info = field_mapping[download_file.column_id]
+                        field_name = field_info["baserow_field"].name
+
+                        for exported_table in exported_tables:
+                            if exported_table["id"] == table_id:
+                                table_name = exported_table["name"]
+                                break
+
+                        if row_id_mapping and table_id in row_id_mapping:
+                            baserow_row_id = row_id_mapping[table_id].get(
+                                download_file.row_id, download_file.row_id
+                            )
+                        break
+
+                import_report.add_failed(
+                    "File",
+                    SCOPE_CELL,
+                    table_name,
+                    ERROR_TYPE_OTHER,
+                    f"Field: {field_name}, Row: {baserow_row_id}, File: {file_name}",
+                )
+                failed_files.append(file_name)
+
+        for file_name in failed_files:
+            files_to_download.pop(file_name, None)
+
         file_archive.add_files(files_to_download)
         progress.increment(state=AIRTABLE_EXPORT_JOB_DOWNLOADING_FILES)
 
@@ -945,18 +1056,28 @@ def to_baserow_database_export(
             **DatabaseExportSerializedStructure.database(tables=exported_tables)
         )
 
+        report_items_count = len(import_report.items)
+
         # After all the tables have been converted to Baserow format, we must
         # download all the user files. Because we first want to the whole conversion to
         # be completed and because we want this to be added to the progress bar, this is
         # done last.
-        user_files_zip = cls.download_files_as_zip(
+        user_files_zip = cls.prepare_downloadable_files(
             files_to_download,
             init_data,
             request_id,
             cookies,
             config,
             progress.create_child_builder(represents_progress=500),
             download_files_buffer,
+            import_report,
+            field_mapping_per_table,
+            exported_tables,
+            row_id_mapping,
+        )
+
+        import_report.append_items_to_exported_table(
+            exported_database, import_report.items[report_items_count:]
         )
 
         return exported_database, user_files_zip

backend/src/baserow/contrib/database/airtable/import_report.py

@@ -13,6 +13,9 @@
 from baserow.contrib.database.views.registries import view_type_registry
 from baserow.core.constants import BASEROW_COLORS
 
+REPORT_TABLE_ID = "report"
+REPORT_TABLE_NAME = "Airtable import report"
+
 SCOPE_FIELD = SelectOption(id="scope_field", value="Field", color="light-blue", order=1)
 SCOPE_CELL = SelectOption(id="scope_cell", value="Cell", color="light-green", order=2)
 SCOPE_VIEW = SelectOption(id="scope_view", value="View", color="light-cyan", order=3)
@@ -161,8 +164,8 @@ def get_baserow_export_table(self, order: int) -> dict:
             exported_rows.append(row)
 
         exported_table = DatabaseExportSerializedStructure.table(
-            id="report",
-            name="Airtable import report",
+            id=REPORT_TABLE_ID,
+            name=REPORT_TABLE_NAME,
             order=order,
             fields=exported_fields,
             views=exported_views,
@@ -172,3 +175,49 @@ def get_baserow_export_table(self, order: int) -> dict:
         )
 
         return exported_table
+
+    def append_items_to_exported_table(
+        self, exported_database: dict, items: list
+    ) -> None:
+        """
+        Appends new items to an existing exported table.
+
+        :param exported_database: The exported database
+        :param items: List of ImportReportFailedItem to append
+        """
+
+        if not items:
+            return
+
+        report_table = next(
+            (
+                table
+                for table in exported_database["tables"]
+                if table["id"] == REPORT_TABLE_ID
+            ),
+            None,
+        )
+        if not report_table:
+            return
+
+        current_row_count = len(report_table["rows"])
+        table_select_options = {
+            opt["value"]: opt for opt in report_table["fields"][2]["select_options"]
+        }
+
+        for index, item in enumerate(items, start=current_row_count + 1):
+            table_select_option = table_select_options.get(item.table)
+            row = DatabaseExportSerializedStructure.row(
+                id=index,
+                order=f"{index}.00000000000000000000",
+                created_on=None,
+                updated_on=None,
+            )
+            row["field_object_name"] = item.object_name
+            row["field_scope"] = item.scope.id
+            row["field_table"] = (
+                table_select_option["id"] if table_select_option else None
+            )
+            row["field_error_type"] = item.error_type.id
+            row["field_message"] = item.message
+            report_table["rows"].append(row)

backend/src/baserow/contrib/database/fields/field_types.py

@@ -3971,12 +3971,19 @@ def set_import_serialized_value(
             if files_zip is None:
                 files.append(file)
             else:
-                with files_zip.open(file["name"]) as stream:
-                    # Try to upload the user file with the original name to make sure
-                    # that if the was already uploaded, it will not be uploaded again.
-                    user_file = user_file_handler.upload_user_file(
-                        None, file["original_name"], stream, storage=storage
-                    )
+                try:
+                    with files_zip.open(file["name"]) as stream:
+                        # Try to upload the user file with the original name
+                        # to make sure that if the was already uploaded, it will
+                        # not be uploaded again.
+                        user_file = user_file_handler.upload_user_file(
+                            None, file["original_name"], stream, storage=storage
+                        )
+                except KeyError:
+                    # File not found in zip archive - skip this file and
+                    # let the import process report handle missing files
+                    # appropriately
+                    continue
 
                 value = user_file.serialize()
                 value["visible_name"] = file["visible_name"]

changelog/entries/unreleased/bug/3581_log_files_that_cannot_be_downloaaded_during_import.json

@@ -0,0 +1,8 @@
+{
+    "type": "bug",
+    "message": "Log files that cannot be downloaaded during import",
+    "domain": "database",
+    "issue_number": 3581,
+    "bullet_points": [],
+    "created_at": "2025-06-10"
+}