diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f5beefa5..69ec8afff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +* Extracted folder/bucket scanning into a new `scan` module + ## [1.16.0] - 2022-04-27 This release contains a preview of replication support. It allows for basic diff --git a/b2sdk/_v3/__init__.py b/b2sdk/_v3/__init__.py index f91669f1b..e3a734de4 100644 --- a/b2sdk/_v3/__init__.py +++ b/b2sdk/_v3/__init__.py @@ -164,14 +164,7 @@ from b2sdk.sync.action import B2HideAction from b2sdk.sync.action import B2UploadAction from b2sdk.sync.action import LocalDeleteAction -from b2sdk.sync.exception import EnvironmentEncodingError from b2sdk.sync.exception import IncompleteSync -from b2sdk.sync.exception import InvalidArgument -from b2sdk.sync.folder import AbstractFolder -from b2sdk.sync.folder import B2Folder -from b2sdk.sync.folder import LocalFolder -from b2sdk.sync.folder_parser import parse_sync_folder -from b2sdk.sync.path import AbstractSyncPath, B2SyncPath, LocalSyncPath from b2sdk.sync.policy import AbstractFileSyncPolicy from b2sdk.sync.policy import CompareVersionMode from b2sdk.sync.policy import NewerFileSyncMode @@ -189,19 +182,30 @@ from b2sdk.sync.policy_manager import POLICY_MANAGER from b2sdk.sync.report import SyncFileReporter from b2sdk.sync.report import SyncReport -from b2sdk.sync.scan_policies import DEFAULT_SCAN_MANAGER -from b2sdk.sync.scan_policies import IntegerRange -from b2sdk.sync.scan_policies import RegexSet -from b2sdk.sync.scan_policies import ScanPoliciesManager -from b2sdk.sync.scan_policies import convert_dir_regex_to_dir_prefix_regex from b2sdk.sync.sync import KeepOrDeleteMode from b2sdk.sync.sync import Synchronizer -from b2sdk.sync.sync import zip_folders from b2sdk.sync.encryption_provider import AbstractSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import BasicSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import ServerDefaultSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER +# scan + +from b2sdk.scan.exception import EnvironmentEncodingError +from b2sdk.scan.exception import InvalidArgument +from b2sdk.scan.folder import AbstractFolder +from b2sdk.scan.folder import B2Folder +from b2sdk.scan.folder import LocalFolder +from b2sdk.scan.folder_parser import parse_folder +from b2sdk.scan.path import AbstractPath, B2Path, LocalPath +from b2sdk.scan.policies import convert_dir_regex_to_dir_prefix_regex +from b2sdk.scan.policies import DEFAULT_SCAN_MANAGER +from b2sdk.scan.policies import IntegerRange +from b2sdk.scan.policies import RegexSet +from b2sdk.scan.policies import ScanPoliciesManager +from b2sdk.scan.report import Report +from b2sdk.scan.scan import zip_folders + # replication from b2sdk.replication.setting import ReplicationConfigurationFactory diff --git a/b2sdk/_v3/exception.py b/b2sdk/_v3/exception.py index 32fea54d7..b2b13b327 100644 --- a/b2sdk/_v3/exception.py +++ b/b2sdk/_v3/exception.py @@ -71,14 +71,14 @@ from b2sdk.exception import SSECKeyError from b2sdk.exception import WrongEncryptionModeForBucketDefault from b2sdk.exception import interpret_b2_error -from b2sdk.sync.exception import EmptyDirectory -from b2sdk.sync.exception import EnvironmentEncodingError from b2sdk.sync.exception import IncompleteSync -from b2sdk.sync.exception import InvalidArgument -from b2sdk.sync.exception import NotADirectory -from b2sdk.sync.exception import UnableToCreateDirectory -from b2sdk.sync.exception import UnSyncableFilename -from b2sdk.sync.exception import check_invalid_argument +from b2sdk.scan.exception import UnableToCreateDirectory +from b2sdk.scan.exception import EmptyDirectory +from b2sdk.scan.exception import EnvironmentEncodingError +from b2sdk.scan.exception import InvalidArgument +from b2sdk.scan.exception import NotADirectory +from b2sdk.scan.exception import UnsupportedFilename +from b2sdk.scan.exception import check_invalid_argument __all__ = ( 'AccessDenied', @@ -144,7 +144,7 @@ 'UnknownHost', 'UnrecognizedBucketType', 'UnableToCreateDirectory', - 'UnSyncableFilename', + 'UnsupportedFilename', 'UnsatisfiableRange', 'UnusableFileName', 'interpret_b2_error', diff --git a/b2sdk/scan/exception.py b/b2sdk/scan/exception.py new file mode 100644 index 000000000..1dbba1fb8 --- /dev/null +++ b/b2sdk/scan/exception.py @@ -0,0 +1,105 @@ +###################################################################### +# +# File: b2sdk/scan/exception.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from contextlib import contextmanager +from typing import Iterator, Type + +from ..exception import B2Error, B2SimpleError + + +class EnvironmentEncodingError(B2Error): + """ + Raised when a file name can not be decoded with system encoding. + """ + + def __init__(self, filename, encoding): + """ + :param filename: an encoded file name + :type filename: str, bytes + :param str encoding: file name encoding + """ + super().__init__() + self.filename = filename + self.encoding = encoding + + def __str__(self): + return """file name %s cannot be decoded with system encoding (%s). +We think this is an environment error which you should workaround by +setting your system encoding properly, for example like this: +export LANG=en_US.UTF-8""" % ( + self.filename, + self.encoding, + ) + + +class InvalidArgument(B2Error): + """ + Raised when one or more arguments are invalid + """ + + def __init__(self, parameter_name, message): + """ + :param parameter_name: name of the function argument + :param message: brief explanation of misconfiguration + """ + super().__init__() + self.parameter_name = parameter_name + self.message = message + + def __str__(self): + return "%s %s" % (self.parameter_name, self.message) + + +class UnsupportedFilename(B2Error): + """ + Raised when a filename is not supported by the scan operation + """ + + def __init__(self, message, filename): + """ + :param message: brief explanation of why the filename was not supported + :param filename: name of the file which is not supported + """ + super().__init__() + self.filename = filename + self.message = message + + def __str__(self): + return "%s: %s" % (self.message, self.filename) + + +@contextmanager +def check_invalid_argument(parameter_name: str, message: str, + *exceptions: Type[Exception]) -> Iterator[None]: + """Raise `InvalidArgument` in case of one of given exception was thrown.""" + try: + yield + except exceptions as exc: + if not message: + message = str(exc) + raise InvalidArgument(parameter_name, message) from exc + + +class BaseDirectoryError(B2SimpleError): + def __init__(self, path): + self.path = path + super().__init__(path) + + +class EmptyDirectory(BaseDirectoryError): + pass + + +class UnableToCreateDirectory(BaseDirectoryError): + pass + + +class NotADirectory(BaseDirectoryError): + pass diff --git a/b2sdk/sync/folder.py b/b2sdk/scan/folder.py similarity index 90% rename from b2sdk/sync/folder.py rename to b2sdk/scan/folder.py index addbc2099..a334eb02b 100644 --- a/b2sdk/sync/folder.py +++ b/b2sdk/scan/folder.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/sync/folder.py +# File: b2sdk/scan/folder.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # @@ -13,13 +13,13 @@ import platform import re import sys - from abc import ABCMeta, abstractmethod -from .exception import EmptyDirectory, EnvironmentEncodingError, UnSyncableFilename, NotADirectory, UnableToCreateDirectory -from .path import B2SyncPath, LocalSyncPath -from .report import SyncReport -from .scan_policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager + from ..utils import fix_windows_path_limit, get_file_mtime, is_file_readable +from .exception import EmptyDirectory, EnvironmentEncodingError, NotADirectory, UnableToCreateDirectory, UnsupportedFilename +from .path import B2Path, LocalPath +from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager +from .report import Report DRIVE_MATCHER = re.compile(r"^([A-Za-z]):([/\\])") ABSOLUTE_PATH_MATCHER = re.compile(r"^(/)|^(\\)") @@ -50,7 +50,7 @@ class AbstractFolder(metaclass=ABCMeta): """ @abstractmethod - def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + def all_files(self, reporter: Report, policies_manager=DEFAULT_SCAN_MANAGER): """ Return an iterator over all of the files in the folder, in the order that B2 uses. @@ -121,7 +121,7 @@ def folder_type(self): """ return 'local' - def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + def all_files(self, reporter: Report, policies_manager=DEFAULT_SCAN_MANAGER): """ Yield all files. @@ -148,7 +148,7 @@ def make_full_path(self, file_name): # Ensure the new full_path is inside the self.root directory if common_prefix != self.root: - raise UnSyncableFilename("illegal file name", full_path) + raise UnsupportedFilename("illegal file name", full_path) return full_path @@ -174,14 +174,14 @@ def ensure_non_empty(self): raise EmptyDirectory(self.root) def _walk_relative_paths( - self, local_dir: str, relative_dir_path: str, reporter: SyncReport, + self, local_dir: str, relative_dir_path: str, reporter, policies_manager: ScanPoliciesManager ): """ Yield a File object for each of the files anywhere under this folder, in the order they would appear in B2, unless the path is excluded by policies manager. - :param relative_dir_path: the path of this dir relative to the sync point, or '' if at sync point + :param relative_dir_path: the path of this dir relative to the scan point, or '' if at scan point """ if not isinstance(local_dir, str): raise ValueError('folder path should be unicode: %s' % repr(local_dir)) @@ -208,15 +208,15 @@ def _walk_relative_paths( name = self._handle_non_unicode_file_name(name) if '/' in name: - raise UnSyncableFilename( - "sync does not support file names that include '/'", + raise UnsupportedFilename( + "scan does not support file names that include '/'", "%s in dir %s" % (name, local_dir) ) local_path = os.path.join(local_dir, name) relative_file_path = join_b2_path( relative_dir_path, name - ) # file path relative to the sync point + ) # file path relative to the scan point # Skip broken symlinks or other inaccessible files if not is_file_readable(local_path, reporter): @@ -251,17 +251,17 @@ def _walk_relative_paths( file_mod_time = get_file_mtime(local_path) file_size = os.path.getsize(local_path) - local_sync_path = LocalSyncPath( + local_scan_path = LocalPath( absolute_path=self.make_full_path(relative_file_path), relative_path=relative_file_path, mod_time=file_mod_time, size=file_size, ) - if policies_manager.should_exclude_local_path(local_sync_path): + if policies_manager.should_exclude_local_path(local_scan_path): continue - yield local_sync_path + yield local_scan_path @classmethod def _handle_non_unicode_file_name(cls, name): @@ -312,7 +312,7 @@ def __init__(self, bucket_name, folder_name, api): self.prefix = '' if self.folder_name == '' else self.folder_name + '/' def all_files( - self, reporter: SyncReport, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER + self, reporter: Report, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER ): """ Yield all files. @@ -346,7 +346,7 @@ def all_files( self._validate_file_name(file_name) if current_name != file_name and current_name is not None and current_versions: - yield B2SyncPath( + yield B2Path( relative_path=current_name, selected_version=current_versions[0], all_versions=current_versions @@ -357,7 +357,7 @@ def all_files( current_versions.append(file_version) if current_name is not None and current_versions: - yield B2SyncPath( + yield B2Path( relative_path=current_name, selected_version=current_versions[0], all_versions=current_versions @@ -374,18 +374,18 @@ def get_file_versions(self): def _validate_file_name(self, file_name): # Do not allow relative paths in file names if RELATIVE_PATH_MATCHER.search(file_name): - raise UnSyncableFilename( - "sync does not support file names that include relative paths", file_name + raise UnsupportedFilename( + "scan does not support file names that include relative paths", file_name ) # Do not allow absolute paths in file names if ABSOLUTE_PATH_MATCHER.search(file_name): - raise UnSyncableFilename( - "sync does not support file names with absolute paths", file_name + raise UnsupportedFilename( + "scan does not support file names with absolute paths", file_name ) # On Windows, do not allow drive letters in file names if platform.system() == "Windows" and DRIVE_MATCHER.search(file_name): - raise UnSyncableFilename( - "sync does not support file names with drive letters", file_name + raise UnsupportedFilename( + "scan does not support file names with drive letters", file_name ) def folder_type(self): diff --git a/b2sdk/sync/folder_parser.py b/b2sdk/scan/folder_parser.py similarity index 87% rename from b2sdk/sync/folder_parser.py rename to b2sdk/scan/folder_parser.py index 9aa4a39fe..1ee3b47af 100644 --- a/b2sdk/sync/folder_parser.py +++ b/b2sdk/scan/folder_parser.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/sync/folder_parser.py +# File: b2sdk/scan/folder_parser.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # @@ -12,13 +12,12 @@ from .folder import B2Folder, LocalFolder -def parse_sync_folder(folder_name, api, local_folder_class=LocalFolder, b2_folder_class=B2Folder): +def parse_folder(folder_name, api, local_folder_class=LocalFolder, b2_folder_class=B2Folder): """ Take either a local path, or a B2 path, and returns a Folder object for it. - B2 paths look like: b2://bucketName/path/name. The '//' is optional, - because the previous sync command didn't use it. + B2 paths look like: b2://bucketName/path/name. The '//' is optional. Anything else is treated like a local folder. @@ -52,4 +51,4 @@ def _parse_bucket_and_folder(bucket_and_path, api, b2_folder_class): (bucket_name, folder_name) = bucket_and_path.split('/', 1) if folder_name.endswith('/'): folder_name = folder_name[:-1] - return b2_folder_class(bucket_name, folder_name, api) + return b2_folder_class(bucket_name, folder_name, api) \ No newline at end of file diff --git a/b2sdk/sync/path.py b/b2sdk/scan/path.py similarity index 94% rename from b2sdk/sync/path.py rename to b2sdk/scan/path.py index a439e7c03..ac9da313e 100644 --- a/b2sdk/sync/path.py +++ b/b2sdk/scan/path.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/sync/path.py +# File: b2sdk/scan/path.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # @@ -14,7 +14,7 @@ from ..file_version import FileVersion -class AbstractSyncPath(ABC): +class AbstractPath(ABC): """ Represent a path in a source or destination folder - be it B2 or local """ @@ -34,7 +34,7 @@ def __repr__(self): ) -class LocalSyncPath(AbstractSyncPath): +class LocalPath(AbstractPath): __slots__ = ['absolute_path', 'relative_path', 'mod_time', 'size'] def __init__(self, absolute_path: str, relative_path: str, mod_time: int, size: int): @@ -52,7 +52,7 @@ def __eq__(self, other): ) -class B2SyncPath(AbstractSyncPath): +class B2Path(AbstractPath): __slots__ = ['relative_path', 'selected_version', 'all_versions'] def __init__( @@ -89,4 +89,4 @@ def __eq__(self, other): self.relative_path == other.relative_path and self.selected_version == other.selected_version and self.all_versions == other.all_versions - ) + ) \ No newline at end of file diff --git a/b2sdk/sync/scan_policies.py b/b2sdk/scan/policies.py similarity index 93% rename from b2sdk/sync/scan_policies.py rename to b2sdk/scan/policies.py index 42d54a28c..931b73b05 100644 --- a/b2sdk/sync/scan_policies.py +++ b/b2sdk/scan/policies.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/sync/scan_policies.py +# File: b2sdk/scan/policies.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # @@ -10,11 +10,12 @@ import logging import re -from typing import Optional, Union, Iterable -from .exception import InvalidArgument, check_invalid_argument -from .path import LocalSyncPath +from typing import Iterable, Optional, Union + from ..file_version import FileVersion +from .exception import InvalidArgument, check_invalid_argument +from .path import LocalPath logger = logging.getLogger(__name__) @@ -107,8 +108,8 @@ def __contains__(self, item): class ScanPoliciesManager: """ - Policy object used when scanning folders for syncing, used to decide - which files to include in the list of files to be synced. + Policy object used when scanning folders, used to decide + which files to include in the list of files. Code that scans through files should at least use should_exclude_file() to decide whether each file should be included; it will check include/exclude @@ -186,9 +187,9 @@ def _should_exclude_relative_path(self, relative_path: str): return False return self._exclude_file_set.matches(relative_path) - def should_exclude_local_path(self, local_path: LocalSyncPath): + def should_exclude_local_path(self, local_path: LocalPath): """ - Whether a local path should be excluded from the Sync or not. + Whether a local path should be excluded from the scan or not. This method assumes that the directory holding the `path_` has already been checked for exclusion. """ @@ -198,7 +199,7 @@ def should_exclude_local_path(self, local_path: LocalSyncPath): def should_exclude_b2_file_version(self, file_version: FileVersion, relative_path: str): """ - Whether a b2 file version should be excluded from the Sync or not. + Whether a b2 file version should be excluded from the scan or not. This method assumes that the directory holding the `path_` has already been checked for exclusion. """ @@ -210,14 +211,14 @@ def should_exclude_b2_file_version(self, file_version: FileVersion, relative_pat def should_exclude_b2_directory(self, dir_path: str): """ - Given the path of a directory, relative to the sync point, + Given the path of a directory, relative to the scan point, decide if all of the files in it should be excluded from the scan. """ return self._exclude_dir_set.matches(dir_path) def should_exclude_local_directory(self, dir_path: str): """ - Given the path of a directory, relative to the sync point, + Given the path of a directory, relative to the scan point, decide if all of the files in it should be excluded from the scan. """ return self._exclude_dir_set.matches(dir_path) diff --git a/b2sdk/scan/report.py b/b2sdk/scan/report.py new file mode 100644 index 000000000..8b0f4326f --- /dev/null +++ b/b2sdk/scan/report.py @@ -0,0 +1,210 @@ +###################################################################### +# +# File: b2sdk/scan/report.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import logging +import threading +import time + +from dataclasses import dataclass +from io import TextIOWrapper + +from ..utils import format_and_scale_number + +logger = logging.getLogger(__name__) + + +@dataclass +class Report: + """ + Handle reporting progress. + + This class is THREAD SAFE, so it can be used from parallel scan threads. + """ + + # Minimum time between displayed updates + UPDATE_INTERVAL = 0.1 + + stdout: TextIOWrapper # standard output file object + no_progress: bool # if True, do not show progress + + def __post_init__(self): + self.start_time = time.time() + + self.count = 0 + self.total_done = False + self.total_count = 0 + + self.closed = False + self.lock = threading.Lock() + self.current_line = '' + self.encoding_warning_was_already_printed = False + self._last_update_time = 0 + self._update_progress() + self.warnings = [] + + def close(self): + """ + Perform a clean-up. + """ + with self.lock: + if not self.no_progress: + self._print_line('', False) + self.closed = True + for warning in self.warnings: + self._print_line(warning, True) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def error(self, message): + """ + Print an error, gracefully interleaving it with a progress bar. + + :param message: an error message + :type message: str + """ + self.print_completion(message) + + def print_completion(self, message): + """ + Remove the progress bar, prints a message, and puts the progress + bar back. + + :param message: an error message + :type message: str + """ + with self.lock: + if not self.closed: + self._print_line(message, True) + self._last_update_time = 0 + self._update_progress() + + def update_count(self, delta: int): + """ + Report that items have been processed. + """ + with self.lock: + self.count += delta + self._update_progress() + + def _update_progress(self): + if self.closed or self.no_progress: + return + + now = time.time() + interval = now - self._last_update_time + if interval < self.UPDATE_INTERVAL: + return + + self._last_update_time = now + time_delta = time.time() - self.start_time + rate = 0 if time_delta == 0 else int(self.count / time_delta) + + message = ' count: %d/%d %s' % ( + self.count, + self.total_count, + format_and_scale_number(rate, '/s') + ) # yapf: disable + + self._print_line(message, False) + + def _print_line(self, line, newline): + """ + Print a line to stdout. + + :param line: a string without a \r or \n in it. + :type line: str + :param newline: True if the output should move to a new line after this one. + :type newline: bool + """ + if len(line) < len(self.current_line): + line += ' ' * (len(self.current_line) - len(line)) + try: + self.stdout.write(line) + except UnicodeEncodeError as encode_error: + if not self.encoding_warning_was_already_printed: + self.encoding_warning_was_already_printed = True + self.stdout.write( + '!WARNING! this terminal cannot properly handle progress reporting. encoding is %s.\n' + % (self.stdout.encoding,) + ) + self.stdout.write(line.encode('ascii', 'backslashreplace').decode()) + logger.warning( + 'could not output the following line with encoding %s on stdout due to %s: %s' % + (self.stdout.encoding, encode_error, line) + ) + if newline: + self.stdout.write('\n') + self.current_line = '' + else: + self.stdout.write('\r') + self.current_line = line + self.stdout.flush() + + def update_total(self, delta): + """ + Report that more files have been found for comparison. + + :param delta: number of files found since the last check + :type delta: int + """ + with self.lock: + self.total_count += delta + self._update_progress() + + def end_total(self): + """ + Total files count is done. Can proceed to step 2. + """ + with self.lock: + self.total_done = True + self._update_progress() + + def local_access_error(self, path): + """ + Add a file access error message to the list of warnings. + + :param path: file path + :type path: str + """ + self.warnings.append('WARNING: %s could not be accessed (broken symlink?)' % (path,)) + + def local_permission_error(self, path): + """ + Add a permission error message to the list of warnings. + + :param path: file path + :type path: str + """ + self.warnings.append( + 'WARNING: %s could not be accessed (no permissions to read?)' % (path,) + ) + + def symlink_skipped(self, path): + pass + + +def sample_report_run(): + """ + Generate a sample report. + """ + import sys + report = Report(sys.stdout, False) + + for i in range(20): + report.update_total(1) + time.sleep(0.2) + if i % 2 == 0: + report.update_count(1) + report.end_total() + report.close() diff --git a/b2sdk/scan/scan.py b/b2sdk/scan/scan.py new file mode 100644 index 000000000..57c68e58a --- /dev/null +++ b/b2sdk/scan/scan.py @@ -0,0 +1,63 @@ +###################################################################### +# +# File: b2sdk/scan/scan.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from typing import Optional, Tuple + +from .folder import AbstractFolder +from .path import AbstractPath +from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager +from .report import Report + + +def zip_folders( + folder_a: AbstractFolder, + folder_b: AbstractFolder, + reporter: Report, + policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER, +) -> Tuple[Optional[AbstractPath], Optional[AbstractPath]]: + """ + Iterate over all of the files in the union of two folders, + matching file names. + + Each item is a pair (file_a, file_b) with the corresponding file + in both folders. Either file (but not both) will be None if the + file is in only one folder. + + :param b2sdk.scan.folder.AbstractFolder folder_a: first folder object. + :param b2sdk.scan.folder.AbstractFolder folder_b: second folder object. + :param reporter: reporter object + :param policies_manager: policies manager object + :return: yields two element tuples + """ + + iter_a = folder_a.all_files(reporter, policies_manager) + iter_b = folder_b.all_files(reporter) + + current_a = next(iter_a, None) + current_b = next(iter_b, None) + + while current_a is not None or current_b is not None: + if current_a is None: + yield (None, current_b) + current_b = next(iter_b, None) + elif current_b is None: + yield (current_a, None) + current_a = next(iter_a, None) + elif current_a.relative_path < current_b.relative_path: + yield (current_a, None) + current_a = next(iter_a, None) + elif current_b.relative_path < current_a.relative_path: + yield (None, current_b) + current_b = next(iter_b, None) + else: + assert current_a.relative_path == current_b.relative_path + yield (current_a, current_b) + current_a = next(iter_a, None) + current_b = next(iter_b, None) diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index 06f8b12af..4a429f005 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -8,16 +8,15 @@ # ###################################################################### -from abc import ABCMeta, abstractmethod - import logging import os -from .encryption_provider import AbstractSyncEncryptionSettingsProvider -from ..bucket import Bucket +from abc import ABCMeta, abstractmethod +from ..bucket import Bucket from ..http_constants import SRC_LAST_MODIFIED_MILLIS +from ..scan.path import B2Path from ..transfer.outbound.upload_source import UploadSourceLocalFile -from .path import B2SyncPath +from .encryption_provider import AbstractSyncEncryptionSettingsProvider from .report import SyncFileReporter logger = logging.getLogger(__name__) @@ -209,13 +208,13 @@ def __str__(self): class B2DownloadAction(AbstractAction): def __init__( self, - source_path: B2SyncPath, + source_path: B2Path, b2_file_name: str, local_full_path: str, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, ): """ - :param b2sdk.v2.B2SyncPath source_path: the file to be downloaded + :param b2sdk.v2.B2Path source_path: the file to be downloaded :param str b2_file_name: b2_file_name :param str local_full_path: a local file path :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider @@ -306,7 +305,7 @@ class B2CopyAction(AbstractAction): def __init__( self, b2_file_name: str, - source_path: B2SyncPath, + source_path: B2Path, dest_b2_file_name, source_bucket: Bucket, destination_bucket: Bucket, @@ -314,7 +313,7 @@ def __init__( ): """ :param str b2_file_name: a b2_file_name - :param b2sdk.v2.B2SyncPath source_path: the file to be copied + :param b2sdk.v2.B2Path source_path: the file to be copied :param str dest_b2_file_name: a name of a destination remote file :param Bucket source_bucket: bucket to copy from :param Bucket destination_bucket: bucket to copy to diff --git a/b2sdk/sync/exception.py b/b2sdk/sync/exception.py index b8c66024d..72de5b722 100644 --- a/b2sdk/sync/exception.py +++ b/b2sdk/sync/exception.py @@ -8,102 +8,9 @@ # ###################################################################### -from contextlib import contextmanager -from typing import Iterator, Type - -from ..exception import B2Error, B2SimpleError - - -class EnvironmentEncodingError(B2Error): - """ - Raised when a file name can not be decoded with system encoding. - """ - - def __init__(self, filename, encoding): - """ - :param filename: an encoded file name - :type filename: str, bytes - :param str encoding: file name encoding - """ - super(EnvironmentEncodingError, self).__init__() - self.filename = filename - self.encoding = encoding - - def __str__(self): - return """file name %s cannot be decoded with system encoding (%s). -We think this is an environment error which you should workaround by -setting your system encoding properly, for example like this: -export LANG=en_US.UTF-8""" % ( - self.filename, - self.encoding, - ) - - -class InvalidArgument(B2Error): - """ - Raised when one or more arguments are invalid - """ - - def __init__(self, parameter_name, message): - """ - :param parameter_name: name of the function argument - :param message: brief explanation of misconfiguration - """ - super(InvalidArgument, self).__init__() - self.parameter_name = parameter_name - self.message = message - - def __str__(self): - return "%s %s" % (self.parameter_name, self.message) +from ..exception import B2SimpleError +from ..scan.exception import BaseDirectoryError class IncompleteSync(B2SimpleError): pass - - -class UnSyncableFilename(B2Error): - """ - Raised when a filename is not supported by the sync operation - """ - - def __init__(self, message, filename): - """ - :param message: brief explanation of why the filename was not supported - :param filename: name of the file which is not supported - """ - super(UnSyncableFilename, self).__init__() - self.filename = filename - self.message = message - - def __str__(self): - return "%s: %s" % (self.message, self.filename) - - -@contextmanager -def check_invalid_argument(parameter_name: str, message: str, - *exceptions: Type[Exception]) -> Iterator[None]: - """Raise `InvalidArgument` in case of one of given exception was thrown.""" - try: - yield - except exceptions as exc: - if not message: - message = str(exc) - raise InvalidArgument(parameter_name, message) from exc - - -class BaseDirectoryError(B2SimpleError): - def __init__(self, path): - self.path = path - super().__init__(path) - - -class EmptyDirectory(BaseDirectoryError): - pass - - -class UnableToCreateDirectory(BaseDirectoryError): - pass - - -class NotADirectory(BaseDirectoryError): - pass diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index 61e96f600..f9a4e688f 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -8,18 +8,18 @@ # ###################################################################### +import logging + from abc import ABCMeta, abstractmethod from enum import Enum, unique from typing import Optional -import logging - from ..exception import DestFileNewer -from .encryption_provider import AbstractSyncEncryptionSettingsProvider, SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER -from .action import LocalDeleteAction, B2CopyAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2UploadAction -from .exception import InvalidArgument -from .folder import AbstractFolder -from .path import AbstractSyncPath +from ..scan.exception import InvalidArgument +from ..scan.folder import AbstractFolder +from ..scan.path import AbstractPath +from .action import B2CopyAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2UploadAction, LocalDeleteAction +from .encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, AbstractSyncEncryptionSettingsProvider ONE_DAY_IN_MS = 24 * 60 * 60 * 1000 @@ -51,9 +51,9 @@ class AbstractFileSyncPolicy(metaclass=ABCMeta): def __init__( self, - source_path: AbstractSyncPath, + source_path: AbstractPath, source_folder: AbstractFolder, - dest_path: AbstractSyncPath, + dest_path: AbstractPath, dest_folder: AbstractFolder, now_millis: int, keep_days: int, @@ -64,9 +64,9 @@ def __init__( AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ - :param b2sdk.v2.AbstractSyncPath source_path: source file object + :param b2sdk.v2.AbstractPath source_path: source file object :param b2sdk.v2.AbstractFolder source_folder: source folder object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder dest_folder: destination folder object :param int now_millis: current time in milliseconds :param int keep_days: days to keep before delete @@ -110,8 +110,8 @@ def _should_transfer(self): @classmethod def files_are_different( cls, - source_path: AbstractSyncPath, - dest_path: AbstractSyncPath, + source_path: AbstractPath, + dest_path: AbstractPath, compare_threshold: Optional[int] = None, compare_version_mode: CompareVersionMode = CompareVersionMode.MODTIME, newer_file_mode: NewerFileSyncMode = NewerFileSyncMode.RAISE_ERROR, @@ -120,8 +120,8 @@ def files_are_different( Compare two files and determine if the the destination file should be replaced by the source file. - :param b2sdk.v2.AbstractSyncPath source_path: source file object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath source_path: source file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param int compare_threshold: compare threshold when comparing by time or size :param b2sdk.v2.CompareVersionMode compare_version_mode: source file version comparator method :param b2sdk.v2.NewerFileSyncMode newer_file_mode: newer destination handling method @@ -385,16 +385,16 @@ def make_b2_delete_note(version, index, transferred): def make_b2_delete_actions( - source_path: AbstractSyncPath, - dest_path: AbstractSyncPath, + source_path: AbstractPath, + dest_path: AbstractPath, dest_folder: AbstractFolder, transferred: bool, ): """ Create the actions to delete files stored on B2, which are not present locally. - :param b2sdk.v2.AbstractSyncPath source_path: source file object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath source_path: source file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder dest_folder: destination folder :param bool transferred: if True, file has been transferred, False otherwise """ @@ -414,8 +414,8 @@ def make_b2_delete_actions( def make_b2_keep_days_actions( - source_path: AbstractSyncPath, - dest_path: AbstractSyncPath, + source_path: AbstractPath, + dest_path: AbstractPath, dest_folder: AbstractFolder, transferred: bool, keep_days: int, @@ -431,8 +431,8 @@ def make_b2_keep_days_actions( only the 25-day old version can be deleted. The 15 day-old version was visible 10 days ago. - :param b2sdk.v2.AbstractSyncPath source_path: source file object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath source_path: source file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder dest_folder: destination folder object :param bool transferred: if True, file has been transferred, False otherwise :param int keep_days: how many days to keep a file diff --git a/b2sdk/sync/policy_manager.py b/b2sdk/sync/policy_manager.py index 2db4115f0..f8f8e0bc5 100644 --- a/b2sdk/sync/policy_manager.py +++ b/b2sdk/sync/policy_manager.py @@ -8,10 +8,10 @@ # ###################################################################### +from ..scan.path import AbstractPath from .policy import CopyAndDeletePolicy, CopyAndKeepDaysPolicy, CopyPolicy, \ DownAndDeletePolicy, DownAndKeepDaysPolicy, DownPolicy, UpAndDeletePolicy, \ UpAndKeepDaysPolicy, UpPolicy -from .path import AbstractSyncPath class SyncPolicyManager: @@ -26,9 +26,9 @@ def __init__(self): def get_policy( self, sync_type, - source_path: AbstractSyncPath, + source_path: AbstractPath, source_folder, - dest_path: AbstractSyncPath, + dest_path: AbstractPath, dest_folder, now_millis, delete, @@ -42,9 +42,9 @@ def get_policy( Return a policy object. :param str sync_type: synchronization type - :param b2sdk.v2.AbstractSyncPath source_path: source file + :param b2sdk.v2.AbstractPath source_path: source file :param str source_folder: a source folder path - :param b2sdk.v2.AbstractSyncPath dest_path: destination file + :param b2sdk.v2.AbstractPath dest_path: destination file :param str dest_folder: a destination folder path :param int now_millis: current time in milliseconds :param bool delete: delete policy diff --git a/b2sdk/sync/report.py b/b2sdk/sync/report.py index dbd291031..296d5a80d 100644 --- a/b2sdk/sync/report.py +++ b/b2sdk/sync/report.py @@ -9,16 +9,19 @@ ###################################################################### import logging -import threading import time +from dataclasses import dataclass + from ..progress import AbstractProgressListener -from ..utils import format_and_scale_number, format_and_scale_fraction +from ..scan.report import Report +from ..utils import format_and_scale_fraction, format_and_scale_number logger = logging.getLogger(__name__) -class SyncReport: +@dataclass +class SyncReport(Report): """ Handle reporting progress for syncing. @@ -33,160 +36,53 @@ class SyncReport: This class is THREAD SAFE, so it can be used from parallel sync threads. """ - # Minimum time between displayed updates - UPDATE_INTERVAL = 0.1 - - def __init__(self, stdout, no_progress): - """ - :param stdout: standard output file object - :param no_progress: if True, do not show progress - :type no_progress: bool - """ - self.stdout = stdout - self.no_progress = no_progress - self.start_time = time.time() - self.total_count = 0 - self.total_done = False + def __post_init__(self): self.compare_done = False self.compare_count = 0 self.total_transfer_files = 0 # set in end_compare() self.total_transfer_bytes = 0 # set in end_compare() self.transfer_files = 0 self.transfer_bytes = 0 - self.current_line = '' - self._last_update_time = 0 - self.closed = False - self.lock = threading.Lock() - self.encoding_warning_was_already_printed = False - self._update_progress() - self.warnings = [] - - def close(self): - """ - Perform a clean-up. - """ - with self.lock: - if not self.no_progress: - self._print_line('', False) - self.closed = True - for warning in self.warnings: - self._print_line(warning, True) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def error(self, message): - """ - Print an error, gracefully interleaving it with a progress bar. - - :param message: an error message - :type message: str - """ - self.print_completion(message) - - def print_completion(self, message): - """ - Remove the progress bar, prints a message, and puts the progress - bar back. - - :param message: an error message - :type message: str - """ - with self.lock: - if not self.closed: - self._print_line(message, True) - self._last_update_time = 0 - self._update_progress() + super().__post_init__() def _update_progress(self): - if not self.closed and not self.no_progress: - now = time.time() - interval = now - self._last_update_time - if self.UPDATE_INTERVAL <= interval: - self._last_update_time = now - time_delta = time.time() - self.start_time - rate = 0 if time_delta == 0 else int(self.transfer_bytes / time_delta) - if not self.total_done: - message = ' count: %d files compare: %d files updated: %d files %s %s' % ( - self.total_count, - self.compare_count, - self.transfer_files, - format_and_scale_number(self.transfer_bytes, 'B'), - format_and_scale_number(rate, 'B/s') - ) # yapf: disable - elif not self.compare_done: - message = ' compare: %d/%d files updated: %d files %s %s' % ( - self.compare_count, - self.total_count, - self.transfer_files, - format_and_scale_number(self.transfer_bytes, 'B'), - format_and_scale_number(rate, 'B/s') - ) # yapf: disable - else: - message = ' compare: %d/%d files updated: %d/%d files %s %s' % ( - self.compare_count, - self.total_count, - self.transfer_files, - self.total_transfer_files, - format_and_scale_fraction(self.transfer_bytes, self.total_transfer_bytes, 'B'), - format_and_scale_number(rate, 'B/s') - ) # yapf: disable - self._print_line(message, False) - - def _print_line(self, line, newline): - """ - Print a line to stdout. - - :param line: a string without a \r or \n in it. - :type line: str - :param newline: True if the output should move to a new line after this one. - :type newline: bool - """ - if len(line) < len(self.current_line): - line += ' ' * (len(self.current_line) - len(line)) - try: - self.stdout.write(line) - except UnicodeEncodeError as encode_error: - if not self.encoding_warning_was_already_printed: - self.encoding_warning_was_already_printed = True - self.stdout.write( - '!WARNING! this terminal cannot properly handle progress reporting. encoding is %s.\n' - % (self.stdout.encoding,) - ) - self.stdout.write(line.encode('ascii', 'backslashreplace').decode()) - logger.warning( - 'could not output the following line with encoding %s on stdout due to %s: %s' % - (self.stdout.encoding, encode_error, line) - ) - if newline: - self.stdout.write('\n') - self.current_line = '' + if self.closed or self.no_progress: + return + + now = time.time() + interval = now - self._last_update_time + if interval < self.UPDATE_INTERVAL: + return + + self._last_update_time = now + time_delta = now - self.start_time + rate = 0 if time_delta == 0 else int(self.transfer_bytes / time_delta) + if not self.total_done: + message = ' count: %d files compare: %d files updated: %d files %s %s' % ( + self.total_count, + self.compare_count, + self.transfer_files, + format_and_scale_number(self.transfer_bytes, 'B'), + format_and_scale_number(rate, 'B/s') + ) # yapf: disable + elif not self.compare_done: + message = ' compare: %d/%d files updated: %d files %s %s' % ( + self.compare_count, + self.total_count, + self.transfer_files, + format_and_scale_number(self.transfer_bytes, 'B'), + format_and_scale_number(rate, 'B/s') + ) # yapf: disable else: - self.stdout.write('\r') - self.current_line = line - self.stdout.flush() - - def update_total(self, delta): - """ - Report that more files have been found for comparison. - - :param delta: number of files found since the last check - :type delta: int - """ - with self.lock: - self.total_count += delta - self._update_progress() - - def end_total(self): - """ - Total files count is done. Can proceed to step 2. - """ - with self.lock: - self.total_done = True - self._update_progress() + message = ' compare: %d/%d files updated: %d/%d files %s %s' % ( + self.compare_count, + self.total_count, + self.transfer_files, + self.total_transfer_files, + format_and_scale_fraction(self.transfer_bytes, self.total_transfer_bytes, 'B'), + format_and_scale_number(rate, 'B/s') + ) # yapf: disable + self._print_line(message, False) def update_compare(self, delta): """ @@ -228,29 +124,6 @@ def update_transfer(self, file_delta, byte_delta): self.transfer_bytes += byte_delta self._update_progress() - def local_access_error(self, path): - """ - Add a file access error message to the list of warnings. - - :param path: file path - :type path: str - """ - self.warnings.append('WARNING: %s could not be accessed (broken symlink?)' % (path,)) - - def local_permission_error(self, path): - """ - Add a permission error message to the list of warnings. - - :param path: file path - :type path: str - """ - self.warnings.append( - 'WARNING: %s could not be accessed (no permissions to read?)' % (path,) - ) - - def symlink_skipped(self, path): - pass - class SyncFileReporter(AbstractProgressListener): """ @@ -261,7 +134,7 @@ def __init__(self, reporter, *args, **kwargs): """ :param reporter: a reporter object """ - super(SyncFileReporter, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.bytes_so_far = 0 self.reporter = reporter diff --git a/b2sdk/sync/sync.py b/b2sdk/sync/sync.py index 8aa9dc3f7..d3df1275f 100644 --- a/b2sdk/sync/sync.py +++ b/b2sdk/sync/sync.py @@ -8,75 +8,26 @@ # ###################################################################### -import logging import concurrent.futures as futures +import logging + from enum import Enum, unique from ..bounded_queue_executor import BoundedQueueExecutor -from .encryption_provider import AbstractSyncEncryptionSettingsProvider, SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER -from .exception import InvalidArgument, IncompleteSync -from .folder import AbstractFolder -from .path import AbstractSyncPath +from ..scan.exception import InvalidArgument +from ..scan.folder import AbstractFolder +from ..scan.path import AbstractPath +from ..scan.policies import DEFAULT_SCAN_MANAGER +from ..scan.scan import zip_folders +from .encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, AbstractSyncEncryptionSettingsProvider +from .exception import IncompleteSync from .policy import CompareVersionMode, NewerFileSyncMode from .policy_manager import POLICY_MANAGER, SyncPolicyManager from .report import SyncReport -from .scan_policies import DEFAULT_SCAN_MANAGER logger = logging.getLogger(__name__) -def next_or_none(iterator): - """ - Return the next item from the iterator, or None if there are no more. - """ - try: - return next(iterator) - except StopIteration: - return None - - -def zip_folders(folder_a, folder_b, reporter, policies_manager=DEFAULT_SCAN_MANAGER): - """ - Iterate over all of the files in the union of two folders, - matching file names. - - Each item is a pair (file_a, file_b) with the corresponding file - in both folders. Either file (but not both) will be None if the - file is in only one folder. - - :param b2sdk.sync.folder.AbstractFolder folder_a: first folder object. - :param b2sdk.sync.folder.AbstractFolder folder_b: second folder object. - :param reporter: reporter object - :param policies_manager: policies manager object - :return: yields two element tuples - """ - - iter_a = folder_a.all_files(reporter, policies_manager) - iter_b = folder_b.all_files(reporter) - - current_a = next_or_none(iter_a) - current_b = next_or_none(iter_b) - - while current_a is not None or current_b is not None: - if current_a is None: - yield (None, current_b) - current_b = next_or_none(iter_b) - elif current_b is None: - yield (current_a, None) - current_a = next_or_none(iter_a) - elif current_a.relative_path < current_b.relative_path: - yield (current_a, None) - current_a = next_or_none(iter_a) - elif current_b.relative_path < current_a.relative_path: - yield (None, current_b) - current_b = next_or_none(iter_b) - else: - assert current_a.relative_path == current_b.relative_path - yield (current_a, current_b) - current_a = next_or_none(iter_a) - current_b = next_or_none(iter_b) - - def count_files(local_folder, reporter, policies_manager): """ Count all of the files in a local folder. @@ -339,8 +290,8 @@ def _make_folder_sync_actions( def _make_file_sync_actions( self, sync_type: str, - source_path: AbstractSyncPath, - dest_path: AbstractSyncPath, + source_path: AbstractPath, + dest_path: AbstractPath, source_folder: AbstractFolder, dest_folder: AbstractFolder, now_millis: int, @@ -351,15 +302,14 @@ def _make_file_sync_actions( Yields the sequence of actions needed to sync the two files :param str sync_type: synchronization type - :param b2sdk.v2.AbstractSyncPath source_path: source file object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath source_path: source file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder source_folder: a source folder object :param b2sdk.v2.AbstractFolder dest_folder: a destination folder object :param int now_millis: current time in milliseconds :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ delete = self.keep_days_or_delete == KeepOrDeleteMode.DELETE - keep_days = self.keep_days policy = self.sync_policy_manager.get_policy( sync_type, @@ -369,7 +319,7 @@ def _make_file_sync_actions( dest_folder, now_millis, delete, - keep_days, + self.keep_days, self.newer_file_mode, self.compare_threshold, self.compare_version_mode, diff --git a/b2sdk/v1/sync/file_to_path_translator.py b/b2sdk/v1/sync/file_to_path_translator.py index 4adf8e5fa..52387be83 100644 --- a/b2sdk/v1/sync/file_to_path_translator.py +++ b/b2sdk/v1/sync/file_to_path_translator.py @@ -11,7 +11,8 @@ from typing import Tuple from b2sdk import v2 -from .file import File, B2File, FileVersion, B2FileVersion + +from .file import B2File, B2FileVersion, File, FileVersion # The goal is to create v1.File objects together with v1.FileVersion objects from v2.SyncPath objects diff --git a/b2sdk/v2/__init__.py b/b2sdk/v2/__init__.py index c895e9f86..9d61375fe 100644 --- a/b2sdk/v2/__init__.py +++ b/b2sdk/v2/__init__.py @@ -9,9 +9,13 @@ ###################################################################### from b2sdk._v3 import * # noqa +from b2sdk._v3 import parse_folder as parse_sync_folder +from b2sdk._v3 import AbstractPath as AbstractSyncPath +from b2sdk._v3 import LocalPath as LocalSyncPath from .api import B2Api from .b2http import B2Http from .bucket import Bucket, BucketFactory from .session import B2Session +from .sync import B2SyncPath from .transfer import DownloadManager, UploadManager diff --git a/b2sdk/v2/exception.py b/b2sdk/v2/exception.py index 5bc9dbd33..2e970afdc 100644 --- a/b2sdk/v2/exception.py +++ b/b2sdk/v2/exception.py @@ -11,6 +11,7 @@ from b2sdk._v3.exception import * # noqa v3BucketIdNotFound = BucketIdNotFound +UnSyncableFilename = UnsupportedFilename # overridden to retain old style isinstance check and attributes diff --git a/b2sdk/v2/sync.py b/b2sdk/v2/sync.py new file mode 100644 index 000000000..4910518c5 --- /dev/null +++ b/b2sdk/v2/sync.py @@ -0,0 +1,13 @@ +###################################################################### +# +# File: b2sdk/v2/sync.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from b2sdk._v3 import B2Path + +B2SyncPath = B2Path diff --git a/doc/source/api/internal/sync/path.rst b/doc/source/api/internal/sync/path.rst index d45b3ae6f..707a888d1 100644 --- a/doc/source/api/internal/sync/path.rst +++ b/doc/source/api/internal/sync/path.rst @@ -1,7 +1,7 @@ -:mod:`b2sdk.sync.path` +:mod:`b2sdk.scan.path` ============================== -.. automodule:: b2sdk.sync.path +.. automodule:: b2sdk.scan.path :members: :undoc-members: :show-inheritance: diff --git a/doc/source/api/sync.rst b/doc/source/api/sync.rst index 212b59420..b36ca0560 100644 --- a/doc/source/api/sync.rst +++ b/doc/source/api/sync.rst @@ -33,7 +33,7 @@ Following are the important optional arguments that can be provided while initia .. code-block:: python >>> from b2sdk.v2 import ScanPoliciesManager - >>> from b2sdk.v2 import parse_sync_folder + >>> from b2sdk.v2 import parse_folder >>> from b2sdk.v2 import Synchronizer >>> from b2sdk.v2 import KeepOrDeleteMode, CompareVersionMode, NewerFileSyncMode >>> import time @@ -42,8 +42,8 @@ Following are the important optional arguments that can be provided while initia >>> source = '/home/user1/b2_example' >>> destination = 'b2://example-mybucket-b2' - >>> source = parse_sync_folder(source, b2_api) - >>> destination = parse_sync_folder(destination, b2_api) + >>> source = parse_folder(source, b2_api) + >>> destination = parse_folder(destination, b2_api) >>> policies_manager = ScanPoliciesManager(exclude_all_symlinks=True) diff --git a/doc/source/quick_start.rst b/doc/source/quick_start.rst index 921fe81ea..f3ec811dc 100644 --- a/doc/source/quick_start.rst +++ b/doc/source/quick_start.rst @@ -28,7 +28,7 @@ Synchronization .. code-block:: python >>> from b2sdk.v2 import ScanPoliciesManager - >>> from b2sdk.v2 import parse_sync_folder + >>> from b2sdk.v2 import parse_folder >>> from b2sdk.v2 import Synchronizer >>> from b2sdk.v2 import SyncReport >>> import time @@ -37,8 +37,8 @@ Synchronization >>> source = '/home/user1/b2_example' >>> destination = 'b2://example-mybucket-b2' - >>> source = parse_sync_folder(source, b2_api) - >>> destination = parse_sync_folder(destination, b2_api) + >>> source = parse_folder(source, b2_api) + >>> destination = parse_folder(destination, b2_api) >>> policies_manager = ScanPoliciesManager(exclude_all_symlinks=True) diff --git a/test/unit/fixtures/folder.py b/test/unit/fixtures/folder.py new file mode 100644 index 000000000..8a92a9042 --- /dev/null +++ b/test/unit/fixtures/folder.py @@ -0,0 +1,101 @@ +###################################################################### +# +# File: test/unit/fixtures/folder.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from unittest import mock + +import apiver_deps +import pytest + +from apiver_deps import B2Folder, LocalFolder, LocalPath +from apiver_deps import DEFAULT_SCAN_MANAGER + + +if apiver_deps.V <= 1: + from apiver_deps import FileVersionInfo as VFileVersion +else: + from apiver_deps import FileVersion as VFileVersion + + +class FakeB2Folder(B2Folder): + def __init__(self, test_files): + self.file_versions = [] + for test_file in test_files: + self.file_versions.extend(self._file_versions(*test_file)) + super().__init__('test-bucket', 'folder', mock.MagicMock()) + + def get_file_versions(self): + yield from iter(self.file_versions) + + def _file_versions(self, name, mod_times, size=10): + """ + Makes FileVersion objects. + + Positive modification times are uploads, and negative modification + times are hides. It's a hack, but it works. + + """ + if apiver_deps.V <= 1: + mandatory_kwargs = {} + else: + mandatory_kwargs = { + 'api': None, + 'account_id': 'account-id', + 'bucket_id': 'bucket-id', + 'content_md5': 'content_md5', + 'server_side_encryption': None, + } + return [ + VFileVersion( + id_='id_%s_%d' % (name[0], abs(mod_time)), + file_name='folder/' + name, + upload_timestamp=abs(mod_time), + action='upload' if 0 < mod_time else 'hide', + size=size, + file_info={'in_b2': 'yes'}, + content_type='text/plain', + content_sha1='content_sha1', + **mandatory_kwargs, + ) for mod_time in mod_times + ] # yapf disable + + +class FakeLocalFolder(LocalFolder): + def __init__(self, test_files): + super().__init__('folder') + self.local_paths = [self._local_path(*test_file) for test_file in test_files] + + def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + for single_path in self.local_paths: + if single_path.relative_path.endswith('/'): + if policies_manager.should_exclude_b2_directory(single_path.relative_path): + continue + else: + if policies_manager.should_exclude_local_path(single_path): + continue + yield single_path + + def make_full_path(self, name): + return '/dir/' + name + + def _local_path(self, name, mod_times, size=10): + """ + Makes a LocalPath object for a local file. + """ + return LocalPath(name, name, mod_times[0], size) + + +@pytest.fixture(scope='session') +def folder_factory(): + def get_folder(f_type, *files): + if f_type == 'b2': + return FakeB2Folder(files) + return FakeLocalFolder(files) + + return get_folder diff --git a/test/unit/scan/__init__.py b/test/unit/scan/__init__.py new file mode 100644 index 000000000..9acce809e --- /dev/null +++ b/test/unit/scan/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: test/unit/scan/__init__.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/test/unit/sync/test_scan_policies.py b/test/unit/scan/test_scan_policies.py similarity index 100% rename from test/unit/sync/test_scan_policies.py rename to test/unit/scan/test_scan_policies.py diff --git a/test/unit/sync/fixtures.py b/test/unit/sync/fixtures.py index a48618048..67f2fa334 100644 --- a/test/unit/sync/fixtures.py +++ b/test/unit/sync/fixtures.py @@ -8,97 +8,9 @@ # ###################################################################### -from unittest import mock - import pytest -import apiver_deps -from apiver_deps import AbstractFolder, B2Folder, LocalFolder, B2SyncPath, LocalSyncPath -from apiver_deps import CompareVersionMode, NewerFileSyncMode, KeepOrDeleteMode -from apiver_deps import DEFAULT_SCAN_MANAGER, POLICY_MANAGER, Synchronizer - -if apiver_deps.V <= 1: - from apiver_deps import FileVersionInfo as VFileVersion -else: - from apiver_deps import FileVersion as VFileVersion - - -class FakeB2Folder(B2Folder): - def __init__(self, test_files): - self.file_versions = [] - for test_file in test_files: - self.file_versions.extend(self._file_versions(*test_file)) - super().__init__('test-bucket', 'folder', mock.MagicMock()) - - def get_file_versions(self): - yield from iter(self.file_versions) - - def _file_versions(self, name, mod_times, size=10): - """ - Makes FileVersion objects. - - Positive modification times are uploads, and negative modification - times are hides. It's a hack, but it works. - - """ - if apiver_deps.V <= 1: - mandatory_kwargs = {} - else: - mandatory_kwargs = { - 'api': None, - 'account_id': 'account-id', - 'bucket_id': 'bucket-id', - 'content_md5': 'content_md5', - 'server_side_encryption': None, - } - return [ - VFileVersion( - id_='id_%s_%d' % (name[0], abs(mod_time)), - file_name='folder/' + name, - upload_timestamp=abs(mod_time), - action='upload' if 0 < mod_time else 'hide', - size=size, - file_info={'in_b2': 'yes'}, - content_type='text/plain', - content_sha1='content_sha1', - **mandatory_kwargs, - ) for mod_time in mod_times - ] # yapf disable - - -class FakeLocalFolder(LocalFolder): - def __init__(self, test_files): - super().__init__('folder') - self.local_sync_paths = [self._local_sync_path(*test_file) for test_file in test_files] - - def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): - for single_path in self.local_sync_paths: - if single_path.relative_path.endswith('/'): - if policies_manager.should_exclude_b2_directory(single_path.relative_path): - continue - else: - if policies_manager.should_exclude_local_path(single_path): - continue - yield single_path - - def make_full_path(self, name): - return '/dir/' + name - - def _local_sync_path(self, name, mod_times, size=10): - """ - Makes a LocalSyncPath object for a local file. - """ - return LocalSyncPath(name, name, mod_times[0], size) - - -@pytest.fixture(scope='session') -def folder_factory(): - def get_folder(f_type, *files): - if f_type == 'b2': - return FakeB2Folder(files) - return FakeLocalFolder(files) - - return get_folder +from apiver_deps import DEFAULT_SCAN_MANAGER, POLICY_MANAGER, CompareVersionMode, KeepOrDeleteMode, NewerFileSyncMode, Synchronizer @pytest.fixture(scope='session') diff --git a/test/unit/sync/test_exception.py b/test/unit/sync/test_exception.py index 9a0efa25f..f2291fa6b 100644 --- a/test/unit/sync/test_exception.py +++ b/test/unit/sync/test_exception.py @@ -14,7 +14,7 @@ EnvironmentEncodingError, InvalidArgument, IncompleteSync, - UnSyncableFilename, + UnsupportedFilename, check_invalid_argument, ) @@ -41,10 +41,10 @@ def test_incomplete_sync(self): except IncompleteSync as e: assert str(e) == 'Incomplete sync: ', str(e) - def test_unsyncablefilename_error(self): + def test_unsupportedfilename_error(self): try: - raise UnSyncableFilename('message', 'filename') - except UnSyncableFilename as e: + raise UnsupportedFilename('message', 'filename') + except UnsupportedFilename as e: assert str(e) == 'message: filename', str(e) diff --git a/test/unit/sync/test_sync.py b/test/unit/sync/test_sync.py index 887d61520..cf7123cac 100644 --- a/test/unit/sync/test_sync.py +++ b/test/unit/sync/test_sync.py @@ -12,10 +12,11 @@ from enum import Enum from functools import partial -from apiver_deps import UpPolicy, B2DownloadAction, B2UploadAction, B2CopyAction, AbstractSyncEncryptionSettingsProvider, UploadSourceLocalFile, SyncPolicyManager +from apiver_deps import UpPolicy, B2DownloadAction, AbstractSyncEncryptionSettingsProvider, UploadSourceLocalFile, SyncPolicyManager from apiver_deps_exception import DestFileNewer, InvalidArgument -from b2sdk.utils import TempDir - +from apiver_deps import KeepOrDeleteMode, NewerFileSyncMode, CompareVersionMode +import pytest +from ..fixtures.folder import * from .fixtures import * DAY = 86400000 # milliseconds diff --git a/test/unit/test_base.py b/test/unit/test_base.py index 332c295a3..820b0c093 100644 --- a/test/unit/test_base.py +++ b/test/unit/test_base.py @@ -8,13 +8,16 @@ # ###################################################################### -from contextlib import contextmanager -from typing import List, Optional import re import unittest +from contextlib import contextmanager +from typing import List, Optional + import apiver_deps + from apiver_deps import B2Api + from b2sdk.v2 import FullApplicationKey diff --git a/test/unit/v0/test_sync.py b/test/unit/v0/test_sync.py index d53d47174..f18b2077c 100644 --- a/test/unit/v0/test_sync.py +++ b/test/unit/v0/test_sync.py @@ -23,7 +23,7 @@ from .deps_exception import UnSyncableFilename, NotADirectory, UnableToCreateDirectory, EmptyDirectory, InvalidArgument, CommandError from .deps import FileVersionInfo -from .deps import AbstractFolder, B2Folder, LocalFolder +from .deps import B2Folder, LocalFolder from .deps import LocalSyncPath, B2SyncPath from .deps import ScanPoliciesManager, DEFAULT_SCAN_MANAGER from .deps import BoundedQueueExecutor, make_folder_sync_actions, zip_folders @@ -447,8 +447,8 @@ def test_multiple_versions(self): self.assertEqual( [ - "B2SyncPath(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", - "B2SyncPath(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" + "B2Path(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", + "B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" ], [ str(f) for f in folder.all_files(self.reporter) if f.relative_path in ('inner/a.txt', 'inner/b.txt') @@ -461,7 +461,7 @@ def test_exclude_modified_multiple_versions(self): ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( - ["B2SyncPath(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ + ["B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ str(f) for f in folder.all_files(self.reporter, policies_manager=polices_manager) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ] diff --git a/test/unit/v1/test_sync.py b/test/unit/v1/test_sync.py index 9eca10d8e..3593ade1c 100644 --- a/test/unit/v1/test_sync.py +++ b/test/unit/v1/test_sync.py @@ -21,7 +21,7 @@ from ..test_base import TestBase -from .deps import AbstractFolder, B2Folder, LocalFolder +from .deps import B2Folder, LocalFolder from .deps import BoundedQueueExecutor, zip_folders from .deps import LocalSyncPath, B2SyncPath from .deps import FileVersionInfo @@ -451,8 +451,8 @@ def test_multiple_versions(self): self.assertEqual( [ - "B2SyncPath(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", - "B2SyncPath(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" + "B2Path(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", + "B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" ], [ str(f) for f in folder.all_files(self.reporter) if f.relative_path in ('inner/a.txt', 'inner/b.txt') @@ -465,7 +465,7 @@ def test_exclude_modified_multiple_versions(self): ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( - ["B2SyncPath(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ + ["B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ str(f) for f in folder.all_files(self.reporter, policies_manager=polices_manager) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ]