From 43dfd5ff09fca6816494a26f25f6504db317cc07 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 17 May 2022 17:23:32 +0300 Subject: [PATCH 01/17] Move chunks to `scan` module --- b2sdk/scan/__init__.py | 9 ++ b2sdk/scan/exception.py | 105 ++++++++++++++++++ b2sdk/{sync => scan}/folder.py | 2 +- b2sdk/{sync => scan}/folder_parser.py | 2 +- b2sdk/{sync => scan}/path.py | 2 +- .../scan_policies.py => scan/policies.py} | 2 +- b2sdk/sync/exception.py | 96 +--------------- 7 files changed, 119 insertions(+), 99 deletions(-) create mode 100644 b2sdk/scan/__init__.py create mode 100644 b2sdk/scan/exception.py rename b2sdk/{sync => scan}/folder.py (99%) rename b2sdk/{sync => scan}/folder_parser.py (97%) rename b2sdk/{sync => scan}/path.py (99%) rename b2sdk/{sync/scan_policies.py => scan/policies.py} (99%) diff --git a/b2sdk/scan/__init__.py b/b2sdk/scan/__init__.py new file mode 100644 index 000000000..672eaf1e6 --- /dev/null +++ b/b2sdk/scan/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: b2sdk/scan/__init__.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/b2sdk/scan/exception.py b/b2sdk/scan/exception.py new file mode 100644 index 000000000..5b5d79e80 --- /dev/null +++ b/b2sdk/scan/exception.py @@ -0,0 +1,105 @@ +###################################################################### +# +# File: b2sdk/sync/exception.py +# +# Copyright 2019 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from contextlib import contextmanager +from typing import Iterator, Type + +from ..exception import B2Error, B2SimpleError + + +class EnvironmentEncodingError(B2Error): + """ + Raised when a file name can not be decoded with system encoding. + """ + + def __init__(self, filename, encoding): + """ + :param filename: an encoded file name + :type filename: str, bytes + :param str encoding: file name encoding + """ + super(EnvironmentEncodingError, self).__init__() + self.filename = filename + self.encoding = encoding + + def __str__(self): + return """file name %s cannot be decoded with system encoding (%s). +We think this is an environment error which you should workaround by +setting your system encoding properly, for example like this: +export LANG=en_US.UTF-8""" % ( + self.filename, + self.encoding, + ) + + +class InvalidArgument(B2Error): + """ + Raised when one or more arguments are invalid + """ + + def __init__(self, parameter_name, message): + """ + :param parameter_name: name of the function argument + :param message: brief explanation of misconfiguration + """ + super(InvalidArgument, self).__init__() + self.parameter_name = parameter_name + self.message = message + + def __str__(self): + return "%s %s" % (self.parameter_name, self.message) + + +class UnSyncableFilename(B2Error): + """ + Raised when a filename is not supported by the sync operation + """ + + def __init__(self, message, filename): + """ + :param message: brief explanation of why the filename was not supported + :param filename: name of the file which is not supported + """ + super(UnSyncableFilename, self).__init__() + self.filename = filename + self.message = message + + def __str__(self): + return "%s: %s" % (self.message, self.filename) + + +@contextmanager +def check_invalid_argument(parameter_name: str, message: str, + *exceptions: Type[Exception]) -> Iterator[None]: + """Raise `InvalidArgument` in case of one of given exception was thrown.""" + try: + yield + except exceptions as exc: + if not message: + message = str(exc) + raise InvalidArgument(parameter_name, message) from exc + + +class BaseDirectoryError(B2SimpleError): + def __init__(self, path): + self.path = path + super().__init__(path) + + +class EmptyDirectory(BaseDirectoryError): + pass + + +class UnableToCreateDirectory(BaseDirectoryError): + pass + + +class NotADirectory(BaseDirectoryError): + pass diff --git a/b2sdk/sync/folder.py b/b2sdk/scan/folder.py similarity index 99% rename from b2sdk/sync/folder.py rename to b2sdk/scan/folder.py index addbc2099..c8ee8d55d 100644 --- a/b2sdk/sync/folder.py +++ b/b2sdk/scan/folder.py @@ -409,4 +409,4 @@ def make_full_path(self, file_name): return self.folder_name + '/' + file_name def __str__(self): - return 'B2Folder(%s, %s)' % (self.bucket_name, self.folder_name) + return 'B2Folder(%s, %s)' % (self.bucket_name, self.folder_name) \ No newline at end of file diff --git a/b2sdk/sync/folder_parser.py b/b2sdk/scan/folder_parser.py similarity index 97% rename from b2sdk/sync/folder_parser.py rename to b2sdk/scan/folder_parser.py index 9aa4a39fe..bb53650c7 100644 --- a/b2sdk/sync/folder_parser.py +++ b/b2sdk/scan/folder_parser.py @@ -52,4 +52,4 @@ def _parse_bucket_and_folder(bucket_and_path, api, b2_folder_class): (bucket_name, folder_name) = bucket_and_path.split('/', 1) if folder_name.endswith('/'): folder_name = folder_name[:-1] - return b2_folder_class(bucket_name, folder_name, api) + return b2_folder_class(bucket_name, folder_name, api) \ No newline at end of file diff --git a/b2sdk/sync/path.py b/b2sdk/scan/path.py similarity index 99% rename from b2sdk/sync/path.py rename to b2sdk/scan/path.py index a439e7c03..cbc93beea 100644 --- a/b2sdk/sync/path.py +++ b/b2sdk/scan/path.py @@ -89,4 +89,4 @@ def __eq__(self, other): self.relative_path == other.relative_path and self.selected_version == other.selected_version and self.all_versions == other.all_versions - ) + ) \ No newline at end of file diff --git a/b2sdk/sync/scan_policies.py b/b2sdk/scan/policies.py similarity index 99% rename from b2sdk/sync/scan_policies.py rename to b2sdk/scan/policies.py index 42d54a28c..3d7c03de5 100644 --- a/b2sdk/sync/scan_policies.py +++ b/b2sdk/scan/policies.py @@ -223,4 +223,4 @@ def should_exclude_local_directory(self, dir_path: str): return self._exclude_dir_set.matches(dir_path) -DEFAULT_SCAN_MANAGER = ScanPoliciesManager() +DEFAULT_SCAN_MANAGER = ScanPoliciesManager() \ No newline at end of file diff --git a/b2sdk/sync/exception.py b/b2sdk/sync/exception.py index b8c66024d..4c6e44192 100644 --- a/b2sdk/sync/exception.py +++ b/b2sdk/sync/exception.py @@ -8,102 +8,8 @@ # ###################################################################### -from contextlib import contextmanager -from typing import Iterator, Type - -from ..exception import B2Error, B2SimpleError - - -class EnvironmentEncodingError(B2Error): - """ - Raised when a file name can not be decoded with system encoding. - """ - - def __init__(self, filename, encoding): - """ - :param filename: an encoded file name - :type filename: str, bytes - :param str encoding: file name encoding - """ - super(EnvironmentEncodingError, self).__init__() - self.filename = filename - self.encoding = encoding - - def __str__(self): - return """file name %s cannot be decoded with system encoding (%s). -We think this is an environment error which you should workaround by -setting your system encoding properly, for example like this: -export LANG=en_US.UTF-8""" % ( - self.filename, - self.encoding, - ) - - -class InvalidArgument(B2Error): - """ - Raised when one or more arguments are invalid - """ - - def __init__(self, parameter_name, message): - """ - :param parameter_name: name of the function argument - :param message: brief explanation of misconfiguration - """ - super(InvalidArgument, self).__init__() - self.parameter_name = parameter_name - self.message = message - - def __str__(self): - return "%s %s" % (self.parameter_name, self.message) +from ..exception import B2SimpleError class IncompleteSync(B2SimpleError): pass - - -class UnSyncableFilename(B2Error): - """ - Raised when a filename is not supported by the sync operation - """ - - def __init__(self, message, filename): - """ - :param message: brief explanation of why the filename was not supported - :param filename: name of the file which is not supported - """ - super(UnSyncableFilename, self).__init__() - self.filename = filename - self.message = message - - def __str__(self): - return "%s: %s" % (self.message, self.filename) - - -@contextmanager -def check_invalid_argument(parameter_name: str, message: str, - *exceptions: Type[Exception]) -> Iterator[None]: - """Raise `InvalidArgument` in case of one of given exception was thrown.""" - try: - yield - except exceptions as exc: - if not message: - message = str(exc) - raise InvalidArgument(parameter_name, message) from exc - - -class BaseDirectoryError(B2SimpleError): - def __init__(self, path): - self.path = path - super().__init__(path) - - -class EmptyDirectory(BaseDirectoryError): - pass - - -class UnableToCreateDirectory(BaseDirectoryError): - pass - - -class NotADirectory(BaseDirectoryError): - pass From ba50848029eca1f43caf1eef55429c23d661f56d Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 17 May 2022 18:21:26 +0300 Subject: [PATCH 02/17] Fix file headers --- b2sdk/scan/exception.py | 4 ++-- b2sdk/scan/folder.py | 2 +- b2sdk/scan/folder_parser.py | 2 +- b2sdk/scan/path.py | 2 +- b2sdk/scan/policies.py | 2 +- doc/source/api/internal/sync/path.rst | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/b2sdk/scan/exception.py b/b2sdk/scan/exception.py index 5b5d79e80..5b111680f 100644 --- a/b2sdk/scan/exception.py +++ b/b2sdk/scan/exception.py @@ -1,8 +1,8 @@ ###################################################################### # -# File: b2sdk/sync/exception.py +# File: b2sdk/scan/exception.py # -# Copyright 2019 Backblaze Inc. All Rights Reserved. +# Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # diff --git a/b2sdk/scan/folder.py b/b2sdk/scan/folder.py index c8ee8d55d..fe6d93513 100644 --- a/b2sdk/scan/folder.py +++ b/b2sdk/scan/folder.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/sync/folder.py +# File: b2sdk/scan/folder.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # diff --git a/b2sdk/scan/folder_parser.py b/b2sdk/scan/folder_parser.py index bb53650c7..feda6a638 100644 --- a/b2sdk/scan/folder_parser.py +++ b/b2sdk/scan/folder_parser.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/sync/folder_parser.py +# File: b2sdk/scan/folder_parser.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # diff --git a/b2sdk/scan/path.py b/b2sdk/scan/path.py index cbc93beea..53d91f5c4 100644 --- a/b2sdk/scan/path.py +++ b/b2sdk/scan/path.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/sync/path.py +# File: b2sdk/scan/path.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # diff --git a/b2sdk/scan/policies.py b/b2sdk/scan/policies.py index 3d7c03de5..7e0692be4 100644 --- a/b2sdk/scan/policies.py +++ b/b2sdk/scan/policies.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/sync/scan_policies.py +# File: b2sdk/scan/policies.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # diff --git a/doc/source/api/internal/sync/path.rst b/doc/source/api/internal/sync/path.rst index d45b3ae6f..707a888d1 100644 --- a/doc/source/api/internal/sync/path.rst +++ b/doc/source/api/internal/sync/path.rst @@ -1,7 +1,7 @@ -:mod:`b2sdk.sync.path` +:mod:`b2sdk.scan.path` ============================== -.. automodule:: b2sdk.sync.path +.. automodule:: b2sdk.scan.path :members: :undoc-members: :show-inheritance: From d1ca3cec5339a8ea70610c25fcb4191862e4d6b4 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 17 May 2022 18:27:29 +0300 Subject: [PATCH 03/17] UnSyncableFilename -> UnsupportedFilename --- b2sdk/_v3/exception.py | 2 +- b2sdk/scan/exception.py | 4 ++-- b2sdk/scan/folder.py | 20 ++++++++++---------- b2sdk/v2/__init__.py | 1 + test/unit/sync/test_exception.py | 8 ++++---- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/b2sdk/_v3/exception.py b/b2sdk/_v3/exception.py index 32fea54d7..86507a683 100644 --- a/b2sdk/_v3/exception.py +++ b/b2sdk/_v3/exception.py @@ -144,7 +144,7 @@ 'UnknownHost', 'UnrecognizedBucketType', 'UnableToCreateDirectory', - 'UnSyncableFilename', + 'UnsupportedFilename', 'UnsatisfiableRange', 'UnusableFileName', 'interpret_b2_error', diff --git a/b2sdk/scan/exception.py b/b2sdk/scan/exception.py index 5b111680f..d68dae3ec 100644 --- a/b2sdk/scan/exception.py +++ b/b2sdk/scan/exception.py @@ -57,9 +57,9 @@ def __str__(self): return "%s %s" % (self.parameter_name, self.message) -class UnSyncableFilename(B2Error): +class UnsupportedFilename(B2Error): """ - Raised when a filename is not supported by the sync operation + Raised when a filename is not supported by the scan operation """ def __init__(self, message, filename): diff --git a/b2sdk/scan/folder.py b/b2sdk/scan/folder.py index fe6d93513..9ee918b1a 100644 --- a/b2sdk/scan/folder.py +++ b/b2sdk/scan/folder.py @@ -15,7 +15,7 @@ import sys from abc import ABCMeta, abstractmethod -from .exception import EmptyDirectory, EnvironmentEncodingError, UnSyncableFilename, NotADirectory, UnableToCreateDirectory +from .exception import EmptyDirectory, EnvironmentEncodingError, UnsupportedFilename, NotADirectory, UnableToCreateDirectory from .path import B2SyncPath, LocalSyncPath from .report import SyncReport from .scan_policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager @@ -148,7 +148,7 @@ def make_full_path(self, file_name): # Ensure the new full_path is inside the self.root directory if common_prefix != self.root: - raise UnSyncableFilename("illegal file name", full_path) + raise UnsupportedFilename("illegal file name", full_path) return full_path @@ -208,8 +208,8 @@ def _walk_relative_paths( name = self._handle_non_unicode_file_name(name) if '/' in name: - raise UnSyncableFilename( - "sync does not support file names that include '/'", + raise UnsupportedFilename( + "scan does not support file names that include '/'", "%s in dir %s" % (name, local_dir) ) @@ -374,18 +374,18 @@ def get_file_versions(self): def _validate_file_name(self, file_name): # Do not allow relative paths in file names if RELATIVE_PATH_MATCHER.search(file_name): - raise UnSyncableFilename( - "sync does not support file names that include relative paths", file_name + raise UnsupportedFilename( + "scan does not support file names that include relative paths", file_name ) # Do not allow absolute paths in file names if ABSOLUTE_PATH_MATCHER.search(file_name): - raise UnSyncableFilename( - "sync does not support file names with absolute paths", file_name + raise UnsupportedFilename( + "scan does not support file names with absolute paths", file_name ) # On Windows, do not allow drive letters in file names if platform.system() == "Windows" and DRIVE_MATCHER.search(file_name): - raise UnSyncableFilename( - "sync does not support file names with drive letters", file_name + raise UnsupportedFilename( + "scan does not support file names with drive letters", file_name ) def folder_type(self): diff --git a/b2sdk/v2/__init__.py b/b2sdk/v2/__init__.py index c895e9f86..9961d8928 100644 --- a/b2sdk/v2/__init__.py +++ b/b2sdk/v2/__init__.py @@ -9,6 +9,7 @@ ###################################################################### from b2sdk._v3 import * # noqa +from b2sdk._v3 import UnsupportedFilename as UnSyncableFilename from .api import B2Api from .b2http import B2Http diff --git a/test/unit/sync/test_exception.py b/test/unit/sync/test_exception.py index 9a0efa25f..f2291fa6b 100644 --- a/test/unit/sync/test_exception.py +++ b/test/unit/sync/test_exception.py @@ -14,7 +14,7 @@ EnvironmentEncodingError, InvalidArgument, IncompleteSync, - UnSyncableFilename, + UnsupportedFilename, check_invalid_argument, ) @@ -41,10 +41,10 @@ def test_incomplete_sync(self): except IncompleteSync as e: assert str(e) == 'Incomplete sync: ', str(e) - def test_unsyncablefilename_error(self): + def test_unsupportedfilename_error(self): try: - raise UnSyncableFilename('message', 'filename') - except UnSyncableFilename as e: + raise UnsupportedFilename('message', 'filename') + except UnsupportedFilename as e: assert str(e) == 'message: filename', str(e) From 11cb325a5949e5b53740e18e3c508bcaf927046c Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 17 May 2022 18:30:06 +0300 Subject: [PATCH 04/17] Refactor super() --- b2sdk/scan/exception.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/b2sdk/scan/exception.py b/b2sdk/scan/exception.py index d68dae3ec..1dbba1fb8 100644 --- a/b2sdk/scan/exception.py +++ b/b2sdk/scan/exception.py @@ -25,7 +25,7 @@ def __init__(self, filename, encoding): :type filename: str, bytes :param str encoding: file name encoding """ - super(EnvironmentEncodingError, self).__init__() + super().__init__() self.filename = filename self.encoding = encoding @@ -49,7 +49,7 @@ def __init__(self, parameter_name, message): :param parameter_name: name of the function argument :param message: brief explanation of misconfiguration """ - super(InvalidArgument, self).__init__() + super().__init__() self.parameter_name = parameter_name self.message = message @@ -67,7 +67,7 @@ def __init__(self, message, filename): :param message: brief explanation of why the filename was not supported :param filename: name of the file which is not supported """ - super(UnSyncableFilename, self).__init__() + super().__init__() self.filename = filename self.message = message From bc3810962b1d4719a59dd113e7a1cae406284579 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 17 May 2022 19:00:06 +0300 Subject: [PATCH 05/17] SyncPath -> ScanPath --- b2sdk/scan/folder.py | 18 +++++++++--------- b2sdk/scan/folder_parser.py | 5 ++--- b2sdk/scan/path.py | 6 +++--- b2sdk/scan/policies.py | 16 ++++++++-------- b2sdk/sync/action.py | 10 +++++----- b2sdk/sync/policy.py | 36 ++++++++++++++++++------------------ b2sdk/sync/policy_manager.py | 10 +++++----- b2sdk/sync/sync.py | 14 +++++++------- b2sdk/v2/__init__.py | 4 ++++ test/unit/sync/fixtures.py | 12 ++++++------ 10 files changed, 67 insertions(+), 64 deletions(-) diff --git a/b2sdk/scan/folder.py b/b2sdk/scan/folder.py index 9ee918b1a..f0c8f07dc 100644 --- a/b2sdk/scan/folder.py +++ b/b2sdk/scan/folder.py @@ -16,9 +16,9 @@ from abc import ABCMeta, abstractmethod from .exception import EmptyDirectory, EnvironmentEncodingError, UnsupportedFilename, NotADirectory, UnableToCreateDirectory -from .path import B2SyncPath, LocalSyncPath +from .path import B2Path, LocalPath from .report import SyncReport -from .scan_policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager +from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from ..utils import fix_windows_path_limit, get_file_mtime, is_file_readable DRIVE_MATCHER = re.compile(r"^([A-Za-z]):([/\\])") @@ -181,7 +181,7 @@ def _walk_relative_paths( Yield a File object for each of the files anywhere under this folder, in the order they would appear in B2, unless the path is excluded by policies manager. - :param relative_dir_path: the path of this dir relative to the sync point, or '' if at sync point + :param relative_dir_path: the path of this dir relative to the scan point, or '' if at scan point """ if not isinstance(local_dir, str): raise ValueError('folder path should be unicode: %s' % repr(local_dir)) @@ -216,7 +216,7 @@ def _walk_relative_paths( local_path = os.path.join(local_dir, name) relative_file_path = join_b2_path( relative_dir_path, name - ) # file path relative to the sync point + ) # file path relative to the scan point # Skip broken symlinks or other inaccessible files if not is_file_readable(local_path, reporter): @@ -251,17 +251,17 @@ def _walk_relative_paths( file_mod_time = get_file_mtime(local_path) file_size = os.path.getsize(local_path) - local_sync_path = LocalSyncPath( + local_scan_path = LocalPath( absolute_path=self.make_full_path(relative_file_path), relative_path=relative_file_path, mod_time=file_mod_time, size=file_size, ) - if policies_manager.should_exclude_local_path(local_sync_path): + if policies_manager.should_exclude_local_path(local_scan_path): continue - yield local_sync_path + yield local_scan_path @classmethod def _handle_non_unicode_file_name(cls, name): @@ -346,7 +346,7 @@ def all_files( self._validate_file_name(file_name) if current_name != file_name and current_name is not None and current_versions: - yield B2SyncPath( + yield B2Path( relative_path=current_name, selected_version=current_versions[0], all_versions=current_versions @@ -357,7 +357,7 @@ def all_files( current_versions.append(file_version) if current_name is not None and current_versions: - yield B2SyncPath( + yield B2Path( relative_path=current_name, selected_version=current_versions[0], all_versions=current_versions diff --git a/b2sdk/scan/folder_parser.py b/b2sdk/scan/folder_parser.py index feda6a638..1ee3b47af 100644 --- a/b2sdk/scan/folder_parser.py +++ b/b2sdk/scan/folder_parser.py @@ -12,13 +12,12 @@ from .folder import B2Folder, LocalFolder -def parse_sync_folder(folder_name, api, local_folder_class=LocalFolder, b2_folder_class=B2Folder): +def parse_folder(folder_name, api, local_folder_class=LocalFolder, b2_folder_class=B2Folder): """ Take either a local path, or a B2 path, and returns a Folder object for it. - B2 paths look like: b2://bucketName/path/name. The '//' is optional, - because the previous sync command didn't use it. + B2 paths look like: b2://bucketName/path/name. The '//' is optional. Anything else is treated like a local folder. diff --git a/b2sdk/scan/path.py b/b2sdk/scan/path.py index 53d91f5c4..ac9da313e 100644 --- a/b2sdk/scan/path.py +++ b/b2sdk/scan/path.py @@ -14,7 +14,7 @@ from ..file_version import FileVersion -class AbstractSyncPath(ABC): +class AbstractPath(ABC): """ Represent a path in a source or destination folder - be it B2 or local """ @@ -34,7 +34,7 @@ def __repr__(self): ) -class LocalSyncPath(AbstractSyncPath): +class LocalPath(AbstractPath): __slots__ = ['absolute_path', 'relative_path', 'mod_time', 'size'] def __init__(self, absolute_path: str, relative_path: str, mod_time: int, size: int): @@ -52,7 +52,7 @@ def __eq__(self, other): ) -class B2SyncPath(AbstractSyncPath): +class B2Path(AbstractPath): __slots__ = ['relative_path', 'selected_version', 'all_versions'] def __init__( diff --git a/b2sdk/scan/policies.py b/b2sdk/scan/policies.py index 7e0692be4..a0d0cc57f 100644 --- a/b2sdk/scan/policies.py +++ b/b2sdk/scan/policies.py @@ -13,7 +13,7 @@ from typing import Optional, Union, Iterable from .exception import InvalidArgument, check_invalid_argument -from .path import LocalSyncPath +from .path import LocalPath from ..file_version import FileVersion logger = logging.getLogger(__name__) @@ -107,8 +107,8 @@ def __contains__(self, item): class ScanPoliciesManager: """ - Policy object used when scanning folders for syncing, used to decide - which files to include in the list of files to be synced. + Policy object used when scanning folders, used to decide + which files to include in the list of files. Code that scans through files should at least use should_exclude_file() to decide whether each file should be included; it will check include/exclude @@ -186,9 +186,9 @@ def _should_exclude_relative_path(self, relative_path: str): return False return self._exclude_file_set.matches(relative_path) - def should_exclude_local_path(self, local_path: LocalSyncPath): + def should_exclude_local_path(self, local_path: LocalPath): """ - Whether a local path should be excluded from the Sync or not. + Whether a local path should be excluded from the scan or not. This method assumes that the directory holding the `path_` has already been checked for exclusion. """ @@ -198,7 +198,7 @@ def should_exclude_local_path(self, local_path: LocalSyncPath): def should_exclude_b2_file_version(self, file_version: FileVersion, relative_path: str): """ - Whether a b2 file version should be excluded from the Sync or not. + Whether a b2 file version should be excluded from the scan or not. This method assumes that the directory holding the `path_` has already been checked for exclusion. """ @@ -210,14 +210,14 @@ def should_exclude_b2_file_version(self, file_version: FileVersion, relative_pat def should_exclude_b2_directory(self, dir_path: str): """ - Given the path of a directory, relative to the sync point, + Given the path of a directory, relative to the scan point, decide if all of the files in it should be excluded from the scan. """ return self._exclude_dir_set.matches(dir_path) def should_exclude_local_directory(self, dir_path: str): """ - Given the path of a directory, relative to the sync point, + Given the path of a directory, relative to the scan point, decide if all of the files in it should be excluded from the scan. """ return self._exclude_dir_set.matches(dir_path) diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index 06f8b12af..75050c8cb 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -17,7 +17,7 @@ from ..http_constants import SRC_LAST_MODIFIED_MILLIS from ..transfer.outbound.upload_source import UploadSourceLocalFile -from .path import B2SyncPath +from ..scan.path import B2Path from .report import SyncFileReporter logger = logging.getLogger(__name__) @@ -209,13 +209,13 @@ def __str__(self): class B2DownloadAction(AbstractAction): def __init__( self, - source_path: B2SyncPath, + source_path: B2Path, b2_file_name: str, local_full_path: str, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, ): """ - :param b2sdk.v2.B2SyncPath source_path: the file to be downloaded + :param b2sdk.v2.B2Path source_path: the file to be downloaded :param str b2_file_name: b2_file_name :param str local_full_path: a local file path :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider @@ -306,7 +306,7 @@ class B2CopyAction(AbstractAction): def __init__( self, b2_file_name: str, - source_path: B2SyncPath, + source_path: B2Path, dest_b2_file_name, source_bucket: Bucket, destination_bucket: Bucket, @@ -314,7 +314,7 @@ def __init__( ): """ :param str b2_file_name: a b2_file_name - :param b2sdk.v2.B2SyncPath source_path: the file to be copied + :param b2sdk.v2.B2Path source_path: the file to be copied :param str dest_b2_file_name: a name of a destination remote file :param Bucket source_bucket: bucket to copy from :param Bucket destination_bucket: bucket to copy to diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index 61e96f600..3ee49ebfc 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -18,8 +18,8 @@ from .encryption_provider import AbstractSyncEncryptionSettingsProvider, SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER from .action import LocalDeleteAction, B2CopyAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2UploadAction from .exception import InvalidArgument -from .folder import AbstractFolder -from .path import AbstractSyncPath +from ..scan.folder import AbstractFolder +from ..scan.path import AbstractPath ONE_DAY_IN_MS = 24 * 60 * 60 * 1000 @@ -51,9 +51,9 @@ class AbstractFileSyncPolicy(metaclass=ABCMeta): def __init__( self, - source_path: AbstractSyncPath, + source_path: AbstractPath, source_folder: AbstractFolder, - dest_path: AbstractSyncPath, + dest_path: AbstractPath, dest_folder: AbstractFolder, now_millis: int, keep_days: int, @@ -64,9 +64,9 @@ def __init__( AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ - :param b2sdk.v2.AbstractSyncPath source_path: source file object + :param b2sdk.v2.AbstractPath source_path: source file object :param b2sdk.v2.AbstractFolder source_folder: source folder object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder dest_folder: destination folder object :param int now_millis: current time in milliseconds :param int keep_days: days to keep before delete @@ -110,8 +110,8 @@ def _should_transfer(self): @classmethod def files_are_different( cls, - source_path: AbstractSyncPath, - dest_path: AbstractSyncPath, + source_path: AbstractPath, + dest_path: AbstractPath, compare_threshold: Optional[int] = None, compare_version_mode: CompareVersionMode = CompareVersionMode.MODTIME, newer_file_mode: NewerFileSyncMode = NewerFileSyncMode.RAISE_ERROR, @@ -120,8 +120,8 @@ def files_are_different( Compare two files and determine if the the destination file should be replaced by the source file. - :param b2sdk.v2.AbstractSyncPath source_path: source file object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath source_path: source file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param int compare_threshold: compare threshold when comparing by time or size :param b2sdk.v2.CompareVersionMode compare_version_mode: source file version comparator method :param b2sdk.v2.NewerFileSyncMode newer_file_mode: newer destination handling method @@ -385,16 +385,16 @@ def make_b2_delete_note(version, index, transferred): def make_b2_delete_actions( - source_path: AbstractSyncPath, - dest_path: AbstractSyncPath, + source_path: AbstractPath, + dest_path: AbstractPath, dest_folder: AbstractFolder, transferred: bool, ): """ Create the actions to delete files stored on B2, which are not present locally. - :param b2sdk.v2.AbstractSyncPath source_path: source file object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath source_path: source file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder dest_folder: destination folder :param bool transferred: if True, file has been transferred, False otherwise """ @@ -414,8 +414,8 @@ def make_b2_delete_actions( def make_b2_keep_days_actions( - source_path: AbstractSyncPath, - dest_path: AbstractSyncPath, + source_path: AbstractPath, + dest_path: AbstractPath, dest_folder: AbstractFolder, transferred: bool, keep_days: int, @@ -431,8 +431,8 @@ def make_b2_keep_days_actions( only the 25-day old version can be deleted. The 15 day-old version was visible 10 days ago. - :param b2sdk.v2.AbstractSyncPath source_path: source file object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath source_path: source file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder dest_folder: destination folder object :param bool transferred: if True, file has been transferred, False otherwise :param int keep_days: how many days to keep a file diff --git a/b2sdk/sync/policy_manager.py b/b2sdk/sync/policy_manager.py index 2db4115f0..f8f8e0bc5 100644 --- a/b2sdk/sync/policy_manager.py +++ b/b2sdk/sync/policy_manager.py @@ -8,10 +8,10 @@ # ###################################################################### +from ..scan.path import AbstractPath from .policy import CopyAndDeletePolicy, CopyAndKeepDaysPolicy, CopyPolicy, \ DownAndDeletePolicy, DownAndKeepDaysPolicy, DownPolicy, UpAndDeletePolicy, \ UpAndKeepDaysPolicy, UpPolicy -from .path import AbstractSyncPath class SyncPolicyManager: @@ -26,9 +26,9 @@ def __init__(self): def get_policy( self, sync_type, - source_path: AbstractSyncPath, + source_path: AbstractPath, source_folder, - dest_path: AbstractSyncPath, + dest_path: AbstractPath, dest_folder, now_millis, delete, @@ -42,9 +42,9 @@ def get_policy( Return a policy object. :param str sync_type: synchronization type - :param b2sdk.v2.AbstractSyncPath source_path: source file + :param b2sdk.v2.AbstractPath source_path: source file :param str source_folder: a source folder path - :param b2sdk.v2.AbstractSyncPath dest_path: destination file + :param b2sdk.v2.AbstractPath dest_path: destination file :param str dest_folder: a destination folder path :param int now_millis: current time in milliseconds :param bool delete: delete policy diff --git a/b2sdk/sync/sync.py b/b2sdk/sync/sync.py index 8aa9dc3f7..a3f2f4e5f 100644 --- a/b2sdk/sync/sync.py +++ b/b2sdk/sync/sync.py @@ -15,12 +15,12 @@ from ..bounded_queue_executor import BoundedQueueExecutor from .encryption_provider import AbstractSyncEncryptionSettingsProvider, SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER from .exception import InvalidArgument, IncompleteSync -from .folder import AbstractFolder -from .path import AbstractSyncPath +from ..scan.folder import AbstractFolder +from ..scan.path import AbstractPath from .policy import CompareVersionMode, NewerFileSyncMode from .policy_manager import POLICY_MANAGER, SyncPolicyManager from .report import SyncReport -from .scan_policies import DEFAULT_SCAN_MANAGER +from ..scan.policies import DEFAULT_SCAN_MANAGER logger = logging.getLogger(__name__) @@ -339,8 +339,8 @@ def _make_folder_sync_actions( def _make_file_sync_actions( self, sync_type: str, - source_path: AbstractSyncPath, - dest_path: AbstractSyncPath, + source_path: AbstractPath, + dest_path: AbstractPath, source_folder: AbstractFolder, dest_folder: AbstractFolder, now_millis: int, @@ -351,8 +351,8 @@ def _make_file_sync_actions( Yields the sequence of actions needed to sync the two files :param str sync_type: synchronization type - :param b2sdk.v2.AbstractSyncPath source_path: source file object - :param b2sdk.v2.AbstractSyncPath dest_path: destination file object + :param b2sdk.v2.AbstractPath source_path: source file object + :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder source_folder: a source folder object :param b2sdk.v2.AbstractFolder dest_folder: a destination folder object :param int now_millis: current time in milliseconds diff --git a/b2sdk/v2/__init__.py b/b2sdk/v2/__init__.py index 9961d8928..444aab031 100644 --- a/b2sdk/v2/__init__.py +++ b/b2sdk/v2/__init__.py @@ -9,6 +9,10 @@ ###################################################################### from b2sdk._v3 import * # noqa +from b2sdk._v3 import parse_folder as parse_sync_folder +from b2sdk._v3 import AbstractPath as AbstractSyncPath +from b2sdk._v3 import B2Path as B2SyncPath +from b2sdk._v3 import LocalPath as LocalSyncPath from b2sdk._v3 import UnsupportedFilename as UnSyncableFilename from .api import B2Api diff --git a/test/unit/sync/fixtures.py b/test/unit/sync/fixtures.py index a48618048..9ca4929ae 100644 --- a/test/unit/sync/fixtures.py +++ b/test/unit/sync/fixtures.py @@ -13,7 +13,7 @@ import pytest import apiver_deps -from apiver_deps import AbstractFolder, B2Folder, LocalFolder, B2SyncPath, LocalSyncPath +from apiver_deps import B2Folder, LocalFolder, LocalPath from apiver_deps import CompareVersionMode, NewerFileSyncMode, KeepOrDeleteMode from apiver_deps import DEFAULT_SCAN_MANAGER, POLICY_MANAGER, Synchronizer @@ -69,10 +69,10 @@ def _file_versions(self, name, mod_times, size=10): class FakeLocalFolder(LocalFolder): def __init__(self, test_files): super().__init__('folder') - self.local_sync_paths = [self._local_sync_path(*test_file) for test_file in test_files] + self.local_paths = [self._local_path(*test_file) for test_file in test_files] def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): - for single_path in self.local_sync_paths: + for single_path in self.local_paths: if single_path.relative_path.endswith('/'): if policies_manager.should_exclude_b2_directory(single_path.relative_path): continue @@ -84,11 +84,11 @@ def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): def make_full_path(self, name): return '/dir/' + name - def _local_sync_path(self, name, mod_times, size=10): + def _local_path(self, name, mod_times, size=10): """ - Makes a LocalSyncPath object for a local file. + Makes a LocalPath object for a local file. """ - return LocalSyncPath(name, name, mod_times[0], size) + return LocalPath(name, name, mod_times[0], size) @pytest.fixture(scope='session') From eddc479de3199b53405e2d2b05978934239ed8c3 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 18 May 2022 00:21:44 +0300 Subject: [PATCH 06/17] Separate Report from SyncReport --- b2sdk/scan/folder.py | 5 +- b2sdk/scan/report.py | 211 +++++++++++++++++++++++++++++++++++++++++ b2sdk/sync/report.py | 218 +++++++++---------------------------------- 3 files changed, 259 insertions(+), 175 deletions(-) create mode 100644 b2sdk/scan/report.py diff --git a/b2sdk/scan/folder.py b/b2sdk/scan/folder.py index f0c8f07dc..cd016df03 100644 --- a/b2sdk/scan/folder.py +++ b/b2sdk/scan/folder.py @@ -17,7 +17,6 @@ from abc import ABCMeta, abstractmethod from .exception import EmptyDirectory, EnvironmentEncodingError, UnsupportedFilename, NotADirectory, UnableToCreateDirectory from .path import B2Path, LocalPath -from .report import SyncReport from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from ..utils import fix_windows_path_limit, get_file_mtime, is_file_readable @@ -174,7 +173,7 @@ def ensure_non_empty(self): raise EmptyDirectory(self.root) def _walk_relative_paths( - self, local_dir: str, relative_dir_path: str, reporter: SyncReport, + self, local_dir: str, relative_dir_path: str, reporter, policies_manager: ScanPoliciesManager ): """ @@ -312,7 +311,7 @@ def __init__(self, bucket_name, folder_name, api): self.prefix = '' if self.folder_name == '' else self.folder_name + '/' def all_files( - self, reporter: SyncReport, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER + self, reporter: Report, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER ): """ Yield all files. diff --git a/b2sdk/scan/report.py b/b2sdk/scan/report.py new file mode 100644 index 000000000..f33c0c5b1 --- /dev/null +++ b/b2sdk/scan/report.py @@ -0,0 +1,211 @@ +###################################################################### +# +# File: b2sdk/scan/report.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import logging +import threading +import time + +from dataclasses import dataclass +from io import TextIOWrapper + +from ..utils import format_and_scale_number + + +logger = logging.getLogger(__name__) + + +@dataclass +class Report: + """ + Handle reporting progress. + + This class is THREAD SAFE, so it can be used from parallel scan threads. + """ + + # Minimum time between displayed updates + UPDATE_INTERVAL = 0.1 + + stdout: TextIOWrapper # standard output file object + no_progress: bool # if True, do not show progress + + def __post_init__(self): + self.start_time = time.time() + + self.count = 0 + self.total_done = False + self.total_count = 0 + + self.closed = False + self.lock = threading.Lock() + self.current_line = '' + self.encoding_warning_was_already_printed = False + self._last_update_time = 0 + self._update_progress() + self.warnings = [] + + def close(self): + """ + Perform a clean-up. + """ + with self.lock: + if not self.no_progress: + self._print_line('', False) + self.closed = True + for warning in self.warnings: + self._print_line(warning, True) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def error(self, message): + """ + Print an error, gracefully interleaving it with a progress bar. + + :param message: an error message + :type message: str + """ + self.print_completion(message) + + def print_completion(self, message): + """ + Remove the progress bar, prints a message, and puts the progress + bar back. + + :param message: an error message + :type message: str + """ + with self.lock: + if not self.closed: + self._print_line(message, True) + self._last_update_time = 0 + self._update_progress() + + def update_count(self, delta: int): + """ + Report that items have been processed. + """ + with self.lock: + self.count += delta + self._update_progress() + + def _update_progress(self): + if self.closed or self.no_progress: + return + + now = time.time() + interval = now - self._last_update_time + if interval < self.UPDATE_INTERVAL: + return + + self._last_update_time = now + time_delta = time.time() - self.start_time + rate = 0 if time_delta == 0 else int(self.count / time_delta) + + message = ' count: %d/%d %s' % ( + self.count, + self.total_count, + format_and_scale_number(rate, '/s') + ) # yapf: disable + + self._print_line(message, False) + + def _print_line(self, line, newline): + """ + Print a line to stdout. + + :param line: a string without a \r or \n in it. + :type line: str + :param newline: True if the output should move to a new line after this one. + :type newline: bool + """ + if len(line) < len(self.current_line): + line += ' ' * (len(self.current_line) - len(line)) + try: + self.stdout.write(line) + except UnicodeEncodeError as encode_error: + if not self.encoding_warning_was_already_printed: + self.encoding_warning_was_already_printed = True + self.stdout.write( + '!WARNING! this terminal cannot properly handle progress reporting. encoding is %s.\n' + % (self.stdout.encoding,) + ) + self.stdout.write(line.encode('ascii', 'backslashreplace').decode()) + logger.warning( + 'could not output the following line with encoding %s on stdout due to %s: %s' % + (self.stdout.encoding, encode_error, line) + ) + if newline: + self.stdout.write('\n') + self.current_line = '' + else: + self.stdout.write('\r') + self.current_line = line + self.stdout.flush() + + def update_total(self, delta): + """ + Report that more files have been found for comparison. + + :param delta: number of files found since the last check + :type delta: int + """ + with self.lock: + self.total_count += delta + self._update_progress() + + def end_total(self): + """ + Total files count is done. Can proceed to step 2. + """ + with self.lock: + self.total_done = True + self._update_progress() + + def local_access_error(self, path): + """ + Add a file access error message to the list of warnings. + + :param path: file path + :type path: str + """ + self.warnings.append('WARNING: %s could not be accessed (broken symlink?)' % (path,)) + + def local_permission_error(self, path): + """ + Add a permission error message to the list of warnings. + + :param path: file path + :type path: str + """ + self.warnings.append( + 'WARNING: %s could not be accessed (no permissions to read?)' % (path,) + ) + + def symlink_skipped(self, path): + pass + + +def sample_report_run(): + """ + Generate a sample report. + """ + import sys + report = Report(sys.stdout, False) + + for i in range(20): + report.update_total(1) + time.sleep(0.2) + if i % 2 == 0: + report.update_count(1) + report.end_total() + report.close() diff --git a/b2sdk/sync/report.py b/b2sdk/sync/report.py index dbd291031..a96f36424 100644 --- a/b2sdk/sync/report.py +++ b/b2sdk/sync/report.py @@ -9,16 +9,20 @@ ###################################################################### import logging -import threading import time +from dataclasses import dataclass + from ..progress import AbstractProgressListener -from ..utils import format_and_scale_number, format_and_scale_fraction +from ..scan.report import Report +from ..utils import format_and_scale_fraction, format_and_scale_number + logger = logging.getLogger(__name__) -class SyncReport: +@dataclass +class SyncReport(Report): """ Handle reporting progress for syncing. @@ -33,160 +37,53 @@ class SyncReport: This class is THREAD SAFE, so it can be used from parallel sync threads. """ - # Minimum time between displayed updates - UPDATE_INTERVAL = 0.1 - - def __init__(self, stdout, no_progress): - """ - :param stdout: standard output file object - :param no_progress: if True, do not show progress - :type no_progress: bool - """ - self.stdout = stdout - self.no_progress = no_progress - self.start_time = time.time() - self.total_count = 0 - self.total_done = False + def __post_init__(self): self.compare_done = False self.compare_count = 0 self.total_transfer_files = 0 # set in end_compare() self.total_transfer_bytes = 0 # set in end_compare() self.transfer_files = 0 self.transfer_bytes = 0 - self.current_line = '' - self._last_update_time = 0 - self.closed = False - self.lock = threading.Lock() - self.encoding_warning_was_already_printed = False - self._update_progress() - self.warnings = [] - - def close(self): - """ - Perform a clean-up. - """ - with self.lock: - if not self.no_progress: - self._print_line('', False) - self.closed = True - for warning in self.warnings: - self._print_line(warning, True) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def error(self, message): - """ - Print an error, gracefully interleaving it with a progress bar. - - :param message: an error message - :type message: str - """ - self.print_completion(message) - - def print_completion(self, message): - """ - Remove the progress bar, prints a message, and puts the progress - bar back. - - :param message: an error message - :type message: str - """ - with self.lock: - if not self.closed: - self._print_line(message, True) - self._last_update_time = 0 - self._update_progress() + super().__post_init__() def _update_progress(self): - if not self.closed and not self.no_progress: - now = time.time() - interval = now - self._last_update_time - if self.UPDATE_INTERVAL <= interval: - self._last_update_time = now - time_delta = time.time() - self.start_time - rate = 0 if time_delta == 0 else int(self.transfer_bytes / time_delta) - if not self.total_done: - message = ' count: %d files compare: %d files updated: %d files %s %s' % ( - self.total_count, - self.compare_count, - self.transfer_files, - format_and_scale_number(self.transfer_bytes, 'B'), - format_and_scale_number(rate, 'B/s') - ) # yapf: disable - elif not self.compare_done: - message = ' compare: %d/%d files updated: %d files %s %s' % ( - self.compare_count, - self.total_count, - self.transfer_files, - format_and_scale_number(self.transfer_bytes, 'B'), - format_and_scale_number(rate, 'B/s') - ) # yapf: disable - else: - message = ' compare: %d/%d files updated: %d/%d files %s %s' % ( - self.compare_count, - self.total_count, - self.transfer_files, - self.total_transfer_files, - format_and_scale_fraction(self.transfer_bytes, self.total_transfer_bytes, 'B'), - format_and_scale_number(rate, 'B/s') - ) # yapf: disable - self._print_line(message, False) - - def _print_line(self, line, newline): - """ - Print a line to stdout. - - :param line: a string without a \r or \n in it. - :type line: str - :param newline: True if the output should move to a new line after this one. - :type newline: bool - """ - if len(line) < len(self.current_line): - line += ' ' * (len(self.current_line) - len(line)) - try: - self.stdout.write(line) - except UnicodeEncodeError as encode_error: - if not self.encoding_warning_was_already_printed: - self.encoding_warning_was_already_printed = True - self.stdout.write( - '!WARNING! this terminal cannot properly handle progress reporting. encoding is %s.\n' - % (self.stdout.encoding,) - ) - self.stdout.write(line.encode('ascii', 'backslashreplace').decode()) - logger.warning( - 'could not output the following line with encoding %s on stdout due to %s: %s' % - (self.stdout.encoding, encode_error, line) - ) - if newline: - self.stdout.write('\n') - self.current_line = '' + if self.closed or self.no_progress: + return + + now = time.time() + interval = now - self._last_update_time + if interval < self.UPDATE_INTERVAL: + return + + self._last_update_time = now + time_delta = time.time() - self.start_time + rate = 0 if time_delta == 0 else int(self.transfer_bytes / time_delta) + if not self.total_done: + message = ' count: %d files compare: %d files updated: %d files %s %s' % ( + self.total_count, + self.compare_count, + self.transfer_files, + format_and_scale_number(self.transfer_bytes, 'B'), + format_and_scale_number(rate, 'B/s') + ) # yapf: disable + elif not self.compare_done: + message = ' compare: %d/%d files updated: %d files %s %s' % ( + self.compare_count, + self.total_count, + self.transfer_files, + format_and_scale_number(self.transfer_bytes, 'B'), + format_and_scale_number(rate, 'B/s') + ) # yapf: disable else: - self.stdout.write('\r') - self.current_line = line - self.stdout.flush() - - def update_total(self, delta): - """ - Report that more files have been found for comparison. - - :param delta: number of files found since the last check - :type delta: int - """ - with self.lock: - self.total_count += delta - self._update_progress() - - def end_total(self): - """ - Total files count is done. Can proceed to step 2. - """ - with self.lock: - self.total_done = True - self._update_progress() + message = ' compare: %d/%d files updated: %d/%d files %s %s' % ( + self.compare_count, + self.total_count, + self.transfer_files, + self.total_transfer_files, + format_and_scale_fraction(self.transfer_bytes, self.total_transfer_bytes, 'B'), + format_and_scale_number(rate, 'B/s') + ) # yapf: disable + self._print_line(message, False) def update_compare(self, delta): """ @@ -228,29 +125,6 @@ def update_transfer(self, file_delta, byte_delta): self.transfer_bytes += byte_delta self._update_progress() - def local_access_error(self, path): - """ - Add a file access error message to the list of warnings. - - :param path: file path - :type path: str - """ - self.warnings.append('WARNING: %s could not be accessed (broken symlink?)' % (path,)) - - def local_permission_error(self, path): - """ - Add a permission error message to the list of warnings. - - :param path: file path - :type path: str - """ - self.warnings.append( - 'WARNING: %s could not be accessed (no permissions to read?)' % (path,) - ) - - def symlink_skipped(self, path): - pass - class SyncFileReporter(AbstractProgressListener): """ @@ -261,7 +135,7 @@ def __init__(self, reporter, *args, **kwargs): """ :param reporter: a reporter object """ - super(SyncFileReporter, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.bytes_so_far = 0 self.reporter = reporter From 48dcf4c724e13b19faab4d1ea2686a670ac80e59 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 18 May 2022 00:24:44 +0300 Subject: [PATCH 07/17] Reformat imports --- b2sdk/scan/policies.py | 6 ++++-- b2sdk/sync/action.py | 11 ++++++----- b2sdk/sync/policy.py | 11 ++++++----- b2sdk/v1/sync/file_to_path_translator.py | 3 ++- test/unit/test_base.py | 7 +++++-- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/b2sdk/scan/policies.py b/b2sdk/scan/policies.py index a0d0cc57f..02150837b 100644 --- a/b2sdk/scan/policies.py +++ b/b2sdk/scan/policies.py @@ -10,11 +10,13 @@ import logging import re -from typing import Optional, Union, Iterable +from typing import Iterable, Optional, Union + +from ..file_version import FileVersion from .exception import InvalidArgument, check_invalid_argument from .path import LocalPath -from ..file_version import FileVersion + logger = logging.getLogger(__name__) diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index 75050c8cb..fc100c745 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -8,18 +8,19 @@ # ###################################################################### -from abc import ABCMeta, abstractmethod - import logging import os -from .encryption_provider import AbstractSyncEncryptionSettingsProvider -from ..bucket import Bucket +from abc import ABCMeta, abstractmethod + +from ..bucket import Bucket from ..http_constants import SRC_LAST_MODIFIED_MILLIS -from ..transfer.outbound.upload_source import UploadSourceLocalFile from ..scan.path import B2Path +from ..transfer.outbound.upload_source import UploadSourceLocalFile +from .encryption_provider import AbstractSyncEncryptionSettingsProvider from .report import SyncFileReporter + logger = logging.getLogger(__name__) diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index 3ee49ebfc..4f1f8eaa0 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -8,18 +8,19 @@ # ###################################################################### +import logging + from abc import ABCMeta, abstractmethod from enum import Enum, unique from typing import Optional -import logging - from ..exception import DestFileNewer -from .encryption_provider import AbstractSyncEncryptionSettingsProvider, SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER -from .action import LocalDeleteAction, B2CopyAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2UploadAction -from .exception import InvalidArgument +from ..scan.exception import InvalidArgument from ..scan.folder import AbstractFolder from ..scan.path import AbstractPath +from .action import B2CopyAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2UploadAction, LocalDeleteAction +from .encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, AbstractSyncEncryptionSettingsProvider + ONE_DAY_IN_MS = 24 * 60 * 60 * 1000 diff --git a/b2sdk/v1/sync/file_to_path_translator.py b/b2sdk/v1/sync/file_to_path_translator.py index 4adf8e5fa..52387be83 100644 --- a/b2sdk/v1/sync/file_to_path_translator.py +++ b/b2sdk/v1/sync/file_to_path_translator.py @@ -11,7 +11,8 @@ from typing import Tuple from b2sdk import v2 -from .file import File, B2File, FileVersion, B2FileVersion + +from .file import B2File, B2FileVersion, File, FileVersion # The goal is to create v1.File objects together with v1.FileVersion objects from v2.SyncPath objects diff --git a/test/unit/test_base.py b/test/unit/test_base.py index 332c295a3..820b0c093 100644 --- a/test/unit/test_base.py +++ b/test/unit/test_base.py @@ -8,13 +8,16 @@ # ###################################################################### -from contextlib import contextmanager -from typing import List, Optional import re import unittest +from contextlib import contextmanager +from typing import List, Optional + import apiver_deps + from apiver_deps import B2Api + from b2sdk.v2 import FullApplicationKey From bf42cb72bc2e5961fb84bc8b0c826d157a82645f Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 18 May 2022 00:25:12 +0300 Subject: [PATCH 08/17] Don't create redundant `keep_days` variable --- b2sdk/sync/sync.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/b2sdk/sync/sync.py b/b2sdk/sync/sync.py index a3f2f4e5f..efb2dc778 100644 --- a/b2sdk/sync/sync.py +++ b/b2sdk/sync/sync.py @@ -359,7 +359,6 @@ def _make_file_sync_actions( :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ delete = self.keep_days_or_delete == KeepOrDeleteMode.DELETE - keep_days = self.keep_days policy = self.sync_policy_manager.get_policy( sync_type, @@ -369,7 +368,7 @@ def _make_file_sync_actions( dest_folder, now_millis, delete, - keep_days, + self.keep_days, self.newer_file_mode, self.compare_threshold, self.compare_version_mode, From 8d0023fa1eb6ba30eec5839658a0b6e77bca6dfc Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 18 May 2022 00:29:31 +0300 Subject: [PATCH 09/17] Move `zip_folders` to `scan/scan.py` --- b2sdk/scan/scan.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++ b2sdk/sync/sync.py | 64 ++++++---------------------------------------- 2 files changed, 72 insertions(+), 56 deletions(-) create mode 100644 b2sdk/scan/scan.py diff --git a/b2sdk/scan/scan.py b/b2sdk/scan/scan.py new file mode 100644 index 000000000..02be35a33 --- /dev/null +++ b/b2sdk/scan/scan.py @@ -0,0 +1,64 @@ +###################################################################### +# +# File: b2sdk/scan/scan.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + + +from typing import Optional, Tuple + +from .folder import AbstractFolder +from .path import AbstractPath +from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager +from .report import Report + + +def zip_folders( + folder_a: AbstractFolder, + folder_b: AbstractFolder, + reporter: Report, + policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER, +) -> Tuple[Optional[AbstractPath], Optional[AbstractPath]]: + """ + Iterate over all of the files in the union of two folders, + matching file names. + + Each item is a pair (file_a, file_b) with the corresponding file + in both folders. Either file (but not both) will be None if the + file is in only one folder. + + :param b2sdk.scan.folder.AbstractFolder folder_a: first folder object. + :param b2sdk.scan.folder.AbstractFolder folder_b: second folder object. + :param reporter: reporter object + :param policies_manager: policies manager object + :return: yields two element tuples + """ + + iter_a = folder_a.all_files(reporter, policies_manager) + iter_b = folder_b.all_files(reporter) + + current_a = next(iter_a, None) + current_b = next(iter_b, None) + + while current_a is not None or current_b is not None: + if current_a is None: + yield (None, current_b) + current_b = next(iter_b, None) + elif current_b is None: + yield (current_a, None) + current_a = next(iter_a, None) + elif current_a.relative_path < current_b.relative_path: + yield (current_a, None) + current_a = next(iter_a, None) + elif current_b.relative_path < current_a.relative_path: + yield (None, current_b) + current_b = next(iter_b, None) + else: + assert current_a.relative_path == current_b.relative_path + yield (current_a, current_b) + current_a = next(iter_a, None) + current_b = next(iter_b, None) diff --git a/b2sdk/sync/sync.py b/b2sdk/sync/sync.py index efb2dc778..e3c9d33a7 100644 --- a/b2sdk/sync/sync.py +++ b/b2sdk/sync/sync.py @@ -8,73 +8,25 @@ # ###################################################################### -import logging import concurrent.futures as futures +import logging + from enum import Enum, unique from ..bounded_queue_executor import BoundedQueueExecutor -from .encryption_provider import AbstractSyncEncryptionSettingsProvider, SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER -from .exception import InvalidArgument, IncompleteSync +from ..scan.exception import InvalidArgument from ..scan.folder import AbstractFolder from ..scan.path import AbstractPath +from ..scan.policies import DEFAULT_SCAN_MANAGER +from ..scan.scan import zip_folders +from .encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, AbstractSyncEncryptionSettingsProvider +from .exception import IncompleteSync from .policy import CompareVersionMode, NewerFileSyncMode from .policy_manager import POLICY_MANAGER, SyncPolicyManager from .report import SyncReport -from ..scan.policies import DEFAULT_SCAN_MANAGER - -logger = logging.getLogger(__name__) - - -def next_or_none(iterator): - """ - Return the next item from the iterator, or None if there are no more. - """ - try: - return next(iterator) - except StopIteration: - return None - - -def zip_folders(folder_a, folder_b, reporter, policies_manager=DEFAULT_SCAN_MANAGER): - """ - Iterate over all of the files in the union of two folders, - matching file names. - Each item is a pair (file_a, file_b) with the corresponding file - in both folders. Either file (but not both) will be None if the - file is in only one folder. - :param b2sdk.sync.folder.AbstractFolder folder_a: first folder object. - :param b2sdk.sync.folder.AbstractFolder folder_b: second folder object. - :param reporter: reporter object - :param policies_manager: policies manager object - :return: yields two element tuples - """ - - iter_a = folder_a.all_files(reporter, policies_manager) - iter_b = folder_b.all_files(reporter) - - current_a = next_or_none(iter_a) - current_b = next_or_none(iter_b) - - while current_a is not None or current_b is not None: - if current_a is None: - yield (None, current_b) - current_b = next_or_none(iter_b) - elif current_b is None: - yield (current_a, None) - current_a = next_or_none(iter_a) - elif current_a.relative_path < current_b.relative_path: - yield (current_a, None) - current_a = next_or_none(iter_a) - elif current_b.relative_path < current_a.relative_path: - yield (None, current_b) - current_b = next_or_none(iter_b) - else: - assert current_a.relative_path == current_b.relative_path - yield (current_a, current_b) - current_a = next_or_none(iter_a) - current_b = next_or_none(iter_b) +logger = logging.getLogger(__name__) def count_files(local_folder, reporter, policies_manager): From 266aa6c3b4ba9ef340025130cded12648c1917ad Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 18 May 2022 00:50:16 +0300 Subject: [PATCH 10/17] Fix imports --- b2sdk/_v3/__init__.py | 30 +++++++++++++++++------------- b2sdk/_v3/exception.py | 14 +++++++------- b2sdk/scan/folder.py | 2 +- b2sdk/sync/exception.py | 1 + doc/source/api/sync.rst | 6 +++--- doc/source/quick_start.rst | 6 +++--- 6 files changed, 32 insertions(+), 27 deletions(-) diff --git a/b2sdk/_v3/__init__.py b/b2sdk/_v3/__init__.py index f91669f1b..e3a734de4 100644 --- a/b2sdk/_v3/__init__.py +++ b/b2sdk/_v3/__init__.py @@ -164,14 +164,7 @@ from b2sdk.sync.action import B2HideAction from b2sdk.sync.action import B2UploadAction from b2sdk.sync.action import LocalDeleteAction -from b2sdk.sync.exception import EnvironmentEncodingError from b2sdk.sync.exception import IncompleteSync -from b2sdk.sync.exception import InvalidArgument -from b2sdk.sync.folder import AbstractFolder -from b2sdk.sync.folder import B2Folder -from b2sdk.sync.folder import LocalFolder -from b2sdk.sync.folder_parser import parse_sync_folder -from b2sdk.sync.path import AbstractSyncPath, B2SyncPath, LocalSyncPath from b2sdk.sync.policy import AbstractFileSyncPolicy from b2sdk.sync.policy import CompareVersionMode from b2sdk.sync.policy import NewerFileSyncMode @@ -189,19 +182,30 @@ from b2sdk.sync.policy_manager import POLICY_MANAGER from b2sdk.sync.report import SyncFileReporter from b2sdk.sync.report import SyncReport -from b2sdk.sync.scan_policies import DEFAULT_SCAN_MANAGER -from b2sdk.sync.scan_policies import IntegerRange -from b2sdk.sync.scan_policies import RegexSet -from b2sdk.sync.scan_policies import ScanPoliciesManager -from b2sdk.sync.scan_policies import convert_dir_regex_to_dir_prefix_regex from b2sdk.sync.sync import KeepOrDeleteMode from b2sdk.sync.sync import Synchronizer -from b2sdk.sync.sync import zip_folders from b2sdk.sync.encryption_provider import AbstractSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import BasicSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import ServerDefaultSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER +# scan + +from b2sdk.scan.exception import EnvironmentEncodingError +from b2sdk.scan.exception import InvalidArgument +from b2sdk.scan.folder import AbstractFolder +from b2sdk.scan.folder import B2Folder +from b2sdk.scan.folder import LocalFolder +from b2sdk.scan.folder_parser import parse_folder +from b2sdk.scan.path import AbstractPath, B2Path, LocalPath +from b2sdk.scan.policies import convert_dir_regex_to_dir_prefix_regex +from b2sdk.scan.policies import DEFAULT_SCAN_MANAGER +from b2sdk.scan.policies import IntegerRange +from b2sdk.scan.policies import RegexSet +from b2sdk.scan.policies import ScanPoliciesManager +from b2sdk.scan.report import Report +from b2sdk.scan.scan import zip_folders + # replication from b2sdk.replication.setting import ReplicationConfigurationFactory diff --git a/b2sdk/_v3/exception.py b/b2sdk/_v3/exception.py index 86507a683..b2b13b327 100644 --- a/b2sdk/_v3/exception.py +++ b/b2sdk/_v3/exception.py @@ -71,14 +71,14 @@ from b2sdk.exception import SSECKeyError from b2sdk.exception import WrongEncryptionModeForBucketDefault from b2sdk.exception import interpret_b2_error -from b2sdk.sync.exception import EmptyDirectory -from b2sdk.sync.exception import EnvironmentEncodingError from b2sdk.sync.exception import IncompleteSync -from b2sdk.sync.exception import InvalidArgument -from b2sdk.sync.exception import NotADirectory -from b2sdk.sync.exception import UnableToCreateDirectory -from b2sdk.sync.exception import UnSyncableFilename -from b2sdk.sync.exception import check_invalid_argument +from b2sdk.scan.exception import UnableToCreateDirectory +from b2sdk.scan.exception import EmptyDirectory +from b2sdk.scan.exception import EnvironmentEncodingError +from b2sdk.scan.exception import InvalidArgument +from b2sdk.scan.exception import NotADirectory +from b2sdk.scan.exception import UnsupportedFilename +from b2sdk.scan.exception import check_invalid_argument __all__ = ( 'AccessDenied', diff --git a/b2sdk/scan/folder.py b/b2sdk/scan/folder.py index cd016df03..73e6223e6 100644 --- a/b2sdk/scan/folder.py +++ b/b2sdk/scan/folder.py @@ -311,7 +311,7 @@ def __init__(self, bucket_name, folder_name, api): self.prefix = '' if self.folder_name == '' else self.folder_name + '/' def all_files( - self, reporter: Report, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER + self, reporter, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER ): """ Yield all files. diff --git a/b2sdk/sync/exception.py b/b2sdk/sync/exception.py index 4c6e44192..72de5b722 100644 --- a/b2sdk/sync/exception.py +++ b/b2sdk/sync/exception.py @@ -9,6 +9,7 @@ ###################################################################### from ..exception import B2SimpleError +from ..scan.exception import BaseDirectoryError class IncompleteSync(B2SimpleError): diff --git a/doc/source/api/sync.rst b/doc/source/api/sync.rst index 212b59420..b36ca0560 100644 --- a/doc/source/api/sync.rst +++ b/doc/source/api/sync.rst @@ -33,7 +33,7 @@ Following are the important optional arguments that can be provided while initia .. code-block:: python >>> from b2sdk.v2 import ScanPoliciesManager - >>> from b2sdk.v2 import parse_sync_folder + >>> from b2sdk.v2 import parse_folder >>> from b2sdk.v2 import Synchronizer >>> from b2sdk.v2 import KeepOrDeleteMode, CompareVersionMode, NewerFileSyncMode >>> import time @@ -42,8 +42,8 @@ Following are the important optional arguments that can be provided while initia >>> source = '/home/user1/b2_example' >>> destination = 'b2://example-mybucket-b2' - >>> source = parse_sync_folder(source, b2_api) - >>> destination = parse_sync_folder(destination, b2_api) + >>> source = parse_folder(source, b2_api) + >>> destination = parse_folder(destination, b2_api) >>> policies_manager = ScanPoliciesManager(exclude_all_symlinks=True) diff --git a/doc/source/quick_start.rst b/doc/source/quick_start.rst index 921fe81ea..f3ec811dc 100644 --- a/doc/source/quick_start.rst +++ b/doc/source/quick_start.rst @@ -28,7 +28,7 @@ Synchronization .. code-block:: python >>> from b2sdk.v2 import ScanPoliciesManager - >>> from b2sdk.v2 import parse_sync_folder + >>> from b2sdk.v2 import parse_folder >>> from b2sdk.v2 import Synchronizer >>> from b2sdk.v2 import SyncReport >>> import time @@ -37,8 +37,8 @@ Synchronization >>> source = '/home/user1/b2_example' >>> destination = 'b2://example-mybucket-b2' - >>> source = parse_sync_folder(source, b2_api) - >>> destination = parse_sync_folder(destination, b2_api) + >>> source = parse_folder(source, b2_api) + >>> destination = parse_folder(destination, b2_api) >>> policies_manager = ScanPoliciesManager(exclude_all_symlinks=True) From 95c39b8c117524a509902139a9336eaaba701e8d Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 18 May 2022 01:32:17 +0300 Subject: [PATCH 11/17] Fix tests --- b2sdk/v2/__init__.py | 3 +-- b2sdk/v2/exception.py | 1 + b2sdk/v2/sync.py | 4 ++++ test/unit/v0/test_sync.py | 8 ++++---- test/unit/v1/test_sync.py | 8 ++++---- 5 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 b2sdk/v2/sync.py diff --git a/b2sdk/v2/__init__.py b/b2sdk/v2/__init__.py index 444aab031..9d61375fe 100644 --- a/b2sdk/v2/__init__.py +++ b/b2sdk/v2/__init__.py @@ -11,12 +11,11 @@ from b2sdk._v3 import * # noqa from b2sdk._v3 import parse_folder as parse_sync_folder from b2sdk._v3 import AbstractPath as AbstractSyncPath -from b2sdk._v3 import B2Path as B2SyncPath from b2sdk._v3 import LocalPath as LocalSyncPath -from b2sdk._v3 import UnsupportedFilename as UnSyncableFilename from .api import B2Api from .b2http import B2Http from .bucket import Bucket, BucketFactory from .session import B2Session +from .sync import B2SyncPath from .transfer import DownloadManager, UploadManager diff --git a/b2sdk/v2/exception.py b/b2sdk/v2/exception.py index 5bc9dbd33..2e970afdc 100644 --- a/b2sdk/v2/exception.py +++ b/b2sdk/v2/exception.py @@ -11,6 +11,7 @@ from b2sdk._v3.exception import * # noqa v3BucketIdNotFound = BucketIdNotFound +UnSyncableFilename = UnsupportedFilename # overridden to retain old style isinstance check and attributes diff --git a/b2sdk/v2/sync.py b/b2sdk/v2/sync.py new file mode 100644 index 000000000..de07d368c --- /dev/null +++ b/b2sdk/v2/sync.py @@ -0,0 +1,4 @@ +from b2sdk._v3 import B2Path + + +B2SyncPath = B2Path diff --git a/test/unit/v0/test_sync.py b/test/unit/v0/test_sync.py index d53d47174..f18b2077c 100644 --- a/test/unit/v0/test_sync.py +++ b/test/unit/v0/test_sync.py @@ -23,7 +23,7 @@ from .deps_exception import UnSyncableFilename, NotADirectory, UnableToCreateDirectory, EmptyDirectory, InvalidArgument, CommandError from .deps import FileVersionInfo -from .deps import AbstractFolder, B2Folder, LocalFolder +from .deps import B2Folder, LocalFolder from .deps import LocalSyncPath, B2SyncPath from .deps import ScanPoliciesManager, DEFAULT_SCAN_MANAGER from .deps import BoundedQueueExecutor, make_folder_sync_actions, zip_folders @@ -447,8 +447,8 @@ def test_multiple_versions(self): self.assertEqual( [ - "B2SyncPath(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", - "B2SyncPath(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" + "B2Path(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", + "B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" ], [ str(f) for f in folder.all_files(self.reporter) if f.relative_path in ('inner/a.txt', 'inner/b.txt') @@ -461,7 +461,7 @@ def test_exclude_modified_multiple_versions(self): ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( - ["B2SyncPath(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ + ["B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ str(f) for f in folder.all_files(self.reporter, policies_manager=polices_manager) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ] diff --git a/test/unit/v1/test_sync.py b/test/unit/v1/test_sync.py index 9eca10d8e..3593ade1c 100644 --- a/test/unit/v1/test_sync.py +++ b/test/unit/v1/test_sync.py @@ -21,7 +21,7 @@ from ..test_base import TestBase -from .deps import AbstractFolder, B2Folder, LocalFolder +from .deps import B2Folder, LocalFolder from .deps import BoundedQueueExecutor, zip_folders from .deps import LocalSyncPath, B2SyncPath from .deps import FileVersionInfo @@ -451,8 +451,8 @@ def test_multiple_versions(self): self.assertEqual( [ - "B2SyncPath(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", - "B2SyncPath(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" + "B2Path(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", + "B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" ], [ str(f) for f in folder.all_files(self.reporter) if f.relative_path in ('inner/a.txt', 'inner/b.txt') @@ -465,7 +465,7 @@ def test_exclude_modified_multiple_versions(self): ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( - ["B2SyncPath(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ + ["B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ str(f) for f in folder.all_files(self.reporter, policies_manager=polices_manager) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ] From 19fb2b58f62895794099989e245e2422ef89d04e Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 18 May 2022 01:35:05 +0300 Subject: [PATCH 12/17] Lint fix --- b2sdk/scan/folder.py | 6 ++---- b2sdk/scan/policies.py | 3 +-- b2sdk/scan/report.py | 1 - b2sdk/scan/scan.py | 1 - b2sdk/sync/action.py | 1 - b2sdk/sync/policy.py | 1 - b2sdk/sync/report.py | 1 - b2sdk/sync/sync.py | 1 - b2sdk/v2/sync.py | 11 ++++++++++- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/b2sdk/scan/folder.py b/b2sdk/scan/folder.py index 73e6223e6..f096734ae 100644 --- a/b2sdk/scan/folder.py +++ b/b2sdk/scan/folder.py @@ -310,9 +310,7 @@ def __init__(self, bucket_name, folder_name, api): self.api = api self.prefix = '' if self.folder_name == '' else self.folder_name + '/' - def all_files( - self, reporter, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER - ): + def all_files(self, reporter, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER): """ Yield all files. """ @@ -408,4 +406,4 @@ def make_full_path(self, file_name): return self.folder_name + '/' + file_name def __str__(self): - return 'B2Folder(%s, %s)' % (self.bucket_name, self.folder_name) \ No newline at end of file + return 'B2Folder(%s, %s)' % (self.bucket_name, self.folder_name) diff --git a/b2sdk/scan/policies.py b/b2sdk/scan/policies.py index 02150837b..931b73b05 100644 --- a/b2sdk/scan/policies.py +++ b/b2sdk/scan/policies.py @@ -17,7 +17,6 @@ from .exception import InvalidArgument, check_invalid_argument from .path import LocalPath - logger = logging.getLogger(__name__) @@ -225,4 +224,4 @@ def should_exclude_local_directory(self, dir_path: str): return self._exclude_dir_set.matches(dir_path) -DEFAULT_SCAN_MANAGER = ScanPoliciesManager() \ No newline at end of file +DEFAULT_SCAN_MANAGER = ScanPoliciesManager() diff --git a/b2sdk/scan/report.py b/b2sdk/scan/report.py index f33c0c5b1..8b0f4326f 100644 --- a/b2sdk/scan/report.py +++ b/b2sdk/scan/report.py @@ -17,7 +17,6 @@ from ..utils import format_and_scale_number - logger = logging.getLogger(__name__) diff --git a/b2sdk/scan/scan.py b/b2sdk/scan/scan.py index 02be35a33..57c68e58a 100644 --- a/b2sdk/scan/scan.py +++ b/b2sdk/scan/scan.py @@ -8,7 +8,6 @@ # ###################################################################### - from typing import Optional, Tuple from .folder import AbstractFolder diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index fc100c745..78b3820f4 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -20,7 +20,6 @@ from .encryption_provider import AbstractSyncEncryptionSettingsProvider from .report import SyncFileReporter - logger = logging.getLogger(__name__) diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index 4f1f8eaa0..f9a4e688f 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -21,7 +21,6 @@ from .action import B2CopyAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2UploadAction, LocalDeleteAction from .encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, AbstractSyncEncryptionSettingsProvider - ONE_DAY_IN_MS = 24 * 60 * 60 * 1000 logger = logging.getLogger(__name__) diff --git a/b2sdk/sync/report.py b/b2sdk/sync/report.py index a96f36424..647a0b2c8 100644 --- a/b2sdk/sync/report.py +++ b/b2sdk/sync/report.py @@ -17,7 +17,6 @@ from ..scan.report import Report from ..utils import format_and_scale_fraction, format_and_scale_number - logger = logging.getLogger(__name__) diff --git a/b2sdk/sync/sync.py b/b2sdk/sync/sync.py index e3c9d33a7..d3df1275f 100644 --- a/b2sdk/sync/sync.py +++ b/b2sdk/sync/sync.py @@ -25,7 +25,6 @@ from .policy_manager import POLICY_MANAGER, SyncPolicyManager from .report import SyncReport - logger = logging.getLogger(__name__) diff --git a/b2sdk/v2/sync.py b/b2sdk/v2/sync.py index de07d368c..4910518c5 100644 --- a/b2sdk/v2/sync.py +++ b/b2sdk/v2/sync.py @@ -1,4 +1,13 @@ -from b2sdk._v3 import B2Path +###################################################################### +# +# File: b2sdk/v2/sync.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from b2sdk._v3 import B2Path B2SyncPath = B2Path From cdba814248a8655c1269e0d21839b3afeea02e56 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 18 May 2022 01:56:25 +0300 Subject: [PATCH 13/17] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f5beefa5..1d411063c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +* Extracted folder/bucket scanning from `sync` module - new `scan` module + ## [1.16.0] - 2022-04-27 This release contains a preview of replication support. It allows for basic From 8ac1500fc994070882d881ed589009de189d3903 Mon Sep 17 00:00:00 2001 From: Alex <73276794+agoncharov-reef@users.noreply.github.com> Date: Thu, 19 May 2022 17:26:49 +0300 Subject: [PATCH 14/17] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Polewicz --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d411063c..69ec8afff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -* Extracted folder/bucket scanning from `sync` module - new `scan` module +### Added +* Extracted folder/bucket scanning into a new `scan` module ## [1.16.0] - 2022-04-27 From df25245e5687906bb816a5adcc6df305e0521202 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Mon, 23 May 2022 13:13:19 +0300 Subject: [PATCH 15/17] Avoid calling time.time() --- b2sdk/sync/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/b2sdk/sync/report.py b/b2sdk/sync/report.py index 647a0b2c8..296d5a80d 100644 --- a/b2sdk/sync/report.py +++ b/b2sdk/sync/report.py @@ -55,7 +55,7 @@ def _update_progress(self): return self._last_update_time = now - time_delta = time.time() - self.start_time + time_delta = now - self.start_time rate = 0 if time_delta == 0 else int(self.transfer_bytes / time_delta) if not self.total_done: message = ' count: %d files compare: %d files updated: %d files %s %s' % ( From 60b064148c22d8979042e2d1784f7eb75df5f569 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Mon, 23 May 2022 13:25:16 +0300 Subject: [PATCH 16/17] Minor fixes --- b2sdk/scan/__init__.py | 9 --------- b2sdk/scan/folder.py | 15 +++++++++------ b2sdk/sync/action.py | 1 - 3 files changed, 9 insertions(+), 16 deletions(-) delete mode 100644 b2sdk/scan/__init__.py diff --git a/b2sdk/scan/__init__.py b/b2sdk/scan/__init__.py deleted file mode 100644 index 672eaf1e6..000000000 --- a/b2sdk/scan/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -###################################################################### -# -# File: b2sdk/scan/__init__.py -# -# Copyright 2022 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### diff --git a/b2sdk/scan/folder.py b/b2sdk/scan/folder.py index f096734ae..a334eb02b 100644 --- a/b2sdk/scan/folder.py +++ b/b2sdk/scan/folder.py @@ -13,12 +13,13 @@ import platform import re import sys - from abc import ABCMeta, abstractmethod -from .exception import EmptyDirectory, EnvironmentEncodingError, UnsupportedFilename, NotADirectory, UnableToCreateDirectory + +from ..utils import fix_windows_path_limit, get_file_mtime, is_file_readable +from .exception import EmptyDirectory, EnvironmentEncodingError, NotADirectory, UnableToCreateDirectory, UnsupportedFilename from .path import B2Path, LocalPath from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager -from ..utils import fix_windows_path_limit, get_file_mtime, is_file_readable +from .report import Report DRIVE_MATCHER = re.compile(r"^([A-Za-z]):([/\\])") ABSOLUTE_PATH_MATCHER = re.compile(r"^(/)|^(\\)") @@ -49,7 +50,7 @@ class AbstractFolder(metaclass=ABCMeta): """ @abstractmethod - def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + def all_files(self, reporter: Report, policies_manager=DEFAULT_SCAN_MANAGER): """ Return an iterator over all of the files in the folder, in the order that B2 uses. @@ -120,7 +121,7 @@ def folder_type(self): """ return 'local' - def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + def all_files(self, reporter: Report, policies_manager=DEFAULT_SCAN_MANAGER): """ Yield all files. @@ -310,7 +311,9 @@ def __init__(self, bucket_name, folder_name, api): self.api = api self.prefix = '' if self.folder_name == '' else self.folder_name + '/' - def all_files(self, reporter, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER): + def all_files( + self, reporter: Report, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER + ): """ Yield all files. """ diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index 78b3820f4..4a429f005 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -10,7 +10,6 @@ import logging import os - from abc import ABCMeta, abstractmethod from ..bucket import Bucket From d8a4c45cc9b3334df261ecdd57241371d7b32f50 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Mon, 23 May 2022 15:10:16 +0300 Subject: [PATCH 17/17] Extract scan tests from sync unit tests --- test/unit/fixtures/folder.py | 101 ++++++++++++++++++ test/unit/scan/__init__.py | 9 ++ .../unit/{sync => scan}/test_scan_policies.py | 0 test/unit/sync/fixtures.py | 90 +--------------- test/unit/sync/test_sync.py | 7 +- 5 files changed, 115 insertions(+), 92 deletions(-) create mode 100644 test/unit/fixtures/folder.py create mode 100644 test/unit/scan/__init__.py rename test/unit/{sync => scan}/test_scan_policies.py (100%) diff --git a/test/unit/fixtures/folder.py b/test/unit/fixtures/folder.py new file mode 100644 index 000000000..8a92a9042 --- /dev/null +++ b/test/unit/fixtures/folder.py @@ -0,0 +1,101 @@ +###################################################################### +# +# File: test/unit/fixtures/folder.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from unittest import mock + +import apiver_deps +import pytest + +from apiver_deps import B2Folder, LocalFolder, LocalPath +from apiver_deps import DEFAULT_SCAN_MANAGER + + +if apiver_deps.V <= 1: + from apiver_deps import FileVersionInfo as VFileVersion +else: + from apiver_deps import FileVersion as VFileVersion + + +class FakeB2Folder(B2Folder): + def __init__(self, test_files): + self.file_versions = [] + for test_file in test_files: + self.file_versions.extend(self._file_versions(*test_file)) + super().__init__('test-bucket', 'folder', mock.MagicMock()) + + def get_file_versions(self): + yield from iter(self.file_versions) + + def _file_versions(self, name, mod_times, size=10): + """ + Makes FileVersion objects. + + Positive modification times are uploads, and negative modification + times are hides. It's a hack, but it works. + + """ + if apiver_deps.V <= 1: + mandatory_kwargs = {} + else: + mandatory_kwargs = { + 'api': None, + 'account_id': 'account-id', + 'bucket_id': 'bucket-id', + 'content_md5': 'content_md5', + 'server_side_encryption': None, + } + return [ + VFileVersion( + id_='id_%s_%d' % (name[0], abs(mod_time)), + file_name='folder/' + name, + upload_timestamp=abs(mod_time), + action='upload' if 0 < mod_time else 'hide', + size=size, + file_info={'in_b2': 'yes'}, + content_type='text/plain', + content_sha1='content_sha1', + **mandatory_kwargs, + ) for mod_time in mod_times + ] # yapf disable + + +class FakeLocalFolder(LocalFolder): + def __init__(self, test_files): + super().__init__('folder') + self.local_paths = [self._local_path(*test_file) for test_file in test_files] + + def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + for single_path in self.local_paths: + if single_path.relative_path.endswith('/'): + if policies_manager.should_exclude_b2_directory(single_path.relative_path): + continue + else: + if policies_manager.should_exclude_local_path(single_path): + continue + yield single_path + + def make_full_path(self, name): + return '/dir/' + name + + def _local_path(self, name, mod_times, size=10): + """ + Makes a LocalPath object for a local file. + """ + return LocalPath(name, name, mod_times[0], size) + + +@pytest.fixture(scope='session') +def folder_factory(): + def get_folder(f_type, *files): + if f_type == 'b2': + return FakeB2Folder(files) + return FakeLocalFolder(files) + + return get_folder diff --git a/test/unit/scan/__init__.py b/test/unit/scan/__init__.py new file mode 100644 index 000000000..9acce809e --- /dev/null +++ b/test/unit/scan/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: test/unit/scan/__init__.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/test/unit/sync/test_scan_policies.py b/test/unit/scan/test_scan_policies.py similarity index 100% rename from test/unit/sync/test_scan_policies.py rename to test/unit/scan/test_scan_policies.py diff --git a/test/unit/sync/fixtures.py b/test/unit/sync/fixtures.py index 9ca4929ae..67f2fa334 100644 --- a/test/unit/sync/fixtures.py +++ b/test/unit/sync/fixtures.py @@ -8,97 +8,9 @@ # ###################################################################### -from unittest import mock - import pytest -import apiver_deps -from apiver_deps import B2Folder, LocalFolder, LocalPath -from apiver_deps import CompareVersionMode, NewerFileSyncMode, KeepOrDeleteMode -from apiver_deps import DEFAULT_SCAN_MANAGER, POLICY_MANAGER, Synchronizer - -if apiver_deps.V <= 1: - from apiver_deps import FileVersionInfo as VFileVersion -else: - from apiver_deps import FileVersion as VFileVersion - - -class FakeB2Folder(B2Folder): - def __init__(self, test_files): - self.file_versions = [] - for test_file in test_files: - self.file_versions.extend(self._file_versions(*test_file)) - super().__init__('test-bucket', 'folder', mock.MagicMock()) - - def get_file_versions(self): - yield from iter(self.file_versions) - - def _file_versions(self, name, mod_times, size=10): - """ - Makes FileVersion objects. - - Positive modification times are uploads, and negative modification - times are hides. It's a hack, but it works. - - """ - if apiver_deps.V <= 1: - mandatory_kwargs = {} - else: - mandatory_kwargs = { - 'api': None, - 'account_id': 'account-id', - 'bucket_id': 'bucket-id', - 'content_md5': 'content_md5', - 'server_side_encryption': None, - } - return [ - VFileVersion( - id_='id_%s_%d' % (name[0], abs(mod_time)), - file_name='folder/' + name, - upload_timestamp=abs(mod_time), - action='upload' if 0 < mod_time else 'hide', - size=size, - file_info={'in_b2': 'yes'}, - content_type='text/plain', - content_sha1='content_sha1', - **mandatory_kwargs, - ) for mod_time in mod_times - ] # yapf disable - - -class FakeLocalFolder(LocalFolder): - def __init__(self, test_files): - super().__init__('folder') - self.local_paths = [self._local_path(*test_file) for test_file in test_files] - - def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): - for single_path in self.local_paths: - if single_path.relative_path.endswith('/'): - if policies_manager.should_exclude_b2_directory(single_path.relative_path): - continue - else: - if policies_manager.should_exclude_local_path(single_path): - continue - yield single_path - - def make_full_path(self, name): - return '/dir/' + name - - def _local_path(self, name, mod_times, size=10): - """ - Makes a LocalPath object for a local file. - """ - return LocalPath(name, name, mod_times[0], size) - - -@pytest.fixture(scope='session') -def folder_factory(): - def get_folder(f_type, *files): - if f_type == 'b2': - return FakeB2Folder(files) - return FakeLocalFolder(files) - - return get_folder +from apiver_deps import DEFAULT_SCAN_MANAGER, POLICY_MANAGER, CompareVersionMode, KeepOrDeleteMode, NewerFileSyncMode, Synchronizer @pytest.fixture(scope='session') diff --git a/test/unit/sync/test_sync.py b/test/unit/sync/test_sync.py index 887d61520..cf7123cac 100644 --- a/test/unit/sync/test_sync.py +++ b/test/unit/sync/test_sync.py @@ -12,10 +12,11 @@ from enum import Enum from functools import partial -from apiver_deps import UpPolicy, B2DownloadAction, B2UploadAction, B2CopyAction, AbstractSyncEncryptionSettingsProvider, UploadSourceLocalFile, SyncPolicyManager +from apiver_deps import UpPolicy, B2DownloadAction, AbstractSyncEncryptionSettingsProvider, UploadSourceLocalFile, SyncPolicyManager from apiver_deps_exception import DestFileNewer, InvalidArgument -from b2sdk.utils import TempDir - +from apiver_deps import KeepOrDeleteMode, NewerFileSyncMode, CompareVersionMode +import pytest +from ..fixtures.folder import * from .fixtures import * DAY = 86400000 # milliseconds