From b5303b2360209441ada9dc80a8b46a8f91dd2a6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Fri, 11 Jun 2021 10:47:10 +0200 Subject: [PATCH 1/8] B2Api.get_file_info return FileVersion --- CHANGELOG.md | 3 +++ b2sdk/api.py | 17 +++++++---------- b2sdk/bucket.py | 2 +- b2sdk/file_lock.py | 22 ++++++++++++++++++++++ b2sdk/file_version.py | 11 ++++++++--- b2sdk/v1/api.py | 10 ++++++++++ b2sdk/v1/bucket.py | 11 ++++++++++- test/unit/api/test_api.py | 37 +++++++++++++++++++++++++++++++++++++ 8 files changed, 98 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f5a7c917..f3817ac18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +* `B2Api.get_file_info` returns a `FileVersion` object in v2 + ## [1.9.0] - 2021-06-07 ### Added diff --git a/b2sdk/api.py b/b2sdk/api.py index 26f48341b..12136d1c7 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -8,13 +8,13 @@ # ###################################################################### -from typing import Any, Dict, Optional +from typing import Optional from .bucket import Bucket, BucketFactory from .encryption.setting import EncryptionSetting from .exception import NonExistentBucket, RestrictedBucket from .file_lock import FileRetentionSetting, LegalHold -from .file_version import FileIdAndName, FileVersionFactory +from .file_version import FileIdAndName, FileVersion, FileVersionFactory from .large_file.services import LargeFileServices from .raw_api import API_VERSION from .session import B2Session @@ -490,18 +490,15 @@ def list_keys(self, start_application_key_id=None): ) # other - def get_file_info(self, file_id: str) -> Dict[str, Any]: + def get_file_info(self, file_id: str) -> FileVersion: """ - Legacy interface which just returns whatever remote API returns. - - .. todo:: - get_file_info() should return a File with .delete(), copy(), rename(), read() and so on + Gets info about file version. :param str file_id: the id of the file who's info will be retrieved. - :return: The parsed response - :rtype: dict """ - return self.session.get_file_info_by_id(file_id) + return self.file_version_factory.from_api_response( + self.session.get_file_info_by_id(file_id) + ) def check_bucket_name_restrictions(self, bucket_name: str): """ diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 2904a5ca7..eb27ea3f2 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -230,7 +230,7 @@ def get_file_info_by_id(self, file_id: str) -> FileVersion: :param str file_id: the id of the file who's info will be retrieved. :rtype: generator[b2sdk.v1.FileVersionInfo] """ - return self.api.file_version_factory.from_api_response(self.api.get_file_info(file_id)) + return self.api.get_file_info(file_id) def get_file_info_by_name(self, file_name: str) -> FileVersion: """ diff --git a/b2sdk/file_lock.py b/b2sdk/file_lock.py index 7044ff89d..f4b517240 100644 --- a/b2sdk/file_lock.py +++ b/b2sdk/file_lock.py @@ -161,6 +161,17 @@ def as_dict(self): "retainUntilTimestamp": self.retain_until, } + def as_dict_with_auth(self): + if self == UNKNOWN_FILE_RETENTION_SETTING: + return { + "isClientAuthorizedToRead": False, + "value": None, + } + return { + "isClientAuthorizedToRead": True, + "value": self.as_dict(), + } + def add_to_to_upload_headers(self, headers): if self.mode is RetentionMode.UNKNOWN: raise ValueError('cannot use an unknown file retention setting in requests') @@ -247,6 +258,17 @@ def to_dict_repr(self): return self.__class__.UNKNOWN.value raise ValueError('Unrepresentable value') + def as_dict_with_auth(self): + if self == self.__class__.UNKNOWN: + return { + "isClientAuthorizedToRead": False, + "value": None, + } + return { + "isClientAuthorizedToRead": True, + "value": self.to_dict_repr(), + } + class BucketRetentionSetting: """Represent bucket's default file retention settings, i.e. whether the files should be retained, in which mode diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index 6414e0df3..426323f7b 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -86,17 +86,22 @@ def __init__( else: self.mod_time_millis = self.upload_timestamp - def as_dict(self): + def as_dict(self, include_auth=False): """ represents the object as a dict which looks almost exactly like the raw api output for upload/list """ result = { 'fileId': self.id_, 'fileName': self.file_name, 'fileInfo': self.file_info, - 'legalHold': self.legal_hold.to_dict_repr() if self.legal_hold is not None else None, 'serverSideEncryption': self.server_side_encryption.as_dict(), - 'fileRetention': self.file_retention.as_dict(), } + if include_auth: + result['legalHold'] = self.legal_hold.as_dict_with_auth() + result['fileRetention'] = self.file_retention.as_dict_with_auth() + else: + result['legalHold'] = self.legal_hold.to_dict_repr() + result['fileRetention'] = self.file_retention.as_dict() + if self.size is not None: result['size'] = self.size if self.upload_timestamp is not None: diff --git a/b2sdk/v1/api.py b/b2sdk/v1/api.py index f19443512..3cb9a369d 100644 --- a/b2sdk/v1/api.py +++ b/b2sdk/v1/api.py @@ -8,6 +8,7 @@ # ###################################################################### +from typing import Any, Dict from b2sdk import _v2 as v2 from .bucket import Bucket, BucketFactory from .file_version import FileVersionInfo, FileVersionInfoFactory, file_version_info_from_id_and_name @@ -18,12 +19,21 @@ # public API method # and to use v1.Bucket # and to retain cancel_large_file return type +# and to retain old style get_file_info return type class B2Api(v2.B2Api): SESSION_CLASS = staticmethod(B2Session) BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) BUCKET_CLASS = staticmethod(Bucket) FILE_VERSION_FACTORY_CLASS = staticmethod(FileVersionInfoFactory) + def get_file_info(self, file_id: str) -> Dict[str, Any]: + """ + Gets info about file version. + + :param str file_id: the id of the file who's info will be retrieved. + """ + return self.session.get_file_info_by_id(file_id) + def get_bucket_by_id(self, bucket_id): """ Return a bucket object with a given ID. Unlike ``get_bucket_by_name``, this method does not need to make any API calls. diff --git a/b2sdk/v1/bucket.py b/b2sdk/v1/bucket.py index a5f0dcc8e..2f218f2d2 100644 --- a/b2sdk/v1/bucket.py +++ b/b2sdk/v1/bucket.py @@ -8,7 +8,7 @@ # ###################################################################### -from .file_version import FileVersionInfoFactory +from .file_version import FileVersionInfo, FileVersionInfoFactory from typing import Optional from b2sdk import _v2 as v2 from b2sdk.utils import validate_b2_file_name @@ -16,6 +16,7 @@ # Overridden to retain the obsolete copy_file and start_large_file methods # and to return old style FILE_VERSION_FACTORY attribute +# and to to adjust to old style B2Api.get_file_info return type class Bucket(v2.Bucket): FILE_VERSION_FACTORY = staticmethod(FileVersionInfoFactory) @@ -89,6 +90,14 @@ def start_large_file( legal_hold=legal_hold, ) + def get_file_info_by_id(self, file_id: str) -> FileVersionInfo: + """ + Gets a file version's by ID. + + :param str file_id: the id of the file who's info will be retrieved. + """ + return self.api.file_version_factory.from_api_response(self.api.get_file_info(file_id)) + class BucketFactory(v2.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) diff --git a/test/unit/api/test_api.py b/test/unit/api/test_api.py index 9f336a93c..a3357446f 100644 --- a/test/unit/api/test_api.py +++ b/test/unit/api/test_api.py @@ -40,6 +40,43 @@ def setUp(self): self.api = B2Api(self.account_info, self.cache, self.raw_api) (self.application_key_id, self.master_key) = self.raw_api.create_account() + def test_get_file_info(self): + self._authorize_account() + bucket = self.api.create_bucket('bucket1', 'allPrivate') + created_file = bucket.upload_bytes(b'hello world', 'file') + + result = self.api.get_file_info(created_file.id_) + + if apiver_deps.V <= 1: + assert result == { + 'accountId': 'account-0', + 'action': 'upload', + 'bucketId': 'bucket_0', + 'contentLength': 11, + 'contentSha1': '2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', + 'contentType': 'b2/x-auto', + 'fileId': '9999', + 'fileInfo': {}, + 'fileName': 'file', + 'fileRetention': { + 'isClientAuthorizedToRead': True, + 'value': { + 'mode': None + } + }, + 'legalHold': { + 'isClientAuthorizedToRead': True, + 'value': None + }, + 'serverSideEncryption': { + 'mode': 'none' + }, + 'uploadTimestamp': 5000 + } + else: + assert isinstance(result, VFileVersion) + assert result == created_file + @pytest.mark.parametrize( 'expected_delete_bucket_output', [ From 4c4ce30c17bfd16a5e15f251d8a45bcfdcf541b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Fri, 11 Jun 2021 10:54:07 +0200 Subject: [PATCH 2/8] irrelevant TODO removed --- b2sdk/account_info/abstract.py | 1 - 1 file changed, 1 deletion(-) diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index 50a1697be..4903784bf 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -320,7 +320,6 @@ def allowed_is_valid(cls, allowed): ('capabilities' in allowed) and ('namePrefix' in allowed) ) - # TODO: make a decorator for set_auth_data() @abstractmethod def _set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, From bd96646b7ae4710a709d0cec537788ab67d8dff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Mon, 14 Jun 2021 10:51:07 +0200 Subject: [PATCH 3/8] B2RawApi renamed to B2RawHTTPApi --- CHANGELOG.md | 1 + b2sdk/_v2/__init__.py | 2 +- b2sdk/api.py | 12 ++++++------ b2sdk/raw_api.py | 8 ++++---- b2sdk/raw_simulator.py | 4 ++-- b2sdk/session.py | 10 +++++----- doc/markup-test.rst | 2 +- test/unit/fixtures/raw_api.py | 4 ++-- test/unit/v0/test_raw_api.py | 8 ++++---- test/unit/v1/test_raw_api.py | 6 +++--- 10 files changed, 29 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3817ac18..688f0ce3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed * `B2Api.get_file_info` returns a `FileVersion` object in v2 +* `B2RawApi` renamed to `B2RawHTTPApi` ## [1.9.0] - 2021-06-07 diff --git a/b2sdk/_v2/__init__.py b/b2sdk/_v2/__init__.py index b1180f630..db9bf9342 100644 --- a/b2sdk/_v2/__init__.py +++ b/b2sdk/_v2/__init__.py @@ -103,7 +103,7 @@ # raw_api from b2sdk.raw_api import AbstractRawApi -from b2sdk.raw_api import B2RawApi +from b2sdk.raw_api import B2RawHTTPApi from b2sdk.raw_api import MetadataDirectiveMode # stream diff --git a/b2sdk/api.py b/b2sdk/api.py index 12136d1c7..946552fb3 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -66,7 +66,7 @@ class B2Api(metaclass=B2TraceMeta): """ Provide file-level access to B2 services. - While :class:`b2sdk.v1.B2RawApi` provides direct access to the B2 web APIs, this + While :class:`b2sdk.v1.B2RawHTTPApi` provides direct access to the B2 web APIs, this class handles several things that simplify the task of uploading and downloading files: @@ -112,11 +112,11 @@ def __init__( default is :class:`~b2sdk.cache.DummyCache` :param raw_api: an instance of one of the following classes: - :class:`~b2sdk.raw_api.B2RawApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, + :class:`~b2sdk.raw_api.B2RawHTTPApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, or any custom class derived from :class:`~b2sdk.raw_api.AbstractRawApi` It makes network-less unit testing simple by using :class:`~b2sdk.raw_simulator.RawSimulator`, - in tests and :class:`~b2sdk.raw_api.B2RawApi` in production. - default is :class:`~b2sdk.raw_api.B2RawApi` + in tests and :class:`~b2sdk.raw_api.B2RawHTTPApi` in production. + default is :class:`~b2sdk.raw_api.B2RawHTTPApi` :param int max_upload_workers: a number of upload threads, default is 10 :param int max_copy_workers: a number of copy threads, default is 10 @@ -141,8 +141,8 @@ def cache(self): def raw_api(self): """ .. warning:: - :class:`~b2sdk.raw_api.B2RawApi` attribute is deprecated. - :class:`~b2sdk.session.B2Session` expose all :class:`~b2sdk.raw_api.B2RawApi` methods now.""" + :class:`~b2sdk.raw_api.B2RawHTTPApi` attribute is deprecated. + :class:`~b2sdk.session.B2Session` expose all :class:`~b2sdk.raw_api.B2RawHTTPApi` methods now.""" return self.session.raw_api def authorize_automatically(self): diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index 813b271bc..1722c5d57 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -344,7 +344,7 @@ def get_download_url_by_name(self, download_url, bucket_name, file_name): return download_url + '/file/' + bucket_name + '/' + b2_url_encode(file_name) -class B2RawApi(AbstractRawApi): +class B2RawHTTPApi(AbstractRawApi): """ Provide access to the B2 web APIs, exactly as they are provided by b2. @@ -1017,11 +1017,11 @@ def copy_part( def test_raw_api(): """ - Exercise the code in B2RawApi by making each call once, just + Exercise the code in B2RawHTTPApi by making each call once, just to make sure the parameters are passed in, and the result is passed back. - The goal is to be a complete test of B2RawApi, so the tests for + The goal is to be a complete test of B2RawHTTPApi, so the tests for the rest of the code can use the simulator. Prints to stdout if things go wrong. @@ -1029,7 +1029,7 @@ def test_raw_api(): :return: 0 on success, non-zero on failure """ try: - raw_api = B2RawApi(B2Http()) + raw_api = B2RawHTTPApi(B2Http()) test_raw_api_helper(raw_api) return 0 except Exception: diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 42fefb62a..4530ece9b 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -978,11 +978,11 @@ def _next_file_id(self): class RawSimulator(AbstractRawApi): """ - Implement the same interface as B2RawApi by simulating all of the + Implement the same interface as B2RawHTTPApi by simulating all of the calls and keeping state in memory. The intended use for this class is for unit tests that test things - built on top of B2RawApi. + built on top of B2RawHTTPApi. """ BUCKET_SIMULATOR_CLASS = BucketSimulator diff --git a/b2sdk/session.py b/b2sdk/session.py index 0908524f2..3621f1e83 100644 --- a/b2sdk/session.py +++ b/b2sdk/session.py @@ -20,7 +20,7 @@ from b2sdk.encryption.setting import EncryptionSetting from b2sdk.exception import (InvalidAuthToken, Unauthorized) from b2sdk.file_lock import BucketRetentionSetting, FileRetentionSetting, LegalHold -from b2sdk.raw_api import ALL_CAPABILITIES, B2RawApi, REALM_URLS +from b2sdk.raw_api import ALL_CAPABILITIES, B2RawHTTPApi, REALM_URLS logger = logging.getLogger(__name__) @@ -58,14 +58,14 @@ def __init__(self, account_info=None, cache=None, raw_api=None): default is :class:`~b2sdk.cache.DummyCache` :param raw_api: an instance of one of the following classes: - :class:`~b2sdk.raw_api.B2RawApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, + :class:`~b2sdk.raw_api.B2RawHTTPApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, or any custom class derived from :class:`~b2sdk.raw_api.AbstractRawApi` It makes network-less unit testing simple by using :class:`~b2sdk.raw_simulator.RawSimulator`, - in tests and :class:`~b2sdk.raw_api.B2RawApi` in production. - default is :class:`~b2sdk.raw_api.B2RawApi` + in tests and :class:`~b2sdk.raw_api.B2RawHTTPApi` in production. + default is :class:`~b2sdk.raw_api.B2RawHTTPApi` """ - self.raw_api = raw_api or B2RawApi(B2Http()) + self.raw_api = raw_api or B2RawHTTPApi(B2Http()) if account_info is None: account_info = self.SQLITE_ACCOUNT_INFO_CLASS() if cache is None: diff --git a/doc/markup-test.rst b/doc/markup-test.rst index b01da2bfe..1fe865ee8 100644 --- a/doc/markup-test.rst +++ b/doc/markup-test.rst @@ -108,7 +108,7 @@ Protected Things which sometimes might be necssary to use that are NOT considered public interface (and may change in a non-major version): * B2Session -* B2RawApi +* B2RawHTTPApi * B2Http .. note:: it is ok for you to use those (better that, than copying our sources), however if you do, please pin your dependencies to middle version. diff --git a/test/unit/fixtures/raw_api.py b/test/unit/fixtures/raw_api.py index d01c4816c..244c35d88 100644 --- a/test/unit/fixtures/raw_api.py +++ b/test/unit/fixtures/raw_api.py @@ -12,7 +12,7 @@ import pytest -from apiver_deps import ALL_CAPABILITIES, B2RawApi +from apiver_deps import ALL_CAPABILITIES, B2RawHTTPApi @pytest.fixture @@ -38,7 +38,7 @@ def fake_b2_raw_api_responses(): @pytest.fixture def fake_b2_raw_api(mocker, fake_b2http, fake_b2_raw_api_responses): - raw_api = mocker.MagicMock(name='FakeB2RawApi', spec=B2RawApi) + raw_api = mocker.MagicMock(name='FakeB2RawHTTPApi', spec=B2RawHTTPApi) raw_api.b2_http = fake_b2http raw_api.authorize_account.return_value = fake_b2_raw_api_responses['authorize_account'] return raw_api diff --git a/test/unit/v0/test_raw_api.py b/test/unit/v0/test_raw_api.py index 2e0bb8b7c..bd09bf766 100644 --- a/test/unit/v0/test_raw_api.py +++ b/test/unit/v0/test_raw_api.py @@ -15,7 +15,7 @@ from .deps import EncryptionKey from .deps import EncryptionMode from .deps import EncryptionSetting -from .deps import B2RawApi +from .deps import B2RawHTTPApi from .deps import B2Http from .deps import BucketRetentionSetting, RetentionPeriod, RetentionMode from .deps_exception import UnusableFileName, WrongEncryptionModeForBucketDefault @@ -30,7 +30,7 @@ class TestRawAPIFilenames(TestBase): """Test that the filename checker passes conforming names and rejects those that don't.""" def setUp(self): - self.raw_api = B2RawApi(B2Http()) + self.raw_api = B2RawHTTPApi(B2Http()) def _should_be_ok(self, filename): """Call with test filenames that follow the filename rules. @@ -104,7 +104,7 @@ class BucketTestBase: @pytest.fixture(autouse=True) def init(self, mocker): b2_http = mocker.MagicMock() - self.raw_api = B2RawApi(b2_http) + self.raw_api = B2RawHTTPApi(b2_http) class TestUpdateBucket(BucketTestBase): @@ -113,7 +113,7 @@ class TestUpdateBucket(BucketTestBase): @pytest.fixture(autouse=True) def init(self, mocker): b2_http = mocker.MagicMock() - self.raw_api = B2RawApi(b2_http) + self.raw_api = B2RawHTTPApi(b2_http) def test_assertion_raises(self): with pytest.raises(AssertionError): diff --git a/test/unit/v1/test_raw_api.py b/test/unit/v1/test_raw_api.py index 0b8fdfb76..155d5a0f5 100644 --- a/test/unit/v1/test_raw_api.py +++ b/test/unit/v1/test_raw_api.py @@ -15,7 +15,7 @@ from .deps import EncryptionKey from .deps import EncryptionMode from .deps import EncryptionSetting -from .deps import B2RawApi +from .deps import B2RawHTTPApi from .deps import B2Http from .deps import BucketRetentionSetting, RetentionPeriod, RetentionMode from .deps_exception import UnusableFileName, WrongEncryptionModeForBucketDefault @@ -30,7 +30,7 @@ class TestRawAPIFilenames(TestBase): """Test that the filename checker passes conforming names and rejects those that don't.""" def setUp(self): - self.raw_api = B2RawApi(B2Http()) + self.raw_api = B2RawHTTPApi(B2Http()) def _should_be_ok(self, filename): """Call with test filenames that follow the filename rules. @@ -104,7 +104,7 @@ class BucketTestBase: @pytest.fixture(autouse=True) def init(self, mocker): b2_http = mocker.MagicMock() - self.raw_api = B2RawApi(b2_http) + self.raw_api = B2RawHTTPApi(b2_http) class TestUpdateBucket(BucketTestBase): From 50e1b8213d917006f904d7e4e818a110759a7089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Mon, 14 Jun 2021 12:18:26 +0200 Subject: [PATCH 4/8] B2HTTP tests are now common --- CHANGELOG.md | 1 + test/unit/b2http/__init__.py | 9 + test/unit/{v0 => b2http}/test_b2http.py | 14 +- test/unit/v1/test_b2http.py | 309 ------------------------ 4 files changed, 17 insertions(+), 316 deletions(-) create mode 100644 test/unit/b2http/__init__.py rename test/unit/{v0 => b2http}/test_b2http.py (96%) delete mode 100644 test/unit/v1/test_b2http.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 688f0ce3a..0e85f7705 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed * `B2Api.get_file_info` returns a `FileVersion` object in v2 * `B2RawApi` renamed to `B2RawHTTPApi` +* `B2HTTP` tests are now common ## [1.9.0] - 2021-06-07 diff --git a/test/unit/b2http/__init__.py b/test/unit/b2http/__init__.py new file mode 100644 index 000000000..4d8876148 --- /dev/null +++ b/test/unit/b2http/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: test/unit/b2http/__init__.py +# +# Copyright 2021 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/test/unit/v0/test_b2http.py b/test/unit/b2http/test_b2http.py similarity index 96% rename from test/unit/v0/test_b2http.py rename to test/unit/b2http/test_b2http.py index 3d36f87d3..b26d78fb6 100644 --- a/test/unit/v0/test_b2http.py +++ b/test/unit/b2http/test_b2http.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: test/unit/v0/test_b2http.py +# File: test/unit/b2http/test_b2http.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # @@ -15,12 +15,12 @@ from ..test_base import TestBase -from .deps_exception import BadDateFormat, BadJson, BrokenPipe, B2ConnectionError, ClockSkew, ConnectionReset, ServiceError, UnknownError, UnknownHost, TooManyRequests -from .deps import USER_AGENT -from .deps import B2Http -from .deps import ClockSkewHook -from .deps import translate_errors as _translate_errors -from .deps import translate_and_retry as _translate_and_retry +from apiver_deps_exception import BadDateFormat, BadJson, BrokenPipe, B2ConnectionError, ClockSkew, ConnectionReset, ServiceError, UnknownError, UnknownHost, TooManyRequests +from apiver_deps import USER_AGENT +from apiver_deps import B2Http +from apiver_deps import ClockSkewHook +from apiver_deps import translate_errors as _translate_errors +from apiver_deps import translate_and_retry as _translate_and_retry if sys.version_info < (3, 3): from mock import call, MagicMock, patch diff --git a/test/unit/v1/test_b2http.py b/test/unit/v1/test_b2http.py deleted file mode 100644 index 86a03dda1..000000000 --- a/test/unit/v1/test_b2http.py +++ /dev/null @@ -1,309 +0,0 @@ -###################################################################### -# -# File: test/unit/v1/test_b2http.py -# -# Copyright 2019 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -import datetime -import requests -import socket -import sys - -from ..test_base import TestBase - -from .deps_exception import BadDateFormat, BadJson, BrokenPipe, B2ConnectionError, ClockSkew, ConnectionReset, ServiceError, UnknownError, UnknownHost, TooManyRequests -from .deps import USER_AGENT -from .deps import B2Http -from .deps import ClockSkewHook -from .deps import translate_errors as _translate_errors -from .deps import translate_and_retry as _translate_and_retry - -if sys.version_info < (3, 3): - from mock import call, MagicMock, patch -else: - from unittest.mock import call, MagicMock, patch - - -class TestTranslateErrors(TestBase): - def test_ok(self): - response = MagicMock() - response.status_code = 200 - actual = _translate_errors(lambda: response) - self.assertTrue(response is actual) # no assertIs until 2.7 - - def test_partial_content(self): - response = MagicMock() - response.status_code = 206 - actual = _translate_errors(lambda: response) - self.assertTrue(response is actual) # no assertIs until 2.7 - - def test_b2_error(self): - response = MagicMock() - response.status_code = 503 - response.content = b'{"status": 503, "code": "server_busy", "message": "busy"}' - with self.assertRaises(ServiceError): - _translate_errors(lambda: response) - - def test_broken_pipe(self): - def fcn(): - raise requests.ConnectionError( - requests.packages.urllib3.exceptions.ProtocolError( - "dummy", socket.error(20, 'Broken pipe') - ) - ) - - with self.assertRaises(BrokenPipe): - _translate_errors(fcn) - - def test_unknown_host(self): - def fcn(): - raise requests.ConnectionError( - requests.packages.urllib3.exceptions.MaxRetryError( - 'AAA nodename nor servname provided, or not known AAA', 'http://example.com' - ) - ) - - with self.assertRaises(UnknownHost): - _translate_errors(fcn) - - def test_connection_error(self): - def fcn(): - raise requests.ConnectionError('a message') - - with self.assertRaises(B2ConnectionError): - _translate_errors(fcn) - - def test_connection_reset(self): - class SysCallError(Exception): - pass - - def fcn(): - raise SysCallError('(104, ECONNRESET)') - - with self.assertRaises(ConnectionReset): - _translate_errors(fcn) - - def test_unknown_error(self): - def fcn(): - raise Exception('a message') - - with self.assertRaises(UnknownError): - _translate_errors(fcn) - - def test_too_many_requests(self): - response = MagicMock() - response.status_code = 429 - response.headers = {'retry-after': 1} - response.content = b'{"status": 429, "code": "Too Many requests", "message": "retry after some time"}' - with self.assertRaises(TooManyRequests): - _translate_errors(lambda: response) - - -class TestTranslateAndRetry(TestBase): - def setUp(self): - self.response = MagicMock() - self.response.status_code = 200 - - def test_works_first_try(self): - fcn = MagicMock() - fcn.side_effect = [self.response] - self.assertTrue(self.response is _translate_and_retry(fcn, 3)) # no assertIs until 2.7 - - def test_non_retryable(self): - with patch('time.sleep') as mock_time: - fcn = MagicMock() - fcn.side_effect = [BadJson('a'), self.response] - # no assertRaises until 2.7 - try: - _translate_and_retry(fcn, 3) - self.fail('should have raised BadJson') - except BadJson: - pass - self.assertEqual([], mock_time.mock_calls) - - def test_works_second_try(self): - with patch('time.sleep') as mock_time: - fcn = MagicMock() - fcn.side_effect = [ServiceError('a'), self.response] - self.assertTrue(self.response is _translate_and_retry(fcn, 3)) # no assertIs until 2.7 - self.assertEqual([call(1.0)], mock_time.mock_calls) - - def test_never_works(self): - with patch('time.sleep') as mock_time: - fcn = MagicMock() - fcn.side_effect = [ - ServiceError('a'), - ServiceError('a'), - ServiceError('a'), self.response - ] - # no assertRaises until 2.7 - try: - _translate_and_retry(fcn, 3) - self.fail('should have raised ServiceError') - except ServiceError: - pass - self.assertEqual([call(1.0), call(1.5)], mock_time.mock_calls) - - def test_too_many_requests_works_after_sleep(self): - with patch('time.sleep') as mock_time: - fcn = MagicMock() - fcn.side_effect = [TooManyRequests(retry_after_seconds=2), self.response] - self.assertIs(self.response, _translate_and_retry(fcn, 3)) - self.assertEqual([call(2)], mock_time.mock_calls) - - def test_too_many_requests_failed_after_sleep(self): - with patch('time.sleep') as mock_time: - fcn = MagicMock() - fcn.side_effect = [ - TooManyRequests(retry_after_seconds=2), - TooManyRequests(retry_after_seconds=5), - ] - with self.assertRaises(TooManyRequests): - _translate_and_retry(fcn, 2) - self.assertEqual([call(2)], mock_time.mock_calls) - - def test_too_many_requests_retry_header_combination_one(self): - # If the first response didn't have a header, second one has, and third one doesn't have, what should happen? - - with patch('time.sleep') as mock_time: - fcn = MagicMock() - fcn.side_effect = [ - TooManyRequests(retry_after_seconds=2), - TooManyRequests(), - TooManyRequests(retry_after_seconds=2), - self.response, - ] - self.assertIs(self.response, _translate_and_retry(fcn, 4)) - self.assertEqual([call(2), call(1.5), call(2)], mock_time.mock_calls) - - def test_too_many_requests_retry_header_combination_two(self): - # If the first response had header, and the second did not, but the third has header again, what should happen? - - with patch('time.sleep') as mock_time: - fcn = MagicMock() - fcn.side_effect = [ - TooManyRequests(), - TooManyRequests(retry_after_seconds=5), - TooManyRequests(), - self.response, - ] - self.assertIs(self.response, _translate_and_retry(fcn, 4)) - self.assertEqual([call(1.0), call(5), call(2.25)], mock_time.mock_calls) - - -class TestB2Http(TestBase): - - URL = 'http://example.com' - UA_APPEND = None - HEADERS = dict(my_header='my_value') - EXPECTED_HEADERS = {'my_header': 'my_value', 'User-Agent': USER_AGENT} - PARAMS = dict(fileSize=100) - PARAMS_JSON_BYTES = b'{"fileSize": 100}' - - def setUp(self): - self.session = MagicMock() - self.response = MagicMock() - - requests = MagicMock() - requests.Session.return_value = self.session - self.b2_http = B2Http( - requests, install_clock_skew_hook=False, user_agent_append=self.UA_APPEND - ) - - def test_post_json_return_json(self): - self.session.post.return_value = self.response - self.response.status_code = 200 - self.response.content = b'{"color": "blue"}' - response_dict = self.b2_http.post_json_return_json(self.URL, self.HEADERS, self.PARAMS) - self.assertEqual({'color': 'blue'}, response_dict) - (pos_args, kw_args) = self.session.post.call_args - self.assertEqual(self.URL, pos_args[0]) - self.assertEqual(self.EXPECTED_HEADERS, kw_args['headers']) - actual_data = kw_args['data'] - actual_data.seek(0) - self.assertEqual(self.PARAMS_JSON_BYTES, actual_data.read()) - - def test_callback(self): - callback = MagicMock() - callback.pre_request = MagicMock() - callback.post_request = MagicMock() - self.b2_http.add_callback(callback) - self.session.post.return_value = self.response - self.response.status_code = 200 - self.response.content = b'{"color": "blue"}' - self.b2_http.post_json_return_json(self.URL, self.HEADERS, self.PARAMS) - callback.pre_request.assert_called_with('POST', 'http://example.com', self.EXPECTED_HEADERS) - callback.post_request.assert_called_with( - 'POST', 'http://example.com', self.EXPECTED_HEADERS, self.response - ) - - def test_get_content(self): - self.session.get.return_value = self.response - self.response.status_code = 200 - with self.b2_http.get_content(self.URL, self.HEADERS) as r: - self.assertTrue(self.response is r) # no assertIs until 2.7 - self.session.get.assert_called_with( - self.URL, headers=self.EXPECTED_HEADERS, stream=True, timeout=B2Http.TIMEOUT - ) - self.response.close.assert_called_with() - - def test_head_content(self): - self.session.head.return_value = self.response - self.response.status_code = 200 - self.response.headers = {"color": "blue"} - - response = self.b2_http.head_content(self.URL, self.HEADERS) - - self.assertEqual({'color': 'blue'}, response.headers) - (pos_args, kw_args) = self.session.head.call_args - self.assertEqual(self.URL, pos_args[0]) - self.assertEqual(self.EXPECTED_HEADERS, kw_args['headers']) - - -class TestB2HttpUserAgentAppend(TestB2Http): - - UA_APPEND = 'ua_extra_string' - EXPECTED_HEADERS = { - **TestB2Http.EXPECTED_HEADERS, 'User-Agent': '%s %s' % (USER_AGENT, UA_APPEND) - } - - -class TestClockSkewHook(TestBase): - def test_bad_format(self): - response = MagicMock() - response.headers = {'Date': 'bad format'} - with self.assertRaises(BadDateFormat): - ClockSkewHook().post_request('POST', 'http://example.com', {}, response) - - def test_bad_month(self): - response = MagicMock() - response.headers = {'Date': 'Fri, 16 XXX 2016 20:52:30 GMT'} - with self.assertRaises(BadDateFormat): - ClockSkewHook().post_request('POST', 'http://example.com', {}, response) - - def test_no_skew(self): - now = datetime.datetime.utcnow() - now_str = now.strftime('%a, %d %b %Y %H:%M:%S GMT') - response = MagicMock() - response.headers = {'Date': now_str} - ClockSkewHook().post_request('POST', 'http://example.com', {}, response) - - def test_positive_skew(self): - now = datetime.datetime.utcnow() + datetime.timedelta(minutes=11) - now_str = now.strftime('%a, %d %b %Y %H:%M:%S GMT') - response = MagicMock() - response.headers = {'Date': now_str} - with self.assertRaises(ClockSkew): - ClockSkewHook().post_request('POST', 'http://example.com', {}, response) - - def test_negative_skew(self): - now = datetime.datetime.utcnow() + datetime.timedelta(minutes=-11) - now_str = now.strftime('%a, %d %b %Y %H:%M:%S GMT') - response = MagicMock() - response.headers = {'Date': now_str} - with self.assertRaises(ClockSkew): - ClockSkewHook().post_request('POST', 'http://example.com', {}, response) From 93f35fa11ba2da5b9962d1c469df5d458c931c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Mon, 14 Jun 2021 12:57:01 +0200 Subject: [PATCH 5/8] B2HttpApiConfig class introduced to provide parameters like user_agent_append to B2Api without using internal classes --- CHANGELOG.md | 1 + b2sdk/_v2/__init__.py | 2 ++ b2sdk/api.py | 40 +++++++++------------- b2sdk/api_config.py | 37 ++++++++++++++++++++ b2sdk/b2http.py | 14 +++----- b2sdk/session.py | 20 ++++++----- b2sdk/v1/__init__.py | 3 ++ b2sdk/v1/api.py | 42 ++++++++++++++++++++++- b2sdk/v1/b2http.py | 60 +++++++++++++++++++++++++++++++++ b2sdk/v1/session.py | 22 ++++++++++++ doc/source/api/api.rst | 5 +++ test/unit/api/test_api.py | 19 +++++++++-- test/unit/b2http/test_b2http.py | 16 +++++++-- test/unit/bucket/test_bucket.py | 6 ++-- test/unit/fixtures/session.py | 4 ++- test/unit/v_all/test_api.py | 3 +- 16 files changed, 242 insertions(+), 52 deletions(-) create mode 100644 b2sdk/api_config.py create mode 100644 b2sdk/v1/b2http.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e85f7705..8396ff232 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `B2Api.get_file_info` returns a `FileVersion` object in v2 * `B2RawApi` renamed to `B2RawHTTPApi` * `B2HTTP` tests are now common +* `B2HttpApiConfig` class introduced to provide parameters like `user_agent_append` to `B2Api` without using internal classes in v2 ## [1.9.0] - 2021-06-07 diff --git a/b2sdk/_v2/__init__.py b/b2sdk/_v2/__init__.py index db9bf9342..16c94261a 100644 --- a/b2sdk/_v2/__init__.py +++ b/b2sdk/_v2/__init__.py @@ -198,6 +198,8 @@ # other from b2sdk.b2http import B2Http +from b2sdk.api_config import B2HttpApiConfig +from b2sdk.api_config import DEFAULT_HTTP_API_CONFIG from b2sdk.b2http import ClockSkewHook from b2sdk.b2http import HttpCallback from b2sdk.b2http import ResponseContextManager diff --git a/b2sdk/api.py b/b2sdk/api.py index 946552fb3..7eb5e7cd0 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -10,6 +10,8 @@ from typing import Optional +from .account_info.abstract import AbstractAccountInfo +from .cache import AbstractCache from .bucket import Bucket, BucketFactory from .encryption.setting import EncryptionSetting from .exception import NonExistentBucket, RestrictedBucket @@ -17,6 +19,7 @@ from .file_version import FileIdAndName, FileVersion, FileVersionFactory from .large_file.services import LargeFileServices from .raw_api import API_VERSION +from .api_config import B2HttpApiConfig, DEFAULT_HTTP_API_CONFIG from .session import B2Session from .transfer import ( CopyManager, @@ -89,39 +92,28 @@ class handles several things that simplify the task of uploading def __init__( self, - account_info=None, - cache=None, - raw_api=None, - max_upload_workers=10, - max_copy_workers=10 + account_info: Optional[AbstractAccountInfo] = None, + cache: Optional[AbstractCache] = None, + max_upload_workers: int = 10, + max_copy_workers: int = 10, + api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG, ): """ Initialize the API using the given account info. - :param account_info: an instance of :class:`~b2sdk.v1.UrlPoolAccountInfo`, - or any custom class derived from - :class:`~b2sdk.v1.AbstractAccountInfo` - To learn more about Account Info objects, see here + :param account_info: To learn more about Account Info objects, see here :class:`~b2sdk.v1.SqliteAccountInfo` - :param cache: an instance of the one of the following classes: - :class:`~b2sdk.cache.DummyCache`, :class:`~b2sdk.cache.InMemoryCache`, - :class:`~b2sdk.cache.AuthInfoCache`, - or any custom class derived from :class:`~b2sdk.cache.AbstractCache` - It is used by B2Api to cache the mapping between bucket name and bucket ids. + :param cache: It is used by B2Api to cache the mapping between bucket name and bucket ids. default is :class:`~b2sdk.cache.DummyCache` - :param raw_api: an instance of one of the following classes: - :class:`~b2sdk.raw_api.B2RawHTTPApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, - or any custom class derived from :class:`~b2sdk.raw_api.AbstractRawApi` - It makes network-less unit testing simple by using :class:`~b2sdk.raw_simulator.RawSimulator`, - in tests and :class:`~b2sdk.raw_api.B2RawHTTPApi` in production. - default is :class:`~b2sdk.raw_api.B2RawHTTPApi` - - :param int max_upload_workers: a number of upload threads, default is 10 - :param int max_copy_workers: a number of copy threads, default is 10 + :param max_upload_workers: a number of upload threads + :param max_copy_workers: a number of copy threads + :param api_config: """ - self.session = self.SESSION_CLASS(account_info=account_info, cache=cache, raw_api=raw_api) + self.session = self.SESSION_CLASS( + account_info=account_info, cache=cache, api_config=api_config + ) self.file_version_factory = self.FILE_VERSION_FACTORY_CLASS(self) self.services = Services( self, diff --git a/b2sdk/api_config.py b/b2sdk/api_config.py new file mode 100644 index 000000000..e4163e185 --- /dev/null +++ b/b2sdk/api_config.py @@ -0,0 +1,37 @@ +###################################################################### +# +# File: b2sdk/api_config.py +# +# Copyright 2021 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import requests + +from typing import Optional + +from types import ModuleType + + +class B2HttpApiConfig: + def __init__( + self, + requests_module: ModuleType = requests, + install_clock_skew_hook: bool = True, + user_agent_append: Optional[str] = None, + ): + """ + A structure with params to be passed to low level API. + + :param requests_module: a reference to requests module + :param bool install_clock_skew_hook: if True, install a clock skew hook + :param str user_agent_append: if provided, the string will be appended to the User-Agent + """ + self.requests_module = requests_module + self.install_clock_skew_hook = install_clock_skew_hook + self.user_agent_append = user_agent_append + + +DEFAULT_HTTP_API_CONFIG = B2HttpApiConfig() diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index e1368899b..9fc7c810a 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -23,6 +23,7 @@ B2Error, B2RequestTimeoutDuringUpload, BadDateFormat, BrokenPipe, B2ConnectionError, B2RequestTimeout, ClockSkew, ConnectionReset, interpret_b2_error, UnknownError, UnknownHost ) +from .api_config import B2HttpApiConfig, DEFAULT_HTTP_API_CONFIG from .version import USER_AGENT logger = logging.getLogger(__name__) @@ -250,20 +251,15 @@ class B2Http(object): # timeout for HTTP GET/POST requests TIMEOUT = 900 # 15 minutes as server-side copy can take time - def __init__(self, requests_module=None, install_clock_skew_hook=True, user_agent_append=None): + def __init__(self, api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG): """ Initialize with a reference to the requests module, which makes it easy to mock for testing. - - :param requests_module: a reference to requests module - :param bool install_clock_skew_hook: if True, install a clock skew hook - :param str user_agent_append: if provided, the string will be appended to the User-Agent """ - requests_to_use = requests_module or requests - self.user_agent = self._get_user_agent(user_agent_append) - self.session = requests_to_use.Session() + self.user_agent = self._get_user_agent(api_config.user_agent_append) + self.session = api_config.requests_module.Session() self.callbacks = [] - if install_clock_skew_hook: + if api_config.install_clock_skew_hook: self.add_callback(ClockSkewHook()) def add_callback(self, callback): diff --git a/b2sdk/session.py b/b2sdk/session.py index 3621f1e83..07a04931b 100644 --- a/b2sdk/session.py +++ b/b2sdk/session.py @@ -13,14 +13,16 @@ from typing import Any, Dict, Optional import logging +from b2sdk.account_info.abstract import AbstractAccountInfo from b2sdk.account_info.sqlite_account_info import SqliteAccountInfo from b2sdk.account_info.exception import MissingAccountData from b2sdk.b2http import B2Http -from b2sdk.cache import AuthInfoCache, DummyCache +from b2sdk.cache import AbstractCache, AuthInfoCache, DummyCache from b2sdk.encryption.setting import EncryptionSetting from b2sdk.exception import (InvalidAuthToken, Unauthorized) from b2sdk.file_lock import BucketRetentionSetting, FileRetentionSetting, LegalHold from b2sdk.raw_api import ALL_CAPABILITIES, B2RawHTTPApi, REALM_URLS +from b2sdk.api_config import B2HttpApiConfig, DEFAULT_HTTP_API_CONFIG logger = logging.getLogger(__name__) @@ -40,7 +42,12 @@ class B2Session(object): """ SQLITE_ACCOUNT_INFO_CLASS = staticmethod(SqliteAccountInfo) - def __init__(self, account_info=None, cache=None, raw_api=None): + def __init__( + self, + account_info: Optional[AbstractAccountInfo] = None, + cache: Optional[AbstractCache] = None, + api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG + ): """ Initialize Session using given account info. @@ -57,15 +64,10 @@ def __init__(self, account_info=None, cache=None, raw_api=None): It is used by B2Api to cache the mapping between bucket name and bucket ids. default is :class:`~b2sdk.cache.DummyCache` - :param raw_api: an instance of one of the following classes: - :class:`~b2sdk.raw_api.B2RawHTTPApi`, :class:`~b2sdk.raw_simulator.RawSimulator`, - or any custom class derived from :class:`~b2sdk.raw_api.AbstractRawApi` - It makes network-less unit testing simple by using :class:`~b2sdk.raw_simulator.RawSimulator`, - in tests and :class:`~b2sdk.raw_api.B2RawHTTPApi` in production. - default is :class:`~b2sdk.raw_api.B2RawHTTPApi` + :param api_config """ - self.raw_api = raw_api or B2RawHTTPApi(B2Http()) + self.raw_api = B2RawHTTPApi(B2Http(api_config)) if account_info is None: account_info = self.SQLITE_ACCOUNT_INFO_CLASS() if cache is None: diff --git a/b2sdk/v1/__init__.py b/b2sdk/v1/__init__.py index c0756f347..960ce796c 100644 --- a/b2sdk/v1/__init__.py +++ b/b2sdk/v1/__init__.py @@ -13,6 +13,7 @@ AbstractAccountInfo, InMemoryAccountInfo, UrlPoolAccountInfo, SqliteAccountInfo, StubAccountInfo ) from b2sdk.v1.api import B2Api +from b2sdk.v1.b2http import B2Http from b2sdk.v1.bucket import Bucket, BucketFactory from b2sdk.v1.cache import AbstractCache from b2sdk.v1.exception import CommandError, DestFileNewer @@ -23,3 +24,5 @@ LocalFolder, B2Folder, parse_sync_folder, File, B2File, FileVersion, AbstractSyncEncryptionSettingsProvider ) + +B2RawApi = B2RawHTTPApi diff --git a/b2sdk/v1/api.py b/b2sdk/v1/api.py index 3cb9a369d..6ecd8f49e 100644 --- a/b2sdk/v1/api.py +++ b/b2sdk/v1/api.py @@ -8,9 +8,12 @@ # ###################################################################### -from typing import Any, Dict +from typing import Any, Dict, Optional from b2sdk import _v2 as v2 +from b2sdk.api import Services +from .account_info import AbstractAccountInfo from .bucket import Bucket, BucketFactory +from .cache import AbstractCache from .file_version import FileVersionInfo, FileVersionInfoFactory, file_version_info_from_id_and_name from .session import B2Session @@ -20,12 +23,49 @@ # and to use v1.Bucket # and to retain cancel_large_file return type # and to retain old style get_file_info return type +# and to accept old-style raw_api argument class B2Api(v2.B2Api): SESSION_CLASS = staticmethod(B2Session) BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) BUCKET_CLASS = staticmethod(Bucket) FILE_VERSION_FACTORY_CLASS = staticmethod(FileVersionInfoFactory) + def __init__( + self, + account_info: Optional[AbstractAccountInfo] = None, + cache: Optional[AbstractCache] = None, + max_upload_workers: int = 10, + max_copy_workers: int = 10, + raw_api: v2.B2RawHTTPApi = None, + api_config: Optional[v2.B2HttpApiConfig] = None, + ): + """ + Initialize the API using the given account info. + + :param account_info: To learn more about Account Info objects, see here + :class:`~b2sdk.v1.SqliteAccountInfo` + + :param cache: It is used by B2Api to cache the mapping between bucket name and bucket ids. + default is :class:`~b2sdk.cache.DummyCache` + + :param max_upload_workers: a number of upload threads + :param max_copy_workers: a number of copy threads + :param raw_api: + :param api_config: + """ + self.session = self.SESSION_CLASS( + account_info=account_info, + cache=cache, + raw_api=raw_api, + api_config=api_config, + ) + self.file_version_factory = self.FILE_VERSION_FACTORY_CLASS(self) + self.services = Services( + self, + max_upload_workers=max_upload_workers, + max_copy_workers=max_copy_workers, + ) + def get_file_info(self, file_id: str) -> Dict[str, Any]: """ Gets info about file version. diff --git a/b2sdk/v1/b2http.py b/b2sdk/v1/b2http.py new file mode 100644 index 000000000..6f257672c --- /dev/null +++ b/b2sdk/v1/b2http.py @@ -0,0 +1,60 @@ +###################################################################### +# +# File: b2sdk/v1/b2http.py +# +# Copyright 2021 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import requests + +from b2sdk import _v2 as v2 + + +# Overridden to retain old-style __init__ signature +class B2Http(v2.B2Http): + """ + A wrapper for the requests module. Provides the operations + needed to access B2, and handles retrying when the returned + status is 503 Service Unavailable, 429 Too Many Requests, etc. + + The operations supported are: + + - post_json_return_json + - post_content_return_json + - get_content + + The methods that return JSON either return a Python dict or + raise a subclass of B2Error. They can be used like this: + + .. code-block:: python + + try: + response_dict = b2_http.post_json_return_json(url, headers, params) + ... + except B2Error as e: + ... + + """ + + # timeout for HTTP GET/POST requests + TIMEOUT = 900 # 15 minutes as server-side copy can take time + + def __init__(self, requests_module=None, install_clock_skew_hook=True, user_agent_append=None): + """ + Initialize with a reference to the requests module, which makes + it easy to mock for testing. + + :param requests_module: a reference to requests module + :param bool install_clock_skew_hook: if True, install a clock skew hook + :param str user_agent_append: if provided, the string will be appended to the User-Agent + """ + super().__init__( + v2.B2HttpApiConfig( + requests_module=requests_module or requests, + install_clock_skew_hook=install_clock_skew_hook, + user_agent_append=user_agent_append + ) + ) diff --git a/b2sdk/v1/session.py b/b2sdk/v1/session.py index 9e49551f2..077b9230f 100644 --- a/b2sdk/v1/session.py +++ b/b2sdk/v1/session.py @@ -8,14 +8,36 @@ # ###################################################################### +from typing import Optional + from b2sdk import _v2 as v2 +from b2sdk._v2.exception import InvalidArgument from .account_info import SqliteAccountInfo # Override to use legacy signature of account_info.set_auth_data, especially the minimum_part_size argument +# and to accept old-style raw_api argument class B2Session(v2.B2Session): SQLITE_ACCOUNT_INFO_CLASS = staticmethod(SqliteAccountInfo) + def __init__( + self, + account_info=None, + cache=None, + raw_api: v2.B2RawHTTPApi = None, + api_config: Optional[v2.B2HttpApiConfig] = None + ): + if raw_api is not None and api_config is not None: + raise InvalidArgument( + 'raw_api,api_config', 'Provide at most one of: raw_api, api_config' + ) + + if api_config is None: + api_config = v2.DEFAULT_HTTP_API_CONFIG + super().__init__(account_info=account_info, cache=cache, api_config=api_config) + if raw_api is not None: + self.raw_api = raw_api + def authorize_account(self, realm, application_key_id, application_key): """ Perform account authorization. diff --git a/doc/source/api/api.rst b/doc/source/api/api.rst index 21be67124..95e173f31 100644 --- a/doc/source/api/api.rst +++ b/doc/source/api/api.rst @@ -4,3 +4,8 @@ B2 Api client .. autoclass:: b2sdk.v1.B2Api() :inherited-members: :special-members: __init__ + + +.. autoclass:: b2sdk.v1.B2HttpApiConfig() + :inherited-members: + :special-members: __init__ diff --git a/test/unit/api/test_api.py b/test/unit/api/test_api.py index a3357446f..ef1c6832d 100644 --- a/test/unit/api/test_api.py +++ b/test/unit/api/test_api.py @@ -12,6 +12,8 @@ import apiver_deps from apiver_deps import B2Api +from apiver_deps import B2HttpApiConfig +from apiver_deps import B2Http from apiver_deps import DummyCache from apiver_deps import EncryptionAlgorithm from apiver_deps import EncryptionMode @@ -23,7 +25,7 @@ from apiver_deps import RawSimulator from apiver_deps import RetentionMode from apiver_deps import NO_RETENTION_FILE_SETTING -from apiver_deps_exception import RestrictedBucket +from apiver_deps_exception import RestrictedBucket, InvalidArgument if apiver_deps.V <= 1: from apiver_deps import FileVersionInfo as VFileVersion @@ -37,7 +39,8 @@ def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = DummyCache() self.raw_api = RawSimulator() - self.api = B2Api(self.account_info, self.cache, self.raw_api) + self.api = B2Api(self.account_info, self.cache) + self.api.session.raw_api = self.raw_api (self.application_key_id, self.master_key) = self.raw_api.create_account() def test_get_file_info(self): @@ -311,3 +314,15 @@ def test_cancel_large_file_v1(self): action='cancel', api=self.api, ) + + @pytest.mark.apiver(to_ver=1) + def test_provide_raw_api_v1(self): + from apiver_deps import B2RawApi # test for legacy name + old_style_api = B2Api(raw_api=B2RawApi(B2Http(user_agent_append='test append'))) + new_style_api = B2Api(api_config=B2HttpApiConfig(user_agent_append='test append')) + assert old_style_api.session.raw_api.b2_http.user_agent == new_style_api.session.raw_api.b2_http.user_agent + with pytest.raises(InvalidArgument): + B2Api( + raw_api=B2RawApi(B2Http(user_agent_append='test append')), + api_config=B2HttpApiConfig(user_agent_append='test append'), + ) diff --git a/test/unit/b2http/test_b2http.py b/test/unit/b2http/test_b2http.py index b26d78fb6..80c6517f2 100644 --- a/test/unit/b2http/test_b2http.py +++ b/test/unit/b2http/test_b2http.py @@ -15,9 +15,11 @@ from ..test_base import TestBase +import apiver_deps from apiver_deps_exception import BadDateFormat, BadJson, BrokenPipe, B2ConnectionError, ClockSkew, ConnectionReset, ServiceError, UnknownError, UnknownHost, TooManyRequests from apiver_deps import USER_AGENT from apiver_deps import B2Http +from apiver_deps import B2HttpApiConfig from apiver_deps import ClockSkewHook from apiver_deps import translate_errors as _translate_errors from apiver_deps import translate_and_retry as _translate_and_retry @@ -210,9 +212,17 @@ def setUp(self): requests = MagicMock() requests.Session.return_value = self.session - self.b2_http = B2Http( - requests, install_clock_skew_hook=False, user_agent_append=self.UA_APPEND - ) + + if apiver_deps.V <= 1: + self.b2_http = B2Http( + requests, install_clock_skew_hook=False, user_agent_append=self.UA_APPEND + ) + else: + self.b2_http = B2Http( + B2HttpApiConfig( + requests, install_clock_skew_hook=False, user_agent_append=self.UA_APPEND + ) + ) def test_post_json_return_json(self): self.session.post.return_value = self.response diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index d91b83750..43b87721c 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -163,7 +163,8 @@ def setUp(self): self.bucket_name = 'my-bucket' self.simulator = self.RAW_SIMULATOR_CLASS() self.account_info = StubAccountInfo() - self.api = B2Api(self.account_info, raw_api=self.simulator) + self.api = B2Api(self.account_info) + self.api.session.raw_api = self.simulator (self.account_id, self.master_key) = self.simulator.create_account() self.api.authorize_account('production', self.account_id, self.master_key) self.api_url = self.account_info.get_api_url() @@ -338,7 +339,8 @@ def test_version_by_name_file_lock(self): self.assertEqual((legal_hold, file_retention), actual) low_perm_account_info = StubAccountInfo() - low_perm_api = B2Api(low_perm_account_info, raw_api=self.simulator) + low_perm_api = B2Api(low_perm_account_info) + low_perm_api.session.raw_api = self.simulator low_perm_key_resp = self.api.create_key( key_name='lowperm', capabilities=[ 'listKeys', diff --git a/test/unit/fixtures/session.py b/test/unit/fixtures/session.py index 4db980d64..9576fb647 100644 --- a/test/unit/fixtures/session.py +++ b/test/unit/fixtures/session.py @@ -15,7 +15,9 @@ @pytest.fixture def b2_session(fake_account_info, fake_cache, fake_b2_raw_api): - return B2Session(account_info=fake_account_info, cache=fake_cache, raw_api=fake_b2_raw_api) + session = B2Session(account_info=fake_account_info, cache=fake_cache) + session.raw_api = fake_b2_raw_api + return session @pytest.fixture diff --git a/test/unit/v_all/test_api.py b/test/unit/v_all/test_api.py index 8d2d924af..61396a967 100644 --- a/test/unit/v_all/test_api.py +++ b/test/unit/v_all/test_api.py @@ -26,7 +26,8 @@ def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = InMemoryCache() self.raw_api = RawSimulator() - self.api = B2Api(self.account_info, self.cache, self.raw_api) + self.api = B2Api(self.account_info, self.cache) + self.api.session.raw_api = self.raw_api (self.application_key_id, self.master_key) = self.raw_api.create_account() def _authorize_account(self): From 1648a6c827074ea5e6433012f08404809a1e7b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Mon, 14 Jun 2021 13:05:27 +0200 Subject: [PATCH 6/8] remove obsolete TODO --- b2sdk/transfer/emerge/planner/planner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/b2sdk/transfer/emerge/planner/planner.py b/b2sdk/transfer/emerge/planner/planner.py index e475ac1ac..22a8eee7c 100644 --- a/b2sdk/transfer/emerge/planner/planner.py +++ b/b2sdk/transfer/emerge/planner/planner.py @@ -104,7 +104,6 @@ def from_account_info( recommended_upload_part_size=None, max_part_size=None ): - # TODO: add support for getting `min_part_size` and `max_part_size` from account info if recommended_upload_part_size is None: recommended_upload_part_size = account_info.get_recommended_part_size() if min_part_size is None and recommended_upload_part_size < cls.DEFAULT_MIN_PART_SIZE: From 1a7c9262184e48a4db8aa416b84d728b419c8e5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Mon, 14 Jun 2021 15:32:39 +0200 Subject: [PATCH 7/8] Bucket.update returns a Bucket object --- CHANGELOG.md | 1 + b2sdk/bucket.py | 34 +++++----- b2sdk/raw_simulator.py | 2 +- b2sdk/v1/bucket.py | 35 ++++++++++ test/unit/bucket/test_bucket.py | 110 +++++++++++++++++++++++++++++++- 5 files changed, 163 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8396ff232..ad1eb8009 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `B2RawApi` renamed to `B2RawHTTPApi` * `B2HTTP` tests are now common * `B2HttpApiConfig` class introduced to provide parameters like `user_agent_append` to `B2Api` without using internal classes in v2 +* `Bucket.update` return s a `Bucket` object in v2 ## [1.9.0] - 2021-06-07 diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index eb27ea3f2..45321a4f9 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -114,17 +114,16 @@ def set_type(self, bucket_type): def update( self, - bucket_type=None, - bucket_info=None, - cors_rules=None, - lifecycle_rules=None, - if_revision_is=None, + bucket_type: Optional[str] = None, + bucket_info: Optional[dict] = None, + cors_rules: Optional[dict] = None, + lifecycle_rules: Optional[dict] = None, + if_revision_is: Optional[int] = None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, ): """ Update various bucket parameters. - For legacy reasons in apiver v1 it returns whatever server returned on b2_update_bucket call, v2 will change that. :param str bucket_type: a bucket type :param dict bucket_info: an info to store with a bucket @@ -135,16 +134,19 @@ def update( :param b2sdk.v1.BucketRetentionSetting default_retention: bucket default retention setting """ account_id = self.api.account_info.get_account_id() - return self.api.session.update_bucket( - account_id, - self.id_, - bucket_type=bucket_type, - bucket_info=bucket_info, - cors_rules=cors_rules, - lifecycle_rules=lifecycle_rules, - if_revision_is=if_revision_is, - default_server_side_encryption=default_server_side_encryption, - default_retention=default_retention, + return BucketFactory.from_api_bucket_dict( + self.api, + self.api.session.update_bucket( + account_id, + self.id_, + bucket_type=bucket_type, + bucket_info=bucket_info, + cors_rules=cors_rules, + lifecycle_rules=lifecycle_rules, + if_revision_is=if_revision_is, + default_server_side_encryption=default_server_side_encryption, + default_retention=default_retention, + ) ) def cancel_large_file(self, file_id): diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 4530ece9b..d817d016c 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -512,7 +512,7 @@ def bucket_dict(self, account_auth_token): 'value': { 'defaultRetention': { 'mode': self.default_retention.mode.value, - 'period': self.default_retention.period, + 'period': self.default_retention.period.as_dict() if self.default_retention.period else None, }, 'isFileLockEnabled': self.is_file_lock_enabled, }, diff --git a/b2sdk/v1/bucket.py b/b2sdk/v1/bucket.py index 2f218f2d2..4fee2d898 100644 --- a/b2sdk/v1/bucket.py +++ b/b2sdk/v1/bucket.py @@ -17,6 +17,7 @@ # Overridden to retain the obsolete copy_file and start_large_file methods # and to return old style FILE_VERSION_FACTORY attribute # and to to adjust to old style B2Api.get_file_info return type +# and to retain old style update return type class Bucket(v2.Bucket): FILE_VERSION_FACTORY = staticmethod(FileVersionInfoFactory) @@ -98,6 +99,40 @@ def get_file_info_by_id(self, file_id: str) -> FileVersionInfo: """ return self.api.file_version_factory.from_api_response(self.api.get_file_info(file_id)) + def update( + self, + bucket_type: Optional[str] = None, + bucket_info: Optional[dict] = None, + cors_rules: Optional[dict] = None, + lifecycle_rules: Optional[dict] = None, + if_revision_is: Optional[int] = None, + default_server_side_encryption: Optional[v2.EncryptionSetting] = None, + default_retention: Optional[v2.BucketRetentionSetting] = None, + ): + """ + Update various bucket parameters. + + :param str bucket_type: a bucket type + :param dict bucket_info: an info to store with a bucket + :param dict cors_rules: CORS rules to store with a bucket + :param dict lifecycle_rules: lifecycle rules to store with a bucket + :param int if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is* + :param b2sdk.v1.EncryptionSetting default_server_side_encryption: default server side encryption settings (``None`` if unknown) + :param b2sdk.v1.BucketRetentionSetting default_retention: bucket default retention setting + """ + account_id = self.api.account_info.get_account_id() + return self.api.session.update_bucket( + account_id, + self.id_, + bucket_type=bucket_type, + bucket_info=bucket_info, + cors_rules=cors_rules, + lifecycle_rules=lifecycle_rules, + if_revision_is=if_revision_is, + default_server_side_encryption=default_server_side_encryption, + default_retention=default_retention, + ) + class BucketFactory(v2.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 43b87721c..479967aab 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -31,6 +31,7 @@ SSECKeyError, ) from apiver_deps import B2Api +from apiver_deps import Bucket from apiver_deps import LargeFileUploadState from apiver_deps import DownloadDestBytes, PreSeekedDownloadDest from apiver_deps import MetadataDirectiveMode @@ -43,8 +44,10 @@ from apiver_deps import hex_sha1_of_bytes, TempDir from apiver_deps import EncryptionAlgorithm, EncryptionSetting, EncryptionMode, EncryptionKey, SSE_NONE, SSE_B2_AES from apiver_deps import CopySource, UploadSourceLocalFile, WriteIntent -from apiver_deps import FileRetentionSetting, LegalHold, RetentionMode, NO_RETENTION_FILE_SETTING +from apiver_deps import BucketRetentionSetting, FileRetentionSetting, LegalHold, RetentionMode, RetentionPeriod, \ + NO_RETENTION_FILE_SETTING import apiver_deps + if apiver_deps.V <= 1: from apiver_deps import FileVersionInfo as VFileVersionInfo else: @@ -814,6 +817,110 @@ def _make_file(self, bucket=None): return actual_bucket.upload_bytes(data, 'hello.txt').id_ +class TestUpdate(TestCaseWithBucket): + def test_update(self): + result = self.bucket.update( + bucket_type='allPrivate', + bucket_info={'info': 'o'}, + cors_rules={'andrea': 'corr'}, + lifecycle_rules={'life': 'is life'}, + default_server_side_encryption=SSE_B2_AES, + default_retention=BucketRetentionSetting( + RetentionMode.COMPLIANCE, RetentionPeriod(years=7) + ), + ) + if apiver_deps.V <= 1: + self.assertEqual( + { + 'accountId': 'account-0', + 'bucketId': 'bucket_0', + 'bucketInfo': { + 'info': 'o' + }, + 'bucketName': 'my-bucket', + 'bucketType': 'allPrivate', + 'corsRules': { + 'andrea': 'corr' + }, + 'defaultServerSideEncryption': + { + 'isClientAuthorizedToRead': True, + 'value': { + 'algorithm': 'AES256', + 'mode': 'SSE-B2' + } + }, + 'fileLockConfiguration': + { + 'isClientAuthorizedToRead': True, + 'value': + { + 'defaultRetention': + { + 'mode': 'compliance', + 'period': { + 'unit': 'years', + 'duration': 7 + } + }, + 'isFileLockEnabled': None + } + }, + 'lifecycleRules': { + 'life': 'is life' + }, + 'options': set(), + 'revision': 2 + }, result + ) + else: + self.assertIsInstance(result, Bucket) + for attr_name, attr_value in { + 'id_': + self.bucket.id_, + 'name': + self.bucket.name, + 'type_': + 'allPrivate', + 'bucket_info': { + 'info': 'o' + }, + 'cors_rules': { + 'andrea': 'corr' + }, + 'lifecycle_rules': { + 'life': 'is life' + }, + 'options_set': + set(), + 'default_server_side_encryption': + SSE_B2_AES, + 'default_retention': + BucketRetentionSetting(RetentionMode.COMPLIANCE, RetentionPeriod(years=7)), + }.items(): + self.assertEqual(attr_value, getattr(result, attr_name), attr_name) + + def test_update_if_revision_is(self): + current_revision = self.bucket.revision + self.bucket.update( + lifecycle_rules={'life': 'is life'}, + if_revision_is=current_revision, + ) + updated_bucket = self.api.get_bucket_by_name(self.bucket.name) + self.assertEqual({'life': 'is life'}, updated_bucket.lifecycle_rules) + + try: + self.bucket.update( + lifecycle_rules={'another': 'life'}, + if_revision_is=current_revision, # this is now the old revision + ) + except Exception: + pass + + not_updated_bucket = self.api.get_bucket_by_name(self.bucket.name) + self.assertEqual({'life': 'is life'}, not_updated_bucket.lifecycle_rules) + + class TestUpload(TestCaseWithBucket): def test_upload_bytes(self): data = b'hello world' @@ -1113,7 +1220,6 @@ def test_create_remote(self): def test_create_remote_encryption(self): for data in [b'hello_world', self._make_data(self.simulator.MIN_PART_SIZE * 3)]: - f1_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES).id_ f2_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES_2).id_ with TempDir() as d: From 7d0279367e546fb265af8c1d7fd5aef90e68530c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Mon, 14 Jun 2021 16:01:25 +0200 Subject: [PATCH 8/8] Bucket.ls argument show_versions renamed to latest_only --- CHANGELOG.md | 1 + b2sdk/bucket.py | 24 +++++++++++++++--------- b2sdk/sync/folder.py | 2 +- b2sdk/v1/bucket.py | 32 ++++++++++++++++++++++++++++++++ b2sdk/v1/sync/folder.py | 8 ++++++++ test/unit/bucket/test_bucket.py | 23 +++++++++++++++++------ 6 files changed, 74 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1eb8009..edf35f642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `B2HTTP` tests are now common * `B2HttpApiConfig` class introduced to provide parameters like `user_agent_append` to `B2Api` without using internal classes in v2 * `Bucket.update` return s a `Bucket` object in v2 +* `Bucket.ls` argument `show_versions` renamed to `latest_only` in v2 ## [1.9.0] - 2021-06-07 diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 45321a4f9..db49f8ca3 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -301,7 +301,13 @@ def list_file_versions(self, file_name, fetch_count=None): if start_file_name is None: return - def ls(self, folder_to_list='', show_versions=False, recursive=False, fetch_count=10000): + def ls( + self, + folder_to_list: str = '', + latest_only: bool = True, + recursive: bool = False, + fetch_count: Optional[int] = 10000 + ): """ Pretend that folders exist and yields the information about the files in a folder. @@ -313,12 +319,12 @@ def ls(self, folder_to_list='', show_versions=False, recursive=False, fetch_coun When the `recursive` flag is set, lists all of the files in the given folder, and all of its sub-folders. - :param str folder_to_list: the name of the folder to list; must not start with "/". + :param folder_to_list: the name of the folder to list; must not start with "/". Empty string means top-level folder - :param bool show_versions: when ``True`` returns info about all versions of a file, - when ``False``, just returns info about the most recent versions - :param bool recursive: if ``True``, list folders recursively - :param int,None fetch_count: how many entries to return or ``None`` to use the default. Acceptable values: 1 - 10000 + :param latest_only: when ``False`` returns info about all versions of a file, + when ``True``, just returns info about the most recent versions + :param recursive: if ``True``, list folders recursively + :param fetch_count: how many entries to return or ``None`` to use the default. Acceptable values: 1 - 10000 :rtype: generator[tuple[b2sdk.v1.FileVersionInfo, str]] :returns: generator of (file_version, folder_name) tuples @@ -344,12 +350,12 @@ def ls(self, folder_to_list='', show_versions=False, recursive=False, fetch_coun start_file_id = None session = self.api.session while True: - if show_versions: + if latest_only: + response = session.list_file_names(self.id_, start_file_name, fetch_count, prefix) + else: response = session.list_file_versions( self.id_, start_file_name, start_file_id, fetch_count, prefix ) - else: - response = session.list_file_names(self.id_, start_file_name, fetch_count, prefix) for entry in response['files']: file_version = self.api.file_version_factory.from_api_response(entry) if not file_version.file_name.startswith(prefix): diff --git a/b2sdk/sync/folder.py b/b2sdk/sync/folder.py index 3f66b27bf..addbc2099 100644 --- a/b2sdk/sync/folder.py +++ b/b2sdk/sync/folder.py @@ -366,7 +366,7 @@ def all_files( def get_file_versions(self): for file_version, _ in self.bucket.ls( self.folder_name, - show_versions=True, + latest_only=False, recursive=True, ): yield file_version diff --git a/b2sdk/v1/bucket.py b/b2sdk/v1/bucket.py index 4fee2d898..b5690fb46 100644 --- a/b2sdk/v1/bucket.py +++ b/b2sdk/v1/bucket.py @@ -133,6 +133,38 @@ def update( default_retention=default_retention, ) + def ls( + self, + folder_to_list: str = '', + show_versions: bool = False, + recursive: bool = False, + fetch_count: Optional[int] = 10000 + ): + """ + Pretend that folders exist and yields the information about the files in a folder. + + B2 has a flat namespace for the files in a bucket, but there is a convention + of using "/" as if there were folders. This method searches through the + flat namespace to find the files and "folders" that live within a given + folder. + + When the `recursive` flag is set, lists all of the files in the given + folder, and all of its sub-folders. + + :param folder_to_list: the name of the folder to list; must not start with "/". + Empty string means top-level folder + :param show_versions: when ``True`` returns info about all versions of a file, + when ``False``, just returns info about the most recent versions + :param recursive: if ``True``, list folders recursively + :param fetch_count: how many entries to return or ``None`` to use the default. Acceptable values: 1 - 10000 + :rtype: generator[tuple[b2sdk.v1.FileVersionInfo, str]] + :returns: generator of (file_version, folder_name) tuples + + .. note:: + In case of `recursive=True`, folder_name is returned only for first file in the folder. + """ + return super().ls(folder_to_list, not show_versions, recursive, fetch_count) + class BucketFactory(v2.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) diff --git a/b2sdk/v1/sync/folder.py b/b2sdk/v1/sync/folder.py index 715b4f237..5c31f76c7 100644 --- a/b2sdk/v1/sync/folder.py +++ b/b2sdk/v1/sync/folder.py @@ -46,6 +46,14 @@ class B2Folder(v2.B2Folder, AbstractFolder): def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): return super().all_files(reporter, wrap_if_necessary(policies_manager)) + def get_file_versions(self): + for file_version, _ in self.bucket.ls( + self.folder_name, + show_versions=True, + recursive=True, + ): + yield file_version + # override to retain "policies_manager" default argument, # translate nice errors to old style Exceptions and CommandError diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 479967aab..88c4120fc 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -159,6 +159,14 @@ def should_retry_upload(self): return self.can_retry +def bucket_ls(bucket, *args, show_versions=False, **kwargs): + if apiver_deps.V <= 1: + ls_all_versions_kwarg = {'show_versions': show_versions} + else: + ls_all_versions_kwarg = {'latest_only': not show_versions} + return bucket.ls(*args, **ls_all_versions_kwarg, **kwargs) + + class TestCaseWithBucket(TestBase): RAW_SIMULATOR_CLASS = RawSimulator @@ -175,13 +183,16 @@ def setUp(self): self.bucket = self.api.create_bucket('my-bucket', 'allPublic') self.bucket_id = self.bucket.id_ + def bucket_ls(self, *args, show_versions=False, **kwargs): + return bucket_ls(self.bucket, *args, show_versions=show_versions, **kwargs) + def assertBucketContents(self, expected, *args, **kwargs): """ - *args and **kwargs are passed to self.bucket.ls() + *args and **kwargs are passed to self.bucket_ls() """ actual = [ (info.file_name, info.size, info.action, folder) - for (info, folder) in self.bucket.ls(*args, **kwargs) + for (info, folder) in self.bucket_ls(*args, **kwargs) ] self.assertEqual(expected, actual) @@ -381,7 +392,7 @@ def test_version_by_id(self): class TestLs(TestCaseWithBucket): def test_empty(self): - self.assertEqual([], list(self.bucket.ls('foo'))) + self.assertEqual([], list(self.bucket_ls('foo'))) def test_one_file_at_root(self): data = b'hello world' @@ -431,7 +442,7 @@ def test_three_files_multiple_versions(self): ] actual = [ (info.id_, info.file_name, info.size, info.action, folder) - for (info, folder) in self.bucket.ls('bb', show_versions=True, fetch_count=1) + for (info, folder) in self.bucket_ls('bb', show_versions=True, fetch_count=1) ] self.assertEqual(expected, actual) @@ -632,7 +643,7 @@ def test_copy_with_replace_metadata(self): ] actual = [ (info.file_name, info.size, info.action, info.content_type, folder) - for (info, folder) in self.bucket.ls(show_versions=True) + for (info, folder) in self.bucket_ls(show_versions=True) ] self.assertEqual(expected, actual) @@ -663,7 +674,7 @@ def test_copy_with_different_bucket(self): def ls(bucket): return [ (info.file_name, info.size, info.action, folder) - for (info, folder) in bucket.ls(show_versions=True) + for (info, folder) in bucket_ls(bucket, show_versions=True) ] expected = [('hello.txt', 11, 'upload', None)]