From 449cc8f8785ddf8a6639d362522c2ac992bb738c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Sun, 17 Apr 2022 22:13:28 +0100 Subject: [PATCH 01/23] BucketStructure introduced --- CHANGELOG.md | 4 + b2sdk/_v3/__init__.py | 2 + b2sdk/bucket.py | 197 +++++++++++++++++++++++-- b2sdk/encryption/setting.py | 3 +- b2sdk/file_lock.py | 9 +- b2sdk/replication/setting.py | 3 +- doc/source/api/bucket.rst | 8 + test/unit/bucket/test_bucket.py | 72 +++++++-- test/unit/bucket/test_bucket_typing.py | 29 ++++ test/unit/v_all/test_api.py | 1 + 10 files changed, 302 insertions(+), 26 deletions(-) create mode 100644 test/unit/bucket/test_bucket_typing.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 16cdd00ff..a43d19f56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +* Add `BucketStructure` to hold info about a bucket +* Add `include_existing_files` parameter to `ReplicationSetupHelper` + ## [1.17.3] - 2022-07-15 ### Fixed diff --git a/b2sdk/_v3/__init__.py b/b2sdk/_v3/__init__.py index 26583f542..da2184b57 100644 --- a/b2sdk/_v3/__init__.py +++ b/b2sdk/_v3/__init__.py @@ -17,6 +17,8 @@ from b2sdk.api import Services from b2sdk.bucket import Bucket from b2sdk.bucket import BucketFactory +from b2sdk.bucket import BucketStructure +from b2sdk.bucket import ValueNotSet from b2sdk.raw_api import ALL_CAPABILITIES, REALM_URLS # encryption diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 2d4f2a9f1..118b95408 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -10,7 +10,10 @@ import logging -from typing import Optional, Tuple +from typing import Optional, Tuple, Union + +if False: + from b2sdk.api import B2Api from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .encryption.types import EncryptionMode @@ -48,16 +51,31 @@ logger = logging.getLogger(__name__) -class Bucket(metaclass=B2TraceMeta): - """ - Provide access to a bucket in B2: listing files, uploading and downloading. - """ +class ValueNotSet: + """Sentry class for signifying no value for a property was supplied""" + pass - DEFAULT_CONTENT_TYPE = AUTO_CONTENT_TYPE + +class BucketStructure(metaclass=B2TraceMeta): + """Structure holding all attributes of a bucket.""" + + id_: Union[str, ValueNotSet] + account_id: Union[str, ValueNotSet] + name: Union[str, ValueNotSet] + type_: Union[str, ValueNotSet] + bucket_info: Union[dict, ValueNotSet] + cors_rules: Union[dict, ValueNotSet] + lifecycle_rules: Union[dict, ValueNotSet] + revision: Union[int, ValueNotSet] + bucket_dict: Union[dict, ValueNotSet] + options_set: Union[set, ValueNotSet] + default_server_side_encryption: Union[EncryptionSetting, ValueNotSet] + default_retention: Union[BucketRetentionSetting, ValueNotSet] + is_file_lock_enabled: Union[Optional[bool], ValueNotSet] + replication: Union[Optional[ReplicationConfiguration], ValueNotSet] def __init__( self, - api, id_, name=None, type_=None, @@ -73,9 +91,10 @@ def __init__( default_retention: BucketRetentionSetting = UNKNOWN_BUCKET_RETENTION, is_file_lock_enabled: Optional[bool] = None, replication: Optional[ReplicationConfiguration] = None, + *, + account_id, ): """ - :param b2sdk.v2.B2Api api: an API object :param str id_: a bucket id :param str name: a bucket name :param str type_: a bucket type @@ -89,9 +108,10 @@ def __init__( :param b2sdk.v2.BucketRetentionSetting default_retention: default retention setting :param bool is_file_lock_enabled: whether file locking is enabled or not :param b2sdk.v2.ReplicationConfiguration replication: replication rules for the bucket + :param str account_id: id of the account owning the bucket """ - self.api = api self.id_ = id_ + self.account_id = account_id self.name = name self.type_ = type_ self.bucket_info = bucket_info or {} @@ -105,6 +125,45 @@ def __init__( self.is_file_lock_enabled = is_file_lock_enabled self.replication = replication + def __repr__(self): + return '%s<%s,%s,%s>' % (type(self).__name__, self.id_, self.name, self.type_) + + +class Bucket(BucketStructure): + """ + Provide access to a bucket in B2: listing files, uploading and downloading. + """ + + api: 'B2Api' + id_: str + account_id: str + name: str + type_: str + bucket_info: dict + cors_rules: dict + lifecycle_rules: dict + revision: int + bucket_dict: dict + options_set: set + default_server_side_encryption: EncryptionSetting + default_retention: BucketRetentionSetting + is_file_lock_enabled: Optional[bool] + replication: Optional[ReplicationConfiguration] + + DEFAULT_CONTENT_TYPE = AUTO_CONTENT_TYPE + + def __init__( + self, + api, + *args, + **kwargs, + ): + """ + :param b2sdk.v2.B2Api api: an API object + """ + self.api = api + super().__init__(*args, account_id=self.api.account_info.get_account_id(), **kwargs) + def get_fresh_state(self) -> 'Bucket': """ Fetch all the information about this bucket and return a new bucket object. @@ -960,9 +1019,6 @@ def as_dict(self): return result - def __repr__(self): - return 'Bucket<%s,%s,%s>' % (self.id_, self.name, self.type_) - class BucketFactory: """ @@ -981,6 +1037,123 @@ def from_api_response(cls, api, response): """ return [cls.from_api_bucket_dict(api, bucket_dict) for bucket_dict in response['buckets']] + @classmethod + def bucket_structure_from_dict(cls, bucket_dict) -> BucketStructure: + """ + Turn a dictionary, like this: + + .. code-block:: python + + { + "bucketType": "allPrivate", + "accountId": "0991231", + "bucketId": "a4ba6a39d8b6b5fd561f0010", + "bucketName": "zsdfrtsazsdfafr", + "accountId": "4aa9865d6f00", + "bucketInfo": {}, + "options": [], + "revision": 1, + "defaultServerSideEncryption": { + "isClientAuthorizedToRead" : true, + "value": { + "algorithm" : "AES256", + "mode" : "SSE-B2" + } + }, + "fileLockConfiguration": { + "isClientAuthorizedToRead": true, + "value": { + "defaultRetention": { + "mode": null, + "period": null + }, + "isFileLockEnabled": false + } + }, + "replicationConfiguration": { + "clientIsAllowedToRead": true, + "value": { + "asReplicationSource": { + "replicationRules": [ + { + "destinationBucketId": "c5f35d53a90a7ea284fb0719", + "fileNamePrefix": "", + "includeExistingFiles": True, + "isEnabled": true, + "priority": 1, + "replicationRuleName": "replication-us-west" + }, + { + "destinationBucketId": "55f34d53a96a7ea284fb0719", + "fileNamePrefix": "", + "includeExistingFiles": True, + "isEnabled": true, + "priority": 2, + "replicationRuleName": "replication-us-west-2" + } + ], + "sourceApplicationKeyId": "10053d55ae26b790000000006" + }, + "asReplicationDestination": { + "sourceToDestinationKeyMapping": { + "10053d55ae26b790000000045": "10053d55ae26b790000000004", + "10053d55ae26b790000000046": "10053d55ae26b790030000004" + } + } + } + } + } + + into a BucketStructure object. + + :param dict bucket_dict: a dictionary with bucket properties + :rtype: BucketStructure + + """ + type_ = bucket_dict.get('bucketType', ValueNotSet()) + bucket_name = bucket_dict.get('bucketName', ValueNotSet()) + bucket_id = bucket_dict.get('bucketId', ValueNotSet()) + bucket_info = bucket_dict.get('bucketInfo', ValueNotSet()) + cors_rules = bucket_dict.get('corsRules', ValueNotSet()) + lifecycle_rules = bucket_dict.get('lifecycleRules', ValueNotSet()) + revision = bucket_dict.get('revision', ValueNotSet()) + options = set(bucket_dict['options']) if 'options' in bucket_dict else ValueNotSet() + account_id = bucket_dict.get('accountId', ValueNotSet()) + + default_server_side_encryption = ( + EncryptionSettingFactory.from_bucket_dict(bucket_dict) + if EncryptionSettingFactory.TOP_LEVEL_KEY in bucket_dict else ValueNotSet() + ) + replication = ( + ReplicationConfigurationFactory.from_bucket_dict(bucket_dict).value + if ReplicationConfigurationFactory.TOP_LEVEL_KEY in bucket_dict else ValueNotSet() + ) + + if FileLockConfiguration.TOP_LEVEL_KEY in bucket_dict: + file_lock_configuration = FileLockConfiguration.from_bucket_dict(bucket_dict) + default_retention = file_lock_configuration.default_retention + is_file_lock_enabled = file_lock_configuration.is_file_lock_enabled + else: + default_retention = ValueNotSet() + is_file_lock_enabled = ValueNotSet() + + return BucketStructure( + bucket_id, + bucket_name, + type_, + bucket_info, + cors_rules, + lifecycle_rules, + revision, + bucket_dict, + options, + default_server_side_encryption, + default_retention, + is_file_lock_enabled, + replication, + account_id=account_id, + ) + @classmethod def from_api_bucket_dict(cls, api, bucket_dict): """ diff --git a/b2sdk/encryption/setting.py b/b2sdk/encryption/setting.py index de4fdbd6d..99493d3fb 100644 --- a/b2sdk/encryption/setting.py +++ b/b2sdk/encryption/setting.py @@ -220,6 +220,7 @@ def __repr__(self): class EncryptionSettingFactory: + TOP_LEVEL_KEY = 'defaultServerSideEncryption' # 2021-03-17: for the bucket the response of the server is: # if authorized to read: # "mode": "none" @@ -301,7 +302,7 @@ def from_bucket_dict(cls, bucket_dict: dict) -> Optional[EncryptionSetting]: """ default_sse = bucket_dict.get( - 'defaultServerSideEncryption', + cls.TOP_LEVEL_KEY, {'isClientAuthorizedToRead': False}, ) diff --git a/b2sdk/file_lock.py b/b2sdk/file_lock.py index 0484376ca..e5db35f09 100644 --- a/b2sdk/file_lock.py +++ b/b2sdk/file_lock.py @@ -284,6 +284,8 @@ def as_dict(self): } if self.period is not None: result['period'] = self.period.as_dict() + else: + result['period'] = None return result def serialize_to_json_for_request(self): @@ -301,6 +303,7 @@ def __repr__(self): class FileLockConfiguration: """Represent bucket's file lock configuration, i.e. whether the file lock mechanism is enabled and default file retention""" + TOP_LEVEL_KEY = 'fileLockConfiguration' def __init__( self, @@ -339,12 +342,12 @@ def from_bucket_dict(cls, bucket_dict): } """ - if not bucket_dict['fileLockConfiguration']['isClientAuthorizedToRead']: + if not bucket_dict[cls.TOP_LEVEL_KEY]['isClientAuthorizedToRead']: return cls(UNKNOWN_BUCKET_RETENTION, None) retention = BucketRetentionSetting.from_bucket_retention_dict( - bucket_dict['fileLockConfiguration']['value']['defaultRetention'] + bucket_dict[cls.TOP_LEVEL_KEY]['value']['defaultRetention'] ) - is_file_lock_enabled = bucket_dict['fileLockConfiguration']['value']['isFileLockEnabled'] + is_file_lock_enabled = bucket_dict[cls.TOP_LEVEL_KEY]['value']['isFileLockEnabled'] return cls(retention, is_file_lock_enabled) def as_dict(self): diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index f80ce99a2..350f2f397 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -192,6 +192,7 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': @dataclass class ReplicationConfigurationFactory: + TOP_LEVEL_KEY = 'replicationConfiguration' is_client_authorized_to_read: bool value: Optional[ReplicationConfiguration] @@ -201,7 +202,7 @@ def from_bucket_dict(cls, bucket_dict: dict) -> 'ReplicationConfigurationFactory Returns ReplicationConfigurationFactory for the given bucket dict retrieved from the api. """ - replication_dict = bucket_dict.get('replicationConfiguration') or {} + replication_dict = bucket_dict.get(cls.TOP_LEVEL_KEY) or {} value_dict = replication_dict.get('value') or {} return cls( diff --git a/doc/source/api/bucket.rst b/doc/source/api/bucket.rst index 0d6110542..143d6e23d 100644 --- a/doc/source/api/bucket.rst +++ b/doc/source/api/bucket.rst @@ -4,3 +4,11 @@ B2 Bucket .. autoclass:: b2sdk.v2.Bucket() :inherited-members: :special-members: __init__ + + +.. autoclass:: b2sdk.v2.BucketStructure() + :inherited-members: + :special-members: __init__ + + +.. autoclass:: b2sdk.v2.ValueNotSet() diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 6d56e33bf..452ac50f8 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -8,14 +8,17 @@ # ###################################################################### import io +import itertools from contextlib import suppress from io import BytesIO import os import platform import unittest.mock as mock +from typing import List import pytest +from .test_bucket_typing import get_all_annotations from ..test_base import TestBase, create_key import apiver_deps @@ -41,7 +44,7 @@ from apiver_deps import FileVersion as VFileVersionInfo from apiver_deps import B2Api from apiver_deps import B2HttpApiConfig -from apiver_deps import Bucket, BucketFactory +from apiver_deps import Bucket, BucketFactory, BucketStructure, ValueNotSet from apiver_deps import DownloadedFile from apiver_deps import DownloadVersion from apiver_deps import LargeFileUploadState @@ -203,6 +206,13 @@ def get_api(self): self.account_info, api_config=B2HttpApiConfig(_raw_api_class=self.RAW_SIMULATOR_CLASS) ) + def new_api_with_new_key(self, capabilities: List[str]) -> B2Api: + new_key = create_key(self.api, capabilities=capabilities, key_name='newtestkey') + new_api = B2Api(StubAccountInfo()) + new_api.session.raw_api = self.simulator + new_api.authorize_account('production', new_key.id_, new_key.application_key) + return new_api + def setUp(self): self.bucket_name = 'my-bucket' self.account_info = StubAccountInfo() @@ -400,12 +410,7 @@ def test_version_by_name_file_lock(self): actual = (file_version.legal_hold, file_version.file_retention) self.assertEqual((legal_hold, file_retention), actual) - low_perm_account_info = StubAccountInfo() - low_perm_api = B2Api(low_perm_account_info) - low_perm_api.session.raw_api = self.simulator - low_perm_key = create_key( - self.api, - key_name='lowperm', + low_perm_api = self.new_api_with_new_key( capabilities=[ 'listKeys', 'listBuckets', @@ -413,8 +418,6 @@ def test_version_by_name_file_lock(self): 'readFiles', ] ) - - low_perm_api.authorize_account('production', low_perm_key.id_, low_perm_key.application_key) low_perm_bucket = low_perm_api.get_bucket_by_name('my-bucket-with-file-lock') file_version = low_perm_bucket.get_file_info_by_name('a') @@ -2122,3 +2125,54 @@ def test_file_info_3(self): def test_file_info_4(self): download_version = self.bucket.get_file_info_by_name('test.txt%253Ffoo%253Dbar') assert download_version.file_name == 'test.txt%253Ffoo%253Dbar' + + +class TestBucketStructure(TestCaseWithBucket): + def test_create_with_all_attributes(self): + recreated_structure = BucketFactory.bucket_structure_from_dict(self.bucket.bucket_dict) + for attr_name in get_all_annotations(BucketStructure): + assert getattr(self.bucket, + attr_name) == getattr(recreated_structure, attr_name), attr_name + + def test_create_with_all_attributes_low_permissions(self): + low_perm_api = self.new_api_with_new_key(capabilities=['listBuckets']) + low_perm_bucket = low_perm_api.get_bucket_by_name(self.bucket.name) + recreated_structure = BucketFactory.bucket_structure_from_dict(low_perm_bucket.bucket_dict) + + comparison_exclusion_list = [ + 'bucket_dict', 'default_server_side_encryption', 'default_retention', + 'is_file_lock_enabled', 'replication' + ] + for attr_name in comparison_exclusion_list: + assert hasattr(self.bucket, attr_name), attr_name + + for attr_name in get_all_annotations(BucketStructure): + assert not isinstance(getattr(recreated_structure, attr_name), ValueNotSet), attr_name + if attr_name not in comparison_exclusion_list: + assert getattr(self.bucket, + attr_name) == getattr(recreated_structure, attr_name), attr_name + + def test_create_with_some_attributes(self): + attributes_to_drop = { + 'cors_rules': 'corsRules', + 'default_server_side_encryption': 'defaultServerSideEncryption', + 'name': 'bucketName' + } + comparison_exclusion_list = ['bucket_dict'] + for attr_name in itertools.chain(attributes_to_drop, comparison_exclusion_list): + assert hasattr(self.bucket, attr_name), attr_name + + new_bucket_dict = self.bucket.bucket_dict.copy() + for key in attributes_to_drop.values(): + new_bucket_dict.pop(key) + + recreated_structure = BucketFactory.bucket_structure_from_dict(new_bucket_dict) + + for attr_name in get_all_annotations(BucketStructure): + if attr_name in comparison_exclusion_list: + assert not isinstance(getattr(recreated_structure, attr_name), ValueNotSet) + elif attr_name in attributes_to_drop: + assert isinstance(getattr(recreated_structure, attr_name), ValueNotSet) + else: + assert getattr(self.bucket, + attr_name) == getattr(recreated_structure, attr_name), attr_name diff --git a/test/unit/bucket/test_bucket_typing.py b/test/unit/bucket/test_bucket_typing.py new file mode 100644 index 000000000..92187282c --- /dev/null +++ b/test/unit/bucket/test_bucket_typing.py @@ -0,0 +1,29 @@ +###################################################################### +# +# File: test/unit/bucket/test_bucket_typing.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import collections +from typing import Union + +from apiver_deps import Bucket, BucketStructure, ValueNotSet + + +def get_all_annotations(class_: type): + return dict( + collections.ChainMap(*(getattr(cls, '__annotations__', {}) for cls in class_.__mro__)) + ) + + +def test_bucket_annotations(): + expected_structure_annotations = {} + for instance_var_name, type_ in get_all_annotations(Bucket).items(): + if instance_var_name == 'api': + continue + expected_structure_annotations[instance_var_name] = Union[type_, ValueNotSet] + assert expected_structure_annotations == get_all_annotations(BucketStructure) diff --git a/test/unit/v_all/test_api.py b/test/unit/v_all/test_api.py index 43ecee13f..89a70c340 100644 --- a/test/unit/v_all/test_api.py +++ b/test/unit/v_all/test_api.py @@ -37,6 +37,7 @@ def _authorize_account(self): @pytest.mark.apiver(to_ver=1) def test_get_bucket_by_id_up_to_v1(self): + self._authorize_account() bucket = self.api.get_bucket_by_id("this id doesn't even exist") assert bucket.id_ == "this id doesn't even exist" for att_name, att_value in [ From af80fc39deaf0d6e846c6e8431e02a1939c97a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Mon, 13 Jun 2022 23:09:49 +0200 Subject: [PATCH 02/23] comment for a peculiar behaviour in BucketFactory --- b2sdk/bucket.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 118b95408..7df034086 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -1120,6 +1120,9 @@ def bucket_structure_from_dict(cls, bucket_dict) -> BucketStructure: options = set(bucket_dict['options']) if 'options' in bucket_dict else ValueNotSet() account_id = bucket_dict.get('accountId', ValueNotSet()) + # The existence of these complex settings is checked below, instead of inside of their respective factory + # classes, because those would either break or return objects indistinguishable from objects representing + # insufficient permission to read set values. default_server_side_encryption = ( EncryptionSettingFactory.from_bucket_dict(bucket_dict) if EncryptionSettingFactory.TOP_LEVEL_KEY in bucket_dict else ValueNotSet() From 00544a0eb50c16740b3e425d57e545b932860bad Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 16 Jun 2022 23:56:19 +0300 Subject: [PATCH 03/23] Replace ValueNotSet() with NOT_SET from module scope --- b2sdk/bucket.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 7df034086..dcca856b1 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -56,6 +56,9 @@ class ValueNotSet: pass +NOT_SET = ValueNotSet() + + class BucketStructure(metaclass=B2TraceMeta): """Structure holding all attributes of a bucket.""" @@ -1110,26 +1113,26 @@ def bucket_structure_from_dict(cls, bucket_dict) -> BucketStructure: :rtype: BucketStructure """ - type_ = bucket_dict.get('bucketType', ValueNotSet()) - bucket_name = bucket_dict.get('bucketName', ValueNotSet()) - bucket_id = bucket_dict.get('bucketId', ValueNotSet()) - bucket_info = bucket_dict.get('bucketInfo', ValueNotSet()) - cors_rules = bucket_dict.get('corsRules', ValueNotSet()) - lifecycle_rules = bucket_dict.get('lifecycleRules', ValueNotSet()) - revision = bucket_dict.get('revision', ValueNotSet()) - options = set(bucket_dict['options']) if 'options' in bucket_dict else ValueNotSet() - account_id = bucket_dict.get('accountId', ValueNotSet()) + type_ = bucket_dict.get('bucketType', cls.NOT_SET) + bucket_name = bucket_dict.get('bucketName', cls.NOT_SET) + bucket_id = bucket_dict.get('bucketId', cls.NOT_SET) + bucket_info = bucket_dict.get('bucketInfo', cls.NOT_SET) + cors_rules = bucket_dict.get('corsRules', cls.NOT_SET) + lifecycle_rules = bucket_dict.get('lifecycleRules', cls.NOT_SET) + revision = bucket_dict.get('revision', cls.NOT_SET) + options = set(bucket_dict['options']) if 'options' in bucket_dict else cls.NOT_SET + account_id = bucket_dict.get('accountId', cls.NOT_SET) # The existence of these complex settings is checked below, instead of inside of their respective factory # classes, because those would either break or return objects indistinguishable from objects representing # insufficient permission to read set values. default_server_side_encryption = ( EncryptionSettingFactory.from_bucket_dict(bucket_dict) - if EncryptionSettingFactory.TOP_LEVEL_KEY in bucket_dict else ValueNotSet() + if EncryptionSettingFactory.TOP_LEVEL_KEY in bucket_dict else cls.NOT_SET ) replication = ( ReplicationConfigurationFactory.from_bucket_dict(bucket_dict).value - if ReplicationConfigurationFactory.TOP_LEVEL_KEY in bucket_dict else ValueNotSet() + if ReplicationConfigurationFactory.TOP_LEVEL_KEY in bucket_dict else cls.NOT_SET ) if FileLockConfiguration.TOP_LEVEL_KEY in bucket_dict: @@ -1137,8 +1140,8 @@ def bucket_structure_from_dict(cls, bucket_dict) -> BucketStructure: default_retention = file_lock_configuration.default_retention is_file_lock_enabled = file_lock_configuration.is_file_lock_enabled else: - default_retention = ValueNotSet() - is_file_lock_enabled = ValueNotSet() + default_retention = cls.NOT_SET + is_file_lock_enabled = cls.NOT_SET return BucketStructure( bucket_id, From 8dd8ab10c6dac1009e38db45f780d121c4168f9d Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Fri, 17 Jun 2022 00:11:59 +0300 Subject: [PATCH 04/23] Make BucketFactory.bucket_structure_from_dict return type configurable via BUCKET_STRUCTURE_CLASS class var --- b2sdk/bucket.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index dcca856b1..d87f1d397 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -10,7 +10,7 @@ import logging -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, Union, Type if False: from b2sdk.api import B2Api @@ -1028,6 +1028,7 @@ class BucketFactory: This is a factory for creating bucket objects from different kind of objects. """ BUCKET_CLASS = staticmethod(Bucket) + BUCKET_STRUCTURE_CLASS = staticmethod(BucketStructure) @classmethod def from_api_response(cls, api, response): @@ -1041,7 +1042,7 @@ def from_api_response(cls, api, response): return [cls.from_api_bucket_dict(api, bucket_dict) for bucket_dict in response['buckets']] @classmethod - def bucket_structure_from_dict(cls, bucket_dict) -> BucketStructure: + def bucket_structure_from_dict(cls, bucket_dict) -> Type[BUCKET_STRUCTURE_CLASS]: """ Turn a dictionary, like this: @@ -1143,7 +1144,7 @@ def bucket_structure_from_dict(cls, bucket_dict) -> BucketStructure: default_retention = cls.NOT_SET is_file_lock_enabled = cls.NOT_SET - return BucketStructure( + return cls.BUCKET_STRUCTURE_CLASS( bucket_id, bucket_name, type_, @@ -1161,7 +1162,7 @@ def bucket_structure_from_dict(cls, bucket_dict) -> BucketStructure: ) @classmethod - def from_api_bucket_dict(cls, api, bucket_dict): + def from_api_bucket_dict(cls, api, bucket_dict) -> Type[BUCKET_CLASS]: """ Turn a dictionary, like this: From 06d25e08362c87e93e874ba81d9858e6304c9806 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Fri, 17 Jun 2022 00:25:10 +0300 Subject: [PATCH 05/23] Allow any Iterable for create_key instead of List --- b2sdk/raw_api.py | 6 +++--- b2sdk/v1/api.py | 4 ++-- test/unit/bucket/test_bucket.py | 4 ++-- test/unit/test_base.py | 7 +++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index ed2c9db57..d6268b223 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -13,7 +13,7 @@ from abc import ABCMeta, abstractmethod from enum import Enum, unique from logging import getLogger -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Iterable from .exception import FileOrBucketNotFound, ResourceNotFound, UnusableFileName, InvalidMetadataDirective, WrongEncryptionModeForBucketDefault, AccessDenied, SSECKeyError, RetentionWriteError from .encryption.setting import EncryptionMode, EncryptionSetting @@ -462,7 +462,7 @@ def create_bucket( ) def create_key( - self, api_url, account_auth_token, account_id, capabilities, key_name, + self, api_url, account_auth_token, account_id, capabilities: Iterable[str], key_name, valid_duration_seconds, bucket_id, name_prefix ): return self._post_json( @@ -470,7 +470,7 @@ def create_key( 'b2_create_key', account_auth_token, accountId=account_id, - capabilities=capabilities, + capabilities=list(capabilities), keyName=key_name, validDurationInSeconds=valid_duration_seconds, bucketId=bucket_id, diff --git a/b2sdk/v1/api.py b/b2sdk/v1/api.py index 516317054..6636e934b 100644 --- a/b2sdk/v1/api.py +++ b/b2sdk/v1/api.py @@ -8,7 +8,7 @@ # ###################################################################### -from typing import Any, Dict, Optional, overload, Tuple, List +from typing import Any, Dict, Optional, overload, Tuple, Iterable from .download_dest import AbstractDownloadDestination from b2sdk import v2 @@ -188,7 +188,7 @@ def list_keys(self, start_application_key_id=None) -> dict: def create_key( self, - capabilities: List[str], + capabilities: Iterable[str], key_name: str, valid_duration_seconds: Optional[int] = None, bucket_id: Optional[str] = None, diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 452ac50f8..16984b5b0 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -14,7 +14,7 @@ import os import platform import unittest.mock as mock -from typing import List +from typing import List, Iterable import pytest @@ -206,7 +206,7 @@ def get_api(self): self.account_info, api_config=B2HttpApiConfig(_raw_api_class=self.RAW_SIMULATOR_CLASS) ) - def new_api_with_new_key(self, capabilities: List[str]) -> B2Api: + def new_api_with_new_key(self, capabilities: Iterable[str]) -> B2Api: new_key = create_key(self.api, capabilities=capabilities, key_name='newtestkey') new_api = B2Api(StubAccountInfo()) new_api.session.raw_api = self.simulator diff --git a/test/unit/test_base.py b/test/unit/test_base.py index 820b0c093..b94424cec 100644 --- a/test/unit/test_base.py +++ b/test/unit/test_base.py @@ -8,12 +8,11 @@ # ###################################################################### +from contextlib import contextmanager +from typing import Optional, Iterable import re import unittest -from contextlib import contextmanager -from typing import List, Optional - import apiver_deps from apiver_deps import B2Api @@ -46,7 +45,7 @@ def assertRaisesRegexp(self, expected_exception, expected_regexp): def create_key( api: B2Api, - capabilities: List[str], + capabilities: Iterable[str], key_name: str, valid_duration_seconds: Optional[int] = None, bucket_id: Optional[str] = None, From 5e35a71768d8a8a05a2638b2405c172137c6802b Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Fri, 17 Jun 2022 12:31:09 +0300 Subject: [PATCH 06/23] Update CHANGELOG.md about making `create_key` accept iterable for `capabilities` --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a43d19f56..0e68c466b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Add `BucketStructure` to hold info about a bucket * Add `include_existing_files` parameter to `ReplicationSetupHelper` +* Modify `create_key` to accept any iterable for `capabilities`, not only list ## [1.17.3] - 2022-07-15 From 96365d513087d09035a9763a1e68d3cdd4805932 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Fri, 17 Jun 2022 15:59:09 +0300 Subject: [PATCH 07/23] Remove staticmethod from BUCKET_CLASS, BUCKET_STRUCTURE_CLASS --- b2sdk/bucket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index d87f1d397..a8596154c 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -1027,8 +1027,8 @@ class BucketFactory: """ This is a factory for creating bucket objects from different kind of objects. """ - BUCKET_CLASS = staticmethod(Bucket) - BUCKET_STRUCTURE_CLASS = staticmethod(BucketStructure) + BUCKET_CLASS = Bucket + BUCKET_STRUCTURE_CLASS = BucketStructure @classmethod def from_api_response(cls, api, response): From 65f35bf85a725b7e879f95283ce1eaf5003a1e51 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Fri, 17 Jun 2022 15:59:40 +0300 Subject: [PATCH 08/23] Fix BucketStructure cls.NOT_SET -> NOT_SET --- b2sdk/bucket.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index a8596154c..28b73a69a 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -1114,26 +1114,26 @@ def bucket_structure_from_dict(cls, bucket_dict) -> Type[BUCKET_STRUCTURE_CLASS] :rtype: BucketStructure """ - type_ = bucket_dict.get('bucketType', cls.NOT_SET) - bucket_name = bucket_dict.get('bucketName', cls.NOT_SET) - bucket_id = bucket_dict.get('bucketId', cls.NOT_SET) - bucket_info = bucket_dict.get('bucketInfo', cls.NOT_SET) - cors_rules = bucket_dict.get('corsRules', cls.NOT_SET) - lifecycle_rules = bucket_dict.get('lifecycleRules', cls.NOT_SET) - revision = bucket_dict.get('revision', cls.NOT_SET) - options = set(bucket_dict['options']) if 'options' in bucket_dict else cls.NOT_SET - account_id = bucket_dict.get('accountId', cls.NOT_SET) + type_ = bucket_dict.get('bucketType', NOT_SET) + bucket_name = bucket_dict.get('bucketName', NOT_SET) + bucket_id = bucket_dict.get('bucketId', NOT_SET) + bucket_info = bucket_dict.get('bucketInfo', NOT_SET) + cors_rules = bucket_dict.get('corsRules', NOT_SET) + lifecycle_rules = bucket_dict.get('lifecycleRules', NOT_SET) + revision = bucket_dict.get('revision', NOT_SET) + options = set(bucket_dict['options']) if 'options' in bucket_dict else NOT_SET + account_id = bucket_dict.get('accountId', NOT_SET) # The existence of these complex settings is checked below, instead of inside of their respective factory # classes, because those would either break or return objects indistinguishable from objects representing # insufficient permission to read set values. default_server_side_encryption = ( EncryptionSettingFactory.from_bucket_dict(bucket_dict) - if EncryptionSettingFactory.TOP_LEVEL_KEY in bucket_dict else cls.NOT_SET + if EncryptionSettingFactory.TOP_LEVEL_KEY in bucket_dict else NOT_SET ) replication = ( ReplicationConfigurationFactory.from_bucket_dict(bucket_dict).value - if ReplicationConfigurationFactory.TOP_LEVEL_KEY in bucket_dict else cls.NOT_SET + if ReplicationConfigurationFactory.TOP_LEVEL_KEY in bucket_dict else NOT_SET ) if FileLockConfiguration.TOP_LEVEL_KEY in bucket_dict: @@ -1141,8 +1141,8 @@ def bucket_structure_from_dict(cls, bucket_dict) -> Type[BUCKET_STRUCTURE_CLASS] default_retention = file_lock_configuration.default_retention is_file_lock_enabled = file_lock_configuration.is_file_lock_enabled else: - default_retention = cls.NOT_SET - is_file_lock_enabled = cls.NOT_SET + default_retention = NOT_SET + is_file_lock_enabled = NOT_SET return cls.BUCKET_STRUCTURE_CLASS( bucket_id, From 8b2d3cf80da1b71b417ca1a6589b1b88f6b43f0e Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Fri, 17 Jun 2022 16:33:29 +0300 Subject: [PATCH 09/23] Improve docstring of BucketStructure --- b2sdk/bucket.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 28b73a69a..7aad092d0 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -60,7 +60,17 @@ class ValueNotSet: class BucketStructure(metaclass=B2TraceMeta): - """Structure holding all attributes of a bucket.""" + """ + Structure holding all attributes of a bucket. + + This structure doesn't hold reference to B2Api, so unlike `Bucket` class + it cannot be used to perform any actions. Instead, this class is used + to only hold Bucket's fields for serializing / deserializing. + + Also important difference from `Bucket` is that this structure + allows storing subset of fields, setting others to `ValueNotSet`, + which preserves from serializing too much information. + """ id_: Union[str, ValueNotSet] account_id: Union[str, ValueNotSet] From 2bdb765edd9fec788e1f793e23e3803e06489921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nowacki?= Date: Sun, 17 Apr 2022 22:15:55 +0100 Subject: [PATCH 10/23] Replication WIP checks --- b2sdk/replication/check.py | 409 +++++++++++++++++++++++++++++++++++++ 1 file changed, 409 insertions(+) create mode 100644 b2sdk/replication/check.py diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py new file mode 100644 index 000000000..c7862ce21 --- /dev/null +++ b/b2sdk/replication/check.py @@ -0,0 +1,409 @@ +import warnings +from typing import Optional, Dict, Union, Tuple + +from b2sdk import version +from b2sdk.api import B2Api +from b2sdk.application_key import ApplicationKey +from b2sdk.bucket import BucketStructure, BucketFactory, Bucket +import enum + +from b2sdk.exception import AccessDenied, BucketIdNotFound + + +class ReplicationFilter: + def __init__(self, source_api: B2Api, destination_api: B2Api, filters: ...): + self.source_api = source_api + self.destination_api = destination_api + + # def get_checks + + +class TwoWayReplicationCheckGenerator: + def __init__( + self, + source_api: B2Api, + destination_api: B2Api, + filter_source_bucket_name: Optional[str], + filter_destination_bucket_name: Optional[str], + filter_replication_rule_name: Optional[str], + file_name_prefix: Optional[str], + ): + self.source_api = source_api + self.destination_api = destination_api + + self.filter_source_bucket_name = filter_source_bucket_name + self.filter_destination_bucket_name = filter_destination_bucket_name + self.filter_replication_rule_name = filter_replication_rule_name + self.file_name_prefix = file_name_prefix + + def get_checks(self): + if self.filter_source_bucket_name is not None: + source_buckets = self.source_api.list_buckets( + bucket_name=self.filter_source_bucket_name + ) + else: + source_buckets = self.source_api.list_buckets() + for source_bucket in source_buckets: + if not source_bucket.replication: + continue + if not source_bucket.replication.as_replication_source: + continue + if not source_bucket.replication.as_replication_source.replication_rules: + continue + source_key = _safe_get_key( + self.source_api, + source_bucket.replication.as_replication_source.source_application_key_id + ) + for rule in source_bucket.replication.as_replication_source.replication_rules: + if ( + self.filter_replication_rule_name is not None and + rule.replication_rule_name != self.filter_replication_rule_name + ): + continue + + if self.file_name_prefix is not None and rule.file_name_prefix != self.file_name_prefix: + continue + + try: + destination_bucket_list = self.destination_api.list_buckets( + bucket_id=rule.destination_bucket_id + ) + if not destination_bucket_list: + raise BucketIdNotFound + except (AccessDenied, BucketIdNotFound): + yield ReplicationSourceCheck(source_bucket, rule.replication_rule_name) + continue + + if ( + self.filter_destination_bucket_name is not None and + destination_bucket_list[0].name != self.filter_destination_bucket_name + ): + continue + + yield TwoWayReplicationCheck( + source_bucket=source_bucket, + replication_rule_name=rule.replication_rule_name, + source_application_key=source_key, + destination_bucket=destination_bucket_list[0], + destination_application_keys=self._get_destination_bucket_keys( + destination_bucket_list[0] + ), + ) + + @classmethod + def _get_destination_bucket_keys(cls, destination_bucket: Bucket) -> \ + Dict[str, Union[None, ApplicationKey, 'AccessDeniedEnum']]: + if not destination_bucket.replication: + return {} + if not destination_bucket.replication.as_replication_destination: + return {} + key_ids = destination_bucket.replication.as_replication_destination.source_to_destination_key_mapping.values( + ) + try: + return {key_id: destination_bucket.api.get_key(key_id) for key_id in key_ids} + except AccessDenied: + return dict.fromkeys(key_ids, AccessDeniedEnum.ACCESS_DENIED) + + +@enum.unique +class CheckState(enum.Enum): + OK = 'ok' + NOT_OK = 'not_ok' + UNKNOWN = 'unknown' + + def is_ok(self): + return self == type(self).OK + + +class AccessDeniedEnum(enum.Enum): + ACCESS_DENIED = 'ACCESS_DENIED' + + +class ReplicationSourceCheck: + """ + key_exists + key_read_capabilities + key_name_prefix_match + is_enabled + + """ + + def __init__(self, bucket: Bucket, rule_name: str): + self.bucket = bucket + self.application_key = _safe_get_key( + self.bucket.api, self.bucket.replication.as_replication_source.source_application_key_id + ) + rules = [ + r for r in self.bucket.replication.as_replication_source.replication_rules + if r.replication_rule_name == rule_name + ] + assert rules + self._rule = rules[0] + + self.is_enabled = self._rule.is_enabled + + ( + self.key_exists, + self.key_bucket_match, + self.key_read_capabilities, + self.key_name_prefix_match, + ) = _check_key(application_key, 'readFiles', self._rule.file_name_prefix, bucket.id_) + + def other_party_data(self): + return OtherPartyReplicationCheckData( + bucket=self.bucket, + keys_mapping={ + self.bucket.replication.as_replication_source.source_application_key_id: + self.application_key + } + ) + + +class ReplicationDestinationCheck: + """ + keys_exist: { + 10053d55ae26b790000000004: True + 10053d55ae26b790030000004: True + 10053d55ae26b790050000004: False + } + keys_write_capabilities: { + 10053d55ae26b790000000004: True + 10053d55ae26b790030000004: False + 10053d55ae26b790050000004: False + } + keys_bucket_match: { + 10053d55ae26b790000000004: True + 10053d55ae26b790030000004: False + 10053d55ae26b790050000004: False + } + """ + + def __init__(self, bucket: Bucket): + self.bucket = bucket + self.keys: Dict[str, Union[Optional[ApplicationKey], AccessDeniedEnum]] = {} + self.keys_exist: Dict[str, CheckState] = {} + self.keys_write_capabilities: Dict[str, CheckState] = {} + self.keys_bucket_match: Dict[str, CheckState] = {} + keys_to_check = bucket.replication.as_replication_destination.source_to_destination_key_mapping.values( + ) + try: + for key_id in keys_to_check: + application_key = self.bucket.api.get_key( + self.bucket.replication.as_replication_source.source_application_key_id + ) + self.keys[key_id] = application_key + if application_key: + self.keys_exist[key_id] = CheckState.OK + self.keys_write_capabilities[ + key_id + ] = CheckState.OK if 'writeFiles' in application_key.capabilities else CheckState.NOT_OK + self.keys_bucket_match[key_id] = ( + CheckState.OK if application_key.bucket_id is None or + application_key.bucket_id == bucket.id_ else CheckState.NOT_OK + ) + else: + self.keys_exist[key_id] = CheckState.NOT_OK + self.keys_write_capabilities[key_id] = CheckState.NOT_OK + self.keys_bucket_match[key_id] = CheckState.NOT_OK + + except AccessDenied: + + self.keys = dict.fromkeys(keys_to_check, None) + self.keys_exist = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) + self.keys_write_capabilities = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) + self.keys_bucket_match = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) + + def other_party_data(self): + return OtherPartyReplicationCheckData( + bucket=self.bucket, + keys_mapping=self.keys, + ) + + +class TwoWayReplicationCheck: + """ + is_enabled + + source_key_exists + source_key_bucket_match + source_key_read_capabilities + source_key_name_prefix_match + + source_key_accepted_in_target_bucket + + destination_key_exists + destination_key_bucket_match + destination_key_write_capabilities + destination_key_name_prefix_match + + file_lock_match + """ + + def __init__( + self, + source_bucket: BucketStructure, + replication_rule_name: str, + source_application_key: Union[Optional[ApplicationKey], AccessDeniedEnum], + destination_bucket: BucketStructure, + destination_application_keys: Dict[str, Union[Optional[ApplicationKey]], AccessDeniedEnum], + ): + rules = [ + r for r in source_bucket.replication.as_replication_source.replication_rules + if r.replication_rule_name == replication_rule_name + ] + assert rules + self._rule = rules[0] + self.is_enabled = CheckState.OK if self._rule.is_enabled else CheckState.NOT_OK + + ( + self.source_key_exists, + self.source_key_bucket_match, + self.source_key_read_capabilities, + self.source_key_name_prefix_match, + ) = self._check_key( + source_application_key, 'readFiles', self._rule.file_name_prefix, source_bucket.id_ + ) + + if destination_bucket.replication is None or destination_bucket.replication.as_replication_destination is None: + destination_application_key_id = None + else: + destination_application_key_id = destination_bucket.replication.as_replication_destination.\ + source_to_destination_key_mapping.get( + source_bucket.replication.as_replication_source.source_application_key_id) + + if destination_application_key_id is None: + self.source_key_accepted_in_target_bucket = CheckState.NOT_OK + else: + self.source_key_accepted_in_target_bucket = CheckState.OK + + destination_application_key = destination_application_keys.get( + destination_application_key_id + ) + + ( + self.destination_key_exists, + self.destination_key_bucket_match, + self.destination_key_read_capabilities, + self.destination_key_key_name_prefix_match, + ) = self._check_key( + destination_application_key, 'writeFiles', self._rule.file_name_prefix, + destination_bucket.id_ + ) + + if destination_bucket.is_file_lock_enabled: + self.file_lock_match = CheckState.OK + elif source_bucket.is_file_lock_enabled == False: + self.file_lock_match = CheckState.OK + elif source_bucket.is_file_lock_enabled is None or destination_bucket.is_file_lock_enabled is None: + self.file_lock_match = CheckState.UNKNOWN + else: + self.file_lock_match = CheckState.NOT_OK + + @classmethod + def _check_key( + cls, + key: Union[Optional[ApplicationKey], AccessDeniedEnum], + capability: str, + replication_name_prefix: str, + bucket_id: str, + ) -> Tuple[CheckState, CheckState, CheckState, CheckState]: + if key == AccessDeniedEnum.ACCESS_DENIED: + return (CheckState.UNKNOWN,) * 4 + if key is None: + return (CheckState.NOT_OK,) * 4 + return ( + CheckState.OK, + CheckState.OK + if key.bucket_id is None or key.bucket_id == bucket_id else CheckState.NOT_OK, + CheckState.OK if capability in key.capabilities else CheckState.NOT_OK, + CheckState.OK if key.name_prefix is None or + replication_name_prefix.startswith(key.name_prefix) else CheckState.NOT_OK, + ) + + +class OtherPartyReplicationCheckData: + b2sdk_version = version.VERSION + + def __init__( + self, + bucket: BucketStructure, + keys_mapping: Dict[str, Union[Optional[ApplicationKey], AccessDeniedEnum]], + b2sdk_version: Optional[str] = None + ): + + self.bucket = bucket + self.keys_mapping = keys_mapping + if b2sdk_version is None: + self.b2sdk_version = type(self).b2sdk_version + else: + self.b2sdk_version = b2sdk_version + + @classmethod + def _dump_key(self, key: Union[Optional[ApplicationKey], AccessDeniedEnum]): + if key is None: + return None + if isinstance(key, AccessDeniedEnum): + return key.value + return key.as_dict() + + @classmethod + def _parse_key(cls, key_representation: Union[None, str, dict] + ) -> Union[Optional[ApplicationKey], AccessDeniedEnum]: + if key_representation is None: + return None + try: + return AccessDeniedEnum(key_representation) + except ValueError: + pass + return ApplicationKey.from_dict(key_representation) + + def as_dict(self): + return { + 'b2sdk_version': self.b2sdk_version, + 'bucket': self.bucket.as_dict(), + 'keys_mapping': {k: self._dump_key(v) + for k, v in self.keys_mapping.items()}, + } + + @classmethod + def from_dict(cls, dict_: dict): + other_party_version = dict_['b2sdk_version'] + if other_party_version != cls.b2sdk_version: + warnings.warn( + f'Other party used a different version of b2sdk ({other_party_version}, this version: ' + f'{cls.b2sdk_version}) when dumping data for checking replication health. Check may not be ' + f'complete.' + ) + + return cls( + b2sdk_version=other_party_version, + bucket=BucketFactory.bucket_structure_from_dict(dict_['bucket']), + keys_mapping={k: cls._parse_key(v) + for k, v in dict_['keys_mapping'].items()} + ) + + +def _safe_get_key(api: B2Api, key_id: str) -> Union[None, AccessDeniedEnum, ApplicationKey]: + try: + return api.get_key(key_id) + except AccessDenied: + return AccessDeniedEnum.ACCESS_DENIED + + +def _check_key( + key: Union[Optional[ApplicationKey], AccessDeniedEnum], + capability: str, + replication_name_prefix: str, + bucket_id: str, +) -> Tuple[CheckState, CheckState, CheckState, CheckState]: + if key == AccessDeniedEnum.ACCESS_DENIED: + return (CheckState.UNKNOWN,) * 4 + if key is None: + return (CheckState.NOT_OK,) * 4 + return ( + CheckState.OK, + CheckState.OK if key.bucket_id is None or key.bucket_id == bucket_id else CheckState.NOT_OK, + CheckState.OK if capability in key.capabilities else CheckState.NOT_OK, + CheckState.OK if key.name_prefix is None or + replication_name_prefix.startswith(key.name_prefix) else CheckState.NOT_OK, + ) From b2250d9d567c25d79be47236b5c22ce7b5190ab5 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 2 Aug 2022 20:49:05 +0300 Subject: [PATCH 11/23] Refactor replication/check.py --- b2sdk/replication/check.py | 212 +++++++++++++++++-------------------- 1 file changed, 97 insertions(+), 115 deletions(-) diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py index c7862ce21..0cc28a697 100644 --- a/b2sdk/replication/check.py +++ b/b2sdk/replication/check.py @@ -1,12 +1,13 @@ +import enum import warnings -from typing import Optional, Dict, Union, Tuple + +from dataclasses import dataclass +from typing import Dict, Generator, Optional, Tuple, Union from b2sdk import version from b2sdk.api import B2Api from b2sdk.application_key import ApplicationKey -from b2sdk.bucket import BucketStructure, BucketFactory, Bucket -import enum - +from b2sdk.bucket import Bucket, BucketFactory, BucketStructure from b2sdk.exception import AccessDenied, BucketIdNotFound @@ -18,77 +19,71 @@ def __init__(self, source_api: B2Api, destination_api: B2Api, filters: ...): # def get_checks +@dataclass class TwoWayReplicationCheckGenerator: - def __init__( - self, - source_api: B2Api, - destination_api: B2Api, - filter_source_bucket_name: Optional[str], - filter_destination_bucket_name: Optional[str], - filter_replication_rule_name: Optional[str], - file_name_prefix: Optional[str], - ): - self.source_api = source_api - self.destination_api = destination_api + source_api: B2Api + destination_api: B2Api + filter_source_bucket_name: Optional[str] = None + filter_destination_bucket_name: Optional[str] = None + filter_replication_rule_name: Optional[str] = None + file_name_prefix: Optional[str] = None + + def get_checks(self) -> Generator['ReplicationCheck']: + source_buckets = self.source_api.list_buckets( + bucket_name=self.filter_source_bucket_name + ) + for source_bucket in source_buckets: + yield from self.get_source_bucket_checks() - self.filter_source_bucket_name = filter_source_bucket_name - self.filter_destination_bucket_name = filter_destination_bucket_name - self.filter_replication_rule_name = filter_replication_rule_name - self.file_name_prefix = file_name_prefix + def get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['ReplicationCheck']: + if not source_bucket.replication: + return - def get_checks(self): - if self.filter_source_bucket_name is not None: - source_buckets = self.source_api.list_buckets( - bucket_name=self.filter_source_bucket_name - ) - else: - source_buckets = self.source_api.list_buckets() - for source_bucket in source_buckets: - if not source_bucket.replication: + if not source_bucket.replication.as_replication_source: + return + + if not source_bucket.replication.as_replication_source.replication_rules: + return + + source_key = _safe_get_key( + self.source_api, + source_bucket.replication.as_replication_source.source_application_key_id + ) + for rule in source_bucket.replication.as_replication_source.replication_rules: + if ( + self.filter_replication_rule_name and + rule.replication_rule_name != self.filter_replication_rule_name + ): + continue + + if self.file_name_prefix and rule.file_name_prefix != self.file_name_prefix: continue - if not source_bucket.replication.as_replication_source: + + try: + destination_bucket_list = self.destination_api.list_buckets( + bucket_id=rule.destination_bucket_id + ) + if not destination_bucket_list: + raise BucketIdNotFound() + except (AccessDenied, BucketIdNotFound): + yield ReplicationSourceCheck(source_bucket, rule.replication_rule_name) continue - if not source_bucket.replication.as_replication_source.replication_rules: + + if ( + self.filter_destination_bucket_name is not None and + destination_bucket_list[0].name != self.filter_destination_bucket_name + ): continue - source_key = _safe_get_key( - self.source_api, - source_bucket.replication.as_replication_source.source_application_key_id + + yield TwoWayReplicationCheck( + source_bucket=source_bucket, + replication_rule_name=rule.replication_rule_name, + source_application_key=source_key, + destination_bucket=destination_bucket_list[0], + destination_application_keys=self._get_destination_bucket_keys( + destination_bucket_list[0] + ), ) - for rule in source_bucket.replication.as_replication_source.replication_rules: - if ( - self.filter_replication_rule_name is not None and - rule.replication_rule_name != self.filter_replication_rule_name - ): - continue - - if self.file_name_prefix is not None and rule.file_name_prefix != self.file_name_prefix: - continue - - try: - destination_bucket_list = self.destination_api.list_buckets( - bucket_id=rule.destination_bucket_id - ) - if not destination_bucket_list: - raise BucketIdNotFound - except (AccessDenied, BucketIdNotFound): - yield ReplicationSourceCheck(source_bucket, rule.replication_rule_name) - continue - - if ( - self.filter_destination_bucket_name is not None and - destination_bucket_list[0].name != self.filter_destination_bucket_name - ): - continue - - yield TwoWayReplicationCheck( - source_bucket=source_bucket, - replication_rule_name=rule.replication_rule_name, - source_application_key=source_key, - destination_bucket=destination_bucket_list[0], - destination_application_keys=self._get_destination_bucket_keys( - destination_bucket_list[0] - ), - ) @classmethod def _get_destination_bucket_keys(cls, destination_bucket: Bucket) -> \ @@ -114,12 +109,39 @@ class CheckState(enum.Enum): def is_ok(self): return self == type(self).OK + @classmethod + def from_bool(cls, value: bool) -> 'CheckState': + return cls.OK if value else cls.NOT_OK + class AccessDeniedEnum(enum.Enum): ACCESS_DENIED = 'ACCESS_DENIED' -class ReplicationSourceCheck: +class ReplicationCheck: + @classmethod + def _check_key( + cls, + key: Union[Optional[ApplicationKey], AccessDeniedEnum], + capability: str, + replication_name_prefix: str, + bucket_id: str, + ) -> Tuple[CheckState, CheckState, CheckState, CheckState]: + if key == AccessDeniedEnum.ACCESS_DENIED: + return (CheckState.UNKNOWN,) * 4 + if key is None: + return (CheckState.NOT_OK,) * 4 + return ( + CheckState.OK, + CheckState.from_bool(key.bucket_id is None or key.bucket_id == bucket_id), + CheckState.from_bool(capability in key.capabilities), + CheckState.from_bool( + key.name_prefix is None or replication_name_prefix.startswith(key.name_prefix) + ), + ) + + +class ReplicationSourceCheck(ReplicationCheck): """ key_exists key_read_capabilities @@ -147,7 +169,7 @@ def __init__(self, bucket: Bucket, rule_name: str): self.key_bucket_match, self.key_read_capabilities, self.key_name_prefix_match, - ) = _check_key(application_key, 'readFiles', self._rule.file_name_prefix, bucket.id_) + ) = self._check_key(self.application_key, 'readFiles', self._rule.file_name_prefix, bucket.id_) def other_party_data(self): return OtherPartyReplicationCheckData( @@ -220,7 +242,7 @@ def other_party_data(self): ) -class TwoWayReplicationCheck: +class TwoWayReplicationCheck(ReplicationCheck): """ is_enabled @@ -269,7 +291,8 @@ def __init__( else: destination_application_key_id = destination_bucket.replication.as_replication_destination.\ source_to_destination_key_mapping.get( - source_bucket.replication.as_replication_source.source_application_key_id) + source_bucket.replication.as_replication_source.source_application_key_id + ) if destination_application_key_id is None: self.source_key_accepted_in_target_bucket = CheckState.NOT_OK @@ -292,34 +315,13 @@ def __init__( if destination_bucket.is_file_lock_enabled: self.file_lock_match = CheckState.OK - elif source_bucket.is_file_lock_enabled == False: + elif source_bucket.is_file_lock_enabled is False: self.file_lock_match = CheckState.OK elif source_bucket.is_file_lock_enabled is None or destination_bucket.is_file_lock_enabled is None: self.file_lock_match = CheckState.UNKNOWN else: self.file_lock_match = CheckState.NOT_OK - @classmethod - def _check_key( - cls, - key: Union[Optional[ApplicationKey], AccessDeniedEnum], - capability: str, - replication_name_prefix: str, - bucket_id: str, - ) -> Tuple[CheckState, CheckState, CheckState, CheckState]: - if key == AccessDeniedEnum.ACCESS_DENIED: - return (CheckState.UNKNOWN,) * 4 - if key is None: - return (CheckState.NOT_OK,) * 4 - return ( - CheckState.OK, - CheckState.OK - if key.bucket_id is None or key.bucket_id == bucket_id else CheckState.NOT_OK, - CheckState.OK if capability in key.capabilities else CheckState.NOT_OK, - CheckState.OK if key.name_prefix is None or - replication_name_prefix.startswith(key.name_prefix) else CheckState.NOT_OK, - ) - class OtherPartyReplicationCheckData: b2sdk_version = version.VERSION @@ -347,8 +349,7 @@ def _dump_key(self, key: Union[Optional[ApplicationKey], AccessDeniedEnum]): return key.as_dict() @classmethod - def _parse_key(cls, key_representation: Union[None, str, dict] - ) -> Union[Optional[ApplicationKey], AccessDeniedEnum]: + def _parse_key(cls, key_representation: Union[None, str, dict]) -> Union[Optional[ApplicationKey], AccessDeniedEnum]: if key_representation is None: return None try: @@ -388,22 +389,3 @@ def _safe_get_key(api: B2Api, key_id: str) -> Union[None, AccessDeniedEnum, Appl return api.get_key(key_id) except AccessDenied: return AccessDeniedEnum.ACCESS_DENIED - - -def _check_key( - key: Union[Optional[ApplicationKey], AccessDeniedEnum], - capability: str, - replication_name_prefix: str, - bucket_id: str, -) -> Tuple[CheckState, CheckState, CheckState, CheckState]: - if key == AccessDeniedEnum.ACCESS_DENIED: - return (CheckState.UNKNOWN,) * 4 - if key is None: - return (CheckState.NOT_OK,) * 4 - return ( - CheckState.OK, - CheckState.OK if key.bucket_id is None or key.bucket_id == bucket_id else CheckState.NOT_OK, - CheckState.OK if capability in key.capabilities else CheckState.NOT_OK, - CheckState.OK if key.name_prefix is None or - replication_name_prefix.startswith(key.name_prefix) else CheckState.NOT_OK, - ) From 85d4f643c4759b98f32a0308224e00c8fb4a20e1 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 3 Aug 2022 17:17:43 +0300 Subject: [PATCH 12/23] Refactor replication check --- b2sdk/replication/check.py | 278 ++++++++++++++++++------------------- 1 file changed, 135 insertions(+), 143 deletions(-) diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py index 0cc28a697..ad7774c3a 100644 --- a/b2sdk/replication/check.py +++ b/b2sdk/replication/check.py @@ -1,3 +1,13 @@ +###################################################################### +# +# File: b2sdk/replication/check.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + import enum import warnings @@ -29,9 +39,7 @@ class TwoWayReplicationCheckGenerator: file_name_prefix: Optional[str] = None def get_checks(self) -> Generator['ReplicationCheck']: - source_buckets = self.source_api.list_buckets( - bucket_name=self.filter_source_bucket_name - ) + source_buckets = self.source_api.list_buckets(bucket_name=self.filter_source_bucket_name) for source_bucket in source_buckets: yield from self.get_source_bucket_checks() @@ -39,20 +47,13 @@ def get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['Replicat if not source_bucket.replication: return - if not source_bucket.replication.as_replication_source: - return - - if not source_bucket.replication.as_replication_source.replication_rules: + if not source_bucket.replication.rules: return - source_key = _safe_get_key( - self.source_api, - source_bucket.replication.as_replication_source.source_application_key_id - ) - for rule in source_bucket.replication.as_replication_source.replication_rules: + source_key = _safe_get_key(self.source_api, source_bucket.replication.source_key_id) + for rule in source_bucket.replication.rules: if ( - self.filter_replication_rule_name and - rule.replication_rule_name != self.filter_replication_rule_name + self.filter_replication_rule_name and rule.name != self.filter_replication_rule_name ): continue @@ -66,7 +67,7 @@ def get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['Replicat if not destination_bucket_list: raise BucketIdNotFound() except (AccessDenied, BucketIdNotFound): - yield ReplicationSourceCheck(source_bucket, rule.replication_rule_name) + yield ReplicationSourceCheck.from_data(source_bucket, rule.replication_rule_name) continue if ( @@ -75,7 +76,7 @@ def get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['Replicat ): continue - yield TwoWayReplicationCheck( + yield TwoWayReplicationCheck.from_data( source_bucket=source_bucket, replication_rule_name=rule.replication_rule_name, source_application_key=source_key, @@ -90,10 +91,8 @@ def _get_destination_bucket_keys(cls, destination_bucket: Bucket) -> \ Dict[str, Union[None, ApplicationKey, 'AccessDeniedEnum']]: if not destination_bucket.replication: return {} - if not destination_bucket.replication.as_replication_destination: - return {} - key_ids = destination_bucket.replication.as_replication_destination.source_to_destination_key_mapping.values( - ) + + key_ids = destination_bucket.replication.source_to_destination_key_mapping.values() try: return {key_id: destination_bucket.api.get_key(key_id) for key_id in key_ids} except AccessDenied: @@ -107,7 +106,7 @@ class CheckState(enum.Enum): UNKNOWN = 'unknown' def is_ok(self): - return self == type(self).OK + return self == self.OK @classmethod def from_bool(cls, value: bool) -> 'CheckState': @@ -141,186 +140,178 @@ def _check_key( ) +@dataclass class ReplicationSourceCheck(ReplicationCheck): - """ - key_exists - key_read_capabilities - key_name_prefix_match - is_enabled + key_exists: CheckState + key_read_capabilities: CheckState + key_name_prefix_match: CheckState + is_enabled: CheckState - """ + _bucket: Bucket + _application_key: Union[None, AccessDeniedEnum, ApplicationKey] - def __init__(self, bucket: Bucket, rule_name: str): - self.bucket = bucket - self.application_key = _safe_get_key( - self.bucket.api, self.bucket.replication.as_replication_source.source_application_key_id - ) - rules = [ - r for r in self.bucket.replication.as_replication_source.replication_rules - if r.replication_rule_name == rule_name - ] + @classmethod + def from_data(cls, bucket: Bucket, rule_name: str) -> 'ReplicationSourceCheck': + kwargs = { + '_bucket': bucket, + } + + application_key = _safe_get_key(bucket.api, bucket.replication.source_key_id) + kwargs['_application_key'] = application_key + + rules = [rule for rule in bucket.replication.rules if rule.name == rule_name] assert rules - self._rule = rules[0] + rule = rules[0] - self.is_enabled = self._rule.is_enabled + kwargs['is_enabled'] = rule.is_enabled ( - self.key_exists, - self.key_bucket_match, - self.key_read_capabilities, - self.key_name_prefix_match, - ) = self._check_key(self.application_key, 'readFiles', self._rule.file_name_prefix, bucket.id_) + kwargs['key_exists'], + _, # kwargs['key_bucket_match'], + kwargs['key_read_capabilities'], + kwargs['key_name_prefix_match'], + ) = cls._check_key(application_key, 'readFiles', rule.file_name_prefix, bucket.id_) + + return cls(**kwargs) def other_party_data(self): return OtherPartyReplicationCheckData( - bucket=self.bucket, - keys_mapping={ - self.bucket.replication.as_replication_source.source_application_key_id: - self.application_key - } + bucket=self._bucket, + keys_mapping={self._bucket.replication.source_key_id: self._application_key}, ) +@dataclass class ReplicationDestinationCheck: - """ - keys_exist: { - 10053d55ae26b790000000004: True - 10053d55ae26b790030000004: True - 10053d55ae26b790050000004: False - } - keys_write_capabilities: { - 10053d55ae26b790000000004: True - 10053d55ae26b790030000004: False - 10053d55ae26b790050000004: False - } - keys_bucket_match: { - 10053d55ae26b790000000004: True - 10053d55ae26b790030000004: False - 10053d55ae26b790050000004: False - } - """ - - def __init__(self, bucket: Bucket): - self.bucket = bucket - self.keys: Dict[str, Union[Optional[ApplicationKey], AccessDeniedEnum]] = {} - self.keys_exist: Dict[str, CheckState] = {} - self.keys_write_capabilities: Dict[str, CheckState] = {} - self.keys_bucket_match: Dict[str, CheckState] = {} - keys_to_check = bucket.replication.as_replication_destination.source_to_destination_key_mapping.values( - ) + key_exist: Dict[str, CheckState] + keys_write_capabilities: Dict[str, CheckState] + keys_bucket_match: Dict[str, CheckState] + + _bucket: Bucket + _keys: Dict[str, Union[Optional[ApplicationKey], AccessDeniedEnum]] + + @classmethod + def from_data(cls, bucket: Bucket) -> 'ReplicationDestinationCheck': + kwargs = { + '_bucket': bucket, + '_keys': {}, + 'key_exist': {}, + 'keys_write_capabilities': {}, + 'keys_bucket_match': {}, + } + + keys_to_check = bucket.replication.source_to_destination_key_mapping.values() try: for key_id in keys_to_check: - application_key = self.bucket.api.get_key( - self.bucket.replication.as_replication_source.source_application_key_id - ) - self.keys[key_id] = application_key + application_key = bucket.api.get_key(bucket.replication.source_key_id) + kwargs['_keys'][key_id] = application_key + if application_key: - self.keys_exist[key_id] = CheckState.OK - self.keys_write_capabilities[ - key_id - ] = CheckState.OK if 'writeFiles' in application_key.capabilities else CheckState.NOT_OK - self.keys_bucket_match[key_id] = ( - CheckState.OK if application_key.bucket_id is None or - application_key.bucket_id == bucket.id_ else CheckState.NOT_OK + kwargs['keys_exist'][key_id] = CheckState.OK + kwargs['keys_write_capabilities'][key_id] = CheckState.from_bool( + 'writeFiles' in application_key.capabilities + ) + kwargs['keys_bucket_match'][key_id] = CheckState.from_bool( + application_key.bucket_id is None or application_key.bucket_id == bucket.id_ ) else: - self.keys_exist[key_id] = CheckState.NOT_OK - self.keys_write_capabilities[key_id] = CheckState.NOT_OK - self.keys_bucket_match[key_id] = CheckState.NOT_OK + kwargs['keys_exist'][key_id] = CheckState.NOT_OK + kwargs['keys_write_capabilities'][key_id] = CheckState.NOT_OK + kwargs['keys_bucket_match'][key_id] = CheckState.NOT_OK except AccessDenied: - self.keys = dict.fromkeys(keys_to_check, None) - self.keys_exist = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) - self.keys_write_capabilities = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) - self.keys_bucket_match = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) + kwargs['_keys'] = dict.fromkeys(keys_to_check, None) + kwargs['keys_exist'] = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) + kwargs['keys_write_capabilities'] = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) + kwargs['keys_bucket_match'] = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) + + return cls(**kwargs) def other_party_data(self): return OtherPartyReplicationCheckData( - bucket=self.bucket, - keys_mapping=self.keys, + bucket=self._bucket, + keys_mapping=self._keys, ) +@dataclass class TwoWayReplicationCheck(ReplicationCheck): - """ - is_enabled + is_enabled: CheckState - source_key_exists - source_key_bucket_match - source_key_read_capabilities - source_key_name_prefix_match + source_key_exists: CheckState + source_key_bucket_match: CheckState + source_key_read_capabilities: CheckState + source_key_name_prefix_match: CheckState - source_key_accepted_in_target_bucket + source_key_accepted_in_target_bucket: CheckState - destination_key_exists - destination_key_bucket_match - destination_key_write_capabilities - destination_key_name_prefix_match + destination_key_exists: CheckState + destination_key_bucket_match: CheckState + destination_key_write_capabilities: CheckState + destination_key_name_prefix_match: CheckState - file_lock_match - """ + file_lock_match: CheckState - def __init__( - self, + @classmethod + def from_data( + cls, source_bucket: BucketStructure, replication_rule_name: str, source_application_key: Union[Optional[ApplicationKey], AccessDeniedEnum], destination_bucket: BucketStructure, destination_application_keys: Dict[str, Union[Optional[ApplicationKey]], AccessDeniedEnum], - ): + ) -> 'TwoWayReplicationCheck': + kwargs = {} + rules = [ - r for r in source_bucket.replication.as_replication_source.replication_rules - if r.replication_rule_name == replication_rule_name + rule for rule in source_bucket.replication.rules if rule.name == replication_rule_name ] assert rules - self._rule = rules[0] - self.is_enabled = CheckState.OK if self._rule.is_enabled else CheckState.NOT_OK + rule = rules[0] + + kwargs['is_enabled'] = CheckState.from_bool(rule.is_enabled), ( - self.source_key_exists, - self.source_key_bucket_match, - self.source_key_read_capabilities, - self.source_key_name_prefix_match, - ) = self._check_key( - source_application_key, 'readFiles', self._rule.file_name_prefix, source_bucket.id_ + kwargs['source_key_exists'], + kwargs['source_key_bucket_match'], + kwargs['source_key_read_capabilities'], + kwargs['source_key_name_prefix_match'], + ) = cls._check_key( + source_application_key, 'readFiles', rule.file_name_prefix, source_bucket.id_ ) - if destination_bucket.replication is None or destination_bucket.replication.as_replication_destination is None: - destination_application_key_id = None - else: - destination_application_key_id = destination_bucket.replication.as_replication_destination.\ - source_to_destination_key_mapping.get( - source_bucket.replication.as_replication_source.source_application_key_id - ) + destination_application_key_id = destination_bucket.replication and destination_bucket.replication.source_to_destination_key_mapping.get( + source_bucket.replication.source_key_id + ) - if destination_application_key_id is None: - self.source_key_accepted_in_target_bucket = CheckState.NOT_OK - else: - self.source_key_accepted_in_target_bucket = CheckState.OK + kwargs['source_key_accepted_in_target_bucket'] = CheckState.from_bool( + destination_application_key_id is not None + ) destination_application_key = destination_application_keys.get( destination_application_key_id ) ( - self.destination_key_exists, - self.destination_key_bucket_match, - self.destination_key_read_capabilities, - self.destination_key_key_name_prefix_match, - ) = self._check_key( - destination_application_key, 'writeFiles', self._rule.file_name_prefix, - destination_bucket.id_ + kwargs['destination_key_exists'], + kwargs['destination_key_bucket_match'], + kwargs['destination_key_read_capabilities'], + kwargs['destination_key_key_name_prefix_match'], + ) = cls._check_key( + destination_application_key, 'writeFiles', rule.file_name_prefix, destination_bucket.id_ ) if destination_bucket.is_file_lock_enabled: - self.file_lock_match = CheckState.OK + kwargs['file_lock_match'] = CheckState.OK elif source_bucket.is_file_lock_enabled is False: - self.file_lock_match = CheckState.OK + kwargs['file_lock_match'] = CheckState.OK elif source_bucket.is_file_lock_enabled is None or destination_bucket.is_file_lock_enabled is None: - self.file_lock_match = CheckState.UNKNOWN + kwargs['file_lock_match'] = CheckState.UNKNOWN else: - self.file_lock_match = CheckState.NOT_OK + kwargs['file_lock_match'] = CheckState.NOT_OK + + return cls(**kwargs) class OtherPartyReplicationCheckData: @@ -349,7 +340,8 @@ def _dump_key(self, key: Union[Optional[ApplicationKey], AccessDeniedEnum]): return key.as_dict() @classmethod - def _parse_key(cls, key_representation: Union[None, str, dict]) -> Union[Optional[ApplicationKey], AccessDeniedEnum]: + def _parse_key(cls, key_representation: Union[None, str, dict] + ) -> Union[Optional[ApplicationKey], AccessDeniedEnum]: if key_representation is None: return None try: From b253a995fa9cf50dd4a5339ec0ebbd24822965f9 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 4 Aug 2022 14:12:58 +0300 Subject: [PATCH 13/23] Even more refactor replication/check.py; update tests --- b2sdk/_v3/__init__.py | 6 + b2sdk/raw_simulator.py | 3 - b2sdk/replication/check.py | 223 +++++++------------ noxfile.py | 1 + test/unit/replication/conftest.py | 74 ++++-- test/unit/replication/test_troubleshooter.py | 84 +++++++ 6 files changed, 232 insertions(+), 159 deletions(-) create mode 100644 test/unit/replication/test_troubleshooter.py diff --git a/b2sdk/_v3/__init__.py b/b2sdk/_v3/__init__.py index da2184b57..55a6f1289 100644 --- a/b2sdk/_v3/__init__.py +++ b/b2sdk/_v3/__init__.py @@ -221,6 +221,12 @@ from b2sdk.replication.monitoring import ReplicationScanResult from b2sdk.replication.monitoring import ReplicationReport from b2sdk.replication.monitoring import ReplicationMonitor +from b2sdk.replication.check import TwoWayReplicationCheckGenerator +from b2sdk.replication.check import ReplicationSourceCheck +from b2sdk.replication.check import ReplicationDestinationCheck +from b2sdk.replication.check import TwoWayReplicationCheck +from b2sdk.replication.check import OtherPartyReplicationCheckData +from b2sdk.replication.check import CheckState # other diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 074808ae4..e8842e95d 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -526,9 +526,6 @@ def __init__( self.is_file_lock_enabled = is_file_lock_enabled self.default_retention = NO_RETENTION_BUCKET_SETTING self.replication = replication - if self.replication is not None: - assert self.replication.asReplicationSource is None or self.replication.asReplicationSource.rules - assert self.replication.asReplicationDestination is None or self.replication.asReplicationDestination.sourceToDestinationKeyMapping def is_allowed_to_read_bucket_encryption_setting(self, account_auth_token): return self._check_capability(account_auth_token, 'readBucketEncryption') diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py index ad7774c3a..2e86ac768 100644 --- a/b2sdk/replication/check.py +++ b/b2sdk/replication/check.py @@ -12,7 +12,7 @@ import warnings from dataclasses import dataclass -from typing import Dict, Generator, Optional, Tuple, Union +from typing import Dict, Generator, Optional, Union from b2sdk import version from b2sdk.api import B2Api @@ -21,12 +21,11 @@ from b2sdk.exception import AccessDenied, BucketIdNotFound -class ReplicationFilter: - def __init__(self, source_api: B2Api, destination_api: B2Api, filters: ...): - self.source_api = source_api - self.destination_api = destination_api - - # def get_checks +def _safe_get_key(api: B2Api, key_id: str) -> Union[None, AccessDenied, ApplicationKey]: + try: + return api.get_key(key_id) + except AccessDenied: + return AccessDenied() @dataclass @@ -38,12 +37,12 @@ class TwoWayReplicationCheckGenerator: filter_replication_rule_name: Optional[str] = None file_name_prefix: Optional[str] = None - def get_checks(self) -> Generator['ReplicationCheck']: + def get_checks(self) -> Generator['ReplicationCheck', None, None]: source_buckets = self.source_api.list_buckets(bucket_name=self.filter_source_bucket_name) for source_bucket in source_buckets: - yield from self.get_source_bucket_checks() + yield from self._get_source_bucket_checks(source_bucket) - def get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['ReplicationCheck']: + def _get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['ReplicationCheck', None, None]: if not source_bucket.replication: return @@ -67,7 +66,7 @@ def get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['Replicat if not destination_bucket_list: raise BucketIdNotFound() except (AccessDenied, BucketIdNotFound): - yield ReplicationSourceCheck.from_data(source_bucket, rule.replication_rule_name) + yield ReplicationSourceCheck.from_data(source_bucket, rule.name) continue if ( @@ -78,7 +77,7 @@ def get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['Replicat yield TwoWayReplicationCheck.from_data( source_bucket=source_bucket, - replication_rule_name=rule.replication_rule_name, + replication_rule_name=rule.name, source_application_key=source_key, destination_bucket=destination_bucket_list[0], destination_application_keys=self._get_destination_bucket_keys( @@ -88,7 +87,7 @@ def get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['Replicat @classmethod def _get_destination_bucket_keys(cls, destination_bucket: Bucket) -> \ - Dict[str, Union[None, ApplicationKey, 'AccessDeniedEnum']]: + Dict[str, Union[None, ApplicationKey, AccessDenied]]: if not destination_bucket.replication: return {} @@ -96,7 +95,7 @@ def _get_destination_bucket_keys(cls, destination_bucket: Bucket) -> \ try: return {key_id: destination_bucket.api.get_key(key_id) for key_id in key_ids} except AccessDenied: - return dict.fromkeys(key_ids, AccessDeniedEnum.ACCESS_DENIED) + return dict.fromkeys(key_ids, AccessDenied()) @enum.unique @@ -113,64 +112,68 @@ def from_bool(cls, value: bool) -> 'CheckState': return cls.OK if value else cls.NOT_OK -class AccessDeniedEnum(enum.Enum): - ACCESS_DENIED = 'ACCESS_DENIED' - - class ReplicationCheck: @classmethod def _check_key( cls, - key: Union[Optional[ApplicationKey], AccessDeniedEnum], + key: Union[None, ApplicationKey, AccessDenied], capability: str, replication_name_prefix: str, bucket_id: str, - ) -> Tuple[CheckState, CheckState, CheckState, CheckState]: - if key == AccessDeniedEnum.ACCESS_DENIED: - return (CheckState.UNKNOWN,) * 4 - if key is None: - return (CheckState.NOT_OK,) * 4 - return ( - CheckState.OK, - CheckState.from_bool(key.bucket_id is None or key.bucket_id == bucket_id), - CheckState.from_bool(capability in key.capabilities), - CheckState.from_bool( - key.name_prefix is None or replication_name_prefix.startswith(key.name_prefix) - ), - ) + ) -> Dict[str, CheckState]: + + result = { + 'key_exists': CheckState.UNKNOWN, + 'key_bucket_match': CheckState.UNKNOWN, + 'key_capabilities': CheckState.UNKNOWN, + 'key_name_prefix_match': CheckState.UNKNOWN, + } + + if isinstance(key, AccessDenied): + pass + + elif key is None: + result = {k: CheckState.NOT_OK for k in result.keys()} + + else: + result.update({ + 'key_exists': CheckState.OK, + 'key_bucket_match': CheckState.from_bool(key.bucket_id is None or key.bucket_id == bucket_id), + 'key_capabilities': CheckState.from_bool(capability in key.capabilities), + 'key_name_prefix_match': CheckState.from_bool( + key.name_prefix is None or replication_name_prefix.startswith(key.name_prefix) + ), + }) + + return result @dataclass class ReplicationSourceCheck(ReplicationCheck): key_exists: CheckState - key_read_capabilities: CheckState + key_bucket_match: CheckState + key_capabilities: CheckState key_name_prefix_match: CheckState + is_enabled: CheckState _bucket: Bucket - _application_key: Union[None, AccessDeniedEnum, ApplicationKey] + _application_key: Union[None, AccessDenied, ApplicationKey] @classmethod def from_data(cls, bucket: Bucket, rule_name: str) -> 'ReplicationSourceCheck': - kwargs = { - '_bucket': bucket, - } - application_key = _safe_get_key(bucket.api, bucket.replication.source_key_id) - kwargs['_application_key'] = application_key rules = [rule for rule in bucket.replication.rules if rule.name == rule_name] assert rules rule = rules[0] - kwargs['is_enabled'] = rule.is_enabled - - ( - kwargs['key_exists'], - _, # kwargs['key_bucket_match'], - kwargs['key_read_capabilities'], - kwargs['key_name_prefix_match'], - ) = cls._check_key(application_key, 'readFiles', rule.file_name_prefix, bucket.id_) + kwargs = { + '_bucket': bucket, + '_application_key': application_key, + 'is_enabled': CheckState.from_bool(rule.is_enabled), + **cls._check_key(application_key, 'readFiles', rule.file_name_prefix, bucket.id_), + } return cls(**kwargs) @@ -182,50 +185,29 @@ def other_party_data(self): @dataclass -class ReplicationDestinationCheck: - key_exist: Dict[str, CheckState] - keys_write_capabilities: Dict[str, CheckState] - keys_bucket_match: Dict[str, CheckState] +class ReplicationDestinationCheck(ReplicationCheck): + key_exists: CheckState + key_capabilities: CheckState + key_bucket_match: CheckState + key_name_prefix_match: CheckState _bucket: Bucket - _keys: Dict[str, Union[Optional[ApplicationKey], AccessDeniedEnum]] + _application_key: Union[None, AccessDenied, ApplicationKey] @classmethod - def from_data(cls, bucket: Bucket) -> 'ReplicationDestinationCheck': + def iter_by_keys(cls, bucket: Bucket) -> Generator['ReplicationDestinationCheck', None, None]: + keys_to_check = bucket.replication.source_to_destination_key_mapping.values() + for key_id in keys_to_check: + yield cls.from_data(bucket=bucket, key_id=key_id) + + @classmethod + def from_data(cls, bucket: Bucket, key_id: str) -> 'ReplicationDestinationCheck': + application_key = _safe_get_key(bucket.api, key_id) kwargs = { '_bucket': bucket, - '_keys': {}, - 'key_exist': {}, - 'keys_write_capabilities': {}, - 'keys_bucket_match': {}, + '_application_key': application_key, + **cls._check_key(application_key, 'writeFiles', '', bucket.id_), } - - keys_to_check = bucket.replication.source_to_destination_key_mapping.values() - try: - for key_id in keys_to_check: - application_key = bucket.api.get_key(bucket.replication.source_key_id) - kwargs['_keys'][key_id] = application_key - - if application_key: - kwargs['keys_exist'][key_id] = CheckState.OK - kwargs['keys_write_capabilities'][key_id] = CheckState.from_bool( - 'writeFiles' in application_key.capabilities - ) - kwargs['keys_bucket_match'][key_id] = CheckState.from_bool( - application_key.bucket_id is None or application_key.bucket_id == bucket.id_ - ) - else: - kwargs['keys_exist'][key_id] = CheckState.NOT_OK - kwargs['keys_write_capabilities'][key_id] = CheckState.NOT_OK - kwargs['keys_bucket_match'][key_id] = CheckState.NOT_OK - - except AccessDenied: - - kwargs['_keys'] = dict.fromkeys(keys_to_check, None) - kwargs['keys_exist'] = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) - kwargs['keys_write_capabilities'] = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) - kwargs['keys_bucket_match'] = dict.fromkeys(keys_to_check, CheckState.UNKNOWN) - return cls(**kwargs) def other_party_data(self): @@ -237,20 +219,9 @@ def other_party_data(self): @dataclass class TwoWayReplicationCheck(ReplicationCheck): - is_enabled: CheckState - - source_key_exists: CheckState - source_key_bucket_match: CheckState - source_key_read_capabilities: CheckState - source_key_name_prefix_match: CheckState - + source: ReplicationSourceCheck + destination: ReplicationDestinationCheck source_key_accepted_in_target_bucket: CheckState - - destination_key_exists: CheckState - destination_key_bucket_match: CheckState - destination_key_write_capabilities: CheckState - destination_key_name_prefix_match: CheckState - file_lock_match: CheckState @classmethod @@ -258,27 +229,15 @@ def from_data( cls, source_bucket: BucketStructure, replication_rule_name: str, - source_application_key: Union[Optional[ApplicationKey], AccessDeniedEnum], + source_application_key: Union[None, ApplicationKey, AccessDenied], destination_bucket: BucketStructure, - destination_application_keys: Dict[str, Union[Optional[ApplicationKey]], AccessDeniedEnum], + destination_application_keys: Dict[str, Union[None, ApplicationKey, AccessDenied]], ) -> 'TwoWayReplicationCheck': kwargs = {} - rules = [ - rule for rule in source_bucket.replication.rules if rule.name == replication_rule_name - ] - assert rules - rule = rules[0] - - kwargs['is_enabled'] = CheckState.from_bool(rule.is_enabled), - - ( - kwargs['source_key_exists'], - kwargs['source_key_bucket_match'], - kwargs['source_key_read_capabilities'], - kwargs['source_key_name_prefix_match'], - ) = cls._check_key( - source_application_key, 'readFiles', rule.file_name_prefix, source_bucket.id_ + kwargs['source'] = ReplicationSourceCheck.from_data( + bucket=source_bucket, + rule_name=replication_rule_name, ) destination_application_key_id = destination_bucket.replication and destination_bucket.replication.source_to_destination_key_mapping.get( @@ -293,13 +252,9 @@ def from_data( destination_application_key_id ) - ( - kwargs['destination_key_exists'], - kwargs['destination_key_bucket_match'], - kwargs['destination_key_read_capabilities'], - kwargs['destination_key_key_name_prefix_match'], - ) = cls._check_key( - destination_application_key, 'writeFiles', rule.file_name_prefix, destination_bucket.id_ + kwargs['destination'] = ReplicationDestinationCheck.from_data( + bucket=destination_bucket, + key_id=destination_application_key.id_, ) if destination_bucket.is_file_lock_enabled: @@ -320,7 +275,7 @@ class OtherPartyReplicationCheckData: def __init__( self, bucket: BucketStructure, - keys_mapping: Dict[str, Union[Optional[ApplicationKey], AccessDeniedEnum]], + keys_mapping: Dict[str, Union[Optional[ApplicationKey], AccessDenied]], b2sdk_version: Optional[str] = None ): @@ -332,22 +287,21 @@ def __init__( self.b2sdk_version = b2sdk_version @classmethod - def _dump_key(self, key: Union[Optional[ApplicationKey], AccessDeniedEnum]): + def _dump_key(self, key: Union[None, ApplicationKey, AccessDenied]): if key is None: return None - if isinstance(key, AccessDeniedEnum): - return key.value + if isinstance(key, AccessDenied): + return key.__class__.__name__ return key.as_dict() @classmethod - def _parse_key(cls, key_representation: Union[None, str, dict] - ) -> Union[Optional[ApplicationKey], AccessDeniedEnum]: + def _parse_key(cls, key_representation: Union[None, str, dict]) -> Union[None, ApplicationKey, AccessDenied]: if key_representation is None: return None - try: - return AccessDeniedEnum(key_representation) - except ValueError: - pass + + if key_representation == AccessDenied.__name__: + return AccessDenied() + return ApplicationKey.from_dict(key_representation) def as_dict(self): @@ -374,10 +328,3 @@ def from_dict(cls, dict_: dict): keys_mapping={k: cls._parse_key(v) for k, v in dict_['keys_mapping'].items()} ) - - -def _safe_get_key(api: B2Api, key_id: str) -> Union[None, AccessDeniedEnum, ApplicationKey]: - try: - return api.get_key(key_id) - except AccessDenied: - return AccessDeniedEnum.ACCESS_DENIED diff --git a/noxfile.py b/noxfile.py index 6fac81a29..e30410a4e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -39,6 +39,7 @@ 'pytest-lazy-fixture==0.6.3', 'pyfakefs==4.5.6', 'pytest-xdist==2.5.0', + 'more_itertools==8.13.0', ] REQUIREMENTS_BUILD = ['setuptools>=20.2'] diff --git a/test/unit/replication/conftest.py b/test/unit/replication/conftest.py index 719f29816..378f7965a 100644 --- a/test/unit/replication/conftest.py +++ b/test/unit/replication/conftest.py @@ -8,9 +8,11 @@ # ###################################################################### +from typing import Union + import pytest -from apiver_deps import B2Api, B2HttpApiConfig, Bucket, RawSimulator, ReplicationConfiguration, ReplicationMonitor, ReplicationRule, StubAccountInfo +from apiver_deps import B2Api, B2HttpApiConfig, Bucket, FullApplicationKey, RawSimulator, ReplicationConfiguration, ReplicationMonitor, ReplicationRule, StubAccountInfo, TwoWayReplicationCheckGenerator @pytest.fixture @@ -24,32 +26,60 @@ def api() -> B2Api: simulator = api.session.raw_api account_id, master_key = simulator.create_account() api.authorize_account('production', account_id, master_key) - # api_url = account_info.get_api_url() - # account_auth_token = account_info.get_account_auth_token()1 return api @pytest.fixture -def destination_bucket(api) -> Bucket: - return api.create_bucket('destination-bucket', 'allPublic') +def destination_key(api) -> Union[FullApplicationKey, dict]: + return api.create_key(capabilities='writeFiles', key_name='destination-key') @pytest.fixture -def source_bucket(api, destination_bucket) -> Bucket: - bucket = api.create_bucket('source-bucket', 'allPublic') - - bucket.replication = ReplicationConfiguration( - rules=[ - ReplicationRule( - destination_bucket_id=destination_bucket.id_, - name='name', - file_name_prefix='folder/', # TODO: is last slash needed? - ), - ], - source_key_id='hoho|trololo', +def destination_key_id(destination_key) -> str: + return destination_key.id_ + + +@pytest.fixture +def source_key(api) -> Union[FullApplicationKey, dict]: + return api.create_key(capabilities='readFiles', key_name='source-key') + + +@pytest.fixture +def source_key_id(source_key) -> str: + return source_key.id_ + + +@pytest.fixture +def destination_bucket(api, source_key_id, destination_key_id) -> Bucket: + return api.create_bucket( + name='destination-bucket', + bucket_type='allPublic', + is_file_lock_enabled=False, + replication=ReplicationConfiguration( + source_to_destination_key_mapping={ + source_key_id: destination_key_id, + }, + ), ) - return bucket + +@pytest.fixture +def source_bucket(api, destination_bucket, source_key_id) -> Bucket: + return api.create_bucket( + name='source-bucket', + bucket_type='allPublic', + is_file_lock_enabled=False, + replication=ReplicationConfiguration( + rules=[ + ReplicationRule( + destination_bucket_id=destination_bucket.id_, + name='name', + file_name_prefix='folder/', # TODO: is last slash needed? + ), + ], + source_key_id=source_key_id, + ), + ) @pytest.fixture @@ -72,3 +102,11 @@ def monitor(source_bucket) -> ReplicationMonitor: source_bucket, rule=source_bucket.replication.rules[0], ) + + +@pytest.fixture +def troubleshooter(source_bucket, destination_bucket) -> TwoWayReplicationCheckGenerator: + return TwoWayReplicationCheckGenerator( + source_api=source_bucket.api, + destination_api=destination_bucket.api, + ) diff --git a/test/unit/replication/test_troubleshooter.py b/test/unit/replication/test_troubleshooter.py new file mode 100644 index 000000000..f5d359307 --- /dev/null +++ b/test/unit/replication/test_troubleshooter.py @@ -0,0 +1,84 @@ +import pytest + +from apiver_deps import CheckState, TwoWayReplicationCheck, TwoWayReplicationCheckGenerator +from more_itertools import one + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_source_bucket_name_filter(api, source_bucket, destination_bucket): + bucket_name = source_bucket.name + + # check original name filter + troubleshooter = TwoWayReplicationCheckGenerator( + source_api=source_bucket.api, + destination_api=destination_bucket.api, + filter_source_bucket_name=bucket_name, + ) + assert len(list(troubleshooter.get_checks())) == 1 + + # check other name filter + troubleshooter = TwoWayReplicationCheckGenerator( + source_api=source_bucket.api, + destination_api=destination_bucket.api, + filter_source_bucket_name=bucket_name + '-other', + ) + assert len(list(troubleshooter.get_checks())) == 0 + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_rule_name_filter(api, source_bucket, destination_bucket): + rule_name = source_bucket.replication.rules[0].name + + # check original name filter + troubleshooter = TwoWayReplicationCheckGenerator( + source_api=source_bucket.api, + destination_api=destination_bucket.api, + filter_replication_rule_name=rule_name, + ) + assert len(list(troubleshooter.get_checks())) == 1 + + # check other name filter + troubleshooter = TwoWayReplicationCheckGenerator( + source_api=source_bucket.api, + destination_api=destination_bucket.api, + filter_replication_rule_name=rule_name + '-other', + ) + assert len(list(troubleshooter.get_checks())) == 0 + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_all_ok(api, source_bucket, troubleshooter): + check = one(troubleshooter.get_checks()) + assert isinstance(check, TwoWayReplicationCheck) + + assert check.source.is_enabled == CheckState.OK + assert check.source.key_exists == CheckState.OK + assert check.source.key_bucket_match == CheckState.OK + assert check.source.key_capabilities == CheckState.OK + assert check.source.key_name_prefix_match == CheckState.OK + assert check.source_key_accepted_in_target_bucket == CheckState.OK + assert check.destination.key_exists == CheckState.OK + assert check.destination.key_bucket_match == CheckState.OK + assert check.destination.key_capabilities == CheckState.OK + assert check.destination.key_name_prefix_match == CheckState.OK + assert check.file_lock_match == CheckState.OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_rule_not_enabled(api, source_bucket, troubleshooter): + replication = source_bucket.replication + replication.rules[0].is_enabled = False + source_bucket.update(replication=replication) + + check = one(troubleshooter.get_checks()) + assert check.source.is_enabled == CheckState.NOT_OK + + +# @pytest.mark.apiver(from_ver=2) +# def test_troubleshooter_rule_not_enabled(api, source_bucket, troubleshooter): +# replication = source_bucket.replication +# replication.rules[0].is_enabled = False +# source_bucket.update(replication=replication) + +# check = one(troubleshooter.get_checks()) +# assert check.source.is_enabled == CheckState.NOT_OK From a9c71595f2a5b97306932c993ee993d9a3c01a69 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 4 Aug 2022 14:18:19 +0300 Subject: [PATCH 14/23] Refactor TwoWayReplicationCheck --- b2sdk/replication/check.py | 44 +++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py index 2e86ac768..9d4a1c58a 100644 --- a/b2sdk/replication/check.py +++ b/b2sdk/replication/check.py @@ -233,38 +233,34 @@ def from_data( destination_bucket: BucketStructure, destination_application_keys: Dict[str, Union[None, ApplicationKey, AccessDenied]], ) -> 'TwoWayReplicationCheck': - kwargs = {} - - kwargs['source'] = ReplicationSourceCheck.from_data( - bucket=source_bucket, - rule_name=replication_rule_name, - ) destination_application_key_id = destination_bucket.replication and destination_bucket.replication.source_to_destination_key_mapping.get( source_bucket.replication.source_key_id ) - kwargs['source_key_accepted_in_target_bucket'] = CheckState.from_bool( - destination_application_key_id is not None - ) - - destination_application_key = destination_application_keys.get( - destination_application_key_id - ) - - kwargs['destination'] = ReplicationDestinationCheck.from_data( - bucket=destination_bucket, - key_id=destination_application_key.id_, - ) - if destination_bucket.is_file_lock_enabled: - kwargs['file_lock_match'] = CheckState.OK + file_lock_match = CheckState.OK elif source_bucket.is_file_lock_enabled is False: - kwargs['file_lock_match'] = CheckState.OK + file_lock_match = CheckState.OK elif source_bucket.is_file_lock_enabled is None or destination_bucket.is_file_lock_enabled is None: - kwargs['file_lock_match'] = CheckState.UNKNOWN + file_lock_match = CheckState.UNKNOWN else: - kwargs['file_lock_match'] = CheckState.NOT_OK + file_lock_match = CheckState.NOT_OK + + kwargs = { + 'source': ReplicationSourceCheck.from_data( + bucket=source_bucket, + rule_name=replication_rule_name, + ), + 'destination': ReplicationDestinationCheck.from_data( + bucket=destination_bucket, + key_id=destination_application_key_id, + ), + 'source_key_accepted_in_target_bucket': CheckState.from_bool( + destination_application_key_id is not None + ), + 'file_lock_match': file_lock_match, + } return cls(**kwargs) @@ -275,7 +271,7 @@ class OtherPartyReplicationCheckData: def __init__( self, bucket: BucketStructure, - keys_mapping: Dict[str, Union[Optional[ApplicationKey], AccessDenied]], + keys_mapping: Dict[str, Union[None, ApplicationKey, AccessDenied]], b2sdk_version: Optional[str] = None ): From 4374c2757d206eeaeb8d56562688dfdc669a8dbf Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 4 Aug 2022 20:37:09 +0300 Subject: [PATCH 15/23] Fix RawSimulator not deleting key from `all_application_keys` in `delete_key()` --- b2sdk/raw_simulator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index e8842e95d..e6fc18f77 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -1397,6 +1397,10 @@ def delete_key(self, api_url, account_auth_token, application_key_id): 'application key does not exist: %s' % (application_key_id,), 'bad_request', ) + self.all_application_keys = [ + key for key in self.all_application_keys + if key.application_key_id != application_key_id + ] return key_sim.as_key() def finish_large_file(self, api_url, account_auth_token, file_id, part_sha1_array): From 92a4cf82f480a60a681f3e58f973d32f5845397a Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 4 Aug 2022 20:40:18 +0300 Subject: [PATCH 16/23] Fix B2Api.get_key() returning wrong key if `key_id` is incorrect --- b2sdk/api.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index 1e3714f6f..121b98e93 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -9,6 +9,7 @@ ###################################################################### from typing import Optional, Tuple, List, Generator +from contextlib import suppress from .account_info.abstract import AbstractAccountInfo from .api_config import B2HttpApiConfig, DEFAULT_HTTP_API_CONFIG @@ -539,10 +540,13 @@ def get_key(self, key_id: str) -> Optional[ApplicationKey]: Raises an exception if profile is not permitted to list keys. """ - return next( - self.list_keys(start_application_key_id=key_id), - None, - ) + with suppress(StopIteration): + key = next(self.list_keys(start_application_key_id=key_id)) + + # list_keys() may return some other key if `key_id` does not exist; + # thus manually check that we retrieved the right key + if key.id_ == key_id: + return key # other def get_file_info(self, file_id: str) -> FileVersion: From 346a24c3ad20df1071a3a0777f7b02edd0d7bb86 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 4 Aug 2022 20:44:36 +0300 Subject: [PATCH 17/23] Fix Pawel using "it's" instead of "its" --- b2sdk/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index 121b98e93..4ddc05e2c 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -534,7 +534,7 @@ def list_keys(self, start_application_key_id: Optional[str] = None def get_key(self, key_id: str) -> Optional[ApplicationKey]: """ - Gets information about a single key: it's capabilities, prefix, name etc + Gets information about a single key: its capabilities, prefix, name etc Returns `None` if the key does not exist. From 774080dffe788294371e2723a70b29ea13c5e306 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 4 Aug 2022 20:46:22 +0300 Subject: [PATCH 18/23] Fix Michal using list(map(lambda...)) instead of list comprehension --- b2sdk/raw_simulator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index e6fc18f77..6ddfa9014 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -1645,8 +1645,8 @@ def list_keys( next_application_key_id = all_keys_sorted[ind + 1].application_key_id break - key_dicts = map(lambda key: key.as_key(), keys) - return dict(keys=list(key_dicts), nextApplicationKeyId=next_application_key_id) + key_dicts = [key.as_key() for key in keys] + return dict(keys=key_dicts, nextApplicationKeyId=next_application_key_id) def list_parts(self, api_url, account_auth_token, file_id, start_part_number, max_part_count): bucket_id = self.file_id_to_bucket_id[file_id] From 3ce142d699709fa21f06a2cdc510e2c8ec7ac3c6 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 4 Aug 2022 20:51:29 +0300 Subject: [PATCH 19/23] Fix lint issues, update tests --- b2sdk/raw_simulator.py | 3 +- b2sdk/replication/check.py | 74 ++++++++++---------- test/unit/replication/test_troubleshooter.py | 25 ++++--- 3 files changed, 55 insertions(+), 47 deletions(-) diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 6ddfa9014..c108a40ea 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -1398,8 +1398,7 @@ def delete_key(self, api_url, account_auth_token, application_key_id): 'bad_request', ) self.all_application_keys = [ - key for key in self.all_application_keys - if key.application_key_id != application_key_id + key for key in self.all_application_keys if key.application_key_id != application_key_id ] return key_sim.as_key() diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py index 9d4a1c58a..629e54a60 100644 --- a/b2sdk/replication/check.py +++ b/b2sdk/replication/check.py @@ -42,7 +42,8 @@ def get_checks(self) -> Generator['ReplicationCheck', None, None]: for source_bucket in source_buckets: yield from self._get_source_bucket_checks(source_bucket) - def _get_source_bucket_checks(self, source_bucket: Bucket) -> Generator['ReplicationCheck', None, None]: + def _get_source_bucket_checks(self, source_bucket: Bucket + ) -> Generator['ReplicationCheck', None, None]: if not source_bucket.replication: return @@ -136,14 +137,21 @@ def _check_key( result = {k: CheckState.NOT_OK for k in result.keys()} else: - result.update({ - 'key_exists': CheckState.OK, - 'key_bucket_match': CheckState.from_bool(key.bucket_id is None or key.bucket_id == bucket_id), - 'key_capabilities': CheckState.from_bool(capability in key.capabilities), - 'key_name_prefix_match': CheckState.from_bool( - key.name_prefix is None or replication_name_prefix.startswith(key.name_prefix) - ), - }) + result.update( + { + 'key_exists': + CheckState.OK, + 'key_bucket_match': + CheckState.from_bool(key.bucket_id is None or key.bucket_id == bucket_id), + 'key_capabilities': + CheckState.from_bool(capability in key.capabilities), + 'key_name_prefix_match': + CheckState.from_bool( + key.name_prefix is None or + replication_name_prefix.startswith(key.name_prefix) + ), + } + ) return result @@ -248,39 +256,30 @@ def from_data( file_lock_match = CheckState.NOT_OK kwargs = { - 'source': ReplicationSourceCheck.from_data( - bucket=source_bucket, - rule_name=replication_rule_name, - ), - 'destination': ReplicationDestinationCheck.from_data( - bucket=destination_bucket, - key_id=destination_application_key_id, - ), - 'source_key_accepted_in_target_bucket': CheckState.from_bool( - destination_application_key_id is not None - ), - 'file_lock_match': file_lock_match, + 'source': + ReplicationSourceCheck.from_data( + bucket=source_bucket, + rule_name=replication_rule_name, + ), + 'destination': + ReplicationDestinationCheck.from_data( + bucket=destination_bucket, + key_id=destination_application_key_id, + ), + 'source_key_accepted_in_target_bucket': + CheckState.from_bool(destination_application_key_id is not None), + 'file_lock_match': + file_lock_match, } return cls(**kwargs) +@dataclass class OtherPartyReplicationCheckData: - b2sdk_version = version.VERSION - - def __init__( - self, - bucket: BucketStructure, - keys_mapping: Dict[str, Union[None, ApplicationKey, AccessDenied]], - b2sdk_version: Optional[str] = None - ): - - self.bucket = bucket - self.keys_mapping = keys_mapping - if b2sdk_version is None: - self.b2sdk_version = type(self).b2sdk_version - else: - self.b2sdk_version = b2sdk_version + bucket: BucketStructure + keys_mapping: Dict[str, Union[None, ApplicationKey, AccessDenied]] + b2sdk_version: version = version.VERSION @classmethod def _dump_key(self, key: Union[None, ApplicationKey, AccessDenied]): @@ -291,7 +290,8 @@ def _dump_key(self, key: Union[None, ApplicationKey, AccessDenied]): return key.as_dict() @classmethod - def _parse_key(cls, key_representation: Union[None, str, dict]) -> Union[None, ApplicationKey, AccessDenied]: + def _parse_key(cls, key_representation: Union[None, str, dict] + ) -> Union[None, ApplicationKey, AccessDenied]: if key_representation is None: return None diff --git a/test/unit/replication/test_troubleshooter.py b/test/unit/replication/test_troubleshooter.py index f5d359307..4cac5d367 100644 --- a/test/unit/replication/test_troubleshooter.py +++ b/test/unit/replication/test_troubleshooter.py @@ -1,3 +1,13 @@ +###################################################################### +# +# File: test/unit/replication/test_troubleshooter.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + import pytest from apiver_deps import CheckState, TwoWayReplicationCheck, TwoWayReplicationCheckGenerator @@ -65,7 +75,7 @@ def test_troubleshooter_all_ok(api, source_bucket, troubleshooter): @pytest.mark.apiver(from_ver=2) -def test_troubleshooter_rule_not_enabled(api, source_bucket, troubleshooter): +def test_troubleshooter_source_not_enabled(api, source_bucket, troubleshooter): replication = source_bucket.replication replication.rules[0].is_enabled = False source_bucket.update(replication=replication) @@ -74,11 +84,10 @@ def test_troubleshooter_rule_not_enabled(api, source_bucket, troubleshooter): assert check.source.is_enabled == CheckState.NOT_OK -# @pytest.mark.apiver(from_ver=2) -# def test_troubleshooter_rule_not_enabled(api, source_bucket, troubleshooter): -# replication = source_bucket.replication -# replication.rules[0].is_enabled = False -# source_bucket.update(replication=replication) +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_source_key_does_not_exist(api, source_bucket, source_key, troubleshooter): + api.delete_key(source_key) + assert not api.get_key(source_key.id_) -# check = one(troubleshooter.get_checks()) -# assert check.source.is_enabled == CheckState.NOT_OK + check = one(troubleshooter.get_checks()) + assert check.source.key_exists == CheckState.NOT_OK From a75267a480588f18a5a63aeaeb3e19dc9c4cba77 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 4 Aug 2022 22:57:27 +0300 Subject: [PATCH 20/23] Bug fixes; add as_dict() --- b2sdk/replication/check.py | 30 ++++++++++++++++---- test/unit/replication/test_troubleshooter.py | 14 ++++----- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py index 629e54a60..ea4db35c5 100644 --- a/b2sdk/replication/check.py +++ b/b2sdk/replication/check.py @@ -11,7 +11,7 @@ import enum import warnings -from dataclasses import dataclass +from dataclasses import dataclass, fields from typing import Dict, Generator, Optional, Union from b2sdk import version @@ -37,13 +37,13 @@ class TwoWayReplicationCheckGenerator: filter_replication_rule_name: Optional[str] = None file_name_prefix: Optional[str] = None - def get_checks(self) -> Generator['ReplicationCheck', None, None]: + def iter_checks(self) -> Generator['ReplicationCheck', None, None]: source_buckets = self.source_api.list_buckets(bucket_name=self.filter_source_bucket_name) for source_bucket in source_buckets: - yield from self._get_source_bucket_checks(source_bucket) + yield from self._iter_source_bucket_checks(source_bucket) - def _get_source_bucket_checks(self, source_bucket: Bucket - ) -> Generator['ReplicationCheck', None, None]: + def _iter_source_bucket_checks(self, source_bucket: Bucket + ) -> Generator['ReplicationCheck', None, None]: if not source_bucket.replication: return @@ -65,7 +65,7 @@ def _get_source_bucket_checks(self, source_bucket: Bucket bucket_id=rule.destination_bucket_id ) if not destination_bucket_list: - raise BucketIdNotFound() + raise BucketIdNotFound(rule.destination_bucket_id) except (AccessDenied, BucketIdNotFound): yield ReplicationSourceCheck.from_data(source_bucket, rule.name) continue @@ -155,6 +155,22 @@ def _check_key( return result + def as_dict(self) -> dict: + result = {} + for field in fields(self): + field_value = getattr(self, field.name) + + if isinstance(field_value, ReplicationCheck): + # source.key_exists = OK ===> {'source_key_exists': OK} + result.update({ + f'{field.name}_{key}' if not key[0] == '_' else f'_{field.name}_{key[1:]}': value + for key, value in field_value.as_dict().items() + }) + else: + result[field.name] = field_value + + return result + @dataclass class ReplicationSourceCheck(ReplicationCheck): @@ -166,6 +182,7 @@ class ReplicationSourceCheck(ReplicationCheck): is_enabled: CheckState _bucket: Bucket + _rule_name: str _application_key: Union[None, AccessDenied, ApplicationKey] @classmethod @@ -178,6 +195,7 @@ def from_data(cls, bucket: Bucket, rule_name: str) -> 'ReplicationSourceCheck': kwargs = { '_bucket': bucket, + '_rule_name': rule_name, '_application_key': application_key, 'is_enabled': CheckState.from_bool(rule.is_enabled), **cls._check_key(application_key, 'readFiles', rule.file_name_prefix, bucket.id_), diff --git a/test/unit/replication/test_troubleshooter.py b/test/unit/replication/test_troubleshooter.py index 4cac5d367..47eea4de1 100644 --- a/test/unit/replication/test_troubleshooter.py +++ b/test/unit/replication/test_troubleshooter.py @@ -24,7 +24,7 @@ def test_troubleshooter_source_bucket_name_filter(api, source_bucket, destinatio destination_api=destination_bucket.api, filter_source_bucket_name=bucket_name, ) - assert len(list(troubleshooter.get_checks())) == 1 + assert len(list(troubleshooter.iter_checks())) == 1 # check other name filter troubleshooter = TwoWayReplicationCheckGenerator( @@ -32,7 +32,7 @@ def test_troubleshooter_source_bucket_name_filter(api, source_bucket, destinatio destination_api=destination_bucket.api, filter_source_bucket_name=bucket_name + '-other', ) - assert len(list(troubleshooter.get_checks())) == 0 + assert len(list(troubleshooter.iter_checks())) == 0 @pytest.mark.apiver(from_ver=2) @@ -45,7 +45,7 @@ def test_troubleshooter_rule_name_filter(api, source_bucket, destination_bucket) destination_api=destination_bucket.api, filter_replication_rule_name=rule_name, ) - assert len(list(troubleshooter.get_checks())) == 1 + assert len(list(troubleshooter.iter_checks())) == 1 # check other name filter troubleshooter = TwoWayReplicationCheckGenerator( @@ -53,12 +53,12 @@ def test_troubleshooter_rule_name_filter(api, source_bucket, destination_bucket) destination_api=destination_bucket.api, filter_replication_rule_name=rule_name + '-other', ) - assert len(list(troubleshooter.get_checks())) == 0 + assert len(list(troubleshooter.iter_checks())) == 0 @pytest.mark.apiver(from_ver=2) def test_troubleshooter_all_ok(api, source_bucket, troubleshooter): - check = one(troubleshooter.get_checks()) + check = one(troubleshooter.iter_checks()) assert isinstance(check, TwoWayReplicationCheck) assert check.source.is_enabled == CheckState.OK @@ -80,7 +80,7 @@ def test_troubleshooter_source_not_enabled(api, source_bucket, troubleshooter): replication.rules[0].is_enabled = False source_bucket.update(replication=replication) - check = one(troubleshooter.get_checks()) + check = one(troubleshooter.iter_checks()) assert check.source.is_enabled == CheckState.NOT_OK @@ -89,5 +89,5 @@ def test_troubleshooter_source_key_does_not_exist(api, source_bucket, source_key api.delete_key(source_key) assert not api.get_key(source_key.id_) - check = one(troubleshooter.get_checks()) + check = one(troubleshooter.iter_checks()) assert check.source.key_exists == CheckState.NOT_OK From f264f386ddfed91d60f133f7a7233eac9b70f484 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Fri, 5 Aug 2022 17:07:00 +0300 Subject: [PATCH 21/23] Lint fix --- b2sdk/replication/check.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py index ea4db35c5..7ba7b947d 100644 --- a/b2sdk/replication/check.py +++ b/b2sdk/replication/check.py @@ -162,10 +162,13 @@ def as_dict(self) -> dict: if isinstance(field_value, ReplicationCheck): # source.key_exists = OK ===> {'source_key_exists': OK} - result.update({ - f'{field.name}_{key}' if not key[0] == '_' else f'_{field.name}_{key[1:]}': value - for key, value in field_value.as_dict().items() - }) + result.update( + { + f'{field.name}_{key}' if not key[0] == '_' else f'_{field.name}_{key[1:]}': + value + for key, value in field_value.as_dict().items() + } + ) else: result[field.name] = field_value From d352d6bf140a7cbc1d0ae7dfa6e0532c5bb409a1 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Mon, 15 Aug 2022 15:14:13 +0300 Subject: [PATCH 22/23] Add is_sse_c_disabled check --- b2sdk/replication/check.py | 3 +++ test/unit/replication/test_troubleshooter.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py index 7ba7b947d..b3ad078d3 100644 --- a/b2sdk/replication/check.py +++ b/b2sdk/replication/check.py @@ -18,6 +18,7 @@ from b2sdk.api import B2Api from b2sdk.application_key import ApplicationKey from b2sdk.bucket import Bucket, BucketFactory, BucketStructure +from b2sdk.encryption.types import EncryptionMode from b2sdk.exception import AccessDenied, BucketIdNotFound @@ -183,6 +184,7 @@ class ReplicationSourceCheck(ReplicationCheck): key_name_prefix_match: CheckState is_enabled: CheckState + is_sse_c_disabled: CheckState _bucket: Bucket _rule_name: str @@ -201,6 +203,7 @@ def from_data(cls, bucket: Bucket, rule_name: str) -> 'ReplicationSourceCheck': '_rule_name': rule_name, '_application_key': application_key, 'is_enabled': CheckState.from_bool(rule.is_enabled), + 'is_sse_c_disabled': CheckState.from_bool(bucket.default_server_side_encryption.mode != EncryptionMode.SSE_C), **cls._check_key(application_key, 'readFiles', rule.file_name_prefix, bucket.id_), } diff --git a/test/unit/replication/test_troubleshooter.py b/test/unit/replication/test_troubleshooter.py index 47eea4de1..962120590 100644 --- a/test/unit/replication/test_troubleshooter.py +++ b/test/unit/replication/test_troubleshooter.py @@ -66,7 +66,10 @@ def test_troubleshooter_all_ok(api, source_bucket, troubleshooter): assert check.source.key_bucket_match == CheckState.OK assert check.source.key_capabilities == CheckState.OK assert check.source.key_name_prefix_match == CheckState.OK + assert check.source.is_sse_c_disabled == CheckState.OK + assert check.source_key_accepted_in_target_bucket == CheckState.OK + assert check.destination.key_exists == CheckState.OK assert check.destination.key_bucket_match == CheckState.OK assert check.destination.key_capabilities == CheckState.OK From fa7aa4b02e7dc175d54b39841ff99c91e95ff8ef Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Mon, 15 Aug 2022 22:31:09 +0300 Subject: [PATCH 23/23] Add troubleshooter tests --- b2sdk/replication/check.py | 17 +- test/unit/replication/test_troubleshooter.py | 180 ++++++++++++++++++- 2 files changed, 191 insertions(+), 6 deletions(-) diff --git a/b2sdk/replication/check.py b/b2sdk/replication/check.py index b3ad078d3..f2aea0ab5 100644 --- a/b2sdk/replication/check.py +++ b/b2sdk/replication/check.py @@ -199,11 +199,18 @@ def from_data(cls, bucket: Bucket, rule_name: str) -> 'ReplicationSourceCheck': rule = rules[0] kwargs = { - '_bucket': bucket, - '_rule_name': rule_name, - '_application_key': application_key, - 'is_enabled': CheckState.from_bool(rule.is_enabled), - 'is_sse_c_disabled': CheckState.from_bool(bucket.default_server_side_encryption.mode != EncryptionMode.SSE_C), + '_bucket': + bucket, + '_rule_name': + rule_name, + '_application_key': + application_key, + 'is_enabled': + CheckState.from_bool(rule.is_enabled), + 'is_sse_c_disabled': + CheckState.from_bool( + bucket.default_server_side_encryption.mode != EncryptionMode.SSE_C + ), **cls._check_key(application_key, 'readFiles', rule.file_name_prefix, bucket.id_), } diff --git a/test/unit/replication/test_troubleshooter.py b/test/unit/replication/test_troubleshooter.py index 962120590..950072f48 100644 --- a/test/unit/replication/test_troubleshooter.py +++ b/test/unit/replication/test_troubleshooter.py @@ -10,7 +10,7 @@ import pytest -from apiver_deps import CheckState, TwoWayReplicationCheck, TwoWayReplicationCheckGenerator +from apiver_deps import CheckState, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, TwoWayReplicationCheck, TwoWayReplicationCheckGenerator from more_itertools import one @@ -94,3 +94,181 @@ def test_troubleshooter_source_key_does_not_exist(api, source_bucket, source_key check = one(troubleshooter.iter_checks()) assert check.source.key_exists == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_source_key_bucket_match(api, source_bucket, source_key, troubleshooter): + key = api.raw_api.key_id_to_key[source_key.id_] + + key.bucket_id_or_none = None + check = one(troubleshooter.iter_checks()) + assert check.source.key_bucket_match == CheckState.OK + + key.bucket_id_or_none = source_bucket.id_ + check = one(troubleshooter.iter_checks()) + assert check.source.key_bucket_match == CheckState.OK + + key.bucket_id_or_none = 'hehe-trololo' + check = one(troubleshooter.iter_checks()) + assert check.source.key_bucket_match == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_source_key_capabilities(api, source_bucket, source_key, troubleshooter): + key = api.raw_api.key_id_to_key[source_key.id_] + + key.capabilities = ['readFilesWithPepper'] + check = one(troubleshooter.iter_checks()) + assert check.source.key_capabilities == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_source_key_name_prefix_match( + api, source_bucket, source_key, troubleshooter +): + key = api.raw_api.key_id_to_key[source_key.id_] + + key.name_prefix_or_none = None + check = one(troubleshooter.iter_checks()) + assert check.source.key_name_prefix_match == CheckState.OK + + key.name_prefix_or_none = 'folder/' + check = one(troubleshooter.iter_checks()) + assert check.source.key_name_prefix_match == CheckState.OK + + key.name_prefix_or_none = 'hoho-trololo/' + check = one(troubleshooter.iter_checks()) + assert check.source.key_name_prefix_match == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_source_sse_c_disabled(api, source_bucket, source_key, troubleshooter): + source_bucket.update( + default_server_side_encryption=EncryptionSetting( + mode=EncryptionMode.SSE_B2, + algorithm=EncryptionAlgorithm.AES256, + ), + replication=source_bucket.replication, + ) + check = one(troubleshooter.iter_checks()) + assert check.source.is_sse_c_disabled == CheckState.OK + + source_bucket.update( + default_server_side_encryption=EncryptionSetting( + mode=EncryptionMode.SSE_C, + algorithm=EncryptionAlgorithm.AES256, + key=EncryptionKey(secret='hoho', key_id='haha'), + ), + replication=source_bucket.replication, + ) + check = one(troubleshooter.iter_checks()) + assert check.source.is_sse_c_disabled == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_source_key_accepted_in_target_bucket( + api, source_bucket, source_key, destination_bucket, troubleshooter +): + destination_replication = destination_bucket.replication + destination_replication.source_to_destination_key_mapping = {} + destination_bucket.update(replication=destination_replication) + + check = one(troubleshooter.iter_checks()) + assert check.source_key_accepted_in_target_bucket == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_file_lock_match( + api, source_bucket, source_key, destination_bucket, troubleshooter +): + source_bucket_obj = source_bucket.api.raw_api.bucket_id_to_bucket[source_bucket.id_] + destination_bucket_obj = destination_bucket.api.raw_api.bucket_id_to_bucket[ + destination_bucket.id_] + + # False, True + source_bucket_obj.is_file_lock_enabled = False + destination_bucket_obj.is_file_lock_enabled = True + + check = one(troubleshooter.iter_checks()) + assert check.file_lock_match == CheckState.OK + + # None, False + source_bucket_obj.is_file_lock_enabled = None + destination_bucket_obj.is_file_lock_enabled = False + + check = one(troubleshooter.iter_checks()) + assert check.file_lock_match == CheckState.UNKNOWN + + # True, None + source_bucket_obj.is_file_lock_enabled = True + destination_bucket_obj.is_file_lock_enabled = None + + check = one(troubleshooter.iter_checks()) + assert check.file_lock_match == CheckState.UNKNOWN + + # True, None + source_bucket_obj.is_file_lock_enabled = True + destination_bucket_obj.is_file_lock_enabled = False + + check = one(troubleshooter.iter_checks()) + assert check.file_lock_match == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_destination_key_exists( + api, destination_bucket, destination_key, troubleshooter +): + api.delete_key(destination_key) + assert not api.get_key(destination_key.id_) + + check = one(troubleshooter.iter_checks()) + assert check.destination.key_exists == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_destination_key_bucket_match( + api, destination_bucket, destination_key, troubleshooter +): + key = api.raw_api.key_id_to_key[destination_key.id_] + + key.bucket_id_or_none = None + check = one(troubleshooter.iter_checks()) + assert check.destination.key_bucket_match == CheckState.OK + + key.bucket_id_or_none = destination_bucket.id_ + check = one(troubleshooter.iter_checks()) + assert check.destination.key_bucket_match == CheckState.OK + + key.bucket_id_or_none = 'hehe-trololo' + check = one(troubleshooter.iter_checks()) + assert check.destination.key_bucket_match == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_destination_key_capabilities( + api, destination_bucket, destination_key, troubleshooter +): + key = api.raw_api.key_id_to_key[destination_key.id_] + + key.capabilities = ['readFilesWithPepper'] + check = one(troubleshooter.iter_checks()) + assert check.destination.key_capabilities == CheckState.NOT_OK + + +@pytest.mark.apiver(from_ver=2) +def test_troubleshooter_destination_key_name_prefix_match( + api, destination_bucket, destination_key, troubleshooter +): + key = api.raw_api.key_id_to_key[destination_key.id_] + + key.name_prefix_or_none = None + check = one(troubleshooter.iter_checks()) + assert check.destination.key_name_prefix_match == CheckState.OK + + key.name_prefix_or_none = '' + check = one(troubleshooter.iter_checks()) + assert check.destination.key_name_prefix_match == CheckState.OK + + key.name_prefix_or_none = 'hoho-trololo/' + check = one(troubleshooter.iter_checks()) + assert check.destination.key_name_prefix_match == CheckState.NOT_OK