From 37480f2dd4e98c4a53d0f67e188d04d5f1c5c2a0 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 9 Dec 2019 16:40:57 +0100 Subject: [PATCH 1/8] WIP --- doc/source/glossary.rst | 3 + doc/source/quick_start.rst | 187 ++++++++++++++++++++++++++++++++++++- 2 files changed, 187 insertions(+), 3 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 63316749c..f2ab8efad 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -4,6 +4,9 @@ Glossary .. glossary:: + absoluteMinimumPartSize + The smallest large file part size, as indicated during authorization process by the server (in 2019 it used to be ``5MB``, but the server can set it dynamincally) + account ID An identifier of the B2 account (not login). Looks like this: ``4ba5845d7aaf``. diff --git a/doc/source/quick_start.rst b/doc/source/quick_start.rst index 7fae4bc31..22316f9fe 100644 --- a/doc/source/quick_start.rst +++ b/doc/source/quick_start.rst @@ -280,8 +280,8 @@ Get file metadata 'uploadTimestamp': 1554361150000} -Copy file -========= +Copy (small) file +================= .. code-block:: python @@ -299,7 +299,7 @@ Copy file 'uploadTimestamp': 1561033728000} -If you want to copy just the part of the file, then you can specify the bytes_range as a tuple. +If you want to copy just the part of the file, then you can specify the bytes_range as a tuple: .. code-block:: python @@ -316,6 +316,11 @@ If you want to copy just the part of the file, then you can specify the bytes_ra 'fileName': 'f2_copy.txt', 'uploadTimestamp': 1561033207000} +Providing the size of the file in `bytes_range` parameter improves efficiency for large files. Please note that `bytes_range` is inclusive, so in order to copy a file if size TODO the parameter is `(0, TODO)` + +.. todo: + fill TODO above + For more information see :meth:`b2sdk.v1.Bucket.copy_file`. @@ -339,3 +344,179 @@ Cancel large file uploads >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> for file_version in bucket.list_unfinished_large_files(): bucket.cancel_large_file(file_version.file_id) + + +************** +Advanced Usage +************** + +Concatenate files +================= + +:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable which *can contain only non-overlapping ranges*. It can be used to glue remote files together, back-to-back, into a new file. + + +Concatenate files (of known size) +--------------------------------- + +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> input_sources = [ + ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, offset_end=200), + ... LocalUploadSource('my_local_path/to_file.txt'), + ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), + ... ] + >>> file_info = {'how': 'good-file'} + >>> bucket.concatenate(input_sources, remote_name, file_info) + >>> bucket.upload_local_file( + local_file=local_file_path, + file_name=b2_file_name, + file_infos=file_info, + ) + + +This method does not allow for checksum verification. If you need that and some ranges overlap, :meth:`b2sdk.v1.Bucket.create_file` may be for you. + +For more information about ``concatenate`` please see :meth:`b2sdk.v1.Bucket.concatenate` and :class:`b2sdk.v1.RemoteUploadSource`. + + +Concatenate files (of unknown size) +----------------------------------- + +Currently it is not supported by **b2sdk**. + + +Update a file efficiently +==================================== + +:meth:`b2sdk.v1.Bucket.create_file` accepts an iterable which *can contain overlapping ranges*. + +Append to the end of a file +--------------------------- + +The assumption here is that the file has been appended to since it was last uploaded to. This assumption is verified by **b2sdk** when possible by recalculating checksums of the overlapping ranges. + +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> input_sources = [ + ... WriteIntent( + ... data=RemoteFileUploadSource( + ... '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', + ... offset=0, + ... length=5000000, + ... ), + ... destination_start=0, + ... destination_end=50000000, + ... ), + ... WriteIntent( + ... data=LocalFileUploadSource('my_local_path/to_file.txt'), + ... destination_start=0, + ... destination_end=60000000, + ... ), + ... ] + >>> file_info = {'how': 'good-file'} + >>> bucket.create_file(input_sources, remote_name, file_info) + + +`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, +as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. + +For more information see :meth:`b2sdk.v1.Bucket.create_file`. + + +Change the middle of the remote file +------------------------------------ + +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> input_sources = [ + ... WriteIntent( + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=500), + ... destination_start=0, + ... destination_end=4000000, + ... ), + ... WriteIntent( + ... LocalFileUploadSource('my_local_path/to_file.txt'), + ... destination_start=4000000, + ... destination_end=4001024, + ... ), + ... WriteIntent( + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=4001024, offset_end=123456789), + ... destination_start=4001024, + ... destination_end=123456789, + ... ), + ... ] + >>> file_info = {'how': 'good-file'} + >>> bucket.create_file(input_sources, remote_name, file_info) + + +`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, +as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. + +For more information see :meth:`b2sdk.v1.Bucket.create_file`. + + +Synthetize a file from local and remote parts +============================================= + +This is useful for advanced usage patterns such as: + - *synthetic backup* + - *reverse synthetic backup* + - mostly-server-side cutting and gluing uncompressed media files such as `wav` and `avi` with rewriting of file headers + - various deduplicated backup scenarios + +Please note that :meth:`b2sdk.v1.Bucket.create_file` accepts **an ordered iterable** which *can contain overlapping ranges*, so the operation does not need to be planned ahead, but can be streamed, which supports very large output objects. + +Scenarios such as below are then possible: + +.. code-block:: + + A C D G + | | | | + | cloud-AC | | cloud-DG | + | | | | + v v v v + ############ ############# + ^ ^ + | | + +---- desired file A-G --------+ + | | + | | + | ######################### | + | ^ ^ | + | | | | + | | local file-BF | | + | | | | + A B C D E F G + +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> def generate_input(): + ... yield WriteIntent( + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, offset_end=offsetC), + ... destination_start=0, + ... destination_end=offsetC, + ... ), + ... yield WriteIntent( + ... LocalFileUploadSource('my_local_path/to_file.txt'), + ... destination_start=offsetB, + ... destination_end=offsetF, + ... ), + ... yield WriteIntent( + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, offset_end=offsetG-offsetD), + ... destination_start=offsetD, + ... destination_end=offsetG, + ... ), + ... + >>> file_info = {'how': 'good-file'} + >>> bucket.create_file(generate_input(), remote_name, file_info) + + + +In such case, if the sizes allow for it (there would be no parts smaller than :term:`absoluteMinimumPartSize`), the only uploaded part will be `C-D`. +Otherwise, more data will be uploaded, but the data transfer will be reduced as much as it can be using a fairly simple algorithm (as cost of finding a perfect solution is NP-hard in some cases). +For more information see :meth:`b2sdk.v1.Bucket.create_file`. From 8a32be4d5b3ffc17f8108856e2b2b7a58b640c2c Mon Sep 17 00:00:00 2001 From: Michal Zukowski Date: Thu, 12 Dec 2019 10:47:47 +0100 Subject: [PATCH 2/8] More advanced usage documentation --- doc/source/quick_start.rst | 224 +++++++++++++++++++++++++++---------- 1 file changed, 162 insertions(+), 62 deletions(-) diff --git a/doc/source/quick_start.rst b/doc/source/quick_start.rst index 22316f9fe..eef0cb8d4 100644 --- a/doc/source/quick_start.rst +++ b/doc/source/quick_start.rst @@ -279,14 +279,15 @@ Get file metadata 'fileName': 'dummy_new.pdf', 'uploadTimestamp': 1554361150000} +Copy file +========= -Copy (small) file -================= +Note that :meth:`b2sdk.v1.Bucket.copy_file` is deprecated. Please switch to :meth:`b2sdk.v2.Bucket.copy`. .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f118df9ba2c5131e8_d20190619_m065809_c002_v0001126_t0040' - >>> bucket.copy_file(file_id, 'f2_copy.txt') + >>> bucket.copy(file_id, 'f2_copy.txt') {'accountId': '451862be08d0', 'action': 'copy', 'bucketId': '5485a1682662eb3e60980d10', @@ -298,28 +299,17 @@ Copy (small) file 'fileName': 'f2_copy.txt', 'uploadTimestamp': 1561033728000} +If the content length is not provided, then if file is larger than server small file limit, copy would not success and error would raise. If length is provided, then file may be splitted on parts and be copied as a large file. Maximum copy part size can be set by ``max_copy_part_size`` - if not set it will default to server maximum part size. If ``max_copy_part_size`` lower than :term:`absoluteMinimumPartSize` file would be copied in single request - this may be used to force copy in single request large file that fits in server small file limit. -If you want to copy just the part of the file, then you can specify the bytes_range as a tuple: +If you want to copy just the part of the file, then you can specify the offset and content length: .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f118df9ba2c5131e8_d20190619_m065809_c002_v0001126_t0040' - >>> bucket.copy_file(file_id, 'f2_copy.txt', bytes_range=(8,15)) - {'accountId': '451862be08d0', - 'action': 'copy', - 'bucketId': '5485a1682662eb3e60980d10', - 'contentLength': 8, - 'contentSha1': '274713be564aecaae8de362acb68658b576d0b40', - 'contentType': 'text/plain', - 'fileId': '4_z5485a1682662eb3e60980d10_f114b0c11b6b6e39e_d20190620_m122007_c002_v0001123_t0004', - 'fileInfo': {'src_last_modified_millis': '1560848707000'}, - 'fileName': 'f2_copy.txt', - 'uploadTimestamp': 1561033207000} + >>> bucket.copy(file_id, 'f2_copy.txt', offset=1024, length=2048) -Providing the size of the file in `bytes_range` parameter improves efficiency for large files. Please note that `bytes_range` is inclusive, so in order to copy a file if size TODO the parameter is `(0, TODO)` +Note that content length is required for offset values other than zero. -.. todo: - fill TODO above For more information see :meth:`b2sdk.v1.Bucket.copy_file`. @@ -353,36 +343,50 @@ Advanced Usage Concatenate files ================= -:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable which *can contain only non-overlapping ranges*. It can be used to glue remote files together, back-to-back, into a new file. - +:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). It can be used to glue remote files together, back-to-back, into a new file. To support automatic continuation, :meth:`b2sdk.v1.Bucket.concatenate` creates plan before it starts to copy/upload anything. :meth:`b2sdk.v1.Bucket.concatenate_stream` can be used to process large input iterator. -Concatenate files (of known size) +Concatenate files of known size --------------------------------- .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ - ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, offset_end=200), + ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), ... LocalUploadSource('my_local_path/to_file.txt'), ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.concatenate(input_sources, remote_name, file_info) - >>> bucket.upload_local_file( - local_file=local_file_path, - file_name=b2_file_name, - file_infos=file_info, - ) -This method does not allow for checksum verification. If you need that and some ranges overlap, :meth:`b2sdk.v1.Bucket.create_file` may be for you. +If one of remote source has length smaller than :term:`absoluteMinimumPartSize` then it cannot be copied into large file part. Such remote source would be downloaded and concatenated locally with local source or with other downloaded remote source. + +Please note that this method only allows checksum verification for local upload sources. Checksum verification for remote sources is available only when local copy is available. In such case :meth:`b2sdk.v1.Bucket.create_file` can be used with overalapping ranges in input. For more information about ``concatenate`` please see :meth:`b2sdk.v1.Bucket.concatenate` and :class:`b2sdk.v1.RemoteUploadSource`. +Concatenate files of known size (streamed version) +-------------------------------------------------- + +:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). The operation would not be planned ahead so it supports very large output objects, but continuation is only possible for local only sources and provided unfinished large file id. See more about continuation in :meth:`b2sdk.v1.Bucket.create_file` paragraph about continuation. -Concatenate files (of unknown size) ------------------------------------ +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> input_sources = [ + ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), + ... LocalUploadSource('my_local_path/to_file.txt'), + ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), + ... ] + >>> file_info = {'how': 'good-file'} + >>> bucket.concatenate_stream(input_sources, remote_name, file_info) + + + + +Concatenate files of unknown size +--------------------------------- Currently it is not supported by **b2sdk**. @@ -390,12 +394,14 @@ Currently it is not supported by **b2sdk**. Update a file efficiently ==================================== -:meth:`b2sdk.v1.Bucket.create_file` accepts an iterable which *can contain overlapping ranges*. +:meth:`b2sdk.v1.Bucket.create_file` accepts an iterable which *can contain overlapping destination ranges*. To support automatic continuation, :meth:`b2sdk.v1.Bucket.create_file` creates plan before it starts to copy/upload anything. :meth:`b2sdk.v1.Bucket.create_file_stream()` can be used to process large input iterator. + +Plase note that all following examples *creates* new file - data in bucket is immutable, but it can create a new file version with the same name as remote source. Old version has to be deleted manually. Append to the end of a file --------------------------- -The assumption here is that the file has been appended to since it was last uploaded to. This assumption is verified by **b2sdk** when possible by recalculating checksums of the overlapping ranges. +The assumption here is that the file has been appended to since it was last uploaded to. This assumption is verified by **b2sdk** when possible by recalculating checksums of the overlapping remote and local ranges. If copied remote part sha does not match with locally available source, file creation process would be interrupted and error raised. .. code-block:: python @@ -407,21 +413,18 @@ The assumption here is that the file has been appended to since it was last uplo ... offset=0, ... length=5000000, ... ), - ... destination_start=0, - ... destination_end=50000000, + ... destination_offset=0, ... ), ... WriteIntent( - ... data=LocalFileUploadSource('my_local_path/to_file.txt'), - ... destination_start=0, - ... destination_end=60000000, + ... data=LocalFileUploadSource('my_local_path/to_file.txt'), # of length 60000000 + ... destination_offset=0, ... ), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.create_file(input_sources, remote_name, file_info) -`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, -as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. +`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range (when concatenating, local source would have destination offset at the end of remote source) For more information see :meth:`b2sdk.v1.Bucket.create_file`. @@ -434,27 +437,23 @@ Change the middle of the remote file >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ ... WriteIntent( - ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=500), - ... destination_start=0, - ... destination_end=4000000, + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=4000000), + ... destination_offset=0, ... ), ... WriteIntent( - ... LocalFileUploadSource('my_local_path/to_file.txt'), - ... destination_start=4000000, - ... destination_end=4001024, + ... LocalFileUploadSource('my_local_path/to_file.txt'), # length 1024 - read from disk + ... destination_offset=4000000, ... ), ... WriteIntent( - ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=4001024, offset_end=123456789), - ... destination_start=4001024, - ... destination_end=123456789, + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=4001024, length=123456789), + ... destination_offset=4001024, ... ), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.create_file(input_sources, remote_name, file_info) -`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, -as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. +`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. For more information see :meth:`b2sdk.v1.Bucket.create_file`. @@ -468,7 +467,7 @@ This is useful for advanced usage patterns such as: - mostly-server-side cutting and gluing uncompressed media files such as `wav` and `avi` with rewriting of file headers - various deduplicated backup scenarios -Please note that :meth:`b2sdk.v1.Bucket.create_file` accepts **an ordered iterable** which *can contain overlapping ranges*, so the operation does not need to be planned ahead, but can be streamed, which supports very large output objects. +Please note that :meth:`b2sdk.v1.Bucket.create_file_stream` accepts **an ordered iterable** which *can contain overlapping ranges*, so the operation does not need to be planned ahead, but can be streamed, which supports very large output objects. Scenarios such as below are then possible: @@ -497,19 +496,16 @@ Scenarios such as below are then possible: >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> def generate_input(): ... yield WriteIntent( - ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, offset_end=offsetC), - ... destination_start=0, - ... destination_end=offsetC, + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=lengthC), + ... destination_offset=0, ... ), ... yield WriteIntent( - ... LocalFileUploadSource('my_local_path/to_file.txt'), - ... destination_start=offsetB, - ... destination_end=offsetF, + ... LocalFileUploadSource('my_local_path/to_file.txt'), # length = offsetF - offsetB + ... destination_offset=offsetB, ... ), ... yield WriteIntent( - ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, offset_end=offsetG-offsetD), - ... destination_start=offsetD, - ... destination_end=offsetG, + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=offsetG-offsetD), + ... destination_offset=offsetD, ... ), ... >>> file_info = {'how': 'good-file'} @@ -517,6 +513,110 @@ Scenarios such as below are then possible: -In such case, if the sizes allow for it (there would be no parts smaller than :term:`absoluteMinimumPartSize`), the only uploaded part will be `C-D`. -Otherwise, more data will be uploaded, but the data transfer will be reduced as much as it can be using a fairly simple algorithm (as cost of finding a perfect solution is NP-hard in some cases). +In such case, if the sizes allow for it (there would be no parts smaller than :term:`absoluteMinimumPartSize`), the only uploaded part will be `C-D`. Otherwise, more data will be uploaded, but the data transfer will be reduced in most cases. :meth:`b2sdk.v1.Bucket.create_file` does not guarantee that outbound transfer usage would be optimal, it would just use greedy algorithm with as small look aheads as possible. For more information see :meth:`b2sdk.v1.Bucket.create_file`. + + +Prioritize remote or local sources +================================== + +:meth:`b2sdk.v1.Bucket.create_file` and streamed version :meth:`b2sdk.v1.Bucket.create_file_stream` supports source origin prioritization so planner would know which sources should be used for overlapping ranges. Supported values are: `local`, `remote`, `local_verification` (default). + +.. code-block:: + + A D G + | | | + | cloud-AD | | + | | | + v v | + ################ | + ^ | + | | + +---- desired file A-G --------+ + | | + | | + | ####### ################# + | ^ ^ ^ | + | | | | | + | | local file BC and DE | + | | | | | + A B C D E + + A=0, B=50M, C=80M, D=100M, E=200 + +.. code-block:: python + + >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local') + # planner parts: cloud[A, B], local[B, C], remote[C, D], local[D, E] + +.. code-block:: python + + >>> planner.create_file(input_sources, remote_name, file_info, prioritize='remote') + # produce parts: cloud[A, D], local[D, E] + +.. code-block:: python + + >>> bucket.create_file(input_sources, remote_name, file_info) + # or + >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local_verification') + # planner parts: cloud[A, B], cloud[B, C], cloud[C, D], local[D, E] + +In `local_verification` prioritization remote range is splitted on three parts to allow local checksum verification. Note that this is only a planner setting - remote part is always verifified if local part exists. + +TODO: prioritization should accept enum + +Continue file update +==================== + +:meth:`b2sdk.v1.Bucket.create_file` supports automatic continuation or manual continuation. :meth:`b2sdk.v1.Bucket.create_file_stream` supports only manual continuation for local source only inputs. The situation looks the same for :meth:`b2sdk.v1.Bucket.concatenate` and :meth:`b2sdk.v1.Bucket.concatenate_stream` (streamed version supports only manual continuation of local sources). Also :meth:`b2sdk.v1.Bucket.upload` and :meth:`b2sdk.v2.Bucket.copy` supports both auto and manual continuation. + +Manual continuation +------------------- + +.. code-block:: python + + >>> def large_file_callback(large_file): + ... # storage is not part of the interface - here only for demonstration purposes + ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) + >>> bucket.create_file(input_sources, remote_name, file_info, large_file_callback=large_file_callback) + # ... + >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] + >>> bucket.create_file(input_sources, remote_name, file_info, large_file_id=large_file_id) + + +Manual continuation (streamed version) +-------------------------------------- + +.. code-block:: python + + >>> def large_file_callback(large_file): + ... # storage is not part of the interface - here only for demonstration purposes + ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) + >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_callback=large_file_callback) + # ... + >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] + >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_id=large_file_id) + +Streams that contains remote sources cannot be continued with :meth:`b2sdk.v1.Bucket.create_file` - internally :meth:`b2sdk.v1.Bucket.create_file` stores plan information in file info for such inputs, and verifies it before any copy/upload and :meth:`b2sdk.v1.Bucket.create_file_stream` cannot store this information. Local source only inputs can be safely continued with :meth:`b2sdk.v1.Bucket.create_file` in auto continue mode or manual continue mode (because plan information is not stored in file info in such case). + +Auto continuation +----------------- + +.. code-block:: python + + >>> bucket.create_file(input_sources, remote_name, file_info) + +For local source only input, :meth:`b2sdk.v1.Bucket.create_file` would try to find matching unfinished large file. It will verify uploaded parts checksums with local sources - the most completed, having all uploaded parts matched candidate would be automatically selected as file to continue. If there is no matching candidate (even if there are unfinished files for the same file name) new large file would be started. + +In other cases plan information would be generated and :meth:`b2sdk.v1.Bucket.create_file` would try to find unfinished large file with matching plan info in its file info. If there is one or more such unfinished large files, :meth:`b2sdk.v1.Bucket.create_file` would verify checksums for all locally available parts and choose any matching candidate. If all candidates fails on uploaded parts checksums verification, process is interrupted and error raises. In such case corrupted unfinished large files should be cancelled manullay and :meth:`b2sdk.v1.Bucket.create_file` should be retried, or auto continuation should be turned off with `auto_continue=False` + + +No continuation +--------------- + +.. code-block:: python + + >>> bucket.create_file(input_sources, remote_name, file_info, auto_continue=False) + + +Note, that this only forces start of a new large file, and it is possible to continue the process with either auto or manual modes. From f55b9429eee13d01636c41e193a31559772939bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Polewicz?= Date: Thu, 12 Dec 2019 14:49:17 +0100 Subject: [PATCH 3/8] Update doc/source/quick_start.rst --- doc/source/quick_start.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/quick_start.rst b/doc/source/quick_start.rst index eef0cb8d4..d9302a77a 100644 --- a/doc/source/quick_start.rst +++ b/doc/source/quick_start.rst @@ -343,7 +343,9 @@ Advanced Usage Concatenate files ================= -:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). It can be used to glue remote files together, back-to-back, into a new file. To support automatic continuation, :meth:`b2sdk.v1.Bucket.concatenate` creates plan before it starts to copy/upload anything. :meth:`b2sdk.v1.Bucket.concatenate_stream` can be used to process large input iterator. +:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). It can be used to glue remote files together, back-to-back, into a new file. To support automatic continuation, :meth:`b2sdk.v1.Bucket.concatenate` creates a plan before it starts to copy/upload anything, saving the hash of that plan in file_info for increased reliability. + +:meth:`b2sdk.v1.Bucket.concatenate_stream` can be used to process a large input iterator with limited continuation reliability - it does not create and validate a plan before starting the transfer. Concatenate files of known size --------------------------------- From a6ccd7e0b22a6133a248320e87b5d571908ceea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Polewicz?= Date: Thu, 12 Dec 2019 14:49:27 +0100 Subject: [PATCH 4/8] Update doc/source/quick_start.rst --- doc/source/quick_start.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/quick_start.rst b/doc/source/quick_start.rst index d9302a77a..89f2a0f80 100644 --- a/doc/source/quick_start.rst +++ b/doc/source/quick_start.rst @@ -299,7 +299,7 @@ Note that :meth:`b2sdk.v1.Bucket.copy_file` is deprecated. Please switch to :me 'fileName': 'f2_copy.txt', 'uploadTimestamp': 1561033728000} -If the content length is not provided, then if file is larger than server small file limit, copy would not success and error would raise. If length is provided, then file may be splitted on parts and be copied as a large file. Maximum copy part size can be set by ``max_copy_part_size`` - if not set it will default to server maximum part size. If ``max_copy_part_size`` lower than :term:`absoluteMinimumPartSize` file would be copied in single request - this may be used to force copy in single request large file that fits in server small file limit. +If the ``content length`` is not provided and the file is larger than 5GB, ``copy`` would not succeed and error would be raised. If length is provided, then the file may be copied as a large file. Maximum copy part size can be set by ``max_copy_part_size`` - if not set, it will default to 5GB. If ``max_copy_part_size`` is lower than :term:`absoluteMinimumPartSize`, file would be copied in single request - this may be used to force copy in single request large file that fits in server small file limit. If you want to copy just the part of the file, then you can specify the offset and content length: From de51b6395c1575af0d9e3790535b4919aa3a75a3 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 18 Dec 2019 17:11:59 +0100 Subject: [PATCH 5/8] Split off the new docs to advanced.rst --- doc/source/advanced.rst | 372 +++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 1 + doc/source/quick_start.rst | 290 +---------------------------- 3 files changed, 375 insertions(+), 288 deletions(-) create mode 100644 doc/source/advanced.rst diff --git a/doc/source/advanced.rst b/doc/source/advanced.rst new file mode 100644 index 000000000..32568894e --- /dev/null +++ b/doc/source/advanced.rst @@ -0,0 +1,372 @@ +######################################### +Advanced usage patterns +######################################### + +B2 server API allows for creation of an object from existing objects. This allows to avoid transferring data from the source machine if the desired outcome can be (at least partially) constructed from what is already on the server. + +The way **b2sdk** exposes this functonality is through a few functions that allow the user to express the desired outcome and then the library takes care of planning and executing the work. Please refer to the table below to compare the support of object creation methods for various usage patterns. + +***************** +Available methods +***************** + +.. |br| raw:: html + +
+ +.. _advanced_methods_support_table: + ++--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ +| Method / supported options | Source | Range |br| overlap | Streaming |br| interface | :ref:`Continuation ` | ++============================================+========+=====================+==========================+====================================+ +| :meth:`b2sdk.v1.Bucket.upload` | local | no | no | automatic | ++--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ +| :meth:`b2sdk.v1.Bucket.copy` | remote | no | no | automatic | ++--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ +| :meth:`b2sdk.v1.Bucket.concatenate` | any | no | no | automatic | ++--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ +| :meth:`b2sdk.v1.Bucket.concatenate_stream` | any | no | yes | manual | ++--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ +| :meth:`b2sdk.v1.Bucket.create_file` | any | yes | no | automatic | ++--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ +| :meth:`b2sdk.v1.Bucket.create_file_stream` | any | yes | yes | manual | ++--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ + +Range overlap +============= + +Some methods support overlapping ranges between local and remote files. **b2sdk** tries to use the remote ranges as much as possible, but due to limitations of ``b2_copy_part`` (specifically the minimum size of a part) that may not be always possible. A possible solutuon for such case is to download a (small) range and then upload it along with another one, to meet the ``b2_copy_part`` requirements. This can be improved if the same data is already available locally - in such case **b2sdk** will use the local range rather than downloading it. + + +Streaming interface +=================== + +Some object creation methods start writing data before reading the whole input (iterator). This can be used to write objects that do not have fully known contents without writing them first locally, so that they could be copied. Such usage pattern can be relevant to small devices which stream data to B2 from an external NAS, where caching large files such as media files or virtual machine images is not an option. + +Please see :ref:`advanced method support table ` to see where streaming interface is supported. + +Continuation +============ + +Please see :ref:`here ` + + +***************** +Concatenate files +***************** + +:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). It can be used to glue remote files together, back-to-back, into a new file. + +:meth:`b2sdk.v1.Bucket.concatenate_stream` does not create and validate a plan before starting the transfer, so it can be used to process a large input iterator, at a cost of limited automated continuation. + + +Concatenate files of known size +================================= + +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> input_sources = [ + ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), + ... LocalUploadSource('my_local_path/to_file.txt'), + ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), + ... ] + >>> file_info = {'how': 'good-file'} + >>> bucket.concatenate(input_sources, remote_name, file_info) + + +If one of remote source has length smaller than :term:`absoluteMinimumPartSize` then it cannot be copied into large file part. Such remote source would be downloaded and concatenated locally with local source or with other downloaded remote source. + +Please note that this method only allows checksum verification for local upload sources. Checksum verification for remote sources is available only when local copy is available. In such case :meth:`b2sdk.v1.Bucket.create_file` can be used with overalapping ranges in input. + +For more information about ``concatenate`` please see :meth:`b2sdk.v1.Bucket.concatenate` and :class:`b2sdk.v1.RemoteUploadSource`. + + +Concatenate files of known size (streamed version) +================================================== + +:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). The operation would not be planned ahead so it supports very large output objects, but continuation is only possible for local only sources and provided unfinished large file id. See more about continuation in :meth:`b2sdk.v1.Bucket.create_file` paragraph about continuation. + +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> input_sources = [ + ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), + ... LocalUploadSource('my_local_path/to_file.txt'), + ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), + ... ] + >>> file_info = {'how': 'good-file'} + >>> bucket.concatenate_stream(input_sources, remote_name, file_info) + + + + +Concatenate files of unknown size +================================= + +While it is supported by B2 server, this pattern is currently not supported by **b2sdk**. + + +********************* +Synthethize an object +********************* + +Using methods described below an object can be created from both local and remote sources while avoiding downloading small ranges when such range is already present on a local drive. + +Update a file efficiently +==================================== + +:meth:`b2sdk.v1.Bucket.create_file` accepts an iterable which *can contain overlapping destination ranges*. + +.. note:: + Following examples *create* new file - data in bucket is immutable, but **b2sdk** can create a new file version with the same name and updated content + + +Append to the end of a file +--------------------------- + +The assumption here is that the file has been appended to since it was last uploaded to. This assumption is verified by **b2sdk** when possible by recalculating checksums of the overlapping remote and local ranges. If copied remote part sha does not match with locally available source, file creation process would be interrupted and an exception would be raised. + +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> input_sources = [ + ... WriteIntent( + ... data=RemoteFileUploadSource( + ... '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', + ... offset=0, + ... length=5000000, + ... ), + ... destination_offset=0, + ... ), + ... WriteIntent( + ... data=LocalFileUploadSource('my_local_path/to_file.txt'), # of length 60000000 + ... destination_offset=0, + ... ), + ... ] + >>> file_info = {'how': 'good-file'} + >>> bucket.create_file(input_sources, remote_name, file_info) + + +`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range (when concatenating, local source would have destination offset at the end of remote source) + +For more information see :meth:`b2sdk.v1.Bucket.create_file`. + + +Change the middle of the remote file +------------------------------------ + +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> input_sources = [ + ... WriteIntent( + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=4000000), + ... destination_offset=0, + ... ), + ... WriteIntent( + ... LocalFileUploadSource('my_local_path/to_file.txt'), # length=1024, here not passed and just checked from local source using seek + ... destination_offset=4000000, + ... ), + ... WriteIntent( + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=4001024, length=123456789), + ... destination_offset=4001024, + ... ), + ... ] + >>> file_info = {'how': 'good-file'} + >>> bucket.create_file(input_sources, remote_name, file_info) + + +`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. + +For more information see :meth:`b2sdk.v1.Bucket.create_file`. + + +Synthetize a file from local and remote parts +============================================= + +This is useful for expert usage patterns such as: + - *synthetic backup* + - *reverse synthetic backup* + - mostly-server-side cutting and gluing uncompressed media files such as `wav` and `avi` with rewriting of file headers + - various deduplicated backup scenarios + +Please note that :meth:`b2sdk.v1.Bucket.create_file_stream` accepts **an ordered iterable** which *can contain overlapping ranges*, so the operation does not need to be planned ahead, but can be streamed, which supports very large output objects. + +Scenarios such as below are then possible: + +.. code-block:: + + A C D G + | | | | + | cloud-AC | | cloud-DG | + | | | | + v v v v + ############ ############# + ^ ^ + | | + +---- desired file A-G --------+ + | | + | | + | ######################### | + | ^ ^ | + | | | | + | | local file-BF | | + | | | | + A B C D E F G + +.. code-block:: python + + >>> bucket = b2_api.get_bucket_by_name(bucket_name) + >>> def generate_input(): + ... yield WriteIntent( + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=lengthC), + ... destination_offset=0, + ... ), + ... yield WriteIntent( + ... LocalFileUploadSource('my_local_path/to_file.txt'), # length = offsetF - offsetB + ... destination_offset=offsetB, + ... ), + ... yield WriteIntent( + ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=offsetG-offsetD), + ... destination_offset=offsetD, + ... ), + ... + >>> file_info = {'how': 'good-file'} + >>> bucket.create_file(generate_input(), remote_name, file_info) + + + +In such case, if the sizes allow for it (there would be no parts smaller than :term:`absoluteMinimumPartSize`), the only uploaded part will be `C-D`. Otherwise, more data will be uploaded, but the data transfer will be reduced in most cases. :meth:`b2sdk.v1.Bucket.create_file` does not guarantee that outbound transfer usage would be optimal, it uses a simple greedy algorithm with as small look-aheads as possible. + +For more information see :meth:`b2sdk.v1.Bucket.create_file`. + + +Prioritize remote or local sources +---------------------------------- + +:meth:`b2sdk.v1.Bucket.create_file` and :meth:`b2sdk.v1.Bucket.create_file_stream` support source/origin prioritization, so that planner would know which sources should be used for overlapping ranges. Supported values are: `local`, `remote` and `local_verification`. + +.. code-block:: + + A D G + | | | + | cloud-AD | | + | | | + v v | + ################ | + ^ | + | | + +---- desired file A-G --------+ + | | + | | + | ####### ################# + | ^ ^ ^ | + | | | | | + | | local file BC and DE | + | | | | | + A B C D E + + A=0, B=50M, C=80M, D=100M, E=200 + +.. code-block:: python + + >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local') + # planner parts: cloud[A, B], local[B, C], remote[C, D], local[D, E] + +Here the planner has only used a remote source where remote range was not available, minimizing downloads. + +.. code-block:: python + + >>> planner.create_file(input_sources, remote_name, file_info, prioritize='remote') + # planner parts: cloud[A, D], local[D, E] + +Here the planner has only used a local source where remote range was not available, minimizing uploads. + +.. code-block:: python + + >>> bucket.create_file(input_sources, remote_name, file_info) + # or + >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local_verification') + # planner parts: cloud[A, B], cloud[B, C], cloud[C, D], local[D, E] + +In `local_verification` mode the remote range was artificially split into three parts to allow for checksum verification against matching local ranges. + +.. note:: + `prioritize` is just a planner setting - remote parts are always verified if matching local parts exists. + +.. TODO:: + prioritization should accept enum, not string + + +.. _continuation: + +************ +Continuation +************ + +Continuation of upload +====================== + +In order to continue a simple upload session, **b2sdk** checks for any available sessions with of the same ``file name``, ``file_info`` and ``media type``, verifying the size of an object as much as possible. + +To support automatic continuation, some advanced methods create a plan before starting copy/upload operations, saving the hash of that plan in ``file_info`` for increased reliability. + +If that is not available, ``large_file_id`` can be extracted via callback during the operation start. It can then be passed into the subsequent call to continue the same task, though the responsibility for passing the exact same input is then on the user of the function. Please see :ref:`advanced method support table ` to see where automatic continuation is supported. ``large_file_id`` can also be passed if automatic continuation is available in order to avoid issues where multiple matchin upload sessions are matching the transfer. + + +Continuation of create/concantenate +=================================== + +:meth:`b2sdk.v1.Bucket.create_file` supports automatic continuation or manual continuation. :meth:`b2sdk.v1.Bucket.create_file_stream` supports only manual continuation for local-only inputs. The situation looks the same for :meth:`b2sdk.v1.Bucket.concatenate` and :meth:`b2sdk.v1.Bucket.concatenate_stream` (streamed version supports only manual continuation of local sources). Also :meth:`b2sdk.v1.Bucket.upload` and :meth:`b2sdk.v2.Bucket.copy` support both automatic and manual continuation. + +Manual continuation +------------------- + +.. code-block:: python + + >>> def large_file_callback(large_file): + ... # storage is not part of the interface - here only for demonstration purposes + ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) + >>> bucket.create_file(input_sources, remote_name, file_info, large_file_callback=large_file_callback) + # ... + >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] + >>> bucket.create_file(input_sources, remote_name, file_info, large_file_id=large_file_id) + + +Manual continuation (streamed version) +-------------------------------------- + +.. code-block:: python + + >>> def large_file_callback(large_file): + ... # storage is not part of the interface - here only for demonstration purposes + ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) + >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_callback=large_file_callback) + # ... + >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] + >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_id=large_file_id) + +Streams that contains remote sources cannot be continued with :meth:`b2sdk.v1.Bucket.create_file` - internally :meth:`b2sdk.v1.Bucket.create_file` stores plan information in file info for such inputs, and verifies it before any copy/upload and :meth:`b2sdk.v1.Bucket.create_file_stream` cannot store this information. Local source only inputs can be safely continued with :meth:`b2sdk.v1.Bucket.create_file` in auto continue mode or manual continue mode (because plan information is not stored in file info in such case). + +Auto continuation +----------------- + +.. code-block:: python + + >>> bucket.create_file(input_sources, remote_name, file_info) + +For local source only input, :meth:`b2sdk.v1.Bucket.create_file` would try to find matching unfinished large file. It will verify uploaded parts checksums with local sources - the most completed, having all uploaded parts matched candidate would be automatically selected as file to continue. If there is no matching candidate (even if there are unfinished files for the same file name) new large file would be started. + +In other cases plan information would be generated and :meth:`b2sdk.v1.Bucket.create_file` would try to find unfinished large file with matching plan info in its file info. If there is one or more such unfinished large files, :meth:`b2sdk.v1.Bucket.create_file` would verify checksums for all locally available parts and choose any matching candidate. If all candidates fails on uploaded parts checksums verification, process is interrupted and error raises. In such case corrupted unfinished large files should be cancelled manullay and :meth:`b2sdk.v1.Bucket.create_file` should be retried, or auto continuation should be turned off with `auto_continue=False` + + +No continuation +--------------- + +.. code-block:: python + + >>> bucket.create_file(input_sources, remote_name, file_info, auto_continue=False) + + +Note, that this only forces start of a new large file - it is still possible to continue the process with either auto or manual modes. diff --git a/doc/source/index.rst b/doc/source/index.rst index 091a1dda4..2f522cab5 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -45,6 +45,7 @@ Documentation index install tutorial quick_start + advanced glossary api_types api_reference diff --git a/doc/source/quick_start.rst b/doc/source/quick_start.rst index 89f2a0f80..5b018c333 100644 --- a/doc/source/quick_start.rst +++ b/doc/source/quick_start.rst @@ -282,7 +282,7 @@ Get file metadata Copy file ========= -Note that :meth:`b2sdk.v1.Bucket.copy_file` is deprecated. Please switch to :meth:`b2sdk.v2.Bucket.copy`. +Please switch to :meth:`b2sdk.v2.Bucket.copy`. .. code-block:: python @@ -311,7 +311,7 @@ If you want to copy just the part of the file, then you can specify the offset a Note that content length is required for offset values other than zero. -For more information see :meth:`b2sdk.v1.Bucket.copy_file`. +For more information see :meth:`b2sdk.v1.Bucket.copy`. Delete file @@ -336,289 +336,3 @@ Cancel large file uploads bucket.cancel_large_file(file_version.file_id) -************** -Advanced Usage -************** - -Concatenate files -================= - -:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). It can be used to glue remote files together, back-to-back, into a new file. To support automatic continuation, :meth:`b2sdk.v1.Bucket.concatenate` creates a plan before it starts to copy/upload anything, saving the hash of that plan in file_info for increased reliability. - -:meth:`b2sdk.v1.Bucket.concatenate_stream` can be used to process a large input iterator with limited continuation reliability - it does not create and validate a plan before starting the transfer. - -Concatenate files of known size ---------------------------------- - -.. code-block:: python - - >>> bucket = b2_api.get_bucket_by_name(bucket_name) - >>> input_sources = [ - ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), - ... LocalUploadSource('my_local_path/to_file.txt'), - ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), - ... ] - >>> file_info = {'how': 'good-file'} - >>> bucket.concatenate(input_sources, remote_name, file_info) - - -If one of remote source has length smaller than :term:`absoluteMinimumPartSize` then it cannot be copied into large file part. Such remote source would be downloaded and concatenated locally with local source or with other downloaded remote source. - -Please note that this method only allows checksum verification for local upload sources. Checksum verification for remote sources is available only when local copy is available. In such case :meth:`b2sdk.v1.Bucket.create_file` can be used with overalapping ranges in input. - -For more information about ``concatenate`` please see :meth:`b2sdk.v1.Bucket.concatenate` and :class:`b2sdk.v1.RemoteUploadSource`. - -Concatenate files of known size (streamed version) --------------------------------------------------- - -:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). The operation would not be planned ahead so it supports very large output objects, but continuation is only possible for local only sources and provided unfinished large file id. See more about continuation in :meth:`b2sdk.v1.Bucket.create_file` paragraph about continuation. - -.. code-block:: python - - >>> bucket = b2_api.get_bucket_by_name(bucket_name) - >>> input_sources = [ - ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), - ... LocalUploadSource('my_local_path/to_file.txt'), - ... RemoteFileUploadSource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), - ... ] - >>> file_info = {'how': 'good-file'} - >>> bucket.concatenate_stream(input_sources, remote_name, file_info) - - - - -Concatenate files of unknown size ---------------------------------- - -Currently it is not supported by **b2sdk**. - - -Update a file efficiently -==================================== - -:meth:`b2sdk.v1.Bucket.create_file` accepts an iterable which *can contain overlapping destination ranges*. To support automatic continuation, :meth:`b2sdk.v1.Bucket.create_file` creates plan before it starts to copy/upload anything. :meth:`b2sdk.v1.Bucket.create_file_stream()` can be used to process large input iterator. - -Plase note that all following examples *creates* new file - data in bucket is immutable, but it can create a new file version with the same name as remote source. Old version has to be deleted manually. - -Append to the end of a file ---------------------------- - -The assumption here is that the file has been appended to since it was last uploaded to. This assumption is verified by **b2sdk** when possible by recalculating checksums of the overlapping remote and local ranges. If copied remote part sha does not match with locally available source, file creation process would be interrupted and error raised. - -.. code-block:: python - - >>> bucket = b2_api.get_bucket_by_name(bucket_name) - >>> input_sources = [ - ... WriteIntent( - ... data=RemoteFileUploadSource( - ... '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', - ... offset=0, - ... length=5000000, - ... ), - ... destination_offset=0, - ... ), - ... WriteIntent( - ... data=LocalFileUploadSource('my_local_path/to_file.txt'), # of length 60000000 - ... destination_offset=0, - ... ), - ... ] - >>> file_info = {'how': 'good-file'} - >>> bucket.create_file(input_sources, remote_name, file_info) - - -`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range (when concatenating, local source would have destination offset at the end of remote source) - -For more information see :meth:`b2sdk.v1.Bucket.create_file`. - - -Change the middle of the remote file ------------------------------------- - -.. code-block:: python - - >>> bucket = b2_api.get_bucket_by_name(bucket_name) - >>> input_sources = [ - ... WriteIntent( - ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=4000000), - ... destination_offset=0, - ... ), - ... WriteIntent( - ... LocalFileUploadSource('my_local_path/to_file.txt'), # length 1024 - read from disk - ... destination_offset=4000000, - ... ), - ... WriteIntent( - ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=4001024, length=123456789), - ... destination_offset=4001024, - ... ), - ... ] - >>> file_info = {'how': 'good-file'} - >>> bucket.create_file(input_sources, remote_name, file_info) - - -`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. - -For more information see :meth:`b2sdk.v1.Bucket.create_file`. - - -Synthetize a file from local and remote parts -============================================= - -This is useful for advanced usage patterns such as: - - *synthetic backup* - - *reverse synthetic backup* - - mostly-server-side cutting and gluing uncompressed media files such as `wav` and `avi` with rewriting of file headers - - various deduplicated backup scenarios - -Please note that :meth:`b2sdk.v1.Bucket.create_file_stream` accepts **an ordered iterable** which *can contain overlapping ranges*, so the operation does not need to be planned ahead, but can be streamed, which supports very large output objects. - -Scenarios such as below are then possible: - -.. code-block:: - - A C D G - | | | | - | cloud-AC | | cloud-DG | - | | | | - v v v v - ############ ############# - ^ ^ - | | - +---- desired file A-G --------+ - | | - | | - | ######################### | - | ^ ^ | - | | | | - | | local file-BF | | - | | | | - A B C D E F G - -.. code-block:: python - - >>> bucket = b2_api.get_bucket_by_name(bucket_name) - >>> def generate_input(): - ... yield WriteIntent( - ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=lengthC), - ... destination_offset=0, - ... ), - ... yield WriteIntent( - ... LocalFileUploadSource('my_local_path/to_file.txt'), # length = offsetF - offsetB - ... destination_offset=offsetB, - ... ), - ... yield WriteIntent( - ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=offsetG-offsetD), - ... destination_offset=offsetD, - ... ), - ... - >>> file_info = {'how': 'good-file'} - >>> bucket.create_file(generate_input(), remote_name, file_info) - - - -In such case, if the sizes allow for it (there would be no parts smaller than :term:`absoluteMinimumPartSize`), the only uploaded part will be `C-D`. Otherwise, more data will be uploaded, but the data transfer will be reduced in most cases. :meth:`b2sdk.v1.Bucket.create_file` does not guarantee that outbound transfer usage would be optimal, it would just use greedy algorithm with as small look aheads as possible. -For more information see :meth:`b2sdk.v1.Bucket.create_file`. - - -Prioritize remote or local sources -================================== - -:meth:`b2sdk.v1.Bucket.create_file` and streamed version :meth:`b2sdk.v1.Bucket.create_file_stream` supports source origin prioritization so planner would know which sources should be used for overlapping ranges. Supported values are: `local`, `remote`, `local_verification` (default). - -.. code-block:: - - A D G - | | | - | cloud-AD | | - | | | - v v | - ################ | - ^ | - | | - +---- desired file A-G --------+ - | | - | | - | ####### ################# - | ^ ^ ^ | - | | | | | - | | local file BC and DE | - | | | | | - A B C D E - - A=0, B=50M, C=80M, D=100M, E=200 - -.. code-block:: python - - >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local') - # planner parts: cloud[A, B], local[B, C], remote[C, D], local[D, E] - -.. code-block:: python - - >>> planner.create_file(input_sources, remote_name, file_info, prioritize='remote') - # produce parts: cloud[A, D], local[D, E] - -.. code-block:: python - - >>> bucket.create_file(input_sources, remote_name, file_info) - # or - >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local_verification') - # planner parts: cloud[A, B], cloud[B, C], cloud[C, D], local[D, E] - -In `local_verification` prioritization remote range is splitted on three parts to allow local checksum verification. Note that this is only a planner setting - remote part is always verifified if local part exists. - -TODO: prioritization should accept enum - -Continue file update -==================== - -:meth:`b2sdk.v1.Bucket.create_file` supports automatic continuation or manual continuation. :meth:`b2sdk.v1.Bucket.create_file_stream` supports only manual continuation for local source only inputs. The situation looks the same for :meth:`b2sdk.v1.Bucket.concatenate` and :meth:`b2sdk.v1.Bucket.concatenate_stream` (streamed version supports only manual continuation of local sources). Also :meth:`b2sdk.v1.Bucket.upload` and :meth:`b2sdk.v2.Bucket.copy` supports both auto and manual continuation. - -Manual continuation -------------------- - -.. code-block:: python - - >>> def large_file_callback(large_file): - ... # storage is not part of the interface - here only for demonstration purposes - ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) - >>> bucket.create_file(input_sources, remote_name, file_info, large_file_callback=large_file_callback) - # ... - >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] - >>> bucket.create_file(input_sources, remote_name, file_info, large_file_id=large_file_id) - - -Manual continuation (streamed version) --------------------------------------- - -.. code-block:: python - - >>> def large_file_callback(large_file): - ... # storage is not part of the interface - here only for demonstration purposes - ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) - >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_callback=large_file_callback) - # ... - >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] - >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_id=large_file_id) - -Streams that contains remote sources cannot be continued with :meth:`b2sdk.v1.Bucket.create_file` - internally :meth:`b2sdk.v1.Bucket.create_file` stores plan information in file info for such inputs, and verifies it before any copy/upload and :meth:`b2sdk.v1.Bucket.create_file_stream` cannot store this information. Local source only inputs can be safely continued with :meth:`b2sdk.v1.Bucket.create_file` in auto continue mode or manual continue mode (because plan information is not stored in file info in such case). - -Auto continuation ------------------ - -.. code-block:: python - - >>> bucket.create_file(input_sources, remote_name, file_info) - -For local source only input, :meth:`b2sdk.v1.Bucket.create_file` would try to find matching unfinished large file. It will verify uploaded parts checksums with local sources - the most completed, having all uploaded parts matched candidate would be automatically selected as file to continue. If there is no matching candidate (even if there are unfinished files for the same file name) new large file would be started. - -In other cases plan information would be generated and :meth:`b2sdk.v1.Bucket.create_file` would try to find unfinished large file with matching plan info in its file info. If there is one or more such unfinished large files, :meth:`b2sdk.v1.Bucket.create_file` would verify checksums for all locally available parts and choose any matching candidate. If all candidates fails on uploaded parts checksums verification, process is interrupted and error raises. In such case corrupted unfinished large files should be cancelled manullay and :meth:`b2sdk.v1.Bucket.create_file` should be retried, or auto continuation should be turned off with `auto_continue=False` - - -No continuation ---------------- - -.. code-block:: python - - >>> bucket.create_file(input_sources, remote_name, file_info, auto_continue=False) - - -Note, that this only forces start of a new large file, and it is possible to continue the process with either auto or manual modes. From d119f4281e36e526142a2f04b1453bdbcc2f543f Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 20 Dec 2019 13:56:48 +0100 Subject: [PATCH 6/8] Remove extra trailing commas from docs --- doc/source/advanced.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/advanced.rst b/doc/source/advanced.rst index 32568894e..cb43c42da 100644 --- a/doc/source/advanced.rst +++ b/doc/source/advanced.rst @@ -222,15 +222,15 @@ Scenarios such as below are then possible: ... yield WriteIntent( ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=lengthC), ... destination_offset=0, - ... ), + ... ) ... yield WriteIntent( ... LocalFileUploadSource('my_local_path/to_file.txt'), # length = offsetF - offsetB ... destination_offset=offsetB, - ... ), + ... ) ... yield WriteIntent( ... RemoteUploadSource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=offsetG-offsetD), ... destination_offset=offsetD, - ... ), + ... ) ... >>> file_info = {'how': 'good-file'} >>> bucket.create_file(generate_input(), remote_name, file_info) From b0e77e0e87e9aa06facd8fa09bbbab7a3acc261d Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 3 Feb 2020 12:24:33 +0100 Subject: [PATCH 7/8] Brush up the quick_start docs --- b2sdk/api.py | 1 + doc/source/quick_start.rst | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index 400f92a68..0f0370fce 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -293,6 +293,7 @@ def delete_bucket(self, bucket): :param b2sdk.v1.Bucket bucket: a :term:`Bucket` to delete :rtype: None + :raises b2sdk.v1.exception.NonExistentBucket: if the bucket does not exist in the account """ account_id = self.account_info.get_account_id() self.session.delete_bucket(account_id, bucket.id_) diff --git a/doc/source/quick_start.rst b/doc/source/quick_start.rst index 5b018c333..738253f04 100644 --- a/doc/source/quick_start.rst +++ b/doc/source/quick_start.rst @@ -18,7 +18,7 @@ Prepare b2sdk >>> b2_api.authorize_account("production", application_key_id, application_key) .. tip:: - Get credentials from B2 website + Get your `keyID` and generate the `application key` itself `here `_ *************** @@ -103,7 +103,7 @@ Delete a bucket >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> b2_api.delete_bucket(bucket) -returns `None` if successful, raises an exception in case of error. +See :meth:`b2sdk.v1.B2Api.delete_bucket` for more information. Update bucket info ================== From 64d8b4c2cbd2d9798260f353c97dd3f670b11c16 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 3 Feb 2020 13:34:48 +0100 Subject: [PATCH 8/8] Update advanced patterns documentation --- doc/source/advanced.rst | 71 +++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/doc/source/advanced.rst b/doc/source/advanced.rst index cb43c42da..e9678d976 100644 --- a/doc/source/advanced.rst +++ b/doc/source/advanced.rst @@ -2,10 +2,14 @@ Advanced usage patterns ######################################### -B2 server API allows for creation of an object from existing objects. This allows to avoid transferring data from the source machine if the desired outcome can be (at least partially) constructed from what is already on the server. +B2 server API allows for creation of an object from existing objects. This allows a client application to avoid transferring data from the source machine if the desired outcome can be (at least partially) constructed from what is already on the server. The way **b2sdk** exposes this functonality is through a few functions that allow the user to express the desired outcome and then the library takes care of planning and executing the work. Please refer to the table below to compare the support of object creation methods for various usage patterns. +.. warning:: + Those functions are not available in the current release of **b2sdk**. The documentation is provided as a preview of what is now being in the last stages of development, so stay tuned and follow progress of `Github issue #77 `_. + + ***************** Available methods ***************** @@ -41,7 +45,7 @@ Some methods support overlapping ranges between local and remote files. **b2sdk* Streaming interface =================== -Some object creation methods start writing data before reading the whole input (iterator). This can be used to write objects that do not have fully known contents without writing them first locally, so that they could be copied. Such usage pattern can be relevant to small devices which stream data to B2 from an external NAS, where caching large files such as media files or virtual machine images is not an option. +Some object creation methods start writing data before reading the whole input (iterator). This can be used to write objects that do not have fully known contents without writing them first locally, so that they could be uploaded. Such usage pattern is common on small devices which stream data to B2 from an external NAS, where temporary storage on the device is very limited. Please see :ref:`advanced method support table ` to see where streaming interface is supported. @@ -75,7 +79,7 @@ Concatenate files of known size >>> bucket.concatenate(input_sources, remote_name, file_info) -If one of remote source has length smaller than :term:`absoluteMinimumPartSize` then it cannot be copied into large file part. Such remote source would be downloaded and concatenated locally with local source or with other downloaded remote source. +If one of remote source has length smaller than :term:`absoluteMinimumPartSize` then it cannot be (server-side) copied into large file part. Such remote source would be downloaded and concatenated with local source or with other downloaded remote source. Please note that this method only allows checksum verification for local upload sources. Checksum verification for remote sources is available only when local copy is available. In such case :meth:`b2sdk.v1.Bucket.create_file` can be used with overalapping ranges in input. @@ -85,7 +89,7 @@ For more information about ``concatenate`` please see :meth:`b2sdk.v1.Bucket.con Concatenate files of known size (streamed version) ================================================== -:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). The operation would not be planned ahead so it supports very large output objects, but continuation is only possible for local only sources and provided unfinished large file id. See more about continuation in :meth:`b2sdk.v1.Bucket.create_file` paragraph about continuation. +:meth:`b2sdk.v1.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). The operation is not be planned ahead, so it supports very large output objects, but continuation is only possible for local only sources or when ``large_file_id`` is provided as an argument. See more notes on continuation :ref:`here ` .. code-block:: python @@ -104,14 +108,15 @@ Concatenate files of known size (streamed version) Concatenate files of unknown size ================================= -While it is supported by B2 server, this pattern is currently not supported by **b2sdk**. +While it is supported by B2 server, this pattern is currently not supported by **b2sdk**. If you think there is a good reason why **b2sdk** should support it, please `open a github issue `_ and describe your use case. ********************* Synthethize an object ********************* -Using methods described below an object can be created from both local and remote sources while avoiding downloading small ranges when such range is already present on a local drive. +Using methods described below an object can be created from both local and remote sources while avoiding downloading small ranges (caused by :term:`absoluteMinimumPartSize` limitation) when such range is already present on a local drive. + Update a file efficiently ==================================== @@ -119,13 +124,13 @@ Update a file efficiently :meth:`b2sdk.v1.Bucket.create_file` accepts an iterable which *can contain overlapping destination ranges*. .. note:: - Following examples *create* new file - data in bucket is immutable, but **b2sdk** can create a new file version with the same name and updated content + Following examples *create* new files - data in bucket is immutable, but **b2sdk** can create a new file version with the same name and updated content Append to the end of a file --------------------------- -The assumption here is that the file has been appended to since it was last uploaded to. This assumption is verified by **b2sdk** when possible by recalculating checksums of the overlapping remote and local ranges. If copied remote part sha does not match with locally available source, file creation process would be interrupted and an exception would be raised. +The assumption here is that the local file has been appended to since it was last uploaded. This assumption is verified by **b2sdk** when possible by recalculating checksums of the overlapping remote and local ranges. If copied remote part checksum does not match the locally available source, file creation process is interrupted and an exception is raised. .. code-block:: python @@ -148,7 +153,9 @@ The assumption here is that the file has been appended to since it was last uplo >>> bucket.create_file(input_sources, remote_name, file_info) -`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range (when concatenating, local source would have destination offset at the end of remote source) +`LocalFileUploadSource` has the size determined automatically in this case. + +This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`. Here **b2sdk** can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to avoid downloading a small range. For more information see :meth:`b2sdk.v1.Bucket.create_file`. @@ -177,7 +184,9 @@ Change the middle of the remote file >>> bucket.create_file(input_sources, remote_name, file_info) -`LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. +`LocalFileUploadSource` has the size determined automatically in this case. + +This is more efficient than :meth:`b2sdk.v1.Bucket.concatenate`. Here **b2sdk** can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to avoid downloading a small range. For more information see :meth:`b2sdk.v1.Bucket.create_file`. @@ -245,7 +254,7 @@ For more information see :meth:`b2sdk.v1.Bucket.create_file`. Prioritize remote or local sources ---------------------------------- -:meth:`b2sdk.v1.Bucket.create_file` and :meth:`b2sdk.v1.Bucket.create_file_stream` support source/origin prioritization, so that planner would know which sources should be used for overlapping ranges. Supported values are: `local`, `remote` and `local_verification`. +:meth:`b2sdk.v1.Bucket.create_file` and :meth:`b2sdk.v1.Bucket.create_file_stream` support source prioritization, so that planner would know which sources should be used for overlapping ranges. Supported values are: `local`, `remote` and `local_verification`. .. code-block:: @@ -267,33 +276,36 @@ Prioritize remote or local sources | | | | | A B C D E - A=0, B=50M, C=80M, D=100M, E=200 + A=0, B=50M, C=80M, D=100M, E=200MB .. code-block:: python - >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local') - # planner parts: cloud[A, B], local[B, C], remote[C, D], local[D, E] + >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='remote') + # planner parts: remote[A, D], local[D, E] -Here the planner has only used a remote source where remote range was not available, minimizing downloads. +Here the planner has only used a local source where remote range was not available, minimizing uploads. .. code-block:: python - >>> planner.create_file(input_sources, remote_name, file_info, prioritize='remote') - # planner parts: cloud[A, D], local[D, E] + >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local') + # planner parts: remote[A, B], local[B, C], remote[C, D], local[D, E] -Here the planner has only used a local source where remote range was not available, minimizing uploads. +Here the planner has only used a remote source where local range was not available. + +.. TODO:: + why do we even support this? Is there any legitimate reason to ever use it? .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info) # or >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local_verification') - # planner parts: cloud[A, B], cloud[B, C], cloud[C, D], local[D, E] + # planner parts: remote[A, B], remote[B, C], remote[C, D], local[D, E] In `local_verification` mode the remote range was artificially split into three parts to allow for checksum verification against matching local ranges. .. note:: - `prioritize` is just a planner setting - remote parts are always verified if matching local parts exists. + `prioritize` is just a planner setting - remote parts are always verified if matching local parts exist .. TODO:: prioritization should accept enum, not string @@ -308,18 +320,13 @@ Continuation Continuation of upload ====================== -In order to continue a simple upload session, **b2sdk** checks for any available sessions with of the same ``file name``, ``file_info`` and ``media type``, verifying the size of an object as much as possible. +In order to continue a simple upload session, **b2sdk** checks for any available sessions with of the same ``file name``, ``file_info`` and ``media_type``, verifying the size of an object as much as possible. To support automatic continuation, some advanced methods create a plan before starting copy/upload operations, saving the hash of that plan in ``file_info`` for increased reliability. -If that is not available, ``large_file_id`` can be extracted via callback during the operation start. It can then be passed into the subsequent call to continue the same task, though the responsibility for passing the exact same input is then on the user of the function. Please see :ref:`advanced method support table ` to see where automatic continuation is supported. ``large_file_id`` can also be passed if automatic continuation is available in order to avoid issues where multiple matchin upload sessions are matching the transfer. +If that is not available, ``large_file_id`` can be extracted via callback during the operation start. It can then be passed into the subsequent call to continue the same task, though the responsibility for passing the exact same input is then on the user of the function. Please see :ref:`advanced method support table ` to see where automatic continuation is supported. ``large_file_id`` can also be passed if automatic continuation is available in order to avoid issues where multiple upload sessions are matching the transfer. -Continuation of create/concantenate -=================================== - -:meth:`b2sdk.v1.Bucket.create_file` supports automatic continuation or manual continuation. :meth:`b2sdk.v1.Bucket.create_file_stream` supports only manual continuation for local-only inputs. The situation looks the same for :meth:`b2sdk.v1.Bucket.concatenate` and :meth:`b2sdk.v1.Bucket.concatenate_stream` (streamed version supports only manual continuation of local sources). Also :meth:`b2sdk.v1.Bucket.upload` and :meth:`b2sdk.v2.Bucket.copy` support both automatic and manual continuation. - Manual continuation ------------------- @@ -347,7 +354,7 @@ Manual continuation (streamed version) >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_id=large_file_id) -Streams that contains remote sources cannot be continued with :meth:`b2sdk.v1.Bucket.create_file` - internally :meth:`b2sdk.v1.Bucket.create_file` stores plan information in file info for such inputs, and verifies it before any copy/upload and :meth:`b2sdk.v1.Bucket.create_file_stream` cannot store this information. Local source only inputs can be safely continued with :meth:`b2sdk.v1.Bucket.create_file` in auto continue mode or manual continue mode (because plan information is not stored in file info in such case). +Streams that contain remote sources cannot be continued with :meth:`b2sdk.v1.Bucket.create_file` - internally :meth:`b2sdk.v1.Bucket.create_file` stores plan information in ``file_info`` for such inputs and verifies it before any copy/upload and :meth:`b2sdk.v1.Bucket.create_file_stream` cannot store this information. Local source only inputs can be safely continued with :meth:`b2sdk.v1.Bucket.create_file` in auto continue mode or manual continue mode (because plan information is not stored in ``file_info`` in such case). Auto continuation ----------------- @@ -356,17 +363,19 @@ Auto continuation >>> bucket.create_file(input_sources, remote_name, file_info) -For local source only input, :meth:`b2sdk.v1.Bucket.create_file` would try to find matching unfinished large file. It will verify uploaded parts checksums with local sources - the most completed, having all uploaded parts matched candidate would be automatically selected as file to continue. If there is no matching candidate (even if there are unfinished files for the same file name) new large file would be started. +For local source only input, :meth:`b2sdk.v1.Bucket.create_file` would try to find matching unfinished large file. It will verify uploaded parts checksums with local sources - the most completed, having all uploaded parts matched candidate would be automatically selected as a session to continue. If there is no matching candidate (even if there are unfinished files for the same file name) new large file would be started. -In other cases plan information would be generated and :meth:`b2sdk.v1.Bucket.create_file` would try to find unfinished large file with matching plan info in its file info. If there is one or more such unfinished large files, :meth:`b2sdk.v1.Bucket.create_file` would verify checksums for all locally available parts and choose any matching candidate. If all candidates fails on uploaded parts checksums verification, process is interrupted and error raises. In such case corrupted unfinished large files should be cancelled manullay and :meth:`b2sdk.v1.Bucket.create_file` should be retried, or auto continuation should be turned off with `auto_continue=False` +In other cases, plan information would be generated and :meth:`b2sdk.v1.Bucket.create_file` would try to find unfinished large file with matching plan info in its ``file_info``. If there is one or more such unfinished large files, :meth:`b2sdk.v1.Bucket.create_file` would verify checksums for all locally available parts and choose a matching candidate. If all candidates fail uploaded parts checksums verification, a new session is created. In such case corrupted unfinished large files could be cancelled manually to prevent the verification taking time until the corrupted session expire. No continuation --------------- +Auto continuation can be turned off with ``auto_continue=False``, for example: + .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info, auto_continue=False) +Please note that this only forces a start of a new large file session and does not delete old sessions, so it is still possible to continue the existing sessions with either auto or manual modes. See :meth:`b2sdk.v1.Bucket.list_unfinished_large_files` and :meth:`b2sdk.v1.Bucket.cancel_large_file` to learn how to find and remove unwanted sessions. -Note, that this only forces start of a new large file - it is still possible to continue the process with either auto or manual modes.